feat(stats): adapt points limit to resolutions (#1050)

stats/README.md

@@ -82,7 +82,7 @@ by enabling word wrapping
 | `STATS__FORCE_​UPDATE_ON_START` | | Fully recalculate all charts on start | `false` |
 | `STATS__CONCURRENT_​START_UPDATES` | | Amount of concurrent charts update on start | `3` |
 | `STATS__​DEFAULT_​SCHEDULE` | | Schedule used for update groups with no config | `"0 0 1 * * * *"` |
-| `STATS__LIMITS__REQUEST_​INTERVAL_LIMIT_DAYS` | | Maximum allowed number of requested points  | `182500` | <-- TODO: change
+| `STATS__LIMITS__REQUESTED_​POINTS_LIMIT` | | Maximum allowed number of requested points  | `182500` |
 
 [anchor]: <> (anchors.envs.end.service)
 

stats/stats-server/src/read_service.rs

@@ -8,7 +8,7 @@ use crate::{
 };
 
 use async_trait::async_trait;
-use chrono::{Duration, NaiveDate, Utc};
+use chrono::{NaiveDate, Utc};
 use proto_v1::stats_service_server::StatsService;
 use sea_orm::{DatabaseConnection, DbErr};
 use stats::{
@@ -17,7 +17,7 @@ use stats::{
         timespans::{Month, Week, Year},
         Timespan,
     },
-    MissingDatePolicy, ReadError, ResolutionKind,
+    ApproxUnsignedDiff, MissingDatePolicy, ReadError, RequestedPointsLimit, ResolutionKind,
 };
 use stats_proto::blockscout::stats::v1::{self as proto_v1, Point};
 use tonic::{Request, Response, Status};
@@ -41,22 +41,22 @@ impl ReadService {
 
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct ReadLimits {
-    /// See [`LimitsSettings::request_interval_limit_days`]
-    pub request_interval_limit: Duration,
+    /// See [`LimitsSettings::requested_points_limit`]
+    pub requested_points_limit: RequestedPointsLimit,
 }
 
 impl From<LimitsSettings> for ReadLimits {
     fn from(value: LimitsSettings) -> Self {
         Self {
-            request_interval_limit: Duration::days(value.request_interval_limit_days.into()),
+            requested_points_limit: RequestedPointsLimit::from_points(value.requested_points_limit),
         }
     }
 }
 
 fn map_read_error(err: ReadError) -> Status {
     match &err {
         ReadError::ChartNotFound(_) => Status::not_found(err.to_string()),
-        ReadError::IntervalLimitExceeded(_) => Status::invalid_argument(err.to_string()),
+        ReadError::IntervalTooLarge(_) => Status::invalid_argument(err.to_string()),
         _ => {
             tracing::error!(err = ?err, "internal read error");
             Status::internal(err.to_string())
@@ -91,12 +91,12 @@ async fn get_serialized_line_chart_data<Resolution>(
     chart_name: String,
     from: Option<NaiveDate>,
     to: Option<NaiveDate>,
-    interval_limit: Option<Duration>,
+    points_limit: Option<RequestedPointsLimit>,
     policy: MissingDatePolicy,
     mark_approx: u64,
 ) -> Result<Vec<Point>, ReadError>
 where
-    Resolution: Timespan + Clone + Ord + Debug,
+    Resolution: Timespan + ApproxUnsignedDiff + Clone + Ord + Debug,
 {
     let from = from.map(|f| Resolution::from_date(f));
     let to = to.map(|t| Resolution::from_date(t));
@@ -105,7 +105,7 @@ where
         &chart_name,
         from,
         to,
-        interval_limit,
+        points_limit,
         policy,
         true,
         mark_approx,
@@ -122,7 +122,7 @@ async fn get_serialized_line_chart_data_resolution_dispatch(
     resolution: ResolutionKind,
     from: Option<NaiveDate>,
     to: Option<NaiveDate>,
-    interval_limit: Option<Duration>,
+    points_limit: Option<RequestedPointsLimit>,
     policy: MissingDatePolicy,
     mark_approx: u64,
 ) -> Result<Vec<Point>, ReadError> {
@@ -133,7 +133,7 @@ async fn get_serialized_line_chart_data_resolution_dispatch(
                 chart_name,
                 from,
                 to,
-                interval_limit,
+                points_limit,
                 policy,
                 mark_approx,
             )
@@ -145,7 +145,7 @@ async fn get_serialized_line_chart_data_resolution_dispatch(
                 chart_name,
                 from,
                 to,
-                interval_limit,
+                points_limit,
                 policy,
                 mark_approx,
             )
@@ -157,7 +157,7 @@ async fn get_serialized_line_chart_data_resolution_dispatch(
                 chart_name,
                 from,
                 to,
-                interval_limit,
+                points_limit,
                 policy,
                 mark_approx,
             )
@@ -169,7 +169,7 @@ async fn get_serialized_line_chart_data_resolution_dispatch(
                 chart_name,
                 from,
                 to,
-                interval_limit,
+                points_limit,
                 policy,
                 mark_approx,
             )
@@ -257,14 +257,14 @@ impl StatsService for ReadService {
         let to = request.to.and_then(|date| NaiveDate::from_str(&date).ok());
         let policy = resolution_info.missing_date_policy;
         let mark_approx = resolution_info.approximate_trailing_points;
-        let interval_limit = Some(self.limits.request_interval_limit);
+        let points_limit = Some(self.limits.requested_points_limit);
         let serialized_chart = get_serialized_line_chart_data_resolution_dispatch(
             &self.db,
             chart_name.clone(),
             resolution,
             from,
             to,
-            interval_limit,
+            points_limit,
             policy,
             mark_approx,
         )

stats/stats-server/src/settings.rs

@@ -77,14 +77,14 @@ pub struct LimitsSettings {
     ///
     /// If start or end of the range is left empty, min/max values
     /// from DB are considered.
-    pub request_interval_limit_days: u32,
+    pub requested_points_limit: u32,
 }
 
 impl Default for LimitsSettings {
     fn default() -> Self {
         Self {
-            // ~500 years seems reasonable
-            request_interval_limit_days: 182500,
+            // ~500 years for days seems reasonable
+            requested_points_limit: 182500,
         }
     }
 }

stats/stats/src/charts/chart.rs

@@ -7,11 +7,13 @@
 use std::fmt::Display;
 
 use crate::{types::Timespan, ReadError};
-use chrono::{DateTime, Duration, Utc};
+use chrono::{DateTime, Utc};
 use entity::sea_orm_active_enums::{ChartResolution, ChartType};
 use sea_orm::prelude::*;
 use thiserror::Error;
 
+use super::db_interaction::read::ApproxUnsignedDiff;
+
 #[derive(Error, Debug)]
 pub enum UpdateError {
     #[error("blockscout database error: {0}")]
@@ -20,8 +22,8 @@ pub enum UpdateError {
     StatsDB(DbErr),
     #[error("chart {0} not found")]
     ChartNotFound(ChartKey),
-    #[error("date interval limit ({limit}) is exceeded; choose smaller time interval.")]
-    IntervalLimitExceeded { limit: Duration },
+    #[error("exceeded limit on requested data points (~{limit}); choose smaller time interval.")]
+    IntervalTooLarge { limit: u32 },
     #[error("internal error: {0}")]
     Internal(String),
 }
@@ -31,7 +33,7 @@ impl From<ReadError> for UpdateError {
         match read {
             ReadError::DB(db) => UpdateError::StatsDB(db),
             ReadError::ChartNotFound(err) => UpdateError::ChartNotFound(err),
-            ReadError::IntervalLimitExceeded(limit) => UpdateError::IntervalLimitExceeded { limit },
+            ReadError::IntervalTooLarge(limit) => UpdateError::IntervalTooLarge { limit },
         }
     }
 }
@@ -142,7 +144,7 @@ impl Display for ChartKey {
 ))]
 pub trait ChartProperties: Sync + Named {
     /// Combination name + resolution must be unique for each chart
-    type Resolution: Timespan;
+    type Resolution: Timespan + ApproxUnsignedDiff;
 
     fn chart_type() -> ChartType;
     fn resolution() -> ResolutionKind {

stats/stats/src/charts/db_interaction/read.rs

@@ -3,14 +3,15 @@ use crate::{
     data_source::kinds::local_db::parameter_traits::QueryBehaviour,
     missing_date::{fill_and_filter_chart, fit_into_range},
     types::{
-        timespans::DateValue, ExtendedTimespanValue, Timespan, TimespanDuration, TimespanValue,
+        timespans::{DateValue, Month, Week, Year},
+        ExtendedTimespanValue, Timespan, TimespanDuration, TimespanValue,
     },
     utils::exclusive_datetime_range_to_inclusive,
     ChartProperties, MissingDatePolicy, UpdateError,
 };
 
 use blockscout_db::entity::blocks;
-use chrono::{DateTime, Duration, NaiveDate, NaiveDateTime, Utc};
+use chrono::{DateTime, NaiveDate, NaiveDateTime, Utc};
 use entity::{chart_data, charts, sea_orm_active_enums::ChartResolution};
 use itertools::Itertools;
 use sea_orm::{
@@ -28,8 +29,8 @@ pub enum ReadError {
     DB(#[from] DbErr),
     #[error("chart {0} not found")]
     ChartNotFound(ChartKey),
-    #[error("date interval limit ({0}) is exceeded; choose smaller time interval.")]
-    IntervalLimitExceeded(Duration),
+    #[error("exceeded limit on requested data points (~{0}); choose smaller time interval.")]
+    IntervalTooLarge(u32),
 }
 
 #[derive(Debug, FromQueryResult)]
@@ -208,7 +209,7 @@ fn relevant_data_until<R: Timespan>(
 ///
 /// `approximate_trailing_points` - number of trailing points to mark as approximate.
 ///
-/// `interval_limit` - max interval [from, to]. If `from` or `to` are none,
+/// `point_limit` - max interval [from, to]. If `from` or `to` are none,
 /// min or max date in DB are taken.
 ///
 /// Note: if some dates within interval `[from, to]` fall on the future, data points
@@ -296,13 +297,13 @@ pub async fn get_line_chart_data<Resolution>(
     chart_name: &String,
     from: Option<Resolution>,
     to: Option<Resolution>,
-    interval_limit: Option<Duration>,
+    point_limit: Option<RequestedPointsLimit>,
     policy: MissingDatePolicy,
     fill_missing_dates: bool,
     approximate_trailing_points: u64,
 ) -> Result<Vec<ExtendedTimespanValue<Resolution, String>>, ReadError>
 where
-    Resolution: Timespan + Debug + Ord + Clone,
+    Resolution: Timespan + ApproxUnsignedDiff + Debug + Ord + Clone,
 {
     let chart = charts::Entity::find()
         .column(charts::Column::Id)
@@ -343,7 +344,7 @@ where
     let data_in_range = fit_into_range(db_data, from.clone(), to.clone(), policy);
 
     let data_unmarked = if fill_missing_dates {
-        fill_and_filter_chart(data_in_range, from, to, policy, interval_limit)?
+        fill_and_filter_chart(data_in_range, from, to, policy, point_limit)?
     } else {
         data_in_range
     };
@@ -563,6 +564,85 @@ where
     Ok(row)
 }
 
+/// May not be exact, but the limit is close to
+/// this number
+#[derive(Clone, Copy, PartialEq, Eq, Debug)]
+pub enum RequestedPointsLimit {
+    Points(u32),
+    NoLimit,
+}
+
+impl RequestedPointsLimit {
+    pub fn from_points(approx_limit: u32) -> Self {
+        Self::Points(approx_limit)
+    }
+
+    pub fn approx_limit(&self) -> Option<u32> {
+        match self {
+            RequestedPointsLimit::Points(p) => Some(*p),
+            RequestedPointsLimit::NoLimit => None,
+        }
+    }
+
+    pub fn fits_in_limit<T: ApproxUnsignedDiff>(&self, from: &T, to: &T) -> bool {
+        let limit = match self {
+            RequestedPointsLimit::Points(p) => *p,
+            RequestedPointsLimit::NoLimit => return true,
+        };
+        to.approx_unsigned_difference(from)
+            .map(|diff| diff <= limit.into())
+            .unwrap_or(true)
+    }
+}
+
+pub trait ApproxUnsignedDiff {
+    /// Approx number of repeats of this timespan to get from `other` to `self`.
+    ///
+    /// `None` if < 0.
+    fn approx_unsigned_difference(&self, other: &Self) -> Option<u64>;
+}
+
+impl ApproxUnsignedDiff for NaiveDate {
+    fn approx_unsigned_difference(&self, other: &Self) -> Option<u64> {
+        self.signed_duration_since(*other)
+            .num_days()
+            .try_into()
+            .ok()
+    }
+}
+
+impl ApproxUnsignedDiff for Week {
+    fn approx_unsigned_difference(&self, other: &Self) -> Option<u64> {
+        self.saturating_first_day()
+            .signed_duration_since(other.saturating_first_day())
+            .num_days()
+            .try_into()
+            .ok()
+            .map(|n: u64| n / 7)
+    }
+}
+
+impl ApproxUnsignedDiff for Month {
+    fn approx_unsigned_difference(&self, other: &Self) -> Option<u64> {
+        self.saturating_first_day()
+            .signed_duration_since(other.saturating_first_day())
+            .num_days()
+            .try_into()
+            .ok()
+            // 30.436875 = mean # of days in month (according to wiki)
+            .map(|n: u64| (n as f64 / 30.436875) as u64)
+    }
+}
+
+impl ApproxUnsignedDiff for Year {
+    fn approx_unsigned_difference(&self, other: &Self) -> Option<u64> {
+        self.number_within_naive_date()
+            .saturating_sub(other.number_within_naive_date())
+            .try_into()
+            .ok()
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

stats/stats/src/lib.rs

@@ -15,7 +15,9 @@ pub use migration;
 
 pub use charts::{
     counters,
-    db_interaction::read::{get_line_chart_data, get_raw_counters, ReadError},
+    db_interaction::read::{
+        get_line_chart_data, get_raw_counters, ApproxUnsignedDiff, ReadError, RequestedPointsLimit,
+    },
     lines, types, ChartKey, ChartProperties, ChartPropertiesObject, MissingDatePolicy, Named,
     ResolutionKind, UpdateError,
 };