feat(stats): delay update instead of startup (#1107)

#1106

* feat(stats): move waiting for indexing to update service

* test for healthcheck when blockscout is not initialized

* test that chart values are empty

stats/stats-server/src/server.rs

@@ -107,15 +107,18 @@ pub async fn stats(mut settings: Settings) -> Result<(), anyhow::Error> {
 
     let blockscout_api_config = init_blockscout_api_client(&settings).await?;
 
-    // Wait for blockscout to index, if necessary.
-    if let Some(config) = blockscout_api_config {
-        wait_for_blockscout_indexing(config, settings.conditional_start).await?;
-    }
-
     let update_service =
         Arc::new(UpdateService::new(db.clone(), blockscout, charts.clone()).await?);
 
     tokio::spawn(async move {
+        // Wait for blockscout to index, if necessary.
+        if let Some(config) = blockscout_api_config {
+            if let Err(e) = wait_for_blockscout_indexing(config, settings.conditional_start).await {
+                tracing::error!(error =? e, "Error starting update service. Failed while waiting for blockscout indexing");
+                return;
+            }
+        }
+
         update_service
             .force_async_update_and_run(
                 settings.concurrent_start_updates,

stats/stats-server/tests/it/counters.rs

@@ -6,7 +6,7 @@ use chrono::NaiveDate;
 
 use stats::tests::{
     init_db::init_db_all,
-    mock_blockscout::{fill_mock_blockscout_data, mock_blockscout_api},
+    mock_blockscout::{default_mock_blockscout_api, fill_mock_blockscout_data},
 };
 use stats_proto::blockscout::stats::v1::Counters;
 use stats_server::{stats, Settings};
@@ -17,7 +17,7 @@ use std::{collections::HashSet, path::PathBuf, str::FromStr};
 #[ignore = "needs database"]
 async fn test_counters_ok() {
     let (stats_db, blockscout_db) = init_db_all("test_counters_ok").await;
-    let blockscout_api = mock_blockscout_api().await;
+    let blockscout_api = default_mock_blockscout_api().await;
     fill_mock_blockscout_data(&blockscout_db, NaiveDate::from_str("2023-03-01").unwrap()).await;
 
     std::env::set_var("STATS__CONFIG", "./tests/config/test.toml");

stats/stats-server/tests/it/indexing_status.rs

@@ -0,0 +1,90 @@
+use blockscout_service_launcher::{
+    launcher::ConfigSettings,
+    test_server::{get_test_server_settings, init_server, send_get_request},
+};
+use pretty_assertions::assert_eq;
+
+use stats::tests::{init_db::init_db_all, mock_blockscout::mock_blockscout_api};
+use stats_proto::blockscout::stats::v1::{
+    health_check_response::ServingStatus, Counters, HealthCheckResponse,
+};
+use stats_server::{stats, Settings};
+use wiremock::ResponseTemplate;
+
+use std::{path::PathBuf, str::FromStr};
+
+use crate::{common::send_arbitrary_request, lines::enabled_resolutions};
+
+#[tokio::test]
+#[ignore = "needs database"]
+async fn test_not_indexed_ok() {
+    // check that when the blockscout is not indexed, the healthcheck still succeeds and
+    // charts don't have any points (i.e. they are not updated)
+    let (stats_db, blockscout_db) = init_db_all("test_healthcheck_ok").await;
+    let blockscout_api = mock_blockscout_api(ResponseTemplate::new(200).set_body_string(
+        r#"{
+            "finished_indexing": false,
+            "finished_indexing_blocks": false,
+            "indexed_blocks_ratio": "0.00",
+            "indexed_internal_transactions_ratio": "0.00"
+        }"#,
+    ))
+    .await;
+
+    std::env::set_var("STATS__CONFIG", "./tests/config/test.toml");
+    let mut settings = Settings::build().expect("Failed to build settings");
+    let (server_settings, base) = get_test_server_settings();
+    settings.server = server_settings;
+    settings.charts_config = PathBuf::from_str("../config/charts.json").unwrap();
+    settings.layout_config = PathBuf::from_str("../config/layout.json").unwrap();
+    settings.update_groups_config = PathBuf::from_str("../config/update_groups.json").unwrap();
+    settings.db_url = stats_db.db_url();
+    settings.blockscout_db_url = blockscout_db.db_url();
+    settings.blockscout_api_url = Some(url::Url::from_str(&blockscout_api.uri()).unwrap());
+
+    init_server(|| stats(settings), &base).await;
+
+    // Sleep until server will start and calculate all values
+    tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+
+    // healthcheck is verified in `init_server`, but we double-check it just in case
+    let request =
+        reqwest::Client::new().request(reqwest::Method::GET, base.join("/health").unwrap());
+    let response = send_arbitrary_request(request).await;
+
+    assert_eq!(
+        response.json::<HealthCheckResponse>().await.unwrap(),
+        HealthCheckResponse {
+            status: ServingStatus::Serving as i32
+        }
+    );
+
+    // check that charts return empty data
+
+    let enabled_resolutions =
+        enabled_resolutions(send_get_request(&base, "/api/v1/lines").await).await;
+    for (line_chart_id, resolutions) in enabled_resolutions {
+        for resolution in resolutions {
+            let chart: serde_json::Value = send_get_request(
+                &base,
+                &format!("/api/v1/lines/{line_chart_id}?resolution={resolution}"),
+            )
+            .await;
+            let chart_data = chart
+                .as_object()
+                .expect("response has to be json object")
+                .get("chart")
+                .expect("response doesn't have 'chart' field")
+                .as_array()
+                .expect("'chart' field has to be json array");
+
+            assert!(
+                chart_data.is_empty(),
+                "chart '{line_chart_id}' '{resolution}' is not empty"
+            );
+        }
+    }
+
+    let counters: Counters = send_get_request(&base, "/api/v1/counters").await;
+    assert!(counters.counters.is_empty())
+}

stats/stats-server/tests/it/lines.rs

@@ -7,19 +7,30 @@ use chrono::NaiveDate;
 use stats::{
     tests::{
         init_db::init_db_all,
-        mock_blockscout::{fill_mock_blockscout_data, mock_blockscout_api},
+        mock_blockscout::{default_mock_blockscout_api, fill_mock_blockscout_data},
     },
     ResolutionKind,
 };
 use stats_server::{stats, Settings};
 
 use std::{collections::HashMap, path::PathBuf, str::FromStr};
 
+pub(crate) async fn enabled_resolutions(
+    line_charts: stats_proto::blockscout::stats::v1::LineCharts,
+) -> HashMap<String, Vec<String>> {
+    line_charts
+        .sections
+        .iter()
+        .flat_map(|sec| sec.charts.clone())
+        .map(|l| (l.id, l.resolutions))
+        .collect()
+}
+
 #[tokio::test]
 #[ignore = "needs database"]
 async fn test_lines_ok() {
     let (stats_db, blockscout_db) = init_db_all("test_lines_ok").await;
-    let blockscout_api = mock_blockscout_api().await;
+    let blockscout_api = default_mock_blockscout_api().await;
     fill_mock_blockscout_data(&blockscout_db, NaiveDate::from_str("2023-03-01").unwrap()).await;
 
     std::env::set_var("STATS__CONFIG", "./tests/config/test.toml");
@@ -40,27 +51,23 @@ async fn test_lines_ok() {
 
     let line_charts: stats_proto::blockscout::stats::v1::LineCharts =
         send_get_request(&base, "/api/v1/lines").await;
-    let sections: Vec<&str> = line_charts
+
+    let section_ids: Vec<&str> = line_charts
         .sections
         .iter()
         .map(|sec| sec.id.as_str())
         .collect();
-    let expected_sections = [
+    let expected_section_ids = [
         "accounts",
         "transactions",
         "blocks",
         "tokens",
         "gas",
         "contracts",
     ];
-    assert_eq!(sections, expected_sections, "wrong sections response");
+    assert_eq!(section_ids, expected_section_ids, "wrong sections response");
 
-    let mut enabled_resolutions: HashMap<String, Vec<String>> = line_charts
-        .sections
-        .iter()
-        .flat_map(|sec| sec.charts.clone())
-        .map(|l| (l.id, l.resolutions))
-        .collect();
+    let mut enabled_resolutions = enabled_resolutions(line_charts).await;
 
     for line_name in [
         "accountsGrowth",

stats/stats-server/tests/it/main.rs

@@ -1,5 +1,6 @@
 mod common;
 
 mod counters;
+mod indexing_status;
 mod lines;
 mod swagger;

stats/stats-server/tests/it/swagger.rs

@@ -4,7 +4,7 @@ use blockscout_service_launcher::{
 };
 use pretty_assertions::assert_eq;
 
-use stats::tests::{init_db::init_db_all, mock_blockscout::mock_blockscout_api};
+use stats::tests::{init_db::init_db_all, mock_blockscout::default_mock_blockscout_api};
 use stats_server::{stats, Settings};
 
 use std::{path::PathBuf, str::FromStr};
@@ -15,7 +15,7 @@ use crate::common::send_arbitrary_request;
 #[ignore = "needs database"]
 async fn test_swagger_ok() {
     let (stats_db, blockscout_db) = init_db_all("test_swagger_ok").await;
-    let blockscout_api = mock_blockscout_api().await;
+    let blockscout_api = default_mock_blockscout_api().await;
 
     std::env::set_var("STATS__CONFIG", "./tests/config/test.toml");
     let mut settings = Settings::build().expect("Failed to build settings");

stats/stats/src/tests/mock_blockscout.rs

@@ -13,18 +13,23 @@ use wiremock::{
     Mock, MockServer, ResponseTemplate,
 };
 
-pub async fn mock_blockscout_api() -> MockServer {
+pub async fn default_mock_blockscout_api() -> MockServer {
+    mock_blockscout_api(ResponseTemplate::new(200).set_body_string(
+        r#"{
+            "finished_indexing": true,
+            "finished_indexing_blocks": true,
+            "indexed_blocks_ratio": "1.00",
+            "indexed_internal_transactions_ratio": "1.00"
+        }"#,
+    ))
+    .await
+}
+
+pub async fn mock_blockscout_api(indexing_status_response: ResponseTemplate) -> MockServer {
     let mock_server = MockServer::start().await;
     Mock::given(method("GET"))
         .and(path("/api/v2/main-page/indexing-status"))
-        .respond_with(ResponseTemplate::new(200).set_body_string(
-            r#"{
-                "finished_indexing": true,
-                "finished_indexing_blocks": true,
-                "indexed_blocks_ratio": "1.00",
-                "indexed_internal_transactions_ratio": "1.00"
-            }"#,
-        ))
+        .respond_with(indexing_status_response)
         .mount(&mock_server)
         .await;
     mock_server