Create test monitor data mart (#16323)

Author: naspirato

.github/scripts/analytics/data_mart_queries/test_monitor_mart.sql

@@ -0,0 +1,41 @@
+SELECT 
+    state_filtered, 
+    test_name, 
+    suite_folder, 
+    full_name, 
+    date_window, 
+    build_type, 
+    branch, 
+    days_ago_window, 
+    pass_count, 
+    mute_count, 
+    fail_count, 
+    skip_count, 
+    owner, 
+    is_muted, 
+    is_test_chunk, 
+    state, 
+    previous_state, 
+    state_change_date, 
+    days_in_state, 
+    previous_mute_state, 
+    mute_state_change_date, 
+    days_in_mute_state, 
+    previous_state_filtered, 
+    state_change_date_filtered, 
+    days_in_state_filtered,
+    CASE 
+        WHEN (state = 'Skipped' AND days_in_state > 14) THEN 'Skipped: > 14 days'
+        WHEN days_in_mute_state >= 30 THEN 'MUTED: delete candidate (>30 days)'
+        ELSE 'MUTED: in sla (<30 days)'
+    END as resolution,
+    String::ReplaceAll(owner, 'TEAM:@ydb-platform/', '') as owner_team,
+    CASE 
+        WHEN is_muted = 1 OR (state = 'Skipped' AND days_in_state > 14) THEN TRUE
+        ELSE FALSE
+    END as is_muted_or_skipped
+FROM `test_results/analytics/tests_monitor`
+WHERE date_window >= CurrentUtcDate() - 30 * Interval("P1D")
+and branch = 'main'
+and build_type ='relwithdebinfo'
+and is_test_chunk = 0

.github/workflows/collect_analytics_fast.yml

@@ -28,6 +28,11 @@ jobs:
       run: |
         python3 -m pip install ydb ydb[yc] codeowners pandas
     - name: Upload new test history to fast table
+      continue-on-error: true
       run: python3 .github/scripts/analytics/test_history_fast.py
     - name: Upload olap perfomance data mart
+      continue-on-error: true
       run: python3 .github/scripts/analytics/data_mart_executor.py --query_path .github/scripts/analytics/data_mart_queries/perfomance_olap_mart.sql --table_path perfomance/olap/fast_results --store_type column --partition_keys Run_start_timestamp --primary_keys Db Suite Test Branch Run_start_timestamp --ttl_min 43200 --ttl_key Run_start_timestamp
+    - name: Upload olap perfomance data mart
+      continue-on-error: true
+      run: python3 .github/scripts/analytics/data_mart_executor.py --query_path .github/scripts/analytics/data_mart_queries/test_monitor_mart.sql --table_path test_results/analytics/test_monitor_mart --store_type column --partition_keys date_window branch build_type owner_team suite_folder --primary_keys date_window owner_team branch build_type suite_folder full_name --ttl_min 43200 --ttl_key date_window