Revert "Bring back cml CI and change what we print (#2605)"

This reverts commit 20398e4.

Author: atvaccaro

.github/workflows/build-dbt.yml

@@ -61,25 +61,39 @@ jobs:
           destination: 'calitp-ci-artifacts/${{github.workflow}}/run_id=${{github.run_id}}/job=${{github.job}}/target/'
       # Only do visualization if we actually changed models and we are merging against main
       - uses: tj-actions/changed-files@v35
-        if: ${{ github.event_name == 'pull_request' }}
+        if: github.event.pull_request.base.ref == 'main'
         id: changed-files-warehouse
         with:
           files: 'warehouse/models/**/*.sql'
-      # install a specific version of node before cml https://github.com/iterative/cml/issues/1377
-      - uses: actions/setup-node@v1
-        with:
-          node-version: '16'
-      - uses: iterative/setup-cml@v1
-        if: steps.changed-files-warehouse.outputs.any_changed == 'true'
-      - name: Create GitHub comment
+      - name: Create report.md
         if: steps.changed-files-warehouse.outputs.any_changed == 'true'
-        env:
-          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           cd warehouse
           gsutil cp -r gs://calitp-dbt-artifacts/latest/ .
-          poetry run python scripts/visualize.py ci-report
-          cml comment update target/report.md
+          echo "Warehouse report 📦" >> report.md
+          echo "### New models 🌱" >> report.md
+          poetry run dbt --no-use-colors ls --resource-type model --select state:new --state ./latest | grep "^calitp_warehouse\." | sed "s/calitp_warehouse\.//g" >> report.md
+          echo "### Changed models 🔀" >> report.md
+          poetry run dbt --no-use-colors ls --resource-type model --select state:modified --exclude state:new --state ./latest | grep "^calitp_warehouse\." | sed "s/calitp_warehouse\.//g" >> report.md
+#          echo "### DAG" >> report.md
+#          poetry run dbt --no-use-colors ls --resource-type model --select state:modified --state ./latest | grep "^calitp_warehouse\." | sed 's/^/--include /g' | xargs poetry run python scripts/visualize.py man --output=target/dag.png
+# TODO: add these back
+#          echo "" >> report.md
+#          echo '![](./target/dag.png "Changed models")' >> report.md
+      - uses: peter-evans/find-comment@v2
+        if: steps.changed-files-warehouse.outputs.any_changed == 'true'
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: 'github-actions[bot]'
+          body-includes: Warehouse report
+      - uses: peter-evans/create-or-update-comment@v3
+        if: steps.changed-files-warehouse.outputs.any_changed == 'true'
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body-path: warehouse/report.md
+          edit-mode: replace
 
   build_push:
     name: package warehouse image

warehouse/macros/gtfs_rt_unnest_column_count_distinct.sql

@@ -1,82 +1,49 @@
 {{
     config(
-        materialized='table',
+        materialized='incremental',
+        incremental_strategy='insert_overwrite',
+        partition_by={
+            'field': 'dt',
+            'data_type': 'date',
+            'granularity': 'day',
+        },
         cluster_by='base64_url',
     )
 }}
 
-
 WITH service_alerts AS (
-    SELECT *,
-            -- https://gtfs.org/realtime/reference/#message-tripdescriptor
+    SELECT * FROM {{ ref('fct_service_alerts_messages_unnested') }}
+    WHERE {{ gtfs_rt_dt_where() }}
+),
+
+fct_service_alerts_trip_summaries AS (
+    SELECT
+        -- https://gtfs.org/realtime/reference/#message-tripdescriptor
         {{ dbt_utils.generate_surrogate_key([
-            'calculated_service_date',
+            'dt',
             'base64_url',
             'trip_id',
+            'trip_route_id',
+            'trip_direction_id',
             'trip_start_time',
+            'trip_start_date',
         ]) }} as key,
-    FROM {{ ref('int_gtfs_rt__service_alerts_trip_day_map_grouping') }}
-),
-
-re_aggregate AS (
-    SELECT
-        key,
-        calculated_service_date,
+        dt,
         base64_url,
         trip_id,
         trip_route_id,
         trip_direction_id,
         trip_start_time,
         trip_start_date,
-        trip_schedule_relationship,
-        feed_timezone,
-        COUNT(DISTINCT unnested_message_ids) AS num_distinct_message_ids,
-        COUNT(DISTINCT unested_header_timestamps) AS num_distinct_header_timestamps,
-        COUNT(DISTINCT unnested_message_keys) AS num_distinct_message_keys,
-        MIN(min_extract_ts) AS min_extract_ts,
-        MAX(max_extract_ts) AS max_extract_ts,
-        MIN(min_header_timestamp) AS min_header_timestamp,
-        MAX(max_header_timestamp) AS max_header_timestamp,
-        ARRAY_AGG(DISTINCT unnested_alert_content)
-            AS alert_content_array
+        COUNT(DISTINCT id) AS num_distinct_message_ids,
+        COUNT(DISTINCT header_timestamp) AS num_distinct_header_timestamps,
+        ARRAY_AGG(DISTINCT service_alert_message_key) AS service_alert_message_keys,
+        MIN(_extract_ts) AS min_extract_ts,
+        MAX(_extract_ts) AS max_extract_ts,
+        MIN(header_timestamp) AS min_header_timestamp,
+        MAX(header_timestamp) AS max_header_timestamp,
     FROM service_alerts
-    LEFT JOIN UNNEST(message_ids_array) AS unnested_message_ids
-    LEFT JOIN UNNEST(header_timestamps_array) AS unested_header_timestamps
-    LEFT JOIN UNNEST(message_keys_array) AS unnested_message_keys
-    LEFT JOIN UNNEST(alert_content_array) AS unnested_alert_content
-    GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10
-),
-
-fct_service_alerts_trip_summaries AS (
-    SELECT
-        key,
-        calculated_service_date,
-        base64_url,
-        trip_id,
-        trip_route_id,
-        trip_direction_id,
-        trip_start_time,
-        trip_start_date,
-        trip_schedule_relationship,
-        feed_timezone,
-        num_distinct_message_ids,
-        num_distinct_header_timestamps,
-        num_distinct_message_keys,
-        min_extract_ts,
-        max_extract_ts,
-        min_header_timestamp,
-        max_header_timestamp,
-        ARRAY_AGG(
-            STRUCT<message_id string, cause string, effect string, header string, description string >
-                (JSON_VALUE(unnested_alert, '$.message_id'),
-                JSON_VALUE(unnested_alert, '$.cause'),
-                JSON_VALUE(unnested_alert, '$.effect'),
-                JSON_VALUE(unnested_alert, '$.header'),
-                JSON_VALUE(unnested_alert, '$.description')))
-        AS alert_content_array
-    FROM re_aggregate
-    LEFT JOIN UNNEST(alert_content_array) AS unnested_alert
-    GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17
-    )
+    GROUP BY 1, 2, 3, 4, 5, 6, 7, 8
+)
 
 SELECT * FROM fct_service_alerts_trip_summaries