switch to jinja for report building; start to tweak output

Author: atvaccaro

Diff for: .github/workflows/build-dbt.yml

@@ -78,16 +78,7 @@ jobs:
         run: |
           cd warehouse
           gsutil cp -r gs://calitp-dbt-artifacts/latest/ .
-          echo "Warehouse report 📦" >> report.md
-          echo "### New models 🌱" >> report.md
-          poetry run dbt --no-use-colors ls --resource-type model --select state:new --state ./latest | grep "^calitp_warehouse\." | sed "s/calitp_warehouse\.//g" >> report.md
-          echo "### Changed incremental models 🔀" >> report.md
-          poetry run dbt --no-use-colors ls --resource-type model --select state:modified,config.materialized:incremental --exclude state:new --state ./latest | grep "^calitp_warehouse\." | sed "s/calitp_warehouse\.//g" >> report.md
-          echo "### DAG" >> report.md
-          poetry run dbt --no-use-colors ls --resource-type model --select state:modified --state ./latest | grep "^calitp_warehouse\." | sed 's/^/--include /g' | xargs poetry run python scripts/visualize.py man --output=target/dag.png
-          echo "" >> report.md
-          echo '![](./target/dag.png "Changed models")' >> report.md
-          echo "> Note: if you have changed any incremental models, you will need to run a full refresh on them at some point, regardless of how `on_schema_change` is configured." >> report.md
+          poetry run python scripts/visualize.py ci-report
           cml comment update report.md
 
   build_push:

Diff for: warehouse/scripts/templates/ci_report.md

@@ -0,0 +1,22 @@
+Warehouse report 📦
+
+### New models 🌱
+
+{% for model in new_models %}
+{{ model }}
+{% endfor %}
+
+### Changed incremental models 🔀
+
+{% for model in changed_incremental_models %}
+{{ model }}
+{% endfor %}
+
+### DAG
+
+![](./dag.png "Changed models")
+
+### Checks/potential follow-ups
+{% if changed_incremental_models %}
+* Full refresh as a follow-up action item
+{% endif %}

Diff for: warehouse/scripts/visualize.py

@@ -2,13 +2,15 @@
 Provide more visualizations than what dbt provides.
 """
 import json
+import os
 import webbrowser
 from pathlib import Path
 from typing import Any, List, Optional, Tuple, Type, Union
 
 import gcsfs  # type: ignore
 import networkx as nx  # type: ignore
 import typer
+from dbt.cli.main import dbtRunner
 from dbt_artifacts import (
     Catalog,
     DbtNode,
@@ -19,6 +21,7 @@
     SeedNode,
     SourceDefinition,
 )
+from jinja2 import Environment, FileSystemLoader, select_autoescape
 
 app = typer.Typer(pretty_exceptions_enable=False)
 
@@ -122,7 +125,11 @@ def build_graph(
         G.add_node(node_or_result.gvrepr, **node_or_result.gvattrs, style="filled")
 
     for node_or_result in nodes:
-        node: DbtNode = node_or_result.node if isinstance(node_or_result, RunResultOutput) else node_or_result  # type: ignore[no-redef]
+        node: DbtNode = (
+            node_or_result.node
+            if isinstance(node_or_result, RunResultOutput)
+            else node_or_result
+        )  # type: ignore[no-redef]
         if not should_display(
             node,
             analyses,
@@ -216,5 +223,71 @@ def viz(
         webbrowser.open(url, new=2)  # open in new tab
 
 
+@app.command()
+def ci_report(
+    latest_dir: str = "./latest",
+    output: str = "target/report.md",
+):
+    dbt = dbtRunner()
+    new_models = dbt.invoke(
+        [
+            "ls",
+            "--resource-type",
+            "model",
+            "--select",
+            "state:new",
+            "--state",
+            latest_dir,
+        ]
+    ).result
+
+    changed_incremental_models = dbt.invoke(
+        [
+            "ls",
+            "--resource-type",
+            "model",
+            "--select",
+            "state:modified,config.materialized:incremental",
+            "--exclude",
+            "state:new",
+            "--state",
+            latest_dir,
+        ]
+    ).result
+
+    modified_models = dbt.invoke(
+        [
+            "ls",
+            "--resource-type",
+            "model",
+            "--select",
+            "state:modified",
+            "--exclude",
+            "state:new",
+            "--state",
+            latest_dir,
+        ]
+    ).result
+    viz(
+        "man",
+        include=modified_models,
+        output=Path("./target/dag.png"),
+    )
+
+    print(os.path.join(__file__, "templates"))
+    env = Environment(
+        loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), "templates")),
+        autoescape=select_autoescape(),
+    )
+    template = env.get_template("ci_report.md")
+    report = template.render(
+        new_models=new_models,
+        changed_incremental_models=changed_incremental_models,
+    )
+    typer.secho(f"Writing to {output}", fg=typer.colors.GREEN)
+    with open(output, "w") as f:
+        f.write(report)
+
+
 if __name__ == "__main__":
     app()