[usage] Add warning alert when usage reconciliation has not been succesfull for more than an hour

Author: easyCZ

operations/observability/mixins/meta/rules/usage.yaml

@@ -14,24 +14,24 @@ spec:
   groups:
   - name: usage
     rules:
-    - alert: GitpodUsageReconcileUsageFailures'
-      expr: sum(increase(grpc_server_handled_total{grpc_service="usage.v1.UsageService", grpc_method="ReconcileUsage", grpc_code!="OK"})) > 1
+    - alert: GitpodUsageReconcileInvoicesFailures'
+      expr: sum(increase(grpc_server_handled_total{grpc_service="usage.v1.BillingService", grpc_method="ReconcileInvoices", grpc_code!="OK"})) > 1
       for: 30m
       labels:
         severity: warning
         team: webapp
       annotations:
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodUsageReconcileUsageFailures.md
-        summary: There are failed usage reconciliations.
-        description: We have accumulated {{ printf "%.2f" $value }} failures. This affects how up-to-date usage data is.
+        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodUsageReconcileInvoicesFailures.md
+        summary: There are failed Stripe invoice reconciliations.
+        description: We have accumulated {{ printf "%.2f" $value }} failures. This affects how much customers will be billed.
 
-    - alert: GitpodUsageReconcileInvoicesFailures'
-      expr: sum(increase(grpc_server_handled_total{grpc_service="usage.v1.BillingService", grpc_method="ReconcileInvoices", grpc_code!="OK"})) > 1
+    - alert: GitpodUsageTooLongSinceLastSuccessfulLedgerReconciliation
+      expr: (time() - gitpod_usage_ledger_last_completed_time{outcome!="success"}) > 60 * 60
       for: 30m
       labels:
         severity: warning
         team: webapp
       annotations:
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodUsageReconcileInvoicesFailures.md
-        summary: There are failed Stripe invoice reconciliations.
-        description: We have accumulated {{ printf "%.2f" $value }} failures. This affects how much customers will be billed.
+        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodUsageTooLongSinceLastSuccessfulLedgerReconciliation.md
+        summary: Usage reconciliation has not run successfully for {{ printf "%.2f" $value }} seconds. Usage data is stale.
+        description: We have not executed scheduled usage reconciliation for {{ printf "%.2f" $value }} seconds. We expect the data to update every 15 minutes to avoid stale usage records and stale invoices.