[usage] Alert on Stripe webhook handling

Author: easyCZ

operations/observability/mixins/meta/rules/usage.yaml

@@ -57,3 +57,14 @@ spec:
         runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodUsageTooLongSinceLastSuccessfulLedgerReconciliation.md
         summary: Usage reconciliation has not run successfully for {{ printf "%.2f" $value }} seconds. Usage data is stale.
         description: We have not executed scheduled usage reconciliation for {{ printf "%.2f" $value }} seconds. We expect the data to update every 15 minutes to avoid stale usage records and stale invoices.
+
+    - alert: GitpodUsageStripeWebhookFailures
+      expr: sum(increase(gitpod_http_request_duration_seconds_count{handler="/stripe/invoices/webhook", code=~"5.*"})) > 0
+      for: 30m
+      labels:
+        severity: warning
+        team: webapp
+      annotations:
+        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodUsageStripeWebhookFailures.md
+        summary: Detected {{ printf "%.2f" $value }} errors handling Stripe webhook.
+        description: Stripe is sending us webhooks but we are failing to handle them. Inconsistent usage data very likely.