Merge branch 'master' into chloedisablenotifyslack

Author: chloeho7

Diff for: bin/send_metrics.py

@@ -0,0 +1,169 @@
+# pylint: skip-file
+# flake8: noqa
+
+"""
+Script that sends generic metrics messages to sentry locally
+
+
+Overview
+
+This script is designed to be used when creating a new use case ID for the first
+time for the generic metrics platform.
+
+
+Usage
+
+
+python send_metrics.py
+
+Without any command line argument, the script will send 3 metrics
+(counter/set/distribution) for each use case ID registered in
+src/sentry/sentry_metrics/use_case_id_registry.py.
+
+
+python send_metrics.py hello world
+
+The script will treat any arguments supplied as a use case ID, and send 3 metrics
+(counter/set/distribution) for each use case ID specified.
+
+"""
+
+import datetime
+import itertools
+import json
+import pprint
+import random
+import string
+import sys
+
+from arroyo.backends.kafka import KafkaPayload, KafkaProducer
+from arroyo.types import Topic
+
+from sentry.sentry_metrics.use_case_id_registry import UseCaseID
+
+BOOTSTRAP_HOST = "127.0.0.1:9092"
+TOPIC_NAME = "ingest-performance-metrics"
+
+conf = {"bootstrap.servers": BOOTSTRAP_HOST}
+
+make_counter_payload = lambda use_case, rand_str: {
+    "name": f"c:{use_case}/{use_case}@none",
+    "tags": {
+        "environment": "production",
+        "session.status": "init",
+        f"gen_metric_e2e_{use_case}_counter_k_{rand_str}": f"gen_metric_e2e_{use_case}_counter_v_{rand_str}",
+    },
+    "timestamp": int(datetime.datetime.now(tz=datetime.timezone.utc).timestamp()),
+    "type": "c",
+    "value": 1,
+    "org_id": 1,
+    "retention_days": 90,
+    "project_id": 3,
+}
+
+make_dist_payload = lambda use_case, rand_str: {
+    "name": f"d:{use_case}/duration@second",
+    "tags": {
+        "environment": "production",
+        "session.status": "healthy",
+        f"gen_metric_e2e_{use_case}_dist_k_{rand_str}": f"gen_metric_e2e_{use_case}_dist_v_{rand_str}",
+    },
+    "timestamp": int(datetime.datetime.now(tz=datetime.timezone.utc).timestamp()),
+    "type": "d",
+    "value": [4, 5, 6],
+    "org_id": 1,
+    "retention_days": 90,
+    "project_id": 3,
+}
+
+make_set_payload = lambda use_case, rand_str: {
+    "name": f"s:{use_case}/error@none",
+    "tags": {
+        "environment": "production",
+        "session.status": "errored",
+        f"gen_metric_e2e_{use_case}_set_k_{rand_str}": f"gen_metric_e2e_{use_case}_set_v_{rand_str}",
+    },
+    "timestamp": int(datetime.datetime.now(tz=datetime.timezone.utc).timestamp()),
+    "type": "s",
+    "value": [3],
+    "org_id": 1,
+    "retention_days": 90,
+    "project_id": 3,
+}
+
+make_psql = (
+    lambda rand_str: f"""
+    SELECT string,
+       organization_id,
+       date_added,
+       use_case_id
+    FROM sentry_perfstringindexer
+    WHERE string ~ 'gen_metric_e2e_.*{rand_str}';
+"""
+)
+
+make_csql = lambda rand_str: "UNION ALL".join(
+    [
+        f"""
+    SELECT use_case_id,
+        org_id,
+        project_id,
+        metric_id,
+        timestamp,
+        tags.key,
+        tags.raw_value
+    FROM {table_name}
+    WHERE arrayExists(v -> match(v, 'gen_metric_e2e_.*{rand_str}'), tags.raw_value)
+    """
+        for table_name in [
+            "generic_metric_counters_raw_local",
+            "generic_metric_distributions_raw_local",
+            "generic_metric_sets_raw_local",
+        ]
+    ]
+)
+
+
+def produce_msgs(messages):
+    producer = KafkaProducer(conf)
+    for i, message in enumerate(messages):
+        print(f"Sending message {i + 1} of {len(messages)}:")
+        pprint.pprint(message)
+        producer.produce(
+            Topic(name=TOPIC_NAME),
+            KafkaPayload(key=None, value=json.dumps(message).encode("utf-8"), headers=[]),
+        )
+        print("Done")
+        print()
+
+    producer.close()
+
+
+if __name__ == "__main__":
+    rand_str = "".join(random.choices(string.ascii_uppercase + string.digits, k=8))
+    use_cases = (
+        [use_case_id.value for use_case_id in UseCaseID if use_case_id is not UseCaseID.SESSIONS]
+        if len(sys.argv) == 1
+        else sys.argv[1:]
+    )
+    messages = list(
+        itertools.chain.from_iterable(
+            (
+                make_counter_payload(use_case, rand_str),
+                make_dist_payload(use_case, rand_str),
+                make_set_payload(use_case, rand_str),
+            )
+            for use_case in use_cases
+        )
+    )
+    random.shuffle(messages)
+
+    produce_msgs(messages)
+    print(
+        f"Use the following SQL to verify postgres, there should be {(strs_per_use_case := 6)} strings for each use cases, {strs_per_use_case * len(use_cases)} in total."
+    )
+    print(make_psql(rand_str))
+    print(
+        f"Use the following SQL to verify clickhouse, there should be {(metrics_per_use_case := 3)} metrics for each use cases, {metrics_per_use_case * len(use_cases)} in total."
+    )
+    print(make_csql(rand_str))

Diff for: codecov.yml

@@ -51,20 +51,22 @@ flags:
     - "static/app/"
     carryforward: true
     # FE uploads 4 coverage reports. This property ensures codecov waits
-    #  for all reports to be uploaded before creating a GitHub status check.
+    # for all reports to be uploaded before creating a GitHub status check.
+    # NOTE: If you change this, make sure to change `comment.after_n_builds` below as well.
     after_n_builds: 4
   backend:
     paths:
     - "src/sentry/**/*.py"
     carryforward: true
-    # Do not send any status checks until N coverage reports are uploaded
+    # Do not send any status checks until n coverage reports are uploaded.
+    # NOTE: If you change this, make sure to change `comment.after_n_builds` below as well.
     after_n_builds: 18
 
 # Read more here: https://docs.codecov.com/docs/pull-request-comments
 comment:
   # This is the addition of carry forward builds and fresh builds, thus, it's the addition
   # of the FE and BE builds
-  after_n_builds: 20
+  after_n_builds: 22
   layout: "diff, files"
   # Update, if comment exists. Otherwise post new.
   behavior: default

Diff for: migrations_lockfile.txt

@@ -6,5 +6,5 @@ To resolve this, rebase against latest master and regenerate your migration. Thi
 will then be regenerated, and you should be able to merge without conflicts.
 
 nodestore: 0002_nodestore_no_dictfield
-sentry: 0519_remove_repo_name_constraint
+sentry: 0521_migrate_world_map_widgets
 social_auth: 0002_default_auto_field

Diff for: src/sentry/api/endpoints/project_performance_issue_settings.py

@@ -58,6 +58,7 @@ class ConfigurableThresholds(Enum):
     RENDER_BLOCKING_ASSET_FCP_RATIO = "render_blocking_fcp_ratio"
     SLOW_DB_QUERY_DURATION = "slow_db_query_duration_threshold"
     N_PLUS_API_CALLS_DURATION = "n_plus_one_api_calls_total_duration_threshold"
+    CONSECUTIVE_HTTP_SPANS_MIN_TIME_SAVED = "consecutive_http_spans_min_time_saved_threshold"
 
 
 internal_only_project_settings_to_group_map: Dict[str, Type[GroupType]] = {
@@ -84,6 +85,7 @@ class ConfigurableThresholds(Enum):
     ConfigurableThresholds.RENDER_BLOCKING_ASSET_FCP_RATIO.value: InternalProjectOptions.RENDER_BLOCKING_ASSET.value,
     ConfigurableThresholds.SLOW_DB_QUERY_DURATION.value: InternalProjectOptions.SLOW_DB_QUERY.value,
     ConfigurableThresholds.N_PLUS_API_CALLS_DURATION.value: InternalProjectOptions.N_PLUS_ONE_API_CALLS.value,
+    ConfigurableThresholds.CONSECUTIVE_HTTP_SPANS_MIN_TIME_SAVED.value: InternalProjectOptions.CONSECUTIVE_HTTP_SPANS.value,
 }
 
 
@@ -125,6 +127,9 @@ class ProjectPerformanceIssueSettingsSerializer(serializers.Serializer):
     n_plus_one_api_calls_total_duration_threshold = serializers.IntegerField(
         required=False, min_value=100, max_value=TEN_SECONDS  # ms
     )
+    consecutive_http_spans_min_time_saved_threshold = serializers.IntegerField(
+        required=False, min_value=1000, max_value=TEN_SECONDS  # ms
+    )
     uncompressed_assets_detection_enabled = serializers.BooleanField(required=False)
     consecutive_http_spans_detection_enabled = serializers.BooleanField(required=False)
     large_http_payload_detection_enabled = serializers.BooleanField(required=False)

Diff for: src/sentry/api/serializers/models/alert_rule.py

@@ -41,7 +41,7 @@ def get_attrs(self, item_list, user, **kwargs):
 
         sentry_app_installations_by_sentry_app_id = app_service.get_related_sentry_app_components(
             organization_ids=[alert_rule.organization_id for alert_rule in alert_rules.values()],
-            sentry_app_ids=trigger_actions.values_list("sentry_app_id", flat=True),
+            sentry_app_ids=list(trigger_actions.values_list("sentry_app_id", flat=True)),
             type="alert-rule-action",
         )
 

Diff for: src/sentry/api_gateway/proxy.py

@@ -53,9 +53,9 @@ def proxy_request(request: HttpRequest, org_slug: str) -> StreamingHttpResponse:
 
     try:
         region = get_region_for_organization(org_slug)
-    except RegionResolutionError:
+    except RegionResolutionError as e:
         logger.info("region_resolution_error", extra={"org_slug": org_slug})
-        raise NotFound
+        raise NotFound from e
 
     target_url = region.to_url(request.path)
     header_dict = clean_proxy_headers(request.headers)

Diff for: src/sentry/conf/server.py

@@ -1639,8 +1639,6 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str:
     "organizations:device-class-synthesis": False,
     # Enable the product selection feature in the getting started docs, regardless of the organization's strategy
     "organizations:getting-started-doc-with-product-selection": False,
-    # Enable the onboarding heartbeat footer on the sdk setup page
-    "organizations:onboarding-heartbeat-footer": False,
     # Enable a new behavior for deleting the freshly created project,
     # if the user clicks on the back button in the onboarding for new orgs
     "organizations:onboarding-project-deletion-on-back-click": False,
@@ -1684,6 +1682,8 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str:
     "projects:alert-filters": True,
     # Enable functionality to specify custom inbound filters on events.
     "projects:custom-inbound-filters": False,
+    # Enable the new flat file indexing system for sourcemaps.
+    "organizations:sourcemaps-bundle-flat-file-indexing": False,
     # Enable data forwarding functionality for projects.
     "projects:data-forwarding": True,
     # Enable functionality to discard groups.