Skip to content

Commit a517de5

Browse files
jangjodijangjodi
authored
chore(similarity): Add longer snuba delay in backfill (#77583)
Increase delay between snuba calls in similarity backfill to help with snuba ratelimit being hit Add worker number to log so we can more easily query which worker numbers are running in backfill
1 parent f5f83ce commit a517de5

File tree

3 files changed

+43
-4
lines changed

3 files changed

+43
-4
lines changed

‎src/sentry/tasks/embeddings_grouping/backfill_seer_grouping_records_for_project.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ def backfill_seer_grouping_records_for_project(
8989
"only_delete": only_delete,
9090
"skip_processed_projects": skip_processed_projects,
9191
"skip_project_ids": skip_project_ids,
92+
"worker_number": worker_number,
9293
},
9394
)
9495

‎src/sentry/tasks/embeddings_grouping/utils.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -304,8 +304,8 @@ def _make_snuba_call(project, snuba_requests, referrer):
304304
bulk_snuba_queries,
305305
snuba_requests,
306306
referrer,
307-
retries=3,
308-
delay=2,
307+
retries=6,
308+
delay=15,
309309
exceptions=SNUBA_RETRY_EXCEPTIONS,
310310
)
311311
except SNUBA_RETRY_EXCEPTIONS as e:

‎tests/sentry/tasks/test_backfill_seer_grouping_records.py

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -448,11 +448,14 @@ def test_get_data_from_snuba(self):
448448
for group_id in group_ids_last_seen:
449449
assert group_id in group_ids_results
450450

451+
@patch("time.sleep", return_value=None)
451452
@patch("sentry.tasks.embeddings_grouping.utils.logger")
452453
@patch(
453454
"sentry.tasks.embeddings_grouping.utils.bulk_snuba_queries", side_effect=RateLimitExceeded
454455
)
455-
def test_get_data_from_snuba_rate_limit_exception(self, mock_bulk_snuba_queries, mock_logger):
456+
def test_get_data_from_snuba_rate_limit_exception(
457+
self, mock_bulk_snuba_queries, mock_logger, mock_sleep
458+
):
456459
group_ids_last_seen = {
457460
group.id: group.last_seen for group in Group.objects.filter(project_id=self.project.id)
458461
}
@@ -467,13 +470,14 @@ def test_get_data_from_snuba_rate_limit_exception(self, mock_bulk_snuba_queries,
467470
},
468471
)
469472

473+
@patch("time.sleep", return_value=None)
470474
@patch("sentry.tasks.embeddings_grouping.utils.logger")
471475
@patch(
472476
"sentry.tasks.embeddings_grouping.utils.bulk_snuba_queries",
473477
side_effect=QueryTooManySimultaneous,
474478
)
475479
def test_get_data_from_snuba_too_many_simultaneous_exception(
476-
self, mock_bulk_snuba_queries, mock_logger
480+
self, mock_bulk_snuba_queries, mock_logger, mock_sleep
477481
):
478482
group_ids_last_seen = {
479483
group.id: group.last_seen for group in Group.objects.filter(project_id=self.project.id)
@@ -568,6 +572,7 @@ def test_backfill_seer_grouping_records_success_cohorts_simple(
568572
"only_delete": False,
569573
"skip_processed_projects": False,
570574
"skip_project_ids": None,
575+
"worker_number": None,
571576
},
572577
),
573578
call(
@@ -580,6 +585,7 @@ def test_backfill_seer_grouping_records_success_cohorts_simple(
580585
"only_delete": False,
581586
"skip_processed_projects": False,
582587
"skip_project_ids": None,
588+
"worker_number": None,
583589
},
584590
),
585591
call(
@@ -592,6 +598,7 @@ def test_backfill_seer_grouping_records_success_cohorts_simple(
592598
"only_delete": False,
593599
"skip_processed_projects": False,
594600
"skip_project_ids": None,
601+
"worker_number": None,
595602
},
596603
),
597604
call(
@@ -604,6 +611,7 @@ def test_backfill_seer_grouping_records_success_cohorts_simple(
604611
"only_delete": False,
605612
"skip_processed_projects": False,
606613
"skip_project_ids": None,
614+
"worker_number": None,
607615
},
608616
),
609617
call(
@@ -653,6 +661,7 @@ def test_backfill_seer_grouping_records_success_cohorts_project_does_not_exist(
653661
"only_delete": False,
654662
"skip_processed_projects": False,
655663
"skip_project_ids": None,
664+
"worker_number": None,
656665
},
657666
),
658667
call(
@@ -669,6 +678,7 @@ def test_backfill_seer_grouping_records_success_cohorts_project_does_not_exist(
669678
"only_delete": False,
670679
"skip_processed_projects": False,
671680
"skip_project_ids": None,
681+
"worker_number": None,
672682
},
673683
),
674684
call(
@@ -681,6 +691,7 @@ def test_backfill_seer_grouping_records_success_cohorts_project_does_not_exist(
681691
"only_delete": False,
682692
"skip_processed_projects": False,
683693
"skip_project_ids": None,
694+
"worker_number": None,
684695
},
685696
),
686697
call(
@@ -756,6 +767,7 @@ def test_backfill_seer_grouping_records_success_cohorts_setting_defined(
756767
"only_delete": False,
757768
"skip_processed_projects": False,
758769
"skip_project_ids": None,
770+
"worker_number": None,
759771
},
760772
),
761773
call(
@@ -768,6 +780,7 @@ def test_backfill_seer_grouping_records_success_cohorts_setting_defined(
768780
"only_delete": False,
769781
"skip_processed_projects": False,
770782
"skip_project_ids": None,
783+
"worker_number": None,
771784
},
772785
),
773786
call(
@@ -780,6 +793,7 @@ def test_backfill_seer_grouping_records_success_cohorts_setting_defined(
780793
"only_delete": False,
781794
"skip_processed_projects": False,
782795
"skip_project_ids": None,
796+
"worker_number": None,
783797
},
784798
),
785799
call(
@@ -792,6 +806,7 @@ def test_backfill_seer_grouping_records_success_cohorts_setting_defined(
792806
"only_delete": False,
793807
"skip_processed_projects": False,
794808
"skip_project_ids": None,
809+
"worker_number": None,
795810
},
796811
),
797812
call(
@@ -1062,6 +1077,7 @@ def test_backfill_seer_grouping_records_multiple_batches(
10621077
"only_delete": False,
10631078
"skip_processed_projects": True,
10641079
"skip_project_ids": None,
1080+
"worker_number": None,
10651081
},
10661082
),
10671083
call(
@@ -1074,6 +1090,7 @@ def test_backfill_seer_grouping_records_multiple_batches(
10741090
"only_delete": False,
10751091
"skip_processed_projects": True,
10761092
"skip_project_ids": None,
1093+
"worker_number": None,
10771094
},
10781095
),
10791096
call(
@@ -1086,6 +1103,7 @@ def test_backfill_seer_grouping_records_multiple_batches(
10861103
"only_delete": False,
10871104
"skip_processed_projects": True,
10881105
"skip_project_ids": None,
1106+
"worker_number": None,
10891107
},
10901108
),
10911109
call("backfill finished, no cohort", extra={"project_id": self.project.id}),
@@ -1235,6 +1253,7 @@ def test_backfill_seer_grouping_records_cohort_only_delete(
12351253
"only_delete": True,
12361254
"skip_processed_projects": False,
12371255
"skip_project_ids": None,
1256+
"worker_number": None,
12381257
},
12391258
),
12401259
call(
@@ -1251,6 +1270,7 @@ def test_backfill_seer_grouping_records_cohort_only_delete(
12511270
"only_delete": True,
12521271
"skip_processed_projects": False,
12531272
"skip_project_ids": None,
1273+
"worker_number": None,
12541274
},
12551275
),
12561276
call(
@@ -1588,6 +1608,7 @@ def test_backfill_seer_grouping_records_skip_project_already_processed(self, moc
15881608
"only_delete": False,
15891609
"skip_processed_projects": True,
15901610
"skip_project_ids": None,
1611+
"worker_number": None,
15911612
},
15921613
),
15931614
call(
@@ -1631,6 +1652,7 @@ def test_backfill_seer_grouping_records_reprocess_project_already_processed(
16311652
"only_delete": False,
16321653
"skip_processed_projects": False,
16331654
"skip_project_ids": None,
1655+
"worker_number": None,
16341656
},
16351657
),
16361658
call(
@@ -1643,6 +1665,7 @@ def test_backfill_seer_grouping_records_reprocess_project_already_processed(
16431665
"only_delete": False,
16441666
"skip_processed_projects": False,
16451667
"skip_project_ids": None,
1668+
"worker_number": None,
16461669
},
16471670
),
16481671
call("backfill finished, no cohort", extra={"project_id": self.project.id}),
@@ -1671,6 +1694,7 @@ def test_backfill_seer_grouping_records_manually_skip_project(self, mock_logger)
16711694
"only_delete": False,
16721695
"skip_processed_projects": False,
16731696
"skip_project_ids": [self.project.id],
1697+
"worker_number": None,
16741698
},
16751699
),
16761700
call(
@@ -1861,6 +1885,7 @@ def test_backfill_seer_grouping_records_gateway_timeout(self, mock_seer_request,
18611885
"only_delete": False,
18621886
"skip_processed_projects": False,
18631887
"skip_project_ids": None,
1888+
"worker_number": None,
18641889
},
18651890
),
18661891
call(
@@ -1882,6 +1907,7 @@ def test_backfill_seer_grouping_records_gateway_timeout(self, mock_seer_request,
18821907
"only_delete": False,
18831908
"skip_processed_projects": False,
18841909
"skip_project_ids": None,
1910+
"worker_number": None,
18851911
},
18861912
),
18871913
call("backfill finished, no cohort", extra={"project_id": self.project.id}),
@@ -1915,6 +1941,7 @@ def test_backfill_seer_grouping_records_internal_error(self, mock_seer_request,
19151941
"only_delete": False,
19161942
"skip_processed_projects": False,
19171943
"skip_project_ids": None,
1944+
"worker_number": None,
19181945
},
19191946
),
19201947
call(
@@ -1993,6 +2020,7 @@ def test_backfill_seer_grouping_records_cohort_creation(
19932020
"only_delete": False,
19942021
"skip_processed_projects": False,
19952022
"skip_project_ids": None,
2023+
"worker_number": worker_number,
19962024
},
19972025
),
19982026
call(
@@ -2005,6 +2033,7 @@ def test_backfill_seer_grouping_records_cohort_creation(
20052033
"only_delete": False,
20062034
"skip_processed_projects": False,
20072035
"skip_project_ids": None,
2036+
"worker_number": worker_number,
20082037
},
20092038
),
20102039
call(
@@ -2017,6 +2046,7 @@ def test_backfill_seer_grouping_records_cohort_creation(
20172046
"only_delete": False,
20182047
"skip_processed_projects": False,
20192048
"skip_project_ids": None,
2049+
"worker_number": worker_number,
20202050
},
20212051
),
20222052
call(
@@ -2029,6 +2059,7 @@ def test_backfill_seer_grouping_records_cohort_creation(
20292059
"only_delete": False,
20302060
"skip_processed_projects": False,
20312061
"skip_project_ids": None,
2062+
"worker_number": worker_number,
20322063
},
20332064
),
20342065
call(
@@ -2099,6 +2130,7 @@ def test_backfill_seer_grouping_records_cohort_creation_not_seer_eligible(
20992130
"only_delete": False,
21002131
"skip_processed_projects": False,
21012132
"skip_project_ids": None,
2133+
"worker_number": worker_number,
21022134
},
21032135
),
21042136
call(
@@ -2111,6 +2143,7 @@ def test_backfill_seer_grouping_records_cohort_creation_not_seer_eligible(
21112143
"only_delete": False,
21122144
"skip_processed_projects": False,
21132145
"skip_project_ids": None,
2146+
"worker_number": worker_number,
21142147
},
21152148
),
21162149
call(
@@ -2123,6 +2156,7 @@ def test_backfill_seer_grouping_records_cohort_creation_not_seer_eligible(
21232156
"only_delete": False,
21242157
"skip_processed_projects": False,
21252158
"skip_project_ids": None,
2159+
"worker_number": worker_number,
21262160
},
21272161
),
21282162
call(
@@ -2198,6 +2232,7 @@ def test_backfill_seer_grouping_records_cohort_creation_multiple_batches(
21982232
"only_delete": False,
21992233
"skip_processed_projects": False,
22002234
"skip_project_ids": None,
2235+
"worker_number": worker_number,
22012236
},
22022237
),
22032238
call(
@@ -2210,6 +2245,7 @@ def test_backfill_seer_grouping_records_cohort_creation_multiple_batches(
22102245
"only_delete": False,
22112246
"skip_processed_projects": False,
22122247
"skip_project_ids": None,
2248+
"worker_number": worker_number,
22132249
},
22142250
),
22152251
call(
@@ -2222,6 +2258,7 @@ def test_backfill_seer_grouping_records_cohort_creation_multiple_batches(
22222258
"only_delete": False,
22232259
"skip_processed_projects": False,
22242260
"skip_project_ids": None,
2261+
"worker_number": worker_number,
22252262
},
22262263
),
22272264
call(
@@ -2234,6 +2271,7 @@ def test_backfill_seer_grouping_records_cohort_creation_multiple_batches(
22342271
"only_delete": False,
22352272
"skip_processed_projects": False,
22362273
"skip_project_ids": None,
2274+
"worker_number": worker_number,
22372275
},
22382276
),
22392277
call(

0 commit comments

Comments
 (0)