Merge branch 'main' into cbeauchesne/green-ci-policy

Author: cbeauchesne

.gitlab-ci.yml

@@ -8,6 +8,8 @@ stages:
 variables:
   REPO_LANG: python # "python" is used everywhere rather than "py"
   REPO_NOTIFICATION_CHANNEL: "#apm-python-release"
+  RELEASE_ALLOW_TEST_FAILURES: false
+  RELEASE_ALLOW_BENCHMARK_FAILURES: false
   # CI_DEBUG_SERVICES: "true"
 
 default:
@@ -49,6 +51,11 @@ tests-gen:
 run-tests-trigger:
  stage: tests
  needs: [ tests-gen ]
+ # Allow the child job to fail if explicitly asked
+ rules:
+   - if: $RELEASE_ALLOW_TEST_FAILURES == "true"
+     allow_failure: true
+   - allow_failure: false
  trigger:
    include:
      - artifact: .gitlab/tests-gen.yml
@@ -58,6 +65,10 @@ run-tests-trigger:
 microbenchmarks:
   stage: benchmarks
   needs: [ "download_ddtrace_artifacts" ]
+  rules:
+    - if: $RELEASE_ALLOW_BENCHMARK_FAILURES == "true"
+      allow_failure: true
+    - allow_failure: false
   trigger:
     include: .gitlab/benchmarks/microbenchmarks.yml
     strategy: depend
@@ -158,7 +169,7 @@ publish-lib-init-pinned-tags:
 configure_system_tests:
   variables:
     SYSTEM_TESTS_SCENARIOS_GROUPS: "simple_onboarding,simple_onboarding_profiling,docker-ssi,lib-injection"
-    
+
 deploy_to_reliability_env:
   needs: []
 

.gitlab/benchmarks/microbenchmarks.yml

@@ -16,7 +16,7 @@ variables:
   tags: ["runner:apm-k8s-tweaked-metal"]
   image: $MICROBENCHMARKS_CI_IMAGE
   interruptible: true
-  timeout: 1h
+  timeout: 30m
   dependencies: [ "baseline:build", "candidate" ]
   script: |
     export REPORTS_DIR="$(pwd)/reports/" && (mkdir "${REPORTS_DIR}" || :)
@@ -149,7 +149,8 @@ microbenchmarks:
         - "appsec_iast_aspects_ospath"
         - "appsec_iast_aspects_re_module"
         - "appsec_iast_aspects_split"
-        - "appsec_iast_django_startup"
+        # Flaky timeouts on starting up
+        # - "appsec_iast_django_startup"
         - "appsec_iast_propagation"
         # Flaky. Timeout errors
         # - "encoder"
@@ -162,6 +163,15 @@ microbenchmarks:
         - "startup"
         - "ddtrace_run"
 
+"microbenchmarks flaky":
+  extends: .benchmarks
+  allow_failure: true
+  parallel:
+    matrix:
+      - SCENARIO:
+        # Flaky timeouts on starting up
+        - "appsec_iast_django_startup"
+
 benchmarks-pr-comment:
   image: $MICROBENCHMARKS_CI_IMAGE
   tags: ["arch:amd64"]

.gitlab/benchmarks/serverless.yml

@@ -7,8 +7,10 @@ benchmark-serverless:
     project: DataDog/serverless-tools
     strategy: depend
   needs: []
-  # TODO: Revert this once the build issue has been resolved
-  allow_failure: true
+  rules:
+    - if: $RELEASE_ALLOW_BENCHMARK_FAILURES == "true"
+      allow_failure: true
+    - allow_failure: false
   variables:
     UPSTREAM_PIPELINE_ID: $CI_PIPELINE_ID
     UPSTREAM_PROJECT_URL: $CI_PROJECT_URL

.gitlab/tests.yml

@@ -1,5 +1,5 @@
 stages:
-  - precheck
+  - setup
   - riot
   - hatch
 
@@ -17,7 +17,7 @@ include:
   extends: .testrunner
   stage: hatch
   # Hatch doesn't use pre-built wheels or venvs so we can start them right away
-  needs: []
+  needs: [ prechecks ]
   parallel: 4
   # DEV: This is the max retries that GitLab currently allows for
   before_script:
@@ -58,7 +58,7 @@ include:
 .test_base_riot:
   extends: .testrunner
   stage: riot
-  needs: [ build_base_venvs ]
+  needs: [ build_base_venvs, prechecks ]
   parallel: 4
   services:
     - !reference [.services, ddagent]

.riot/requirements/d449a90.txt renamed to .riot/requirements/10ba61c.txt

@@ -2,26 +2,30 @@
 # This file is autogenerated by pip-compile with Python 3.13
 # by the following command:
 #
-#    pip-compile --allow-unsafe --no-annotate .riot/requirements/d449a90.in
+#    pip-compile --allow-unsafe --no-annotate .riot/requirements/10ba61c.in
 #
-attrs==25.1.0
-coverage[toml]==7.6.12
+attrs==25.3.0
+coverage[toml]==7.8.0
 gunicorn==23.0.0
 hypothesis==6.45.0
-iniconfig==2.0.0
-lz4==4.4.3
+iniconfig==2.1.0
+jsonschema==4.23.0
+jsonschema-specifications==2024.10.1
+lz4==4.4.4
 mock==5.2.0
 opentracing==2.4.0
 packaging==24.2
 pluggy==1.5.0
-protobuf==6.30.0
+protobuf==6.30.2
 py-cpuinfo==8.0.0
 pytest==8.3.5
 pytest-asyncio==0.21.1
 pytest-benchmark==5.1.0
-pytest-cov==6.0.0
+pytest-cov==6.1.1
 pytest-cpp==2.6.0
 pytest-mock==3.14.0
 pytest-randomly==3.16.0
+referencing==0.36.2
+rpds-py==0.24.0
 sortedcontainers==2.4.0
-uwsgi==2.0.28
+uwsgi==2.0.29

.riot/requirements/cb9a5da.txt renamed to .riot/requirements/137ea60.txt

@@ -2,14 +2,16 @@
 # This file is autogenerated by pip-compile with Python 3.13
 # by the following command:
 #
-#    pip-compile --allow-unsafe --no-annotate .riot/requirements/cb9a5da.in
+#    pip-compile --allow-unsafe --no-annotate .riot/requirements/137ea60.in
 #
-attrs==25.1.0
-coverage[toml]==7.6.12
+attrs==25.3.0
+coverage[toml]==7.8.0
 gunicorn==23.0.0
 hypothesis==6.45.0
-iniconfig==2.0.0
-lz4==4.4.3
+iniconfig==2.1.0
+jsonschema==4.23.0
+jsonschema-specifications==2024.10.1
+lz4==4.4.4
 mock==5.2.0
 opentracing==2.4.0
 packaging==24.2
@@ -19,9 +21,11 @@ py-cpuinfo==8.0.0
 pytest==8.3.5
 pytest-asyncio==0.21.1
 pytest-benchmark==5.1.0
-pytest-cov==6.0.0
+pytest-cov==6.1.1
 pytest-cpp==2.6.0
 pytest-mock==3.14.0
 pytest-randomly==3.16.0
+referencing==0.36.2
+rpds-py==0.24.0
 sortedcontainers==2.4.0
-uwsgi==2.0.28
+uwsgi==2.0.29

.riot/requirements/b83f7ca.txt renamed to .riot/requirements/1387420.txt

@@ -2,36 +2,41 @@
 # This file is autogenerated by pip-compile with Python 3.9
 # by the following command:
 #
-#    pip-compile --allow-unsafe --no-annotate .riot/requirements/b83f7ca.in
+#    pip-compile --allow-unsafe --no-annotate .riot/requirements/1387420.in
 #
-attrs==24.3.0
-coverage[toml]==7.6.10
+attrs==25.3.0
+coverage[toml]==7.8.0
 exceptiongroup==1.2.2
 gevent==24.11.1
 greenlet==3.1.1
 gunicorn[gevent]==23.0.0
 hypothesis==6.45.0
-importlib-metadata==8.5.0
-iniconfig==2.0.0
-lz4==4.3.3
-mock==5.1.0
+importlib-metadata==8.6.1
+iniconfig==2.1.0
+jsonschema==4.23.0
+jsonschema-specifications==2024.10.1
+lz4==4.4.4
+mock==5.2.0
 opentracing==2.4.0
 packaging==24.2
 pluggy==1.5.0
 py-cpuinfo==8.0.0
-pytest==8.3.4
+pytest==8.3.5
 pytest-asyncio==0.21.1
 pytest-benchmark==5.1.0
-pytest-cov==6.0.0
+pytest-cov==6.1.1
 pytest-cpp==2.6.0
 pytest-mock==3.14.0
 pytest-randomly==3.16.0
+referencing==0.36.2
+rpds-py==0.24.0
 sortedcontainers==2.4.0
 tomli==2.2.1
-uwsgi==2.0.28
+typing-extensions==4.13.2
+uwsgi==2.0.29
 zipp==3.21.0
 zope-event==5.0
 zope-interface==7.2
 
 # The following packages are considered to be unsafe in a requirements file:
-setuptools==75.7.0
+setuptools==78.1.0

.riot/requirements/2e36381.txt renamed to .riot/requirements/17d21ee.txt

@@ -2,32 +2,37 @@
 # This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
-#    pip-compile --allow-unsafe --no-annotate .riot/requirements/2e36381.in
+#    pip-compile --allow-unsafe --no-annotate .riot/requirements/17d21ee.in
 #
-attrs==24.3.0
-coverage[toml]==7.6.10
+attrs==25.3.0
+coverage[toml]==7.8.0
 gevent==24.11.1
 greenlet==3.1.1
 gunicorn[gevent]==23.0.0
 hypothesis==6.45.0
-iniconfig==2.0.0
-lz4==4.3.3
-mock==5.1.0
+iniconfig==2.1.0
+jsonschema==4.23.0
+jsonschema-specifications==2024.10.1
+lz4==4.4.4
+mock==5.2.0
 opentracing==2.4.0
 packaging==24.2
 pluggy==1.5.0
 py-cpuinfo==8.0.0
-pytest==8.3.4
+pytest==8.3.5
 pytest-asyncio==0.21.1
 pytest-benchmark==5.1.0
-pytest-cov==6.0.0
+pytest-cov==6.1.1
 pytest-cpp==2.6.0
 pytest-mock==3.14.0
 pytest-randomly==3.16.0
+referencing==0.36.2
+rpds-py==0.24.0
 sortedcontainers==2.4.0
-uwsgi==2.0.28
+typing-extensions==4.13.2
+uwsgi==2.0.29
 zope-event==5.0
 zope-interface==7.2
 
 # The following packages are considered to be unsafe in a requirements file:
-setuptools==75.7.0
+setuptools==78.1.0

.riot/requirements/18008a7.txt

@@ -2,34 +2,39 @@
 # This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
-#    pip-compile --allow-unsafe --no-annotate .riot/requirements/1359ebb.in
+#    pip-compile --allow-unsafe --no-annotate .riot/requirements/187c22a.in
 #
-attrs==24.3.0
-coverage[toml]==7.6.10
+attrs==25.3.0
+coverage[toml]==7.8.0
 exceptiongroup==1.2.2
 gevent==24.11.1
 greenlet==3.1.1
 gunicorn[gevent]==23.0.0
 hypothesis==6.45.0
-iniconfig==2.0.0
-lz4==4.3.3
-mock==5.1.0
+iniconfig==2.1.0
+jsonschema==4.23.0
+jsonschema-specifications==2024.10.1
+lz4==4.4.4
+mock==5.2.0
 opentracing==2.4.0
 packaging==24.2
 pluggy==1.5.0
 py-cpuinfo==8.0.0
-pytest==8.3.4
+pytest==8.3.5
 pytest-asyncio==0.21.1
 pytest-benchmark==5.1.0
-pytest-cov==6.0.0
+pytest-cov==6.1.1
 pytest-cpp==2.6.0
 pytest-mock==3.14.0
 pytest-randomly==3.16.0
+referencing==0.36.2
+rpds-py==0.24.0
 sortedcontainers==2.4.0
 tomli==2.2.1
-uwsgi==2.0.28
+typing-extensions==4.13.2
+uwsgi==2.0.29
 zope-event==5.0
 zope-interface==7.2
 
 # The following packages are considered to be unsafe in a requirements file:
-setuptools==75.7.0
+setuptools==78.1.0