Merge branch 'master' into mtoff/stable_config

Author: mtoffl01

.circleci/config.continue.yml.j2

@@ -858,6 +858,9 @@ jobs:
               APPSEC_RASP
               APPSEC_RUNTIME_ACTIVATION
               APPSEC_WAF_TELEMETRY
+              APPSEC_STANDALONE_V2
+              IAST_STANDALONE_V2
+              SCA_STANDALONE_V2
               REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD
               "
             fi

.github/CODEOWNERS

@@ -65,3 +65,10 @@ dd-java-agent/instrumentation/spring-security-5/ @DataDog/asm-java
 # @DataDog/data-jobs-monitoring
 dd-java-agent/instrumentation/spark/            @DataDog/data-jobs-monitoring
 dd-java-agent/instrumentation/spark-executor/   @DataDog/data-jobs-monitoring
+
+# @DataDog/data-streams-monitoring
+dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/datastreams  @DataDog/data-streams-monitoring
+dd-trace-core/src/main/java/datadog/trace/core/datastreams                  @DataDog/data-streams-monitoring
+dd-trace-core/src/test/groovy/datadog/trace/core/datastreams                @DataDog/data-streams-monitoring
+internal-api/src/main/java/datadog/trace/api/datastreams                    @DataDog/data-streams-monitoring
+internal-api/src/test/groovy/datadog/trace/api/datastreams                  @DataDog/data-streams-monitoring

.gitlab-ci.yml

@@ -23,15 +23,24 @@ variables:
   DEPENDENCY_CACHE_POLICY: pull
   BUILD_CACHE_POLICY: pull
   GRADLE_VERSION: "8.4" # must match gradle-wrapper.properties
+  JAVA_BUILD_IMAGE_VERSION: "v25.01"
 
 default:
   tags: [ "arch:amd64" ]
 
+.fan_in:
+  stage: tests
+  image: registry.ddbuild.io/images/base/gbi-ubuntu_2204-slim:release
+  script:
+    - echo "done"
+
 .gradle_build: &gradle_build
-  image: ghcr.io/datadog/dd-trace-java-docker-build:v25.01-base
+  image: ghcr.io/datadog/dd-trace-java-docker-build:${JAVA_BUILD_IMAGE_VERSION}-base
+  stage: build
   variables:
-    GRADLE_OPTS: "-Dorg.gradle.jvmargs='-Xmx2560M -Xms2560M'"
-    GRADLE_ARGS: " -PskipTests --build-cache --stacktrace --no-daemon --parallel --max-workers=2"
+    GRADLE_OPTS: "-Dorg.gradle.jvmargs='-Xmx2560M -Xms2560M -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
+    MAVEN_OPTS: "-Xms64M -Xmx512M"
+    GRADLE_WORKERS: 2
     KUBERNETES_CPU_REQUEST: 8
     KUBERNETES_MEMORY_REQUEST: 6Gi
   cache:
@@ -42,14 +51,15 @@ default:
         - .gradle/caches
         - .gradle/notifications
       policy: $DEPENDENCY_CACHE_POLICY
-    - key: $CI_PIPELINE_ID # Incremental build cache. Shared by all jobs in the pipeline
+    - key: $CI_PIPELINE_ID-$BUILD_CACHE_TYPE # Incremental build cache. Shared by all jobs in the pipeline of the same type
       paths:
         - .gradle/caches/$GRADLE_VERSION
         - .gradle/$GRADLE_VERSION/executionHistory
         - workspace
       policy: $BUILD_CACHE_POLICY
   before_script:
     - export GRADLE_USER_HOME=`pwd`/.gradle
+    - export GRADLE_ARGS=" --build-cache --stacktrace --no-daemon --parallel --max-workers=$GRADLE_WORKERS"
     # for weird reasons, gradle will always "chmod 700" the .gradle folder
     # with Gitlab caching, .gradle is always owned by root and thus gradle's chmod invocation fails
     # This dance is a hack to have .gradle owned by the Gitlab runner user
@@ -61,21 +71,24 @@ default:
 
 build:
   extends: .gradle_build
-  stage: build
   variables:
     BUILD_CACHE_POLICY: push
+    BUILD_CACHE_TYPE: lib
     DEPENDENCY_CACHE_POLICY: pull
   script:
-    - ./gradlew clean :dd-java-agent:shadowJar :dd-trace-api:jar :dd-trace-ot:shadowJar $GRADLE_ARGS
+    - if [ $CI_PIPELINE_SOURCE == "schedule" ] ; then ./gradlew resolveAndLockAll --write-locks; fi
+    - ./gradlew clean :dd-java-agent:shadowJar :dd-trace-api:jar :dd-trace-ot:shadowJar -PskipTests $GRADLE_ARGS
     - echo UPSTREAM_TRACER_VERSION=$(java -jar workspace/dd-java-agent/build/libs/*.jar) >> upstream.env
     - echo "BUILD_JOB_NAME=$CI_JOB_NAME" >> build.env
     - echo "BUILD_JOB_ID=$CI_JOB_ID" >> build.env
   artifacts:
+    when: always
     paths:
       - 'workspace/dd-java-agent/build/libs/*.jar'
       - 'workspace/dd-trace-api/build/libs/*.jar'
       - 'workspace/dd-trace-ot/build/libs/*.jar'
       - 'upstream.env'
+      - '.gradle/daemon/*/*.out.log'
     reports:
       dotenv: build.env
 
@@ -90,6 +103,94 @@ build_and_populate_dep_cache:
     - when: manual
       allow_failure: true
 
+spotless:
+  extends: .gradle_build
+  stage: tests
+  needs: []
+  script:
+    - export JAVA_HOME=$JAVA_11_HOME
+    - ./gradlew spotlessCheck $GRADLE_ARGS
+
+test_published_artifacts:
+  extends: .gradle_build
+  image: ghcr.io/datadog/dd-trace-java-docker-build:${JAVA_BUILD_IMAGE_VERSION}-7 # Needs Java7 for some tests
+  stage: tests
+  needs: [ build ]
+  rules:
+    - if: '$POPULATE_CACHE'
+      when: never
+    - when: on_success
+  variables:
+    BUILD_CACHE_TYPE: lib
+  script:
+    - mvn_local_repo=$(./mvnw help:evaluate -Dexpression=settings.localRepository -q -DforceStdout)
+    - rm -rf "${mvn_local_repo}/com/datadoghq"
+    - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text)
+    - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text)
+    - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
+    - ./gradlew publishToMavenLocal $GRADLE_ARGS
+    - cd test-published-dependencies
+    - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx512M -Xms512M -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
+    - ./gradlew check --info $GRADLE_ARGS
+  after_script:
+    - .circleci/collect_reports.sh
+  artifacts:
+    when: always
+    paths:
+      - ./check_reports
+
+muzzle:
+  extends: .gradle_build
+  needs: [ build ]
+  stage: tests
+  parallel: 8
+  rules:
+    - if: '$POPULATE_CACHE'
+      when: never
+    - when: on_success
+  variables:
+    BUILD_CACHE_TYPE: test
+  script:
+    - export SKIP_BUILDSCAN="true"
+    - echo CI_NODE_INDEX=$CI_NODE_INDEX
+    - echo CI_NODE_TOTAL=CI_NODE_TOTAL
+    - ./gradlew writeMuzzleTasksToFile $GRADLE_ARGS
+    - sort workspace/build/muzzleTasks > sortedMuzzleTasks
+    - split --number=l/$CI_NODE_TOTAL --suffix-length=1 --numeric-suffixes sortedMuzzleTasks muzzleSplit
+    - export NODE_ZERO_INDEX=$((CI_NODE_INDEX - 1))
+    - ./gradlew `cat muzzleSplit${NODE_ZERO_INDEX} | xargs` $GRADLE_ARGS
+  after_script:
+    - .circleci/collect_reports.sh
+  artifacts:
+    when: always
+    paths:
+      - ./reports
+      - '.gradle/daemon/*/*.out.log'
+
+muzzle-dep-report:
+  extends: .gradle_build
+  needs: [ build ]
+  stage: tests
+  variables:
+    BUILD_CACHE_TYPE: test
+  script:
+    - export SKIP_BUILDSCAN="true"
+    - ./gradlew generateMuzzleReport muzzleInstrumentationReport $GRADLE_ARGS
+  after_script:
+    - .circleci/collect_muzzle_deps.sh
+  artifacts:
+    when: always
+    paths:
+      - ./reports
+      - '.gradle/daemon/*/*.out.log'
+
+required:
+  extends: .fan_in
+  needs:
+    - spotless
+    - muzzle
+    - test_published_artifacts
+
 deploy_to_profiling_backend:
   stage: publish
   needs: [ build ]
@@ -165,6 +266,8 @@ deploy_to_sonatype:
   extends: .gradle_build
   stage: publish
   needs: [ build ]
+  variables:
+    BUILD_CACHE_TYPE: lib
   rules:
     - if: '$POPULATE_CACHE'
       when: never
@@ -180,7 +283,7 @@ deploy_to_sonatype:
     - export SONATYPE_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.sonatype_password --with-decryption --query "Parameter.Value" --out text)
     - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text)
     - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text)
-    - ./gradlew -PbuildInfo.build.number=$CI_JOB_ID publishToSonatype closeSonatypeStagingRepository $GRADLE_ARGS
+    - ./gradlew -PbuildInfo.build.number=$CI_JOB_ID publishToSonatype closeSonatypeStagingRepository -PskipTests $GRADLE_ARGS
   artifacts:
     paths:
       - 'workspace/dd-java-agent/build/libs/*.jar'

benchmark/load/petclinic/benchmark.json

@@ -41,6 +41,12 @@
         "VARIANT": "iast",
         "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.iast.enabled=true"
       }
+    },
+    "code_origins": {
+      "env": {
+        "VARIANT": "code_origins",
+        "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.code.origin.for.spans.enabled=true"
+      }
     }
   }
 }

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpServerDecorator.java

@@ -1,6 +1,7 @@
 package datadog.trace.bootstrap.instrumentation.decorator;
 
 import static datadog.trace.api.cache.RadixTreeCache.UNSET_STATUS;
+import static datadog.trace.api.datastreams.DataStreamsContext.fromTags;
 import static datadog.trace.api.gateway.Events.EVENTS;
 import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.traceConfig;
 import static datadog.trace.bootstrap.instrumentation.decorator.http.HttpResourceDecorator.HTTP_RESOURCE_DECORATOR;
@@ -148,7 +149,7 @@ public AgentSpan startSpan(
     }
     AgentPropagation.ContextVisitor<REQUEST_CARRIER> getter = getter();
     if (null != carrier && null != getter) {
-      tracer().getDataStreamsMonitoring().setCheckpoint(span, SERVER_PATHWAY_EDGE_TAGS, 0, 0);
+      tracer().getDataStreamsMonitoring().setCheckpoint(span, fromTags(SERVER_PATHWAY_EDGE_TAGS));
     }
     return span;
   }

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/ExecutorInstrumentationUtils.java

@@ -1,10 +1,10 @@
 package datadog.trace.bootstrap.instrumentation.java.concurrent;
 
-import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeScope;
+import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan;
+import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.isAsyncPropagationEnabled;
 import static datadog.trace.bootstrap.instrumentation.java.concurrent.ExcludeFilter.ExcludeType;
 
 import datadog.trace.bootstrap.ContextStore;
-import datadog.trace.bootstrap.instrumentation.api.AgentScope;
 import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
 import java.util.concurrent.Executor;
 import org.slf4j.Logger;
@@ -26,14 +26,10 @@ public static boolean shouldAttachStateToTask(final Object task, final Executor
     if (task == null) {
       return false;
     }
-
     if (ExcludeFilter.exclude(ExcludeType.EXECUTOR, task)) {
       return false;
     }
-
-    final AgentScope scope = activeScope();
-
-    return scope != null && scope.isAsyncPropagating();
+    return activeSpan() != null && isAsyncPropagationEnabled();
   }
 
   /**

dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/OOMENotifierScriptInitializer.java

@@ -116,7 +116,7 @@ private static boolean copyOOMEscript(Path scriptPath) {
   private static class ScriptCleanupVisitor implements FileVisitor<Path> {
     private static final Pattern PID_PATTERN = Pattern.compile(".*?" + PID_PREFIX + "(\\d+)");
 
-    private final Set<String> pidSet = PidHelper.getJavaPids();
+    private Set<String> pidSet;
 
     static void run(Path dir) {
       try {
@@ -145,9 +145,14 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO
       Matcher matcher = PID_PATTERN.matcher(fileName);
       if (matcher.find()) {
         String pid = matcher.group(1);
-        if (pid != null && !pid.equals(PidHelper.getPid()) && !this.pidSet.contains(pid)) {
-          LOG.debug("Cleaning process specific file {}", file);
-          Files.delete(file);
+        if (pid != null && !pid.equals(PidHelper.getPid())) {
+          if (this.pidSet == null) {
+            this.pidSet = PidHelper.getJavaPids(); // only fork jps when required
+          }
+          if (!this.pidSet.contains(pid)) {
+            LOG.debug("Cleaning process specific file {}", file);
+            Files.delete(file);
+          }
         }
       }
       return CONTINUE;

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/codeorigin/DefaultCodeOriginRecorder.java

@@ -121,7 +121,7 @@ private CodeOriginProbe createProbe(
 
     // i think this check is unnecessary at this point time but leaving for now to be safe
     if (installed == null) {
-      if (Config.get().isDynamicInstrumentationEnabled()) {
+      if (Config.get().isDistributedDebuggerEnabled()) {
         registerLogProbe(probe);
       }
       installProbes();

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/probe/LogProbe.java

@@ -701,15 +701,19 @@ public void commit(CapturedContext lineContext, int line) {
       shouldCommit = true;
     }
     if (shouldCommit) {
-      if (isCaptureSnapshot()) {
-        // freeze context just before commit because line probes have only one context
-        Duration timeout =
-            Duration.of(Config.get().getDynamicInstrumentationCaptureTimeout(), ChronoUnit.MILLIS);
-        lineContext.freeze(new TimeoutChecker(timeout));
-        snapshot.addLine(lineContext, line);
+      incrementBudget();
+      if (inBudget()) {
+        if (isCaptureSnapshot()) {
+          // freeze context just before commit because line probes have only one context
+          Duration timeout =
+              Duration.of(
+                  Config.get().getDynamicInstrumentationCaptureTimeout(), ChronoUnit.MILLIS);
+          lineContext.freeze(new TimeoutChecker(timeout));
+          snapshot.addLine(lineContext, line);
+        }
+        commitSnapshot(snapshot, sink);
+        return;
       }
-      commitSnapshot(snapshot, sink);
-      return;
     }
     sink.skipSnapshot(id, DebuggerContext.SkipCause.CONDITION);
   }

dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/origin/CodeOriginTest.java

@@ -88,6 +88,7 @@ public void before() {
     tracer.addTraceInterceptor(traceInterceptor);
 
     setFieldInConfig(Config.get(), "debuggerCodeOriginEnabled", true);
+    setFieldInConfig(Config.get(), "distributedDebuggerEnabled", true);
     setFieldInConfig(InstrumenterConfig.get(), "codeOriginEnabled", true);
   }