Merge branch 'master' into mattalp/additional-jvmstat-telemetry

Author: MattAlp

.github/workflows/system-tests.yaml

@@ -0,0 +1,64 @@
+name: system-tests
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+# Cancel long-running jobs when a new commit is pushed
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on:
+      group: APM Larger Runners
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
+        with:
+          submodules: 'recursive'
+          fetch-depth: 0
+
+      - name: Cache Gradle dependencies
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
+          restore-keys: |
+            ${{ runner.os }}-gradle-
+      
+      - name: Build dd-trace-java
+        run: |
+          GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G'" \
+          JAVA_HOME=$JAVA_HOME_8_X64 \
+          JAVA_8_HOME=$JAVA_HOME_8_X64 \
+          JAVA_11_HOME=$JAVA_HOME_11_X64 \
+          JAVA_17_HOME=$JAVA_HOME_17_X64 \
+          JAVA_21_HOME=$JAVA_HOME_21_X64 \
+          ./gradlew clean :dd-java-agent:shadowJar \
+            --build-cache --parallel --stacktrace --no-daemon --max-workers=4
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        with:
+          name: binaries
+          path: workspace/dd-java-agent/build/libs/
+
+  main:
+    needs:
+    - build
+    uses:  DataDog/system-tests/.github/workflows/system-tests.yml@main
+    secrets: inherit
+    permissions:
+      contents: read
+      packages: write
+    with:
+      library: java
+      binaries_artifact: binaries
+      desired_execution_time: 900  # 15 minutes
+      scenarios_groups: tracer-release
+      excluded_scenarios: CROSSED_TRACING_LIBRARIES,INTEGRATIONS_AWS  # require AWS credentials
+      skip_empty_scenarios: true

buildSrc/build.gradle.kts

@@ -30,15 +30,15 @@ dependencies {
   implementation(gradleApi())
   implementation(localGroovy())
 
-  implementation("net.bytebuddy", "byte-buddy-gradle-plugin", "1.14.18")
+  implementation("net.bytebuddy", "byte-buddy-gradle-plugin", "1.15.11")
 
   implementation("org.eclipse.aether", "aether-connector-basic", "1.1.0")
   implementation("org.eclipse.aether", "aether-transport-http", "1.1.0")
   implementation("org.apache.maven", "maven-aether-provider", "3.3.9")
 
   implementation("com.google.guava", "guava", "20.0")
-  implementation("org.ow2.asm", "asm", "9.7.1")
-  implementation("org.ow2.asm", "asm-tree", "9.7.1")
+  implementation("org.ow2.asm", "asm", "9.8")
+  implementation("org.ow2.asm", "asm-tree", "9.8")
 
   testImplementation("org.spockframework", "spock-core", "2.2-groovy-3.0")
   testImplementation("org.codehaus.groovy", "groovy-all", "3.0.17")

buildSrc/call-site-instrumentation-plugin/build.gradle.kts

@@ -32,11 +32,11 @@ dependencies {
   compileOnly("com.google.code.findbugs", "jsr305", "3.0.2")
 
   implementation("org.freemarker", "freemarker", "2.3.30")
-  implementation("org.ow2.asm", "asm", "9.7.1")
-  implementation("org.ow2.asm", "asm-tree", "9.7.1")
+  implementation("org.ow2.asm", "asm", "9.8")
+  implementation("org.ow2.asm", "asm-tree", "9.8")
   implementation("com.github.javaparser", "javaparser-symbol-solver-core", "3.24.4")
 
-  testImplementation("net.bytebuddy", "byte-buddy", "1.14.18")
+  testImplementation("net.bytebuddy", "byte-buddy", "1.15.11")
   testImplementation("org.spockframework", "spock-core", "2.0-groovy-3.0")
   testImplementation("org.objenesis", "objenesis", "3.0.1")
   testImplementation("org.codehaus.groovy", "groovy-all", "3.0.17")

buildSrc/src/test/groovy/CallSiteInstrumentationPluginTest.groovy

@@ -27,7 +27,7 @@ class CallSiteInstrumentationPluginTest extends Specification {
     }
     
     dependencies {
-      implementation group: 'net.bytebuddy', name: 'byte-buddy', version: '1.14.18'
+      implementation group: 'net.bytebuddy', name: 'byte-buddy', version: '1.15.11'
       implementation group: 'com.google.auto.service', name: 'auto-service-annotations', version: '1.0-rc7'
     }
   '''

buildSrc/src/test/groovy/InstrumentPluginTest.groovy

@@ -23,7 +23,7 @@ class InstrumentPluginTest extends Specification {
     }
 
     dependencies {
-      compileOnly group: 'net.bytebuddy', name: 'byte-buddy', version: '1.14.18' // just to build TestPlugin
+      compileOnly group: 'net.bytebuddy', name: 'byte-buddy', version: '1.15.11' // just to build TestPlugin
     }
 
     apply plugin: 'instrument'

dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/CrashUploader.java

@@ -121,6 +121,7 @@ public void upload(@Nonnull List<Path> files) throws IOException {
     }
   }
 
+  @SuppressForbidden
   boolean uploadToLogs(@Nonnull Path file) {
     try {
       uploadToLogs(new String(Files.readAllBytes(file), StandardCharsets.UTF_8), System.out);
@@ -156,6 +157,7 @@ void uploadToLogs(@Nonnull String message, @Nonnull PrintStream out) throws IOEx
   }
 
   // @VisibleForTesting
+  @SuppressForbidden
   static String extractErrorKind(String fileContent) {
     Matcher matcher = ERROR_MESSAGE_PATTERN.matcher(fileContent);
     if (!matcher.find()) {

dd-java-agent/agent-debugger/debugger-bootstrap/src/main/java/datadog/trace/bootstrap/debugger/el/ValueReferences.java

@@ -14,10 +14,14 @@ public final class ValueReferences {
   public static String RETURN_EXTENSION_NAME = "return";
   public static String ITERATOR_EXTENSION_NAME = "it";
   public static String EXCEPTION_EXTENSION_NAME = "exception";
+  public static String KEY_EXTENSION_NAME = "key";
+  public static String VALUE_EXTENSION_NAME = "value";
   public static String DURATION_REF = SYNTHETIC_PREFIX + DURATION_EXTENSION_NAME;
   public static String RETURN_REF = SYNTHETIC_PREFIX + RETURN_EXTENSION_NAME;
   public static String ITERATOR_REF = SYNTHETIC_PREFIX + ITERATOR_EXTENSION_NAME;
   public static String EXCEPTION_REF = SYNTHETIC_PREFIX + EXCEPTION_EXTENSION_NAME;
+  public static String KEY_REF = SYNTHETIC_PREFIX + KEY_EXTENSION_NAME;
+  public static String VALUE_REF = SYNTHETIC_PREFIX + VALUE_EXTENSION_NAME;
 
   public static String synthetic(String name) {
     return SYNTHETIC_PREFIX + name;

dd-java-agent/agent-debugger/debugger-el/src/main/java/com/datadog/debugger/el/PrettyPrintVisitor.java

@@ -89,27 +89,27 @@ public String visit(EndsWithExpression endsWithExpression) {
   public String visit(FilterCollectionExpression filterCollectionExpression) {
     return "filter("
         + nullSafeAccept(filterCollectionExpression.getSource())
-        + ", "
+        + ", {"
         + nullSafeAccept(filterCollectionExpression.getFilterExpression())
-        + ")";
+        + "})";
   }
 
   @Override
   public String visit(HasAllExpression hasAllExpression) {
     return "all("
         + nullSafeAccept(hasAllExpression.getValueExpression())
-        + ", "
+        + ", {"
         + nullSafeAccept(hasAllExpression.getFilterPredicateExpression())
-        + ")";
+        + "})";
   }
 
   @Override
   public String visit(HasAnyExpression hasAnyExpression) {
     return "any("
         + nullSafeAccept(hasAnyExpression.getValueExpression())
-        + ", "
+        + ", {"
         + nullSafeAccept(hasAnyExpression.getFilterPredicateExpression())
-        + ")";
+        + "})";
   }
 
   @Override

dd-java-agent/agent-debugger/debugger-el/src/main/java/com/datadog/debugger/el/expressions/FilterCollectionExpression.java

@@ -58,13 +58,14 @@ public CollectionValue<?> evaluate(ValueReferenceResolver valueRefResolver) {
     } else if (collectionValue instanceof MapValue) {
       MapValue materialized = (MapValue) collectionValue;
       Map<Object, Object> filtered = new HashMap<>();
-
       for (Value<?> key : materialized.getKeys()) {
         Value<?> value = key.isUndefined() ? Value.undefinedValue() : materialized.get(key);
-        if (filterExpression.evaluate(
-            valueRefResolver.withExtensions(
-                Collections.singletonMap(
-                    ValueReferences.ITERATOR_EXTENSION_NAME, new MapValue.Entry(key, value))))) {
+        Map<String, Object> valueRefExtensions = new HashMap<>();
+        valueRefExtensions.put(ValueReferences.KEY_EXTENSION_NAME, key);
+        valueRefExtensions.put(ValueReferences.VALUE_EXTENSION_NAME, value);
+        valueRefExtensions.put(
+            ValueReferences.ITERATOR_EXTENSION_NAME, new MapValue.Entry(key, value));
+        if (filterExpression.evaluate(valueRefResolver.withExtensions(valueRefExtensions))) {
           filtered.put(key.getValue(), value.getValue());
         }
       }

dd-java-agent/agent-debugger/debugger-el/src/main/java/com/datadog/debugger/el/expressions/HasAllExpression.java

@@ -11,6 +11,8 @@
 import datadog.trace.bootstrap.debugger.el.ValueReferences;
 import datadog.trace.bootstrap.debugger.util.WellKnownClasses;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Set;
 
 /**
@@ -65,10 +67,13 @@ public Boolean evaluate(ValueReferenceResolver valueRefResolver) {
       }
       for (Value<?> key : map.getKeys()) {
         Value<?> val = key.isUndefined() ? Value.undefinedValue() : map.get(key);
+        Map<String, Object> valueRefExtensions = new HashMap<>();
+        valueRefExtensions.put(ValueReferences.KEY_EXTENSION_NAME, key);
+        valueRefExtensions.put(ValueReferences.VALUE_EXTENSION_NAME, val);
+        valueRefExtensions.put(
+            ValueReferences.ITERATOR_EXTENSION_NAME, new MapValue.Entry(key, val));
         if (!filterPredicateExpression.evaluate(
-            valueRefResolver.withExtensions(
-                Collections.singletonMap(
-                    ValueReferences.ITERATOR_EXTENSION_NAME, new MapValue.Entry(key, val))))) {
+            valueRefResolver.withExtensions(valueRefExtensions))) {
           return Boolean.FALSE;
         }
       }

dd-java-agent/agent-debugger/debugger-el/src/main/java/com/datadog/debugger/el/expressions/HasAnyExpression.java

@@ -11,6 +11,8 @@
 import datadog.trace.bootstrap.debugger.el.ValueReferences;
 import datadog.trace.bootstrap.debugger.util.WellKnownClasses;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Set;
 
 /**
@@ -65,11 +67,13 @@ public Boolean evaluate(ValueReferenceResolver valueRefResolver) {
       }
       for (Value<?> key : map.getKeys()) {
         Value<?> val = key.isUndefined() ? Value.undefinedValue() : map.get(key);
-
+        Map<String, Object> valueRefExtensions = new HashMap<>();
+        valueRefExtensions.put(ValueReferences.KEY_EXTENSION_NAME, key);
+        valueRefExtensions.put(ValueReferences.VALUE_EXTENSION_NAME, val);
+        valueRefExtensions.put(
+            ValueReferences.ITERATOR_EXTENSION_NAME, new MapValue.Entry(key, val));
         if (filterPredicateExpression.evaluate(
-            valueRefResolver.withExtensions(
-                Collections.singletonMap(
-                    ValueReferences.ITERATOR_EXTENSION_NAME, new MapValue.Entry(key, val))))) {
+            valueRefResolver.withExtensions(valueRefExtensions))) {
           return Boolean.TRUE;
         }
       }

dd-java-agent/agent-debugger/debugger-el/src/test/java/com/datadog/debugger/el/expressions/FilterCollectionExpressionTest.java

@@ -28,7 +28,7 @@ void testMatchingList() {
     assertFalse(filtered.isEmpty());
     assertFalse(filtered.isNull());
     assertFalse(filtered.isUndefined());
-    assertEquals("filter(int[], @it < 2)", print(expression));
+    assertEquals("filter(int[], {@it < 2})", print(expression));
   }
 
   @Test
@@ -41,7 +41,7 @@ void testEmptyList() {
     assertTrue(filtered.isEmpty());
     assertFalse(filtered.isNull());
     assertFalse(filtered.isUndefined());
-    assertEquals("filter(int[], @it < 2)", print(expression));
+    assertEquals("filter(int[], {@it < 2})", print(expression));
   }
 
   @Test
@@ -52,7 +52,7 @@ void testNullList() {
     CollectionValue<?> filtered = expression.evaluate(RefResolverHelper.createResolver(this));
     assertEquals(collection, filtered);
     assertTrue(filtered.isNull());
-    assertEquals("filter(null, @it < 2)", print(expression));
+    assertEquals("filter(null, {@it < 2})", print(expression));
   }
 
   @Test
@@ -63,7 +63,7 @@ void testNullObjectList() {
     CollectionValue<?> filtered = expression.evaluate(RefResolverHelper.createResolver(this));
     assertEquals(collection, filtered);
     assertTrue(filtered.isNull());
-    assertEquals("filter(null, @it < 2)", print(expression));
+    assertEquals("filter(null, {@it < 2})", print(expression));
   }
 
   @Test
@@ -74,7 +74,7 @@ void testUndefinedList() {
     CollectionValue<?> filtered = expression.evaluate(RefResolverHelper.createResolver(this));
     assertEquals(collection, filtered);
     assertTrue(filtered.isUndefined());
-    assertEquals("filter(null, @it < 2)", print(expression));
+    assertEquals("filter(null, {@it < 2})", print(expression));
   }
 
   @Test
@@ -104,7 +104,7 @@ void testMatchingMap() {
     assertFalse(filtered.isEmpty());
     assertFalse(filtered.isNull());
     assertFalse(filtered.isUndefined());
-    assertEquals("filter(Map, @it.value < 2)", print(expression));
+    assertEquals("filter(Map, {@it.value < 2})", print(expression));
   }
 
   @Test
@@ -117,7 +117,7 @@ void testEmptyMap() {
     assertTrue(filtered.isEmpty());
     assertFalse(filtered.isNull());
     assertFalse(filtered.isUndefined());
-    assertEquals("filter(Map, @it < 2)", print(expression));
+    assertEquals("filter(Map, {@it < 2})", print(expression));
   }
 
   @Test
@@ -128,7 +128,7 @@ void testNullMap() {
     CollectionValue<?> filtered = expression.evaluate(RefResolverHelper.createResolver(this));
     assertEquals(collection, filtered);
     assertTrue(filtered.isNull());
-    assertEquals("filter(null, @it < 2)", print(expression));
+    assertEquals("filter(null, {@it < 2})", print(expression));
   }
 
   @Test
@@ -139,7 +139,7 @@ void testNullObjectMap() {
     CollectionValue<?> filtered = expression.evaluate(RefResolverHelper.createResolver(this));
     assertEquals(collection, filtered);
     assertTrue(filtered.isNull());
-    assertEquals("filter(null, @it < 2)", print(expression));
+    assertEquals("filter(null, {@it < 2})", print(expression));
   }
 
   @Test
@@ -150,6 +150,29 @@ void testUndefinedMap() {
     CollectionValue<?> filtered = expression.evaluate(RefResolverHelper.createResolver(this));
     assertEquals(collection, filtered);
     assertTrue(filtered.isUndefined());
-    assertEquals("filter(null, @it < 2)", print(expression));
+    assertEquals("filter(null, {@it < 2})", print(expression));
+  }
+
+  @Test
+  void keyValueMap() {
+    Map<String, Integer> map = new HashMap<>();
+    map.put("a", 1);
+    map.put("b", 2);
+    map.put("c", 3);
+    MapValue collection = new MapValue(map);
+
+    FilterCollectionExpression expression =
+        new FilterCollectionExpression(collection, eq(ref(ValueReferences.KEY_REF), value("b")));
+    CollectionValue<?> filtered = expression.evaluate(RefResolverHelper.createResolver(this));
+    assertNotEquals(collection, filtered);
+    assertEquals(1, filtered.count());
+    assertEquals("filter(Map, {@key == \"b\"})", print(expression));
+
+    expression =
+        new FilterCollectionExpression(collection, eq(ref(ValueReferences.VALUE_REF), value(2)));
+    filtered = expression.evaluate(RefResolverHelper.createResolver(this));
+    assertNotEquals(collection, filtered);
+    assertEquals(1, filtered.count());
+    assertEquals("filter(Map, {@value == 2})", print(expression));
   }
 }