Merge branch 'master' into daniel.mohedano/suppress-repo-index-resolution-exception

Author: daniel-mohedano

.circleci/config.continue.yml.j2

@@ -36,7 +36,7 @@ instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation
 debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
 profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
 
-default_system_tests_commit: &default_system_tests_commit 1de04c42cbd9783432258004e46eb5982bbc9fe5
+default_system_tests_commit: &default_system_tests_commit 55cc62fb333b8de1466f8a0ecfdfa55fe9f57031
 
 parameters:
   nightly:

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java

@@ -250,6 +250,11 @@ public static void start(
       setSystemPropertyDefault(
           propertyNameToSystemPropertyName("integration.kafka.enabled"), "true");
 
+      if (Config.get().isDataJobsOpenLineageEnabled()) {
+        setSystemPropertyDefault(
+            propertyNameToSystemPropertyName("integration.spark-openlineage.enabled"), "true");
+      }
+
       String javaCommand = System.getProperty("sun.java.command");
       String dataJobsCommandPattern = Config.get().getDataJobsCommandPattern();
       if (!isDataJobsSupported(javaCommand, dataJobsCommandPattern)) {

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/AsyncResultExtensions.java

@@ -61,17 +61,17 @@ public Object apply(Object result, AgentSpan span) {
       }
       return null;
     }
+  }
 
-    private <T> BiConsumer<T, Throwable> finishSpan(AgentSpan span) {
-      return (o, throwable) -> {
-        if (throwable != null) {
-          span.addThrowable(
-              throwable instanceof ExecutionException || throwable instanceof CompletionException
-                  ? throwable.getCause()
-                  : throwable);
-        }
-        span.finish();
-      };
-    }
+  public static <T> BiConsumer<T, Throwable> finishSpan(AgentSpan span) {
+    return (o, throwable) -> {
+      if (throwable != null) {
+        span.addThrowable(
+            throwable instanceof ExecutionException || throwable instanceof CompletionException
+                ? throwable.getCause()
+                : throwable);
+      }
+      span.finish();
+    };
   }
 }

dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/context/OtelContext.java

@@ -1,170 +1,109 @@
 package datadog.opentelemetry.shim.context;
 
 import datadog.opentelemetry.shim.trace.OtelSpan;
-import datadog.trace.bootstrap.instrumentation.api.AgentScope;
 import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
-import datadog.trace.bootstrap.instrumentation.api.AgentTracer;
 import datadog.trace.bootstrap.instrumentation.api.AttachableWrapper;
-import io.opentelemetry.api.trace.Span;
 import io.opentelemetry.context.Context;
 import io.opentelemetry.context.ContextKey;
 import io.opentelemetry.context.Scope;
-import java.util.Arrays;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 import javax.annotation.Nullable;
 import javax.annotation.ParametersAreNonnullByDefault;
 
+@SuppressWarnings({"rawtypes", "unchecked"})
 @ParametersAreNonnullByDefault
 public class OtelContext implements Context {
-  private static final Object[] NO_ENTRIES = {};
 
   /** Overridden root context. */
-  public static final Context ROOT = new OtelContext(OtelSpan.invalid(), OtelSpan.invalid());
+  public static final Context ROOT = new OtelContext(datadog.context.Context.root());
 
   private static final String OTEL_CONTEXT_SPAN_KEY = "opentelemetry-trace-span-key";
   private static final String OTEL_CONTEXT_ROOT_SPAN_KEY = "opentelemetry-traces-local-root-span";
 
-  /** Keep track of propagated context that has not been captured on the scope stack. */
-  private static final ThreadLocal<OtelContext> lastPropagated = new ThreadLocal<>();
+  /** Records the keys needed to access the delegate context, mapped by key name. */
+  private static final Map<ContextKey<?>, datadog.context.ContextKey<?>> DELEGATE_KEYS =
+      new ConcurrentHashMap<>();
 
-  private final Span currentSpan;
-  private final Span rootSpan;
+  private final datadog.context.Context delegate;
 
-  private final Object[] entries;
+  public OtelContext(datadog.context.Context delegate) {
+    this.delegate = delegate;
+  }
 
-  public OtelContext(Span currentSpan, Span rootSpan) {
-    this(currentSpan, rootSpan, NO_ENTRIES);
+  public static Context current() {
+    return new OtelContext(datadog.context.Context.current());
   }
 
-  public OtelContext(Span currentSpan, Span rootSpan, Object[] entries) {
-    this.currentSpan = currentSpan;
-    this.rootSpan = rootSpan;
-    this.entries = entries;
+  @Override
+  public Scope makeCurrent() {
+    return new OtelScope(Context.super.makeCurrent(), delegate.attach());
   }
 
   @Nullable
   @Override
-  @SuppressWarnings("unchecked")
   public <V> V get(ContextKey<V> key) {
     if (OTEL_CONTEXT_SPAN_KEY.equals(key.toString())) {
-      return (V) this.currentSpan;
+      AgentSpan span = AgentSpan.fromContext(delegate);
+      if (span != null) {
+        return (V) toOtelSpan(span);
+      }
+      // fall-through and check for non-datadog span data
     } else if (OTEL_CONTEXT_ROOT_SPAN_KEY.equals(key.toString())) {
-      return (V) this.rootSpan;
-    }
-    for (int i = 0; i < this.entries.length; i += 2) {
-      if (this.entries[i] == key) {
-        return (V) this.entries[i + 1];
+      AgentSpan span = AgentSpan.fromContext(delegate);
+      if (span != null) {
+        return (V) toOtelSpan(span.getLocalRootSpan());
       }
+      // fall-through and check for non-datadog span data
     }
-    return null;
+    return (V) delegate.get(delegateKey(key));
   }
 
   @Override
   public <V> Context with(ContextKey<V> key, V value) {
     if (OTEL_CONTEXT_SPAN_KEY.equals(key.toString())) {
-      return new OtelContext((Span) value, this.rootSpan, this.entries);
-    } else if (OTEL_CONTEXT_ROOT_SPAN_KEY.equals(key.toString())) {
-      return new OtelContext(this.currentSpan, (Span) value, this.entries);
-    }
-    Object[] newEntries = null;
-    int oldEntriesLength = this.entries.length;
-    for (int i = 0; i < oldEntriesLength; i += 2) {
-      if (this.entries[i] == key) {
-        if (this.entries[i + 1] == value) {
-          return this;
-        }
-        newEntries = this.entries.clone();
-        newEntries[i + 1] = value;
-        break;
+      if (value instanceof OtelSpan) {
+        AgentSpan span = ((OtelSpan) value).asAgentSpan();
+        return new OtelContext(delegate.with(span));
       }
+      // fall-through and store as non-datadog span data
     }
-    if (null == newEntries) {
-      newEntries = Arrays.copyOf(this.entries, oldEntriesLength + 2);
-      newEntries[oldEntriesLength] = key;
-      newEntries[oldEntriesLength + 1] = value;
-    }
-    return new OtelContext(this.currentSpan, this.rootSpan, newEntries);
+    return new OtelContext(delegate.with(delegateKey(key), value));
   }
 
   @Override
-  public Scope makeCurrent() {
-    final Scope scope = Context.super.makeCurrent();
-    if (this.currentSpan instanceof OtelSpan) {
-      // only keep propagated context until next span activation
-      lastPropagated.remove();
-      AgentScope agentScope = ((OtelSpan) this.currentSpan).activate();
-      return new OtelScope(scope, agentScope, this.entries);
-    } else {
-      // propagated context not on the scope stack, capture it here
-      lastPropagated.set(this);
-      return new Scope() {
-        @Override
-        public void close() {
-          lastPropagated.remove();
-          scope.close();
-        }
-      };
-    }
-  }
-
-  public static Context current() {
-    // Check for propagated context not on the scope stack
-    Context context = lastPropagated.get();
-    if (null != context) {
-      return context;
-    }
-    // Check empty context
-    AgentScope agentCurrentScope = AgentTracer.activeScope();
-    if (null == agentCurrentScope) {
-      return OtelContext.ROOT;
-    }
-    // Get OTel current span
-    Span otelCurrentSpan = null;
-    AgentSpan agentCurrentSpan = agentCurrentScope.span();
-    if (agentCurrentSpan instanceof AttachableWrapper) {
-      Object wrapper = ((AttachableWrapper) agentCurrentSpan).getWrapper();
-      if (wrapper instanceof OtelSpan) {
-        otelCurrentSpan = (OtelSpan) wrapper;
-      }
+  public boolean equals(Object o) {
+    if (o == null || getClass() != o.getClass()) {
+      return false;
     }
-    if (otelCurrentSpan == null) {
-      otelCurrentSpan = new OtelSpan(agentCurrentSpan);
-    }
-    // Get OTel root span
-    Span otelRootSpan = null;
-    AgentSpan agentRootSpan = agentCurrentSpan.getLocalRootSpan();
-    if (agentRootSpan instanceof AttachableWrapper) {
-      Object wrapper = ((AttachableWrapper) agentRootSpan).getWrapper();
-      if (wrapper instanceof OtelSpan) {
-        otelRootSpan = (OtelSpan) wrapper;
-      }
-    }
-    if (otelRootSpan == null) {
-      otelRootSpan = new OtelSpan(agentRootSpan);
-    }
-    // Get OTel custom context entries
-    Object[] contextEntries = NO_ENTRIES;
-    if (agentCurrentScope instanceof AttachableWrapper) {
-      Object wrapper = ((AttachableWrapper) agentCurrentScope).getWrapper();
-      if (wrapper instanceof OtelScope) {
-        contextEntries = ((OtelScope) wrapper).contextEntries();
-      }
-    }
-    return new OtelContext(otelCurrentSpan, otelRootSpan, contextEntries);
+    return delegate.equals(((OtelContext) o).delegate);
   }
 
-  /** Last propagated context not on the scope stack; {@code null} if there's no such context. */
-  @Nullable
-  public static Context lastPropagated() {
-    return lastPropagated.get();
+  @Override
+  public int hashCode() {
+    return delegate.hashCode();
   }
 
   @Override
   public String toString() {
-    return "OtelContext{"
-        + "currentSpan="
-        + this.currentSpan.getSpanContext()
-        + ", rootSpan="
-        + this.rootSpan.getSpanContext()
-        + '}';
+    return "OtelContext{" + "delegate=" + delegate + '}';
+  }
+
+  private static datadog.context.ContextKey delegateKey(ContextKey key) {
+    return DELEGATE_KEYS.computeIfAbsent(key, OtelContext::mapByKeyName);
+  }
+
+  private static datadog.context.ContextKey mapByKeyName(ContextKey key) {
+    return datadog.context.ContextKey.named(key.toString());
+  }
+
+  private static OtelSpan toOtelSpan(AgentSpan span) {
+    if (span instanceof AttachableWrapper) {
+      Object wrapper = ((AttachableWrapper) span).getWrapper();
+      if (wrapper instanceof OtelSpan) {
+        return (OtelSpan) wrapper;
+      }
+    }
+    return new OtelSpan(span);
   }
 }

dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/context/OtelScope.java

@@ -1,31 +1,20 @@
 package datadog.opentelemetry.shim.context;
 
-import datadog.trace.bootstrap.instrumentation.api.AgentScope;
-import datadog.trace.bootstrap.instrumentation.api.AttachableWrapper;
+import datadog.context.ContextScope;
 import io.opentelemetry.context.Scope;
 
 public class OtelScope implements Scope {
   private final Scope scope;
-  private final AgentScope delegate;
-  private final Object[] contextEntries;
+  private final ContextScope delegate;
 
-  public OtelScope(Scope scope, AgentScope delegate, Object[] contextEntries) {
+  public OtelScope(Scope scope, ContextScope delegate) {
     this.scope = scope;
     this.delegate = delegate;
-    this.contextEntries = contextEntries;
-    if (delegate instanceof AttachableWrapper) {
-      ((AttachableWrapper) delegate).attachWrapper(this);
-    }
-  }
-
-  /** Context entries from {@link OtelContext}, captured when the context was made current. */
-  Object[] contextEntries() {
-    return contextEntries;
   }
 
   @Override
   public void close() {
-    this.delegate.close();
-    this.scope.close();
+    delegate.close();
+    scope.close();
   }
 }

dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/context/propagation/AgentTextMapPropagator.java

@@ -7,7 +7,6 @@
 
 import datadog.opentelemetry.shim.context.OtelContext;
 import datadog.opentelemetry.shim.trace.OtelExtractedContext;
-import datadog.opentelemetry.shim.trace.OtelSpan;
 import datadog.trace.api.TracePropagationStyle;
 import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
 import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext;
@@ -59,7 +58,7 @@ public <C> Context extract(Context context, @Nullable C carrier, TextMapGetter<C
     } else {
       TraceState traceState = extractTraceState(extracted, carrier, getter);
       SpanContext spanContext = fromRemote(extracted, traceState);
-      return new OtelContext(Span.wrap(spanContext), OtelSpan.invalid());
+      return Span.wrap(spanContext).storeInContext(OtelContext.ROOT);
     }
   }
 

dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/trace/OtelSpanBuilder.java

@@ -11,7 +11,6 @@
 import static java.lang.Boolean.parseBoolean;
 import static java.util.Locale.ROOT;
 
-import datadog.opentelemetry.shim.context.OtelContext;
 import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
 import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext;
 import datadog.trace.bootstrap.instrumentation.api.AgentTracer;
@@ -202,11 +201,7 @@ public Span startSpan() {
       setSpanKind(INTERNAL);
     }
     if (!this.ignoreActiveSpan) {
-      // support automatic parenting from propagated context not on the scope stack
-      Context context = OtelContext.lastPropagated();
-      if (null != context) {
-        setParent(context);
-      }
+      setParent(Context.current());
     }
     AgentSpan delegate = this.delegate.start();
     // Apply overrides

dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie

@@ -357,6 +357,7 @@
 0 org.springframework.util.StreamUtils
 # Included for IAST Spring mvc unvalidated redirect and xss vulnerability
 0 org.springframework.web.method.support.HandlerMethodReturnValueHandlerComposite
+0 org.springframework.web.method.support.InvocableHandlerMethod
 2 org.xml.*
 2 org.yaml.snakeyaml.*
 # Need for IAST sink

dd-java-agent/appsec/build.gradle

@@ -15,7 +15,7 @@ dependencies {
   implementation project(':internal-api')
   implementation project(':communication')
   implementation project(':telemetry')
-  implementation group: 'io.sqreen', name: 'libsqreen', version: '12.0.0'
+  implementation group: 'io.sqreen', name: 'libsqreen', version: '13.0.1'
   implementation libs.moshi
 
   testImplementation libs.bytebuddy
@@ -68,8 +68,8 @@ ext {
   minimumInstructionCoverage = 0.8
   excludedClassesCoverage = [
     'com.datadog.appsec.config.MergedAsmData.InvalidAsmDataException',
-    'com.datadog.appsec.powerwaf.LibSqreenInitialization',
-    'com.datadog.appsec.powerwaf.PowerWAFModule.PowerWAFDataCallback',
+    'com.datadog.appsec.ddwaf.WafInitialization',
+    'com.datadog.appsec.ddwaf.WAFModule.WAFDataCallback',
     'com.datadog.appsec.report.*',
     'com.datadog.appsec.config.AppSecConfigServiceImpl.SubscribeFleetServiceRunnable.1',
     'com.datadog.appsec.util.StandardizedLogging',
@@ -90,7 +90,6 @@ ext {
     'com.datadog.appsec.config.CurrentAppSecConfig',
     // equals() / hashCode() are not well covered
     'com.datadog.appsec.config.AppSecConfig.Helper',
-    'com.datadog.appsec.powerwaf.PowerWAFModule.PowerWAFEventsCallback',
     // assert never fails
     'com.datadog.appsec.util.StandardizedLogging',
     'com.datadog.appsec.util.AbortStartupException',