Allow login events SDK to be used with appsec disabled (#8464)

Allow login event SDK to be used with appsec disabled

Author: manuel-alvarez-alvarez

.circleci/config.continue.yml.j2

@@ -862,6 +862,8 @@ jobs:
               APPSEC_API_SECURITY
               APPSEC_API_SECURITY_RC
               APPSEC_API_SECURITY_WITH_SAMPLING
+              APPSEC_AUTO_EVENTS_RC
+              APPSEC_AUTO_EVENTS_EXTENDED
               APPSEC_WAF_TELEMETRY
               APPSEC_STANDALONE_V2
               IAST_STANDALONE_V2

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java

@@ -16,6 +16,7 @@
 import datadog.trace.api.Platform;
 import datadog.trace.api.StatsDClientManager;
 import datadog.trace.api.WithGlobalTracer;
+import datadog.trace.api.appsec.AppSecEventTracker;
 import datadog.trace.api.config.AppSecConfig;
 import datadog.trace.api.config.CiVisibilityConfig;
 import datadog.trace.api.config.CwsConfig;
@@ -816,6 +817,14 @@ private static StatsDClientManager statsDClientManager() throws Exception {
   }
 
   private static void maybeStartAppSec(Class<?> scoClass, Object o) {
+
+    try {
+      // event tracking SDK must be available for customers even if AppSec is fully disabled
+      AppSecEventTracker.install();
+    } catch (final Exception e) {
+      log.debug("Error starting AppSec Event Tracker", e);
+    }
+
     if (!(appSecEnabled || (remoteConfigEnabled && !appSecFullyDisabled))) {
       return;
     }

dd-java-agent/appsec/src/main/java/com/datadog/appsec/AppSecSystem.java

@@ -17,7 +17,6 @@
 import datadog.remoteconfig.ConfigurationPoller;
 import datadog.trace.api.Config;
 import datadog.trace.api.ProductActivation;
-import datadog.trace.api.appsec.AppSecEventTracker;
 import datadog.trace.api.gateway.SubscriptionService;
 import datadog.trace.api.telemetry.ProductChange;
 import datadog.trace.api.telemetry.ProductChangeCollector;
@@ -97,8 +96,6 @@ private static void doStart(SubscriptionService gw, SharedCommunicationObjects s
 
     Blocking.setBlockingService(new BlockingServiceImpl(REPLACEABLE_EVENT_PRODUCER));
 
-    AppSecEventTracker.setEventTracker(new AppSecEventTracker());
-
     STARTED.set(true);
 
     String startedAppSecModules = String.join(", ", STARTED_MODULES_INFO.values());

dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/AppSecRequestContext.java

@@ -8,7 +8,6 @@
 import com.datadog.appsec.report.AppSecEvent;
 import com.datadog.appsec.util.StandardizedLogging;
 import datadog.trace.api.Config;
-import datadog.trace.api.UserIdCollectionMode;
 import datadog.trace.api.http.StoredBodySupplier;
 import datadog.trace.api.internal.TraceSegment;
 import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
@@ -136,10 +135,8 @@ public class AppSecRequestContext implements DataBundle, Closeable {
 
   // keep a reference to the last published usr.id
   private volatile String userId;
-  private volatile UserIdCollectionMode userIdSource;
   // keep a reference to the last published usr.login
   private volatile String userLogin;
-  private volatile UserIdCollectionMode userLoginSource;
   // keep a reference to the last published usr.session_id
   private volatile String sessionId;
 
@@ -536,36 +533,30 @@ public void setRespDataPublished(boolean respDataPublished) {
     this.respDataPublished = respDataPublished;
   }
 
-  public String getUserId() {
-    return userId;
-  }
-
-  public void setUserId(String userId) {
+  /**
+   * Updates the current used usr.id
+   *
+   * @return {@code false} if the user id has not been updated
+   */
+  public boolean updateUserId(String userId) {
+    if (Objects.equals(this.userId, userId)) {
+      return false;
+    }
     this.userId = userId;
+    return true;
   }
 
-  public UserIdCollectionMode getUserIdSource() {
-    return userIdSource;
-  }
-
-  public void setUserIdSource(UserIdCollectionMode userIdSource) {
-    this.userIdSource = userIdSource;
-  }
-
-  public String getUserLogin() {
-    return userLogin;
-  }
-
-  public void setUserLogin(String userLogin) {
+  /**
+   * Updates current used usr.login
+   *
+   * @return {@code false} if the user login has not been updated
+   */
+  public boolean updateUserLogin(String userLogin) {
+    if (Objects.equals(this.userLogin, userLogin)) {
+      return false;
+    }
     this.userLogin = userLogin;
-  }
-
-  public UserIdCollectionMode getUserLoginSource() {
-    return userLoginSource;
-  }
-
-  public void setUserLoginSource(UserIdCollectionMode userLoginSource) {
-    this.userLoginSource = userLoginSource;
+    return true;
   }
 
   public void setSessionId(String sessionId) {

dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/GatewayBridge.java

@@ -1,16 +1,10 @@
 package com.datadog.appsec.gateway;
 
 import static com.datadog.appsec.event.data.MapDataBundle.Builder.CAPACITY_0_2;
-import static com.datadog.appsec.event.data.MapDataBundle.Builder.CAPACITY_3_4;
 import static com.datadog.appsec.event.data.MapDataBundle.Builder.CAPACITY_6_10;
 import static com.datadog.appsec.gateway.AppSecRequestContext.DEFAULT_REQUEST_HEADERS_ALLOW_LIST;
 import static com.datadog.appsec.gateway.AppSecRequestContext.REQUEST_HEADERS_ALLOW_LIST;
 import static com.datadog.appsec.gateway.AppSecRequestContext.RESPONSE_HEADERS_ALLOW_LIST;
-import static datadog.trace.api.UserIdCollectionMode.ANONYMIZATION;
-import static datadog.trace.api.UserIdCollectionMode.DISABLED;
-import static datadog.trace.api.UserIdCollectionMode.SDK;
-import static datadog.trace.api.telemetry.LogCollector.SEND_TELEMETRY;
-import static datadog.trace.util.Strings.toHexString;
 
 import com.datadog.appsec.AppSecSystem;
 import com.datadog.appsec.api.security.ApiSecurityRequestSampler;
@@ -28,7 +22,6 @@
 import com.datadog.appsec.report.AppSecEventWrapper;
 import datadog.trace.api.Config;
 import datadog.trace.api.ProductTraceSource;
-import datadog.trace.api.UserIdCollectionMode;
 import datadog.trace.api.gateway.Events;
 import datadog.trace.api.gateway.Flow;
 import datadog.trace.api.gateway.IGSpanInfo;
@@ -48,19 +41,17 @@
 import java.net.URISyntaxException;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.EnumMap;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import org.slf4j.Logger;
@@ -76,21 +67,16 @@ public class GatewayBridge {
   private static final Pattern QUERY_PARAM_SPLITTER = Pattern.compile("&");
   private static final Map<String, List<String>> EMPTY_QUERY_PARAMS = Collections.emptyMap();
 
-  private static final int HASH_SIZE_BYTES = 16; // 128 bits
-  private static final String ANON_PREFIX = "anon_";
-  private static final AtomicBoolean SHA_MISSING_REPORTED = new AtomicBoolean(false);
-
   /** User tracking tags that will force the collection of request headers */
   private static final String[] USER_TRACKING_TAGS = {
     "appsec.events.users.login.success.track", "appsec.events.users.login.failure.track"
   };
 
-  private static final Map<String, LoginEvent> EVENT_MAPPINGS = new HashMap<>();
+  private static final Map<LoginEvent, Address<?>> EVENT_MAPPINGS = new EnumMap<>(LoginEvent.class);
 
   static {
-    EVENT_MAPPINGS.put("users.login.success", LoginEvent.LOGIN_SUCCESS);
-    EVENT_MAPPINGS.put("users.login.failure", LoginEvent.LOGIN_FAILURE);
-    EVENT_MAPPINGS.put("users.signup", LoginEvent.SIGN_UP);
+    EVENT_MAPPINGS.put(LoginEvent.LOGIN_SUCCESS, KnownAddresses.LOGIN_SUCCESS);
+    EVENT_MAPPINGS.put(LoginEvent.LOGIN_FAILURE, KnownAddresses.LOGIN_FAILURE);
   }
 
   private static final String METASTRUCT_EXPLOIT = "exploit";
@@ -198,44 +184,16 @@ public void reset() {
     shellCmdSubInfo = null;
   }
 
-  private Flow<Void> onUser(
-      final RequestContext ctx_, final UserIdCollectionMode mode, final String originalUser) {
-    if (mode == DISABLED) {
-      return NoopFlow.INSTANCE;
-    }
-    final String user = anonymizeUser(mode, originalUser);
-    if (user == null) {
-      return NoopFlow.INSTANCE;
-    }
+  private Flow<Void> onUser(final RequestContext ctx_, final String user) {
     final AppSecRequestContext ctx = ctx_.getData(RequestContextSlot.APPSEC);
     if (ctx == null) {
       return NoopFlow.INSTANCE;
     }
-    final TraceSegment segment = ctx_.getTraceSegment();
-
-    // span with ASM data
-    segment.setTagTop(Tags.ASM_KEEP, true);
-    segment.setTagTop(Tags.PROPAGATED_TRACE_SOURCE, ProductTraceSource.ASM);
-
-    // skip event if we have an SDK one
-    if (mode != SDK) {
-      segment.setTagTop("_dd.appsec.usr.id", user);
-      if (ctx.getUserIdSource() == SDK) {
-        return NoopFlow.INSTANCE;
-      }
-    }
-
-    // update span tags
-    segment.setTagTop("usr.id", user);
-    segment.setTagTop("_dd.appsec.user.collection_mode", mode.fullName());
 
     // update current context with new user id
-    ctx.setUserIdSource(mode);
-    final boolean newUserId = !user.equals(ctx.getUserId());
-    if (!newUserId) {
+    if (!ctx.updateUserId(user)) {
       return NoopFlow.INSTANCE;
     }
-    ctx.setUserId(user);
 
     // call waf if we have a new user id
     while (true) {
@@ -259,96 +217,29 @@ private Flow<Void> onUser(
   }
 
   private Flow<Void> onLoginEvent(
-      final RequestContext ctx_,
-      final UserIdCollectionMode mode,
-      final String eventName,
-      final Boolean exists,
-      final String originalUser,
-      final Map<String, String> metadata) {
-    if (mode == DISABLED) {
-      return NoopFlow.INSTANCE;
-    }
+      final RequestContext ctx_, final LoginEvent event, final String login) {
     final AppSecRequestContext ctx = ctx_.getData(RequestContextSlot.APPSEC);
     if (ctx == null) {
       return NoopFlow.INSTANCE;
     }
-    final TraceSegment segment = ctx_.getTraceSegment();
-
-    // span with ASM data
-    segment.setTagTop(Tags.ASM_KEEP, true);
-    segment.setTagTop(Tags.PROPAGATED_TRACE_SOURCE, ProductTraceSource.ASM);
-
-    // update span tags
-    segment.setTagTop("appsec.events." + eventName + ".track", true, true);
-    if (metadata != null && !metadata.isEmpty()) {
-      segment.setTagTop("appsec.events." + eventName, metadata, true);
-    }
-    if (mode == SDK) {
-      segment.setTagTop("_dd.appsec.events." + eventName + ".sdk", true, true);
-    } else {
-      segment.setTagTop("_dd.appsec.events." + eventName + ".auto.mode", mode.fullName(), true);
-    }
-
-    if (exists != null) {
-      if (mode == SDK || ctx.getUserLoginSource() != SDK) {
-        segment.setTagTop("appsec.events." + eventName + ".usr.exists", exists, true);
-      }
-    }
-
-    final String user = anonymizeUser(mode, originalUser);
-    if (user == null) {
-      // can happen in custom events
-      return NoopFlow.INSTANCE;
-    }
-
-    // parse the event (might be null for custom events sent via the SDK)
-    final LoginEvent sourceEvent = EVENT_MAPPINGS.get(eventName);
-
-    // skip event if we have an SDK one
-    if (mode != SDK) {
-      segment.setTagTop("_dd.appsec.usr.login", user);
-      if (ctx.getUserLoginSource() == SDK) {
-        return NoopFlow.INSTANCE;
-      }
-    } else {
-      if (sourceEvent == LoginEvent.LOGIN_SUCCESS) {
-        segment.setTagTop("usr.id", user, false);
-      } else {
-        segment.setTagTop("appsec.events." + eventName + ".usr.id", user, true);
-      }
-      segment.setTagTop("_dd.appsec.user.collection_mode", mode.fullName());
-    }
-
-    // update user span tags
-    segment.setTagTop("appsec.events." + eventName + ".usr.login", user, true);
 
     // update current context with new user login
-    ctx.setUserLoginSource(mode);
-    if (mode == SDK) {
-      ctx.setUserIdSource(mode); // we are setting the usr.id through the SDK
-    }
-    final boolean newUserLogin = !user.equals(ctx.getUserLogin());
-    if (!newUserLogin) {
+    if (!ctx.updateUserLogin(login)) {
       return NoopFlow.INSTANCE;
     }
-    ctx.setUserLogin(user);
 
     // call waf if we have a new user login
-    final List<Address<?>> addresses = new ArrayList<>(3);
-    final MapDataBundle.Builder bundleBuilder = new MapDataBundle.Builder(CAPACITY_3_4);
+    final List<Address<?>> addresses = new ArrayList<>(2);
+    final MapDataBundle.Builder bundleBuilder = new MapDataBundle.Builder(CAPACITY_0_2);
     addresses.add(KnownAddresses.USER_LOGIN);
-    bundleBuilder.add(KnownAddresses.USER_LOGIN, user);
-    if (mode == SDK) {
-      addresses.add(KnownAddresses.USER_ID);
-      bundleBuilder.add(KnownAddresses.USER_ID, user);
-    }
-    // we don't support null values for the address so we use an invalid placeholder here
-    if (sourceEvent == LoginEvent.LOGIN_SUCCESS) {
-      addresses.add(KnownAddresses.LOGIN_SUCCESS);
-      bundleBuilder.add(KnownAddresses.LOGIN_SUCCESS, "invalid");
-    } else if (sourceEvent == LoginEvent.LOGIN_FAILURE) {
-      addresses.add(KnownAddresses.LOGIN_FAILURE);
-      bundleBuilder.add(KnownAddresses.LOGIN_FAILURE, "invalid");
+    bundleBuilder.add(KnownAddresses.USER_LOGIN, login);
+
+    // parse the event
+    Address<?> address = EVENT_MAPPINGS.get(event);
+    if (address != null) {
+      addresses.add(address);
+      // we don't support null values for the address so we use an invalid placeholder here
+      bundleBuilder.add(address, "invalid");
     }
     final DataBundle bundle = bundleBuilder.build();
     final String subInfoKey =
@@ -1159,33 +1050,6 @@ private static int byteToDigit(byte b) {
     return -1;
   }
 
-  protected static String anonymizeUser(final UserIdCollectionMode mode, final String userId) {
-    if (mode != ANONYMIZATION || userId == null) {
-      return userId;
-    }
-    MessageDigest digest;
-    try {
-      // TODO avoid lookup a new instance every time
-      digest = MessageDigest.getInstance("SHA-256");
-    } catch (NoSuchAlgorithmException e) {
-      if (!SHA_MISSING_REPORTED.getAndSet(true)) {
-        log.error(
-            SEND_TELEMETRY,
-            "Missing SHA-256 digest, user collection in 'anon' mode cannot continue",
-            e);
-      }
-      return null;
-    }
-    digest.update(userId.getBytes());
-    byte[] hash = digest.digest();
-    if (hash.length > HASH_SIZE_BYTES) {
-      byte[] temp = new byte[HASH_SIZE_BYTES];
-      System.arraycopy(hash, 0, temp, 0, temp.length);
-      hash = temp;
-    }
-    return ANON_PREFIX + toHexString(hash);
-  }
-
   private static class IGAppSecEventDependencies {
 
     private static final Map<Address<?>, Collection<datadog.trace.api.gateway.EventType<?>>>