Skip to content

Commit a62edd3

Browse files
mtoffl01mtoffl01
authored
Merge branch 'master' into mtoff/stable_cfg_2
2 parents a99d016 + 60f8046 commit a62edd3

File tree

73 files changed

+2552
-452
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

73 files changed

+2552
-452
lines changed

‎.circleci/config.continue.yml.j2

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation
3636
debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
3737
profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
3838

39-
default_system_tests_commit: &default_system_tests_commit 69a5e874384dd256e2e3f42fc1c95807a67efbe6
39+
default_system_tests_commit: &default_system_tests_commit 1ef00a34ad1f83ae999887e510ef1ea1c27b151b
4040

4141
parameters:
4242
nightly:
@@ -414,7 +414,7 @@ jobs:
414414
name: Check Project
415415
command: >-
416416
MAVEN_OPTS="-Xms64M -Xmx256M"
417-
GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
417+
GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx3G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
418418
./gradlew
419419
<< parameters.gradleTarget >>
420420
-PskipTests

‎dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/ThirdPartyLibraries.java

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,12 @@
77
import java.io.IOException;
88
import java.io.InputStream;
99
import java.io.InputStreamReader;
10+
import java.util.Arrays;
1011
import java.util.HashSet;
1112
import java.util.List;
1213
import java.util.Set;
1314
import java.util.stream.Collectors;
15+
import java.util.stream.Stream;
1416
import org.slf4j.Logger;
1517
import org.slf4j.LoggerFactory;
1618

@@ -22,6 +24,20 @@ public class ThirdPartyLibraries {
2224
private static final JsonAdapter<InternalConfig> ADAPTER =
2325
new Moshi.Builder().build().adapter(InternalConfig.class);
2426
private static final String FILE_NAME = "/third_party_libraries.json";
27+
private static final Set<String> DEFAULT_SHADING_IDENTIFIERS =
28+
new HashSet<>(
29+
Arrays.asList(
30+
"shaded",
31+
"thirdparty",
32+
"dependencies",
33+
"relocated",
34+
"bundled",
35+
"embedded",
36+
"vendor",
37+
"repackaged",
38+
"shadow",
39+
"shim",
40+
"wrapper"));
2541

2642
private ThirdPartyLibraries() {}
2743

@@ -46,6 +62,13 @@ public Set<String> getThirdPartyExcludes(Config config) {
4662
.collect(Collectors.toSet());
4763
}
4864

65+
public Set<String> getShadingIdentifiers(Config config) {
66+
Stream<String> configStream =
67+
config.getThirdPartyShadingIdentifiers().stream().filter(s -> !s.isEmpty());
68+
return Stream.concat(configStream, DEFAULT_SHADING_IDENTIFIERS.stream())
69+
.collect(Collectors.toSet());
70+
}
71+
4972
// Add a*, b*, c*, ..., z* to the exclude trie in ClassNameFiltering. Simply adding * does not
5073
// work.
5174
private static Set<String> getExcludeAll() {

‎dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/SymbolExtractionTransformer.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package com.datadog.debugger.symbol;
22

33
import datadog.trace.bootstrap.debugger.DebuggerContext.ClassNameFilter;
4+
import datadog.trace.util.Strings;
45
import java.lang.instrument.ClassFileTransformer;
56
import java.security.ProtectionDomain;
67
import org.slf4j.Logger;
@@ -34,7 +35,7 @@ public byte[] transform(
3435
// Don't parse our own classes to avoid duplicate class definition
3536
return null;
3637
}
37-
if (classNameFiltering.isExcluded(className)) {
38+
if (classNameFiltering.isExcluded(Strings.getClassName(className))) {
3839
return null;
3940
}
4041
symbolAggregator.parseClass(className, classfileBuffer, protectionDomain);

‎dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/util/ClassNameFiltering.java

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,36 +14,60 @@ public class ClassNameFiltering implements ClassNameFilter {
1414

1515
private final ClassNameTrie includeTrie;
1616
private final ClassNameTrie excludeTrie;
17+
private final ClassNameTrie shadingTrie;
1718

1819
public ClassNameFiltering(Config config) {
1920
this(
2021
ThirdPartyLibraries.INSTANCE.getThirdPartyLibraries(config),
21-
ThirdPartyLibraries.INSTANCE.getThirdPartyExcludes(config));
22+
ThirdPartyLibraries.INSTANCE.getThirdPartyExcludes(config),
23+
ThirdPartyLibraries.INSTANCE.getShadingIdentifiers(config));
2224
}
2325

2426
public ClassNameFiltering(Set<String> excludes) {
25-
this(excludes, Collections.emptySet());
27+
this(excludes, Collections.emptySet(), Collections.emptySet());
2628
}
2729

28-
public ClassNameFiltering(Set<String> excludes, Set<String> includes) {
30+
public ClassNameFiltering(
31+
Set<String> excludes, Set<String> includes, Set<String> shadingIdentifiers) {
2932
ClassNameTrie.Builder excludeBuilder = new ClassNameTrie.Builder();
3033
excludes.forEach(s -> excludeBuilder.put(s + "*", 1));
3134
this.excludeTrie = excludeBuilder.buildTrie();
3235
ClassNameTrie.Builder includeBuilder = new ClassNameTrie.Builder();
3336
includes.forEach(s -> includeBuilder.put(s + "*", 1));
3437
this.includeTrie = includeBuilder.buildTrie();
38+
ClassNameTrie.Builder shadingBuilder = new ClassNameTrie.Builder();
39+
shadingIdentifiers.forEach(s -> shadingBuilder.put(s + "*", 1));
40+
this.shadingTrie = shadingBuilder.buildTrie();
3541
}
3642

43+
// className is the fully qualified class name with '.' (Java type) notation
3744
public boolean isExcluded(String className) {
38-
return (includeTrie.apply(className) < 0 && excludeTrie.apply(className) > 0)
45+
int shadedIdx = shadedIndexOf(className);
46+
shadedIdx = Math.max(shadedIdx, 0);
47+
return (includeTrie.apply(className, shadedIdx) < 0
48+
&& excludeTrie.apply(className, shadedIdx) > 0)
3949
|| isLambdaProxyClass(className);
4050
}
4151

4252
static boolean isLambdaProxyClass(String className) {
4353
return LAMBDA_PROXY_CLASS_PATTERN.matcher(className).matches();
4454
}
4555

56+
int shadedIndexOf(String className) {
57+
int idx = 0;
58+
int previousIdx = 0;
59+
while ((idx = className.indexOf('.', previousIdx)) > 0) {
60+
if (shadingTrie.apply(className, previousIdx) > 0) {
61+
return idx + 1;
62+
}
63+
idx++;
64+
previousIdx = idx;
65+
}
66+
return -1;
67+
}
68+
4669
public static ClassNameFiltering allowAll() {
47-
return new ClassNameFiltering(Collections.emptySet(), Collections.emptySet());
70+
return new ClassNameFiltering(
71+
Collections.emptySet(), Collections.emptySet(), Collections.emptySet());
4872
}
4973
}

‎dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/ThirdPartyLibrariesTest.java

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package com.datadog.debugger.agent;
22

3+
import static org.junit.jupiter.api.Assertions.assertEquals;
34
import static org.junit.jupiter.api.Assertions.assertFalse;
45
import static org.junit.jupiter.api.Assertions.assertNotNull;
56
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -20,6 +21,7 @@ class ThirdPartyLibrariesTest {
2021
void setUp() {
2122
when(mockConfig.getThirdPartyIncludes()).thenReturn(Collections.emptySet());
2223
when(mockConfig.getThirdPartyExcludes()).thenReturn(Collections.emptySet());
24+
when(mockConfig.getThirdPartyShadingIdentifiers()).thenReturn(Collections.emptySet());
2325
}
2426

2527
@Test
@@ -29,7 +31,6 @@ void testGetExcludesContainsDefaultExclude() {
2931

3032
@Test
3133
void testGetExcludesWithExplicitExclude() {
32-
3334
when(mockConfig.getThirdPartyIncludes())
3435
.thenReturn(Collections.singleton("com.datadog.debugger"));
3536
assertTrue(
@@ -72,4 +73,22 @@ void testGetExcludeAll() {
7273
Set<String> excludeAll = ThirdPartyLibraries.INSTANCE.getThirdPartyLibraries(null);
7374
for (char c : ThirdPartyLibraries.ALPHABET) assertTrue(excludeAll.contains(String.valueOf(c)));
7475
}
76+
77+
@Test
78+
void testEmptyStrings() {
79+
int expectedIncludeDefaultSize =
80+
ThirdPartyLibraries.INSTANCE.getThirdPartyLibraries(mockConfig).size();
81+
int expectedShadingDefaultSize =
82+
ThirdPartyLibraries.INSTANCE.getShadingIdentifiers(mockConfig).size();
83+
when(mockConfig.getThirdPartyIncludes()).thenReturn(Collections.singleton(""));
84+
when(mockConfig.getThirdPartyExcludes()).thenReturn(Collections.singleton(""));
85+
when(mockConfig.getThirdPartyShadingIdentifiers()).thenReturn(Collections.singleton(""));
86+
assertEquals(
87+
expectedIncludeDefaultSize,
88+
ThirdPartyLibraries.INSTANCE.getThirdPartyLibraries(mockConfig).size());
89+
assertTrue(ThirdPartyLibraries.INSTANCE.getThirdPartyExcludes(mockConfig).isEmpty());
90+
assertEquals(
91+
expectedShadingDefaultSize,
92+
ThirdPartyLibraries.INSTANCE.getShadingIdentifiers(mockConfig).size());
93+
}
7594
}

‎dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/symbol/SymDBEnablementTest.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,8 @@ public void noDuplicateSymbolExtraction() {
169169
ClassNameFiltering classNameFiltering =
170170
new ClassNameFiltering(
171171
Collections.singleton("org.springframework."),
172-
Collections.singleton("com.datadog.debugger."));
172+
Collections.singleton("com.datadog.debugger."),
173+
Collections.emptySet());
173174
SymbolAggregator symbolAggregator = new SymbolAggregator(classNameFiltering, mockSymbolSink, 1);
174175
SymDBEnablement symDBEnablement =
175176
new SymDBEnablement(instr, config, symbolAggregator, classNameFiltering);

‎dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/symbol/SymbolExtractionTransformerTest.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -982,7 +982,8 @@ private SymbolExtractionTransformer createTransformer(
982982
return createTransformer(
983983
symbolSink,
984984
symbolFlushThreshold,
985-
new ClassNameFiltering(TRANSFORMER_EXCLUDES, Collections.singleton(SYMBOL_PACKAGE)));
985+
new ClassNameFiltering(
986+
TRANSFORMER_EXCLUDES, Collections.singleton(SYMBOL_PACKAGE), Collections.emptySet()));
986987
}
987988

988989
private SymbolExtractionTransformer createTransformer(

‎dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/util/ClassNameFilteringTest.java

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@
66
import static org.mockito.Mockito.mock;
77
import static org.mockito.Mockito.when;
88

9-
import com.datadog.debugger.agent.ThirdPartyLibraries;
109
import datadog.trace.api.Config;
10+
import java.util.Arrays;
1111
import java.util.Collections;
1212
import java.util.HashSet;
1313
import java.util.Set;
@@ -52,15 +52,18 @@ public void testIncludeOverridesExclude() {
5252
ClassNameFiltering classNameFiltering =
5353
new ClassNameFiltering(
5454
Collections.singleton("com.datadog.debugger"),
55-
Collections.singleton("com.datadog.debugger"));
55+
Collections.singleton("com.datadog.debugger"),
56+
Collections.emptySet());
5657
assertFalse(classNameFiltering.isExcluded("com.datadog.debugger.FooBar"));
5758
}
5859

5960
@Test
6061
public void testIncludePrefixOverridesExclude() {
6162
ClassNameFiltering classNameFiltering =
6263
new ClassNameFiltering(
63-
Collections.singleton("com.datadog.debugger"), Collections.singleton("com.datadog"));
64+
Collections.singleton("com.datadog.debugger"),
65+
Collections.singleton("com.datadog"),
66+
Collections.emptySet());
6467
assertFalse(classNameFiltering.isExcluded("com.datadog.debugger.FooBar"));
6568
}
6669

@@ -69,7 +72,8 @@ public void testIncludeSomeExcludeSome() {
6972
ClassNameFiltering classNameFiltering =
7073
new ClassNameFiltering(
7174
Stream.of("com.datadog.debugger", "org.junit").collect(Collectors.toSet()),
72-
Collections.singleton("com.datadog.debugger"));
75+
Collections.singleton("com.datadog.debugger"),
76+
Collections.emptySet());
7377
assertFalse(classNameFiltering.isExcluded("com.datadog.debugger.FooBar"));
7478
assertTrue(classNameFiltering.isExcluded("org.junit.FooBar"));
7579
}
@@ -88,11 +92,25 @@ public void testExcludeDefaults(String input) {
8892
Config config = mock(Config.class);
8993
when(config.getThirdPartyExcludes()).thenReturn(Collections.emptySet());
9094
when(config.getThirdPartyIncludes()).thenReturn(Collections.emptySet());
91-
ClassNameFiltering classNameFiltering =
92-
new ClassNameFiltering(ThirdPartyLibraries.INSTANCE.getThirdPartyLibraries(config));
95+
when(config.getThirdPartyShadingIdentifiers()).thenReturn(Collections.emptySet());
96+
ClassNameFiltering classNameFiltering = new ClassNameFiltering(config);
9397
assertTrue(classNameFiltering.isExcluded(input));
9498
}
9599

100+
@Test
101+
public void testShaded() {
102+
Config config = mock(Config.class);
103+
when(config.getThirdPartyExcludes()).thenReturn(Collections.emptySet());
104+
when(config.getThirdPartyIncludes())
105+
.thenReturn(new HashSet<>(Arrays.asList("com.google", "org.junit")));
106+
when(config.getThirdPartyShadingIdentifiers()).thenReturn(Collections.emptySet());
107+
ClassNameFiltering classNameFiltering = new ClassNameFiltering(config);
108+
assertTrue(classNameFiltering.isExcluded("com.google.FooBar"));
109+
assertTrue(classNameFiltering.isExcluded("shaded.com.google.FooBar"));
110+
assertFalse(classNameFiltering.isExcluded("com.example.shaded.com.example.FooBar"));
111+
assertTrue(classNameFiltering.isExcluded("com.example.shaded.com.google.FooBar"));
112+
}
113+
96114
@Test
97115
void lambdaProxyClasses() {
98116
// jdk8: at

‎dd-java-agent/appsec/build.gradle

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ ext {
8282
'com.datadog.appsec.config.AppSecFeatures.ApiSecurity',
8383
'com.datadog.appsec.config.AppSecFeatures.AutoUserInstrum',
8484
'com.datadog.appsec.event.ReplaceableEventProducerService',
85+
'com.datadog.appsec.api.security.ApiSecuritySampler.NoOp',
8586
]
8687
excludedClassesBranchCoverage = [
8788
'com.datadog.appsec.gateway.GatewayBridge',

‎dd-java-agent/appsec/src/main/java/com/datadog/appsec/AppSecSystem.java

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package com.datadog.appsec;
22

3-
import com.datadog.appsec.api.security.ApiSecurityRequestSampler;
3+
import com.datadog.appsec.api.security.ApiSecuritySampler;
4+
import com.datadog.appsec.api.security.ApiSecuritySamplerImpl;
5+
import com.datadog.appsec.api.security.AppSecSpanPostProcessor;
46
import com.datadog.appsec.blocking.BlockingServiceImpl;
57
import com.datadog.appsec.config.AppSecConfigService;
68
import com.datadog.appsec.config.AppSecConfigServiceImpl;
@@ -21,6 +23,7 @@
2123
import datadog.trace.api.telemetry.ProductChange;
2224
import datadog.trace.api.telemetry.ProductChangeCollector;
2325
import datadog.trace.bootstrap.ActiveSubsystems;
26+
import datadog.trace.bootstrap.instrumentation.api.SpanPostProcessor;
2427
import java.util.Collections;
2528
import java.util.HashMap;
2629
import java.util.List;
@@ -66,7 +69,17 @@ private static void doStart(SubscriptionService gw, SharedCommunicationObjects s
6669
EventDispatcher eventDispatcher = new EventDispatcher();
6770
REPLACEABLE_EVENT_PRODUCER.replaceEventProducerService(eventDispatcher);
6871

69-
ApiSecurityRequestSampler requestSampler = new ApiSecurityRequestSampler(config);
72+
ApiSecuritySampler requestSampler;
73+
if (Config.get().isApiSecurityEnabled()) {
74+
requestSampler = new ApiSecuritySamplerImpl();
75+
// When DD_API_SECURITY_ENABLED=true, ths post-processor is set even when AppSec is inactive.
76+
// This should be low overhead since the post-processor exits early if there's no AppSec
77+
// context.
78+
SpanPostProcessor.Holder.INSTANCE =
79+
new AppSecSpanPostProcessor(requestSampler, REPLACEABLE_EVENT_PRODUCER);
80+
} else {
81+
requestSampler = new ApiSecuritySampler.NoOp();
82+
}
7083

7184
ConfigurationPoller configurationPoller = sco.configurationPoller(config);
7285
// may throw and abort startup

‎dd-java-agent/appsec/src/main/java/com/datadog/appsec/api/security/ApiSecurityRequestSampler.java

Lines changed: 0 additions & 56 deletions
This file was deleted.

0 commit comments

Comments
 (0)