Skip to content

Commit dcf73eb

Browse files
daniel-mohedanodaniel-mohedano
authored
Implement test disabling (#8377)
1 parent f8c95c8 commit dcf73eb

File tree

131 files changed

+2834
-215
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

131 files changed

+2834
-215
lines changed

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestFrameworkModule.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ TestSuiteImpl testSuiteStart(
3333

3434
boolean isQuarantined(TestIdentifier test);
3535

36+
boolean isDisabled(TestIdentifier test);
37+
3638
/**
3739
* Returns the reason for skipping a test, IF it can be skipped.
3840
*

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestImpl.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import datadog.trace.api.civisibility.telemetry.tag.BrowserDriver;
2020
import datadog.trace.api.civisibility.telemetry.tag.EventType;
2121
import datadog.trace.api.civisibility.telemetry.tag.HasFailedAllRetries;
22+
import datadog.trace.api.civisibility.telemetry.tag.IsDisabled;
2223
import datadog.trace.api.civisibility.telemetry.tag.IsModified;
2324
import datadog.trace.api.civisibility.telemetry.tag.IsNew;
2425
import datadog.trace.api.civisibility.telemetry.tag.IsQuarantined;
@@ -277,7 +278,8 @@ public void end(@Nullable Long endTime) {
277278
EventType.TEST,
278279
span.getTag(Tags.TEST_IS_NEW) != null ? IsNew.TRUE : null,
279280
span.getTag(Tags.TEST_IS_MODIFIED) != null ? IsModified.TRUE : null,
280-
span.getTag(Tags.TEST_MANAGEMENT_IS_QUARANTINED) != null ? IsQuarantined.TRUE : null,
281+
span.getTag(Tags.TEST_TEST_MANAGEMENT_IS_QUARANTINED) != null ? IsQuarantined.TRUE : null,
282+
span.getTag(Tags.TEST_TEST_MANAGEMENT_IS_TEST_DISABLED) != null ? IsDisabled.TRUE : null,
281283
span.getTag(Tags.TEST_IS_RETRY) != null ? IsRetry.TRUE : null,
282284
span.getTag(Tags.TEST_HAS_FAILED_ALL_RETRIES) != null ? HasFailedAllRetries.TRUE : null,
283285
retryReason instanceof TagValue ? (TagValue) retryReason : null,

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,11 @@ public boolean isQuarantined(TestIdentifier test) {
109109
return executionStrategy.isQuarantined(test);
110110
}
111111

112+
@Override
113+
public boolean isDisabled(TestIdentifier test) {
114+
return executionStrategy.isDisabled(test);
115+
}
116+
112117
@Nullable
113118
@Override
114119
public SkipReason skipReason(TestIdentifier test) {

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,11 @@ public boolean isQuarantined(TestIdentifier test) {
9494
return executionStrategy.isQuarantined(test);
9595
}
9696

97+
@Override
98+
public boolean isDisabled(TestIdentifier test) {
99+
return executionStrategy.isDisabled(test);
100+
}
101+
97102
@Nullable
98103
@Override
99104
public SkipReason skipReason(TestIdentifier test) {

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,11 @@ public void onTestStart(
168168
}
169169

170170
if (testModule.isQuarantined(thisTest)) {
171-
test.setTag(Tags.TEST_MANAGEMENT_IS_QUARANTINED, true);
171+
test.setTag(Tags.TEST_TEST_MANAGEMENT_IS_QUARANTINED, true);
172+
}
173+
174+
if (testModule.isDisabled(thisTest)) {
175+
test.setTag(Tags.TEST_TEST_MANAGEMENT_IS_TEST_DISABLED, true);
172176
}
173177

174178
if (testExecutionHistory != null) {

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,9 +90,15 @@ public SkipReason skipReason(TestIdentifier test) {
9090
if (test == null) {
9191
return null;
9292
}
93+
94+
if (isDisabled(test)) {
95+
return SkipReason.DISABLED;
96+
}
97+
9398
if (!executionSettings.isTestSkippingEnabled()) {
9499
return null;
95100
}
101+
96102
Map<TestIdentifier, TestMetadata> skippableTests = executionSettings.getSkippableTests();
97103
TestMetadata testMetadata = skippableTests.get(test);
98104
if (testMetadata == null) {

‎dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -294,14 +294,17 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
294294

295295
def givenQuarantinedTests(List<TestFQN> tests) {
296296
quarantinedTests.addAll(tests)
297+
testManagementEnabled = true
297298
}
298299

299300
def givenDisabledTests(List<TestFQN> tests) {
300301
disabledTests.addAll(tests)
302+
testManagementEnabled = true
301303
}
302304

303305
def givenAttemptToFixTests(List<TestFQN> tests) {
304306
attemptToFixTests.addAll(tests)
307+
testManagementEnabled = true
305308
}
306309

307310
def givenDiff(Diff diff) {
@@ -320,10 +323,6 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
320323
this.impactedTestsDetectionEnabled = impactedTestsDetectionEnabled
321324
}
322325

323-
def givenTestManagementEnabled(boolean testManagementEnabled) {
324-
this.testManagementEnabled = testManagementEnabled
325-
}
326-
327326
def givenTestsOrder(String testsOrder) {
328327
injectSysConfig(CiVisibilityConfig.CIVISIBILITY_TEST_ORDER, testsOrder)
329328
}

‎dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/JUnit4CucumberSkipInstrumentation.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,9 @@ public JUnit4CucumberSkipInstrumentation() {
3434

3535
@Override
3636
public boolean isApplicable(Set<TargetSystem> enabledSystems) {
37-
return super.isApplicable(enabledSystems) && Config.get().isCiVisibilityTestSkippingEnabled();
37+
return super.isApplicable(enabledSystems)
38+
&& (Config.get().isCiVisibilityTestSkippingEnabled()
39+
|| Config.get().isCiVisibilityTestManagementEnabled());
3840
}
3941

4042
@Override

‎dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,6 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
9191
}
9292

9393
def "test quarantined #testcaseName"() {
94-
givenTestManagementEnabled(true)
9594
givenQuarantinedTests(quarantined)
9695

9796
runFeatures(features, true)
@@ -106,7 +105,6 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
106105
}
107106

108107
def "test quarantined auto-retries #testcaseName"() {
109-
givenTestManagementEnabled(true)
110108
givenQuarantinedTests(quarantined)
111109

112110
givenFlakyRetryEnabled(true)
@@ -118,16 +116,15 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
118116
assertSpansData(testcaseName)
119117

120118
where:
121-
testcaseName | features | quarantined | retried
119+
testcaseName | features | quarantined | retried
122120
"test-quarantined-failed-atr" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
123121
new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition")
124-
] | [
122+
] | [
125123
new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition")
126124
]
127125
}
128126

129127
def "test quarantined early flakiness detection #testcaseName"() {
130-
givenTestManagementEnabled(true)
131128
givenQuarantinedTests(quarantined)
132129

133130
givenEarlyFlakinessDetectionEnabled(true)
@@ -139,15 +136,29 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
139136
assertSpansData(testcaseName)
140137

141138
where:
142-
testcaseName | features | quarantined | known
139+
testcaseName | features | quarantined | known
143140
"test-quarantined-failed-known" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
144141
new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition")
145-
] | [
142+
] | [
146143
new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition")
147144
]
148145
"test-quarantined-failed-efd" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
149146
new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition")
150-
] | []
147+
] | []
148+
}
149+
150+
def "test disabled #testcaseName"() {
151+
givenDisabledTests(disabled)
152+
153+
runFeatures(features, true)
154+
155+
assertSpansData(testcaseName)
156+
157+
where:
158+
testcaseName | features | disabled
159+
"test-disabled-failed" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
160+
new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition")
161+
]
151162
}
152163

153164
private String version() {

‎dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-disabled-failed/coverages.ftl

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[ ]

‎dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-disabled-failed/events.ftl

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
[ {
2+
"content" : {
3+
"duration" : ${content_duration},
4+
"error" : 0,
5+
"meta" : {
6+
"_dd.p.tid" : ${content_meta__dd_p_tid},
7+
"component" : "junit",
8+
"dummy_ci_tag" : "dummy_ci_tag_value",
9+
"env" : "none",
10+
"library_version" : ${content_meta_library_version},
11+
"span.kind" : "test_suite_end",
12+
"test.framework" : "cucumber",
13+
"test.framework_version" : ${content_meta_test_framework_version},
14+
"test.module" : "cucumber-junit-4",
15+
"test.status" : "skip",
16+
"test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic",
17+
"test.type" : "test",
18+
"test_session.name" : "session-name"
19+
},
20+
"metrics" : {
21+
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}
22+
},
23+
"name" : "junit.test_suite",
24+
"resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic",
25+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
26+
"start" : ${content_start},
27+
"test_module_id" : ${content_test_module_id},
28+
"test_session_id" : ${content_test_session_id},
29+
"test_suite_id" : ${content_test_suite_id}
30+
},
31+
"type" : "test_suite_end",
32+
"version" : 1
33+
}, {
34+
"content" : {
35+
"duration" : ${content_duration_2},
36+
"error" : 0,
37+
"meta" : {
38+
"_dd.profiling.ctx" : "test",
39+
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
40+
"component" : "junit",
41+
"dummy_ci_tag" : "dummy_ci_tag_value",
42+
"env" : "none",
43+
"language" : "jvm",
44+
"library_version" : ${content_meta_library_version},
45+
"runtime-id" : ${content_meta_runtime_id},
46+
"span.kind" : "test",
47+
"test.framework" : "cucumber",
48+
"test.framework_version" : ${content_meta_test_framework_version},
49+
"test.module" : "cucumber-junit-4",
50+
"test.name" : "Addition",
51+
"test.skip_reason" : "Flaky test is disabled by Datadog",
52+
"test.status" : "skip",
53+
"test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic",
54+
"test.test_management.is_test_disabled" : "true",
55+
"test.traits" : "{\"category\":[\"foo\"]}",
56+
"test.type" : "test",
57+
"test_session.name" : "session-name"
58+
},
59+
"metrics" : {
60+
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2},
61+
"_dd.profiling.enabled" : 0,
62+
"_dd.trace_span_attribute_schema" : 0,
63+
"process_id" : ${content_metrics_process_id}
64+
},
65+
"name" : "junit.test",
66+
"parent_id" : ${content_parent_id},
67+
"resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic.Addition",
68+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
69+
"span_id" : ${content_span_id},
70+
"start" : ${content_start_2},
71+
"test_module_id" : ${content_test_module_id},
72+
"test_session_id" : ${content_test_session_id},
73+
"test_suite_id" : ${content_test_suite_id},
74+
"trace_id" : ${content_trace_id}
75+
},
76+
"type" : "test",
77+
"version" : 2
78+
}, {
79+
"content" : {
80+
"duration" : ${content_duration_3},
81+
"error" : 0,
82+
"meta" : {
83+
"_dd.p.tid" : ${content_meta__dd_p_tid_2},
84+
"_dd.profiling.ctx" : "test",
85+
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
86+
"component" : "junit",
87+
"dummy_ci_tag" : "dummy_ci_tag_value",
88+
"env" : "none",
89+
"language" : "jvm",
90+
"library_version" : ${content_meta_library_version},
91+
"runtime-id" : ${content_meta_runtime_id},
92+
"span.kind" : "test_session_end",
93+
"test.command" : "cucumber-junit-4",
94+
"test.framework" : "cucumber",
95+
"test.framework_version" : ${content_meta_test_framework_version},
96+
"test.status" : "skip",
97+
"test.test_management.enabled" : "true",
98+
"test.type" : "test",
99+
"test_session.name" : "session-name"
100+
},
101+
"metrics" : {
102+
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3},
103+
"_dd.profiling.enabled" : 0,
104+
"_dd.trace_span_attribute_schema" : 0,
105+
"process_id" : ${content_metrics_process_id}
106+
},
107+
"name" : "junit.test_session",
108+
"resource" : "cucumber-junit-4",
109+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
110+
"start" : ${content_start_3},
111+
"test_session_id" : ${content_test_session_id}
112+
},
113+
"type" : "test_session_end",
114+
"version" : 1
115+
}, {
116+
"content" : {
117+
"duration" : ${content_duration_4},
118+
"error" : 0,
119+
"meta" : {
120+
"_dd.p.tid" : ${content_meta__dd_p_tid_3},
121+
"component" : "junit",
122+
"dummy_ci_tag" : "dummy_ci_tag_value",
123+
"env" : "none",
124+
"library_version" : ${content_meta_library_version},
125+
"span.kind" : "test_module_end",
126+
"test.framework" : "cucumber",
127+
"test.framework_version" : ${content_meta_test_framework_version},
128+
"test.module" : "cucumber-junit-4",
129+
"test.status" : "skip",
130+
"test.test_management.enabled" : "true",
131+
"test.type" : "test",
132+
"test_session.name" : "session-name"
133+
},
134+
"metrics" : {
135+
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}
136+
},
137+
"name" : "junit.test_module",
138+
"resource" : "cucumber-junit-4",
139+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
140+
"start" : ${content_start_4},
141+
"test_module_id" : ${content_test_module_id},
142+
"test_session_id" : ${content_test_session_id}
143+
},
144+
"type" : "test_module_end",
145+
"version" : 1
146+
} ]

‎dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-itr-skipping/events.ftl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@
4949
"test.framework_version" : ${content_meta_test_framework_version},
5050
"test.module" : "cucumber-junit-4",
5151
"test.name" : "Addition",
52-
"test.skip_reason" : "Skipped by Datadog Intelligent Test Runner",
52+
"test.skip_reason" : "Skipped by Datadog Test Impact Analysis",
5353
"test.skipped_by_itr" : "true",
5454
"test.status" : "skip",
5555
"test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic.feature:Basic Arithmetic",

0 commit comments

Comments
 (0)