Implement test disabling

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestFrameworkModule.java

@@ -33,6 +33,8 @@ TestSuiteImpl testSuiteStart(
 
   boolean isQuarantined(TestIdentifier test);
 
+  boolean isDisabled(TestIdentifier test);
+
   /**
    * Returns the reason for skipping a test, IF it can be skipped.
    *

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestImpl.java

@@ -19,6 +19,7 @@
 import datadog.trace.api.civisibility.telemetry.tag.BrowserDriver;
 import datadog.trace.api.civisibility.telemetry.tag.EventType;
 import datadog.trace.api.civisibility.telemetry.tag.HasFailedAllRetries;
+import datadog.trace.api.civisibility.telemetry.tag.IsDisabled;
 import datadog.trace.api.civisibility.telemetry.tag.IsModified;
 import datadog.trace.api.civisibility.telemetry.tag.IsNew;
 import datadog.trace.api.civisibility.telemetry.tag.IsQuarantined;
@@ -277,7 +278,8 @@ public void end(@Nullable Long endTime) {
         EventType.TEST,
         span.getTag(Tags.TEST_IS_NEW) != null ? IsNew.TRUE : null,
         span.getTag(Tags.TEST_IS_MODIFIED) != null ? IsModified.TRUE : null,
-        span.getTag(Tags.TEST_MANAGEMENT_IS_QUARANTINED) != null ? IsQuarantined.TRUE : null,
+        span.getTag(Tags.TEST_TEST_MANAGEMENT_IS_QUARANTINED) != null ? IsQuarantined.TRUE : null,
+        span.getTag(Tags.TEST_TEST_MANAGEMENT_IS_TEST_DISABLED) != null ? IsDisabled.TRUE : null,
         span.getTag(Tags.TEST_IS_RETRY) != null ? IsRetry.TRUE : null,
         span.getTag(Tags.TEST_HAS_FAILED_ALL_RETRIES) != null ? HasFailedAllRetries.TRUE : null,
         retryReason instanceof TagValue ? (TagValue) retryReason : null,

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java

@@ -109,6 +109,11 @@ public boolean isQuarantined(TestIdentifier test) {
     return executionStrategy.isQuarantined(test);
   }
 
+  @Override
+  public boolean isDisabled(TestIdentifier test) {
+    return executionStrategy.isDisabled(test);
+  }
+
   @Nullable
   @Override
   public SkipReason skipReason(TestIdentifier test) {

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java

@@ -94,6 +94,11 @@ public boolean isQuarantined(TestIdentifier test) {
     return executionStrategy.isQuarantined(test);
   }
 
+  @Override
+  public boolean isDisabled(TestIdentifier test) {
+    return executionStrategy.isDisabled(test);
+  }
+
   @Nullable
   @Override
   public SkipReason skipReason(TestIdentifier test) {

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java

@@ -168,7 +168,11 @@ public void onTestStart(
     }
 
     if (testModule.isQuarantined(thisTest)) {
-      test.setTag(Tags.TEST_MANAGEMENT_IS_QUARANTINED, true);
+      test.setTag(Tags.TEST_TEST_MANAGEMENT_IS_QUARANTINED, true);
+    }
+
+    if (testModule.isDisabled(thisTest)) {
+      test.setTag(Tags.TEST_TEST_MANAGEMENT_IS_TEST_DISABLED, true);
     }
 
     if (testExecutionHistory != null) {

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java

@@ -90,9 +90,15 @@ public SkipReason skipReason(TestIdentifier test) {
     if (test == null) {
       return null;
     }
+
+    if (isDisabled(test)) {
+      return SkipReason.DISABLED;
+    }
+
     if (!executionSettings.isTestSkippingEnabled()) {
       return null;
     }
+
     Map<TestIdentifier, TestMetadata> skippableTests = executionSettings.getSkippableTests();
     TestMetadata testMetadata = skippableTests.get(test);
     if (testMetadata == null) {

dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy

@@ -294,14 +294,17 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
 
   def givenQuarantinedTests(List<TestFQN> tests) {
     quarantinedTests.addAll(tests)
+    testManagementEnabled = true
   }
 
   def givenDisabledTests(List<TestFQN> tests) {
     disabledTests.addAll(tests)
+    testManagementEnabled = true
   }
 
   def givenAttemptToFixTests(List<TestFQN> tests) {
     attemptToFixTests.addAll(tests)
+    testManagementEnabled = true
   }
 
   def givenDiff(Diff diff) {
@@ -320,10 +323,6 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
     this.impactedTestsDetectionEnabled = impactedTestsDetectionEnabled
   }
 
-  def givenTestManagementEnabled(boolean testManagementEnabled) {
-    this.testManagementEnabled = testManagementEnabled
-  }
-
   def givenTestsOrder(String testsOrder) {
     injectSysConfig(CiVisibilityConfig.CIVISIBILITY_TEST_ORDER, testsOrder)
   }

dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/JUnit4CucumberSkipInstrumentation.java

@@ -34,7 +34,9 @@ public JUnit4CucumberSkipInstrumentation() {
 
   @Override
   public boolean isApplicable(Set<TargetSystem> enabledSystems) {
-    return super.isApplicable(enabledSystems) && Config.get().isCiVisibilityTestSkippingEnabled();
+    return super.isApplicable(enabledSystems)
+        && (Config.get().isCiVisibilityTestSkippingEnabled()
+            || Config.get().isCiVisibilityTestManagementEnabled());
   }
 
   @Override

dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy

@@ -91,7 +91,6 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
   }
 
   def "test quarantined #testcaseName"() {
-    givenTestManagementEnabled(true)
     givenQuarantinedTests(quarantined)
 
     runFeatures(features, true)
@@ -106,7 +105,6 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
   }
 
   def "test quarantined auto-retries #testcaseName"() {
-    givenTestManagementEnabled(true)
     givenQuarantinedTests(quarantined)
 
     givenFlakyRetryEnabled(true)
@@ -118,16 +116,15 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
     assertSpansData(testcaseName)
 
     where:
-    testcaseName                  | features                                                            | quarantined                    | retried
+    testcaseName                  | features                                                            | quarantined       | retried
     "test-quarantined-failed-atr" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
       new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition")
-    ]                                                                                                                                    | [
+    ]                                                                                                                       | [
       new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition")
     ]
   }
 
   def "test quarantined early flakiness detection #testcaseName"() {
-    givenTestManagementEnabled(true)
     givenQuarantinedTests(quarantined)
 
     givenEarlyFlakinessDetectionEnabled(true)
@@ -139,15 +136,29 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
     assertSpansData(testcaseName)
 
     where:
-    testcaseName                    | features                                                            | quarantined                  | known
+    testcaseName                    | features                                                            | quarantined     | known
     "test-quarantined-failed-known" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
       new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition")
-    ]                                                                                                                                    | [
+    ]                                                                                                                       | [
       new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition")
     ]
     "test-quarantined-failed-efd"   | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
       new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition")
-    ]                                                                                                                                    | []
+    ]                                                                                                                       | []
+  }
+
+  def "test disabled #testcaseName"() {
+    givenDisabledTests(disabled)
+
+    runFeatures(features, true)
+
+    assertSpansData(testcaseName)
+
+    where:
+    testcaseName           | features                                                            | disabled
+    "test-disabled-failed" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
+      new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition")
+    ]
   }
 
   private String version() {

dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-disabled-failed/coverages.ftl

@@ -0,0 +1 @@
+[ ]

dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-disabled-failed/events.ftl

@@ -0,0 +1,146 @@
+[ {
+  "content" : {
+    "duration" : ${content_duration},
+    "error" : 0,
+    "meta" : {
+      "_dd.p.tid" : ${content_meta__dd_p_tid},
+      "component" : "junit",
+      "dummy_ci_tag" : "dummy_ci_tag_value",
+      "env" : "none",
+      "library_version" : ${content_meta_library_version},
+      "span.kind" : "test_suite_end",
+      "test.framework" : "cucumber",
+      "test.framework_version" : ${content_meta_test_framework_version},
+      "test.module" : "cucumber-junit-4",
+      "test.status" : "skip",
+      "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic",
+      "test.type" : "test",
+      "test_session.name" : "session-name"
+    },
+    "metrics" : {
+      "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}
+    },
+    "name" : "junit.test_suite",
+    "resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic",
+    "service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
+    "start" : ${content_start},
+    "test_module_id" : ${content_test_module_id},
+    "test_session_id" : ${content_test_session_id},
+    "test_suite_id" : ${content_test_suite_id}
+  },
+  "type" : "test_suite_end",
+  "version" : 1
+}, {
+  "content" : {
+    "duration" : ${content_duration_2},
+    "error" : 0,
+    "meta" : {
+      "_dd.profiling.ctx" : "test",
+      "_dd.tracer_host" : ${content_meta__dd_tracer_host},
+      "component" : "junit",
+      "dummy_ci_tag" : "dummy_ci_tag_value",
+      "env" : "none",
+      "language" : "jvm",
+      "library_version" : ${content_meta_library_version},
+      "runtime-id" : ${content_meta_runtime_id},
+      "span.kind" : "test",
+      "test.framework" : "cucumber",
+      "test.framework_version" : ${content_meta_test_framework_version},
+      "test.module" : "cucumber-junit-4",
+      "test.name" : "Addition",
+      "test.skip_reason" : "Flaky test is disabled by Datadog",
+      "test.status" : "skip",
+      "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic",
+      "test.test_management.is_test_disabled" : "true",
+      "test.traits" : "{\"category\":[\"foo\"]}",
+      "test.type" : "test",
+      "test_session.name" : "session-name"
+    },
+    "metrics" : {
+      "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2},
+      "_dd.profiling.enabled" : 0,
+      "_dd.trace_span_attribute_schema" : 0,
+      "process_id" : ${content_metrics_process_id}
+    },
+    "name" : "junit.test",
+    "parent_id" : ${content_parent_id},
+    "resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic.Addition",
+    "service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
+    "span_id" : ${content_span_id},
+    "start" : ${content_start_2},
+    "test_module_id" : ${content_test_module_id},
+    "test_session_id" : ${content_test_session_id},
+    "test_suite_id" : ${content_test_suite_id},
+    "trace_id" : ${content_trace_id}
+  },
+  "type" : "test",
+  "version" : 2
+}, {
+  "content" : {
+    "duration" : ${content_duration_3},
+    "error" : 0,
+    "meta" : {
+      "_dd.p.tid" : ${content_meta__dd_p_tid_2},
+      "_dd.profiling.ctx" : "test",
+      "_dd.tracer_host" : ${content_meta__dd_tracer_host},
+      "component" : "junit",
+      "dummy_ci_tag" : "dummy_ci_tag_value",
+      "env" : "none",
+      "language" : "jvm",
+      "library_version" : ${content_meta_library_version},
+      "runtime-id" : ${content_meta_runtime_id},
+      "span.kind" : "test_session_end",
+      "test.command" : "cucumber-junit-4",
+      "test.framework" : "cucumber",
+      "test.framework_version" : ${content_meta_test_framework_version},
+      "test.status" : "skip",
+      "test.test_management.enabled" : "true",
+      "test.type" : "test",
+      "test_session.name" : "session-name"
+    },
+    "metrics" : {
+      "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3},
+      "_dd.profiling.enabled" : 0,
+      "_dd.trace_span_attribute_schema" : 0,
+      "process_id" : ${content_metrics_process_id}
+    },
+    "name" : "junit.test_session",
+    "resource" : "cucumber-junit-4",
+    "service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
+    "start" : ${content_start_3},
+    "test_session_id" : ${content_test_session_id}
+  },
+  "type" : "test_session_end",
+  "version" : 1
+}, {
+  "content" : {
+    "duration" : ${content_duration_4},
+    "error" : 0,
+    "meta" : {
+      "_dd.p.tid" : ${content_meta__dd_p_tid_3},
+      "component" : "junit",
+      "dummy_ci_tag" : "dummy_ci_tag_value",
+      "env" : "none",
+      "library_version" : ${content_meta_library_version},
+      "span.kind" : "test_module_end",
+      "test.framework" : "cucumber",
+      "test.framework_version" : ${content_meta_test_framework_version},
+      "test.module" : "cucumber-junit-4",
+      "test.status" : "skip",
+      "test.test_management.enabled" : "true",
+      "test.type" : "test",
+      "test_session.name" : "session-name"
+    },
+    "metrics" : {
+      "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}
+    },
+    "name" : "junit.test_module",
+    "resource" : "cucumber-junit-4",
+    "service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
+    "start" : ${content_start_4},
+    "test_module_id" : ${content_test_module_id},
+    "test_session_id" : ${content_test_session_id}
+  },
+  "type" : "test_module_end",
+  "version" : 1
+} ]

dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-itr-skipping/events.ftl

@@ -49,7 +49,7 @@
       "test.framework_version" : ${content_meta_test_framework_version},
       "test.module" : "cucumber-junit-4",
       "test.name" : "Addition",
-      "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner",
+      "test.skip_reason" : "Skipped by Datadog Test Impact Analysis",
       "test.skipped_by_itr" : "true",
       "test.status" : "skip",
       "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic.feature:Basic Arithmetic",