diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettings.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettings.java index 52cd5e0deb4..4d0ed23a2f8 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettings.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettings.java @@ -30,6 +30,7 @@ public class ExecutionSettings { Collections.emptyMap(), Collections.emptyList(), null, + null, LineDiff.EMPTY); private final boolean itrEnabled; @@ -41,6 +42,7 @@ public class ExecutionSettings { @Nullable private final String itrCorrelationId; @Nonnull private final Map skippableTests; @Nonnull private final Map skippableTestsCoverage; + @Nonnull private final Collection quarantinedTests; @Nullable private final Collection flakyTests; @Nullable private final Collection knownTests; @Nonnull private final Diff pullRequestDiff; @@ -55,6 +57,7 @@ public ExecutionSettings( @Nullable String itrCorrelationId, @Nonnull Map skippableTests, @Nonnull Map skippableTestsCoverage, + @Nonnull Collection quarantinedTests, @Nullable Collection flakyTests, @Nullable Collection knownTests, @Nonnull Diff pullRequestDiff) { @@ -67,6 +70,7 @@ public ExecutionSettings( this.itrCorrelationId = itrCorrelationId; this.skippableTests = skippableTests; this.skippableTestsCoverage = skippableTestsCoverage; + this.quarantinedTests = quarantinedTests; this.flakyTests = flakyTests; this.knownTests = knownTests; this.pullRequestDiff = pullRequestDiff; @@ -117,6 +121,11 @@ public Map getSkippableTests() { return skippableTests; } + @Nonnull + public Collection getQuarantinedTests() { + return quarantinedTests; + } + /** * @return the list of known tests for the given module (can be empty), or {@code null} if known * tests could not be obtained @@ -156,6 +165,7 @@ public boolean equals(Object o) { && Objects.equals(itrCorrelationId, that.itrCorrelationId) && Objects.equals(skippableTests, that.skippableTests) && Objects.equals(skippableTestsCoverage, that.skippableTestsCoverage) + && Objects.equals(quarantinedTests, that.quarantinedTests) && Objects.equals(flakyTests, that.flakyTests) && Objects.equals(knownTests, that.knownTests) && Objects.equals(pullRequestDiff, that.pullRequestDiff); @@ -171,6 +181,7 @@ public int hashCode() { itrCorrelationId, skippableTests, skippableTestsCoverage, + quarantinedTests, flakyTests, knownTests, pullRequestDiff); @@ -207,6 +218,7 @@ public static ByteBuffer serialize(ExecutionSettings settings) { TestMetadataSerializer::serialize); s.write(settings.skippableTestsCoverage, Serializer::write, Serializer::write); + s.write(settings.quarantinedTests, TestIdentifierSerializer::serialize); s.write(settings.flakyTests, TestIdentifierSerializer::serialize); s.write(settings.knownTests, TestIdentifierSerializer::serialize); @@ -234,6 +246,8 @@ public static ExecutionSettings deserialize(ByteBuffer buffer) { Map skippableTestsCoverage = Serializer.readMap(buffer, Serializer::readString, Serializer::readBitSet); + Collection quarantinedTests = + Serializer.readSet(buffer, TestIdentifierSerializer::deserialize); Collection flakyTests = Serializer.readSet(buffer, TestIdentifierSerializer::deserialize); Collection knownTests = @@ -251,6 +265,7 @@ public static ExecutionSettings deserialize(ByteBuffer buffer) { itrCorrelationId, skippableTests, skippableTestsCoverage, + quarantinedTests, flakyTests, knownTests, diff); diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettingsFactoryImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettingsFactoryImpl.java index 36f6d899a3a..d6604544ee1 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettingsFactoryImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettingsFactoryImpl.java @@ -233,6 +233,7 @@ private Map doCreate( .getIdentifiersByModule() .getOrDefault(moduleName, Collections.emptyMap()), skippableTests.getCoveredLinesByRelativeSourcePath(), + Collections.emptyList(), // FIXME implement retrieving quarantined tests flakyTestsByModule != null ? flakyTestsByModule.getOrDefault(moduleName, Collections.emptyList()) : null, diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/RetryUntilSuccessful.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/RetryUntilSuccessful.java index 9b3b256bf30..2a5463f4738 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/RetryUntilSuccessful.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/RetryUntilSuccessful.java @@ -9,29 +9,34 @@ public class RetryUntilSuccessful implements TestExecutionPolicy { private final int maxExecutions; + private final boolean suppressFailures; private int executions; /** Total execution counter that is shared by all retry policies */ private final AtomicInteger totalExecutions; - public RetryUntilSuccessful(int maxExecutions, AtomicInteger totalExecutions) { + public RetryUntilSuccessful( + int maxExecutions, boolean suppressFailures, AtomicInteger totalExecutions) { this.maxExecutions = maxExecutions; + this.suppressFailures = suppressFailures; this.totalExecutions = totalExecutions; this.executions = 0; } @Override public boolean applicable() { - // the last execution is not altered by the policy - // (no retries, no exceptions suppressing) - return executions < maxExecutions - 1; + return currentExecutionIsNotLast() || suppressFailures; } @Override public boolean suppressFailures() { - // if this isn't the last execution, - // possible failures should be suppressed - return applicable(); + // do not suppress failures for last execution + // (unless flag to suppress all failures is set) + return currentExecutionIsNotLast() || suppressFailures; + } + + private boolean currentExecutionIsNotLast() { + return executions < maxExecutions - 1; } @Override diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/RunNTimes.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/RunNTimes.java index a79e43c8786..79144f1d303 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/RunNTimes.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/RunNTimes.java @@ -9,25 +9,30 @@ public class RunNTimes implements TestExecutionPolicy { private final EarlyFlakeDetectionSettings earlyFlakeDetectionSettings; + private final boolean suppressFailures; private int executions; private int maxExecutions; - public RunNTimes(EarlyFlakeDetectionSettings earlyFlakeDetectionSettings) { + public RunNTimes( + EarlyFlakeDetectionSettings earlyFlakeDetectionSettings, boolean suppressFailures) { this.earlyFlakeDetectionSettings = earlyFlakeDetectionSettings; + this.suppressFailures = suppressFailures; this.executions = 0; this.maxExecutions = earlyFlakeDetectionSettings.getExecutions(0); } @Override public boolean applicable() { - // the last execution is not altered by the policy - // (no retries, no exceptions suppressing) + return currentExecutionIsNotLast() || suppressFailures(); + } + + private boolean currentExecutionIsNotLast() { return executions < maxExecutions - 1; } @Override public boolean suppressFailures() { - return false; + return suppressFailures; } @Override diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/RunOnceIgnoreOutcome.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/RunOnceIgnoreOutcome.java new file mode 100644 index 00000000000..29edc8a3e02 --- /dev/null +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/RunOnceIgnoreOutcome.java @@ -0,0 +1,36 @@ +package datadog.trace.civisibility.execution; + +import datadog.trace.api.civisibility.execution.TestExecutionPolicy; +import datadog.trace.api.civisibility.telemetry.tag.RetryReason; +import org.jetbrains.annotations.Nullable; + +/** + * Runs a test case once. If it fails - suppresses the failure so that the build status is not + * affected. + */ +public class RunOnceIgnoreOutcome implements TestExecutionPolicy { + + private boolean testExecuted; + + @Override + public boolean applicable() { + return !testExecuted; + } + + @Override + public boolean suppressFailures() { + return true; + } + + @Override + public boolean retry(boolean successful, long durationMillis) { + testExecuted = true; + return false; + } + + @Nullable + @Override + public RetryReason currentExecutionRetryReason() { + return null; + } +} diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java index 42b33ed921e..6764615386b 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java @@ -11,6 +11,7 @@ import datadog.trace.civisibility.execution.Regular; import datadog.trace.civisibility.execution.RetryUntilSuccessful; import datadog.trace.civisibility.execution.RunNTimes; +import datadog.trace.civisibility.execution.RunOnceIgnoreOutcome; import datadog.trace.civisibility.source.LinesResolver; import datadog.trace.civisibility.source.SourcePathResolver; import java.lang.reflect.Method; @@ -60,6 +61,11 @@ public boolean isFlaky(TestIdentifier test) { return flakyTests != null && flakyTests.contains(test.withoutParameters()); } + public boolean isQuarantined(TestIdentifier test) { + Collection quarantinedTests = executionSettings.getQuarantinedTests(); + return quarantinedTests.contains(test.withoutParameters()); + } + @Nullable public SkipReason skipReason(TestIdentifier test) { if (test == null) { @@ -85,29 +91,44 @@ public TestExecutionPolicy executionPolicy(TestIdentifier test, TestSourceData t return Regular.INSTANCE; } - EarlyFlakeDetectionSettings efdSettings = executionSettings.getEarlyFlakeDetectionSettings(); - if (efdSettings.isEnabled() && !isEFDLimitReached()) { - if (isNew(test) || isModified(testSource)) { - // check-then-act with "earlyFlakeDetectionsUsed" is not atomic here, - // but we don't care if we go "a bit" over the limit, it does not have to be precise - earlyFlakeDetectionsUsed.incrementAndGet(); - return new RunNTimes(efdSettings); - } + if (isEFDApplicable(test, testSource)) { + // check-then-act with "earlyFlakeDetectionsUsed" is not atomic here, + // but we don't care if we go "a bit" over the limit, it does not have to be precise + earlyFlakeDetectionsUsed.incrementAndGet(); + return new RunNTimes(executionSettings.getEarlyFlakeDetectionSettings(), isQuarantined(test)); } - if (executionSettings.isFlakyTestRetriesEnabled()) { - Collection flakyTests = executionSettings.getFlakyTests(); - if ((flakyTests == null || flakyTests.contains(test.withoutParameters())) - && autoRetriesUsed.get() < config.getCiVisibilityTotalFlakyRetryCount()) { - // check-then-act with "autoRetriesUsed" is not atomic here, - // but we don't care if we go "a bit" over the limit, it does not have to be precise - return new RetryUntilSuccessful(config.getCiVisibilityFlakyRetryCount(), autoRetriesUsed); - } + if (isAutoRetryApplicable(test)) { + // check-then-act with "autoRetriesUsed" is not atomic here, + // but we don't care if we go "a bit" over the limit, it does not have to be precise + return new RetryUntilSuccessful( + config.getCiVisibilityFlakyRetryCount(), isQuarantined(test), autoRetriesUsed); + } + + if (isQuarantined(test)) { + return new RunOnceIgnoreOutcome(); } return Regular.INSTANCE; } + private boolean isAutoRetryApplicable(TestIdentifier test) { + if (!executionSettings.isFlakyTestRetriesEnabled()) { + return false; + } + + Collection flakyTests = executionSettings.getFlakyTests(); + return (flakyTests == null || flakyTests.contains(test.withoutParameters())) + && autoRetriesUsed.get() < config.getCiVisibilityTotalFlakyRetryCount(); + } + + private boolean isEFDApplicable(TestIdentifier test, TestSourceData testSource) { + EarlyFlakeDetectionSettings efdSettings = executionSettings.getEarlyFlakeDetectionSettings(); + return efdSettings.isEnabled() + && !isEFDLimitReached() + && (isNew(test) || isModified(testSource)); + } + public boolean isEFDLimitReached() { Collection knownTests = executionSettings.getKnownTests(); if (knownTests == null) { diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ExecutionSettingsTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ExecutionSettingsTest.groovy index 76846445f30..e1380576d31 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ExecutionSettingsTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ExecutionSettingsTest.groovy @@ -29,6 +29,7 @@ class ExecutionSettingsTest extends Specification { null, [:], [:], + new HashSet<>([]), null, new HashSet<>([]), LineDiff.EMPTY), @@ -41,8 +42,9 @@ class ExecutionSettingsTest extends Specification { true, new EarlyFlakeDetectionSettings(true, [], 10), "", - [new TestIdentifier("bc", "def", "g"): new TestMetadata(true), new TestIdentifier("de", "f", null): new TestMetadata(false)], + [(new TestIdentifier("bc", "def", "g")): new TestMetadata(true), (new TestIdentifier("de", "f", null)): new TestMetadata(false)], [:], + new HashSet<>([new TestIdentifier("suite", "quarantined", null)]), new HashSet<>([new TestIdentifier("name", null, null)]), new HashSet<>([new TestIdentifier("b", "c", "g")]), new LineDiff(["path": lines()]) @@ -62,6 +64,7 @@ class ExecutionSettingsTest extends Specification { }), "cov2": BitSet.valueOf(new byte[]{ 4, 5, 6 })], + new HashSet<>([new TestIdentifier("suite", "quarantined", null), new TestIdentifier("another", "another-quarantined", null)]), new HashSet<>([new TestIdentifier("name", null, "g"), new TestIdentifier("b", "c", null)]), new HashSet<>([new TestIdentifier("b", "c", null), new TestIdentifier("bb", "cc", null)]), new LineDiff(["path": lines(1, 2, 3)]), @@ -75,12 +78,13 @@ class ExecutionSettingsTest extends Specification { true, new EarlyFlakeDetectionSettings(true, [new EarlyFlakeDetectionSettings.ExecutionsByDuration(10, 20), new EarlyFlakeDetectionSettings.ExecutionsByDuration(30, 40)], 10), "itrCorrelationId", - [new TestIdentifier("bc", "def", null): new TestMetadata(true), new TestIdentifier("de", "f", null): new TestMetadata(true)], + [(new TestIdentifier("bc", "def", null)): new TestMetadata(true), (new TestIdentifier("de", "f", null)): new TestMetadata(true)], ["cov" : BitSet.valueOf(new byte[]{ 1, 2, 3 }), "cov2": BitSet.valueOf(new byte[]{ 4, 5, 6 })], + new HashSet<>([new TestIdentifier("suite", "quarantined", null), new TestIdentifier("another", "another-quarantined", null)]), new HashSet<>([]), new HashSet<>([new TestIdentifier("b", "c", null), new TestIdentifier("bb", "cc", "g")]), new LineDiff(["path": lines(1, 2, 3), "path-b": lines(1, 2, 128, 257, 999)]), diff --git a/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy b/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy index 3c4cc39186b..99d0e873fb4 100644 --- a/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy @@ -71,6 +71,7 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner { private static Path agentKeyFile private static final List skippableTests = new ArrayList<>() + private static final List quarantinedTests = new ArrayList<>() private static final List flakyTests = new ArrayList<>() private static final List knownTests = new ArrayList<>() private static volatile Diff diff = LineDiff.EMPTY @@ -79,6 +80,7 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner { private static volatile boolean flakyRetryEnabled = false private static volatile boolean earlyFlakinessDetectionEnabled = false private static volatile boolean impactedTestsDetectionEnabled = false + private static volatile boolean quarantineEnabled = false public static final int SLOW_TEST_THRESHOLD_MILLIS = 1000 public static final int VERY_SLOW_TEST_THRESHOLD_MILLIS = 2000 @@ -130,6 +132,7 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner { itrEnabled ? "itrCorrelationId" : null, skippableTestsWithMetadata, [:], + quarantinedTests, flakyTests, earlyFlakinessDetectionEnabled || CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(Config.get().ciVisibilityTestOrder) ? knownTests : null, diff) @@ -251,10 +254,12 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner { @Override void setup() { skippableTests.clear() + quarantinedTests.clear() flakyTests.clear() knownTests.clear() diff = LineDiff.EMPTY itrEnabled = false + quarantineEnabled = false flakyRetryEnabled = false earlyFlakinessDetectionEnabled = false impactedTestsDetectionEnabled = false @@ -275,10 +280,18 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner { knownTests.addAll(tests) } + def givenQuarantinedTests(List tests) { + quarantinedTests.addAll(tests) + } + def givenDiff(Diff diff) { this.diff = diff } + def givenQuarantineEnabled(boolean quarantineEnabled) { + this.quarantineEnabled = quarantineEnabled + } + def givenFlakyRetryEnabled(boolean flakyRetryEnabled) { this.flakyRetryEnabled = flakyRetryEnabled } diff --git a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy index ab090c83c1a..3a2b0804c36 100644 --- a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy +++ b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy @@ -89,6 +89,66 @@ class CucumberTest extends CiVisibilityInstrumentationTest { "test-efd-new-slow-test" | ["org/example/cucumber/calculator/basic_arithmetic_slow.feature"] | [] } + def "test quarantined #testcaseName"() { + givenQuarantineEnabled(true) + givenQuarantinedTests(quarantined) + + runFeatures(features, true) + + assertSpansData(testcaseName) + + where: + testcaseName | features | quarantined + "test-failed" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ + new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null) + ] + } + + def "test quarantined auto-retries #testcaseName"() { + givenQuarantineEnabled(true) + givenQuarantinedTests(quarantined) + + givenFlakyRetryEnabled(true) + givenFlakyTests(retried) + + // every test retry fails, but the build status is successful + runFeatures(features, true) + + assertSpansData(testcaseName) + + where: + testcaseName | features | quarantined | retried + "test-retry-failed" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ + new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null) + ] | [ + new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null) + ] + } + + def "test quarantined early flakiness detection #testcaseName"() { + givenQuarantineEnabled(true) + givenQuarantinedTests(quarantined) + + givenEarlyFlakinessDetectionEnabled(true) + givenKnownTests(known) + + // every retry fails, but the build status is successful + runFeatures(features, true) + + assertSpansData(testcaseName) + + where: + testcaseName | features | quarantined | known + "test-failed" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ + new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null) + ] | [ + new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null) + ] + "test-failed-efd" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ + new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null) + ] | [] + } + private String version() { return CucumberTracingListener.FRAMEWORK_VERSION < "7" ? CucumberTracingListener.FRAMEWORK_VERSION : "latest" } diff --git a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-failed-efd/coverages.ftl b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-failed-efd/coverages.ftl new file mode 100644 index 00000000000..48a7dff56d9 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-failed-efd/coverages.ftl @@ -0,0 +1,22 @@ +[ { + "files" : [ { + "filename" : "org/example/cucumber/calculator/basic_arithmetic_failed.feature" + } ], + "span_id" : ${content_span_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} +}, { + "files" : [ { + "filename" : "org/example/cucumber/calculator/basic_arithmetic_failed.feature" + } ], + "span_id" : ${content_span_id_2}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} +}, { + "files" : [ { + "filename" : "org/example/cucumber/calculator/basic_arithmetic_failed.feature" + } ], + "span_id" : ${content_span_id_3}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-failed-efd/events.ftl b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-failed-efd/events.ftl new file mode 100644 index 00000000000..ac77c4dec0f --- /dev/null +++ b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-failed-efd/events.ftl @@ -0,0 +1,246 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "cucumber-junit-4", + "test.status" : "fail", + "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count} + }, + "name" : "junit.test_suite", + "resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack}, + "error.type" : "java.lang.AssertionError", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.module" : "cucumber-junit-4", + "test.name" : "Addition", + "test.status" : "fail", + "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.traits" : "{\"category\":[\"foo\"]}", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic.Addition", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack_2}, + "error.type" : "java.lang.AssertionError", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.is_retry" : "true", + "test.module" : "cucumber-junit-4", + "test.name" : "Addition", + "test.retry_reason" : "efd", + "test.status" : "fail", + "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.traits" : "{\"category\":[\"foo\"]}", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic.Addition", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack_3}, + "error.type" : "java.lang.AssertionError", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.is_retry" : "true", + "test.module" : "cucumber-junit-4", + "test.name" : "Addition", + "test.retry_reason" : "efd", + "test.status" : "fail", + "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.traits" : "{\"category\":[\"foo\"]}", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic.Addition", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_3}, + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_3} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "cucumber-junit-4", + "test.early_flake.enabled" : "true", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test_session", + "resource" : "cucumber-junit-4", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_5}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_6}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.early_flake.enabled" : "true", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "cucumber-junit-4", + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_6} + }, + "name" : "junit.test_module", + "resource" : "cucumber-junit-4", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_6}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-failed/coverages.ftl b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-failed/coverages.ftl new file mode 100644 index 00000000000..e48f884bdbe --- /dev/null +++ b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-failed/coverages.ftl @@ -0,0 +1,8 @@ +[ { + "files" : [ { + "filename" : "org/example/cucumber/calculator/basic_arithmetic_failed.feature" + } ], + "span_id" : ${content_span_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-failed/events.ftl b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-failed/events.ftl new file mode 100644 index 00000000000..64dc9c7e7e9 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-failed/events.ftl @@ -0,0 +1,145 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "cucumber-junit-4", + "test.status" : "fail", + "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count} + }, + "name" : "junit.test_suite", + "resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack}, + "error.type" : "java.lang.AssertionError", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "cucumber-junit-4", + "test.name" : "Addition", + "test.status" : "fail", + "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.traits" : "{\"category\":[\"foo\"]}", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic.Addition", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "cucumber-junit-4", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test_session", + "resource" : "cucumber-junit-4", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "cucumber-junit-4", + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4} + }, + "name" : "junit.test_module", + "resource" : "cucumber-junit-4", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] diff --git a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-retry-failed/coverages.ftl b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-retry-failed/coverages.ftl new file mode 100644 index 00000000000..aed9df93b31 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-retry-failed/coverages.ftl @@ -0,0 +1,36 @@ +[ { + "files" : [ { + "filename" : "org/example/cucumber/calculator/basic_arithmetic_failed.feature" + } ], + "span_id" : ${content_span_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} +}, { + "files" : [ { + "filename" : "org/example/cucumber/calculator/basic_arithmetic_failed.feature" + } ], + "span_id" : ${content_span_id_2}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} +}, { + "files" : [ { + "filename" : "org/example/cucumber/calculator/basic_arithmetic_failed.feature" + } ], + "span_id" : ${content_span_id_3}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} +}, { + "files" : [ { + "filename" : "org/example/cucumber/calculator/basic_arithmetic_failed.feature" + } ], + "span_id" : ${content_span_id_4}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} +}, { + "files" : [ { + "filename" : "org/example/cucumber/calculator/basic_arithmetic_failed.feature" + } ], + "span_id" : ${content_span_id_5}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-retry-failed/events.ftl b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-retry-failed/events.ftl new file mode 100644 index 00000000000..a28cbaf5b8a --- /dev/null +++ b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-retry-failed/events.ftl @@ -0,0 +1,337 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "cucumber-junit-4", + "test.status" : "fail", + "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count} + }, + "name" : "junit.test_suite", + "resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack}, + "error.type" : "java.lang.AssertionError", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "cucumber-junit-4", + "test.name" : "Addition", + "test.status" : "fail", + "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.traits" : "{\"category\":[\"foo\"]}", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic.Addition", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack_2}, + "error.type" : "java.lang.AssertionError", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_retry" : "true", + "test.module" : "cucumber-junit-4", + "test.name" : "Addition", + "test.retry_reason" : "atr", + "test.status" : "fail", + "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.traits" : "{\"category\":[\"foo\"]}", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic.Addition", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack_3}, + "error.type" : "java.lang.AssertionError", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_retry" : "true", + "test.module" : "cucumber-junit-4", + "test.name" : "Addition", + "test.retry_reason" : "atr", + "test.status" : "fail", + "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.traits" : "{\"category\":[\"foo\"]}", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic.Addition", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_3}, + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_3} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack_4}, + "error.type" : "java.lang.AssertionError", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_retry" : "true", + "test.module" : "cucumber-junit-4", + "test.name" : "Addition", + "test.retry_reason" : "atr", + "test.status" : "fail", + "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.traits" : "{\"category\":[\"foo\"]}", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic.Addition", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_4}, + "start" : ${content_start_5}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_4} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_6}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack_5}, + "error.type" : "java.lang.AssertionError", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_retry" : "true", + "test.module" : "cucumber-junit-4", + "test.name" : "Addition", + "test.retry_reason" : "atr", + "test.status" : "fail", + "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.traits" : "{\"category\":[\"foo\"]}", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_6}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic.Addition", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_5}, + "start" : ${content_start_6}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_5} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_7}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "cucumber-junit-4", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_7}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test_session", + "resource" : "cucumber-junit-4", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_7}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_8}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "cucumber-junit-4", + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_8} + }, + "name" : "junit.test_module", + "resource" : "cucumber-junit-4", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_8}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy index 7f26ff3b58c..0b4fbcb1a4f 100644 --- a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy +++ b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy @@ -79,6 +79,54 @@ class MUnitTest extends CiVisibilityInstrumentationTest { "test-succeed-impacted" | [TestSucceedMUnit] | new LineDiff([(DUMMY_SOURCE_PATH): lines(DUMMY_TEST_METHOD_START)]) } + def "test quarantined #testcaseName"() { + givenQuarantineEnabled(true) + givenQuarantinedTests(quarantined) + + runTests(tests, true) + + assertSpansData(testcaseName) + + where: + testcaseName | tests | quarantined + "test-failed" | [TestFailedMUnit] | [new TestIdentifier("org.example.TestFailedMUnit", "Calculator.add", null)] + } + + def "test quarantined auto-retries #testcaseName"() { + givenQuarantineEnabled(true) + givenQuarantinedTests(quarantined) + + givenFlakyRetryEnabled(true) + givenFlakyTests(retried) + + // every test retry fails, but the build status is successful + runTests(tests, true) + + assertSpansData(testcaseName) + + where: + testcaseName | tests | quarantined | retried + "test-retry-failed" | [TestFailedMUnit] | [new TestIdentifier("org.example.TestFailedMUnit", "Calculator.add", null)] | [new TestIdentifier("org.example.TestFailedMUnit", "Calculator.add", null)] + } + + def "test quarantined early flakiness detection #testcaseName"() { + givenQuarantineEnabled(true) + givenQuarantinedTests(quarantined) + + givenEarlyFlakinessDetectionEnabled(true) + givenKnownTests(known) + + // every retry fails, but the build status is successful + runTests(tests, true) + + assertSpansData(testcaseName) + + where: + testcaseName | tests | quarantined | known + "test-failed" | [TestFailedMUnit] | [new TestIdentifier("org.example.TestFailedMUnit", "Calculator.add", null)] | [new TestIdentifier("org.example.TestFailedMUnit", "Calculator.add", null)] + "test-failed-efd" | [TestFailedMUnit] | [new TestIdentifier("org.example.TestFailedMUnit", "Calculator.add", null)] | [] + } + private void runTests(Collection> tests, boolean expectSuccess = true) { TestEventsHandlerHolder.start() try { diff --git a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-failed-efd/coverages.ftl b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-failed-efd/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-failed-efd/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-failed-efd/events.ftl b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-failed-efd/events.ftl new file mode 100644 index 00000000000..c5e94d96146 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-failed-efd/events.ftl @@ -0,0 +1,256 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "munit-junit-4", + "test.early_flake.enabled" : "true", + "test.framework" : "munit", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test_session", + "resource" : "munit-junit-4", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.early_flake.enabled" : "true", + "test.framework" : "munit", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "munit-junit-4", + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2} + }, + "name" : "junit.test_module", + "resource" : "munit-junit-4", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "munit", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "munit-junit-4", + "test.source.file" : "dummy_source_path", + "test.status" : "fail", + "test.suite" : "org.example.TestFailedMUnit", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "test.source.end" : 19, + "test.source.start" : 11 + }, + "name" : "junit.test_suite", + "resource" : "org.example.TestFailedMUnit", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack}, + "error.type" : "munit.ComparisonFailException", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "munit", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.module" : "munit-junit-4", + "test.name" : "Calculator.add", + "test.source.file" : "dummy_source_path", + "test.status" : "fail", + "test.suite" : "org.example.TestFailedMUnit", + "test.traits" : "{\"category\":[\"myTag\"]}", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailedMUnit.Calculator.add", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack_2}, + "error.type" : "munit.ComparisonFailException", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "munit", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.is_retry" : "true", + "test.module" : "munit-junit-4", + "test.name" : "Calculator.add", + "test.retry_reason" : "efd", + "test.source.file" : "dummy_source_path", + "test.status" : "fail", + "test.suite" : "org.example.TestFailedMUnit", + "test.traits" : "{\"category\":[\"myTag\"]}", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailedMUnit.Calculator.add", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_5}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_6}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack_3}, + "error.type" : "munit.ComparisonFailException", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "munit", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.is_retry" : "true", + "test.module" : "munit-junit-4", + "test.name" : "Calculator.add", + "test.retry_reason" : "efd", + "test.source.file" : "dummy_source_path", + "test.status" : "fail", + "test.suite" : "org.example.TestFailedMUnit", + "test.traits" : "{\"category\":[\"myTag\"]}", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_6}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailedMUnit.Calculator.add", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_3}, + "start" : ${content_start_6}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_3} + }, + "type" : "test", + "version" : 2 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy b/dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy index 3b3c228476b..fa27adcfab9 100644 --- a/dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy +++ b/dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy @@ -126,6 +126,55 @@ class JUnit4Test extends CiVisibilityInstrumentationTest { "test-succeed-impacted" | [TestSucceed] | new LineDiff([(DUMMY_SOURCE_PATH): lines(DUMMY_TEST_METHOD_START)]) } + def "test quarantined #testcaseName"() { + givenQuarantineEnabled(true) + givenQuarantinedTests(quarantined) + + runTests(tests, true) + + assertSpansData(testcaseName) + + where: + testcaseName | tests | quarantined + "test-failed" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] + "test-failed-parameterized" | [TestFailedParameterized] | [new TestIdentifier("org.example.TestFailedParameterized", "test_failed_parameterized", null)] + } + + def "test quarantined auto-retries #testcaseName"() { + givenQuarantineEnabled(true) + givenQuarantinedTests(quarantined) + + givenFlakyRetryEnabled(true) + givenFlakyTests(retried) + + // every test retry fails, but the build status is successful + runTests(tests, true) + + assertSpansData(testcaseName) + + where: + testcaseName | tests | quarantined | retried + "test-retry-failed" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] + } + + def "test quarantined early flakiness detection #testcaseName"() { + givenQuarantineEnabled(true) + givenQuarantinedTests(quarantined) + + givenEarlyFlakinessDetectionEnabled(true) + givenKnownTests(known) + + // every retry fails, but the build status is successful + runTests(tests, true) + + assertSpansData(testcaseName) + + where: + testcaseName | tests | quarantined | known + "test-failed" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] + "test-failed-efd" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] | [] + } + private void runTests(Collection> tests, boolean expectSuccess = true) { TestEventsHandlerHolder.start() try { diff --git a/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-failed-efd/coverages.ftl b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-failed-efd/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-failed-efd/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-failed-efd/events.ftl b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-failed-efd/events.ftl new file mode 100644 index 00000000000..b9bc04bc9ba --- /dev/null +++ b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-failed-efd/events.ftl @@ -0,0 +1,259 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "junit-4.10", + "test.early_flake.enabled" : "true", + "test.framework" : "junit4", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test_session", + "resource" : "junit-4.10", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.early_flake.enabled" : "true", + "test.framework" : "junit4", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "junit-4.10", + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2} + }, + "name" : "junit.test_module", + "resource" : "junit-4.10", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "junit4", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "junit-4.10", + "test.source.file" : "dummy_source_path", + "test.status" : "fail", + "test.suite" : "org.example.TestFailed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "test.source.end" : 19, + "test.source.start" : 11 + }, + "name" : "junit.test_suite", + "resource" : "org.example.TestFailed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.stack" : ${content_meta_error_stack}, + "error.type" : "java.lang.AssertionError", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "junit4", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.module" : "junit-4.10", + "test.name" : "test_failed", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test_failed()V", + "test.status" : "fail", + "test.suite" : "org.example.TestFailed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailed.test_failed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.stack" : ${content_meta_error_stack_2}, + "error.type" : "java.lang.AssertionError", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "junit4", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.is_retry" : "true", + "test.module" : "junit-4.10", + "test.name" : "test_failed", + "test.retry_reason" : "efd", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test_failed()V", + "test.status" : "fail", + "test.suite" : "org.example.TestFailed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailed.test_failed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_5}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_6}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.stack" : ${content_meta_error_stack_3}, + "error.type" : "java.lang.AssertionError", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "junit4", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.is_retry" : "true", + "test.module" : "junit-4.10", + "test.name" : "test_failed", + "test.retry_reason" : "efd", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test_failed()V", + "test.status" : "fail", + "test.suite" : "org.example.TestFailed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_6}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailed.test_failed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_3}, + "start" : ${content_start_6}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_3} + }, + "type" : "test", + "version" : 2 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-failed-parameterized/coverages.ftl b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-failed-parameterized/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-failed-parameterized/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-failed-parameterized/events.ftl b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-failed-parameterized/events.ftl new file mode 100644 index 00000000000..6ab10b0fb6f --- /dev/null +++ b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-failed-parameterized/events.ftl @@ -0,0 +1,202 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "junit-4.10", + "test.framework" : "junit4", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test_session", + "resource" : "junit-4.10", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "junit4", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "junit-4.10", + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2} + }, + "name" : "junit.test_module", + "resource" : "junit-4.10", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "junit4", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "junit-4.10", + "test.source.file" : "dummy_source_path", + "test.status" : "fail", + "test.suite" : "org.example.TestFailedParameterized", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "test.source.end" : 19, + "test.source.start" : 11 + }, + "name" : "junit.test_suite", + "resource" : "org.example.TestFailedParameterized", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack}, + "error.type" : "org.opentest4j.AssertionFailedError", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "junit4", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "junit-4.10", + "test.name" : "test_failed_parameterized", + "test.parameters" : "{\"metadata\":{\"test_name\":\"test_failed_parameterized[0 0 42]\"}}", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test_failed_parameterized()V", + "test.status" : "fail", + "test.suite" : "org.example.TestFailedParameterized", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailedParameterized.test_failed_parameterized", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "junit4", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "junit-4.10", + "test.name" : "test_failed_parameterized", + "test.parameters" : "{\"metadata\":{\"test_name\":\"test_failed_parameterized[21 21 42]\"}}", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test_failed_parameterized()V", + "test.status" : "pass", + "test.suite" : "org.example.TestFailedParameterized", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailedParameterized.test_failed_parameterized", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_5}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "test", + "version" : 2 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/groovy/CucumberTest.groovy b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/groovy/CucumberTest.groovy index 756322286da..5f2f57f5bb3 100644 --- a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/groovy/CucumberTest.groovy +++ b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/groovy/CucumberTest.groovy @@ -68,15 +68,15 @@ class CucumberTest extends CiVisibilityInstrumentationTest { assertSpansData(testcaseName) where: - testcaseName | success | features | retriedTests - "test-failed" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [] - "test-retry-failed" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ + testcaseName | success | features | retriedTests + "test-failed" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [] + "test-retry-failed" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null) ] - "test-failed-then-succeed" | true | ["org/example/cucumber/calculator/basic_arithmetic_failed_then_succeed.feature"] | [ + "test-failed-then-succeed" | true | ["org/example/cucumber/calculator/basic_arithmetic_failed_then_succeed.feature"] | [ new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed_then_succeed.feature:Basic Arithmetic", "Addition", null) ] - "test-failed-scenario-outline-${version()}" | false | ["org/example/cucumber/calculator/basic_arithmetic_with_failed_examples.feature"] | [ + "test-retry-failed-scenario-outline-${version()}" | false | ["org/example/cucumber/calculator/basic_arithmetic_with_failed_examples.feature"] | [ new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_with_failed_examples.feature:Basic Arithmetic With Examples", "Many additions.Single digits.${parameterizedTestNameSuffix()}", null) ] } @@ -99,6 +99,66 @@ class CucumberTest extends CiVisibilityInstrumentationTest { "test-efd-new-slow-test" | ["org/example/cucumber/calculator/basic_arithmetic_slow.feature"] | [] } + def "test quarantined #testcaseName"() { + givenQuarantineEnabled(true) + givenQuarantinedTests(quarantined) + + runFeatures(features, false, true) + + assertSpansData(testcaseName) + + where: + testcaseName | features | quarantined + "test-failed" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ + new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null) + ] + } + + def "test quarantined auto-retries #testcaseName"() { + givenQuarantineEnabled(true) + givenQuarantinedTests(quarantined) + + givenFlakyRetryEnabled(true) + givenFlakyTests(retried) + + // every test retry fails, but the build status is successful + runFeatures(features, false, true) + + assertSpansData(testcaseName) + + where: + testcaseName | features | quarantined | retried + "test-retry-failed" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ + new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null) + ] | [ + new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null) + ] + } + + def "test quarantined early flakiness detection #testcaseName"() { + givenQuarantineEnabled(true) + givenQuarantinedTests(quarantined) + + givenEarlyFlakinessDetectionEnabled(true) + givenKnownTests(known) + + // every retry fails, but the build status is successful + runFeatures(features, false, true) + + assertSpansData(testcaseName) + + where: + testcaseName | features | quarantined | known + "test-failed" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ + new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null) + ] | [ + new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null) + ] + "test-failed-efd" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ + new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null) + ] | [] + } + private String parameterizedTestNameSuffix() { // older releases report different example names version() == "5.4.0" ? "Example #1" : "Example #1.1" diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-failed-efd/coverages.ftl b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-failed-efd/coverages.ftl new file mode 100644 index 00000000000..48a7dff56d9 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-failed-efd/coverages.ftl @@ -0,0 +1,22 @@ +[ { + "files" : [ { + "filename" : "org/example/cucumber/calculator/basic_arithmetic_failed.feature" + } ], + "span_id" : ${content_span_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} +}, { + "files" : [ { + "filename" : "org/example/cucumber/calculator/basic_arithmetic_failed.feature" + } ], + "span_id" : ${content_span_id_2}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} +}, { + "files" : [ { + "filename" : "org/example/cucumber/calculator/basic_arithmetic_failed.feature" + } ], + "span_id" : ${content_span_id_3}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-failed-efd/events.ftl b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-failed-efd/events.ftl new file mode 100644 index 00000000000..4acbe339494 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-failed-efd/events.ftl @@ -0,0 +1,246 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "cucumber-junit-5", + "test.status" : "fail", + "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count} + }, + "name" : "junit.test_suite", + "resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack}, + "error.type" : "java.lang.AssertionError", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.module" : "cucumber-junit-5", + "test.name" : "Addition", + "test.status" : "fail", + "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.traits" : "{\"category\":[\"foo\"]}", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic.Addition", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack_2}, + "error.type" : "java.lang.AssertionError", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.is_retry" : "true", + "test.module" : "cucumber-junit-5", + "test.name" : "Addition", + "test.retry_reason" : "efd", + "test.status" : "fail", + "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.traits" : "{\"category\":[\"foo\"]}", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic.Addition", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack_3}, + "error.type" : "java.lang.AssertionError", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.is_retry" : "true", + "test.module" : "cucumber-junit-5", + "test.name" : "Addition", + "test.retry_reason" : "efd", + "test.status" : "fail", + "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.traits" : "{\"category\":[\"foo\"]}", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic.Addition", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_3}, + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_3} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "cucumber-junit-5", + "test.early_flake.enabled" : "true", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test_session", + "resource" : "cucumber-junit-5", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_5}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_6}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.early_flake.enabled" : "true", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "cucumber-junit-5", + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_6} + }, + "name" : "junit.test_module", + "resource" : "cucumber-junit-5", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_6}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-failed-scenario-outline-5.4.0/coverages.ftl b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-retry-failed-scenario-outline-5.4.0/coverages.ftl similarity index 100% rename from dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-failed-scenario-outline-5.4.0/coverages.ftl rename to dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-retry-failed-scenario-outline-5.4.0/coverages.ftl diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-failed-scenario-outline-5.4.0/events.ftl b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-retry-failed-scenario-outline-5.4.0/events.ftl similarity index 100% rename from dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-failed-scenario-outline-5.4.0/events.ftl rename to dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-retry-failed-scenario-outline-5.4.0/events.ftl diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-failed-scenario-outline-latest/coverages.ftl b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-retry-failed-scenario-outline-latest/coverages.ftl similarity index 100% rename from dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-failed-scenario-outline-latest/coverages.ftl rename to dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-retry-failed-scenario-outline-latest/coverages.ftl diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-failed-scenario-outline-latest/events.ftl b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-retry-failed-scenario-outline-latest/events.ftl similarity index 100% rename from dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-failed-scenario-outline-latest/events.ftl rename to dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-retry-failed-scenario-outline-latest/events.ftl diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy index bc2899c7a06..e7ea2814d38 100644 --- a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy @@ -133,6 +133,55 @@ class SpockTest extends CiVisibilityInstrumentationTest { "test-succeed-impacted" | [TestSucceedSpock] | new LineDiff([(DUMMY_SOURCE_PATH): lines(DUMMY_TEST_METHOD_START)]) } + def "test quarantined #testcaseName"() { + givenQuarantineEnabled(true) + givenQuarantinedTests(quarantined) + + runTests(tests, true) + + assertSpansData(testcaseName) + + where: + testcaseName | tests | quarantined + "test-failed" | [TestFailedSpock] | [new TestIdentifier("org.example.TestFailedSpock", "test failed", null)] + "test-failed-parameterized" | [TestFailedParameterizedSpock] | [new TestIdentifier("org.example.TestFailedParameterizedSpock", "test add 4 and 4", null)] + } + + def "test quarantined auto-retries #testcaseName"() { + givenQuarantineEnabled(true) + givenQuarantinedTests(quarantined) + + givenFlakyRetryEnabled(true) + givenFlakyTests(retried) + + // every test retry fails, but the build status is successful + runTests(tests, true) + + assertSpansData(testcaseName) + + where: + testcaseName | tests | quarantined | retried + "test-retry-failed" | [TestFailedSpock] | [new TestIdentifier("org.example.TestFailedSpock", "test failed", null)] | [new TestIdentifier("org.example.TestFailedSpock", "test failed", null)] + } + + def "test quarantined early flakiness detection #testcaseName"() { + givenQuarantineEnabled(true) + givenQuarantinedTests(quarantined) + + givenEarlyFlakinessDetectionEnabled(true) + givenKnownTests(known) + + // every retry fails, but the build status is successful + runTests(tests, true) + + assertSpansData(testcaseName) + + where: + testcaseName | tests | quarantined | known + "test-failed" | [TestFailedSpock] | [new TestIdentifier("org.example.TestFailedSpock", "test failed", null)] | [new TestIdentifier("org.example.TestFailedSpock", "test failed", null)] + "test-failed-efd" | [TestFailedSpock] | [new TestIdentifier("org.example.TestFailedSpock", "test failed", null)] | [] + } + private static void runTests(List> classes, boolean expectSuccess = true) { TestEventsHandlerHolder.startForcefully() diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-failed-efd/coverages.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-failed-efd/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-failed-efd/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-failed-efd/events.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-failed-efd/events.ftl new file mode 100644 index 00000000000..7939af650fb --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-failed-efd/events.ftl @@ -0,0 +1,262 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "spock", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "spock-junit-5", + "test.source.file" : "dummy_source_path", + "test.status" : "fail", + "test.suite" : "org.example.TestFailedSpock", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, + "test.source.end" : 19, + "test.source.start" : 11 + }, + "name" : "junit.test_suite", + "resource" : "org.example.TestFailedSpock", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack}, + "error.type" : "org.spockframework.runtime.SpockComparisonFailure", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "spock", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.module" : "spock-junit-5", + "test.name" : "test failed", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test failed()V", + "test.status" : "fail", + "test.suite" : "org.example.TestFailedSpock", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailedSpock.test failed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack_2}, + "error.type" : "org.spockframework.runtime.SpockComparisonFailure", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "spock", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.is_retry" : "true", + "test.module" : "spock-junit-5", + "test.name" : "test failed", + "test.retry_reason" : "efd", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test failed()V", + "test.status" : "fail", + "test.suite" : "org.example.TestFailedSpock", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailedSpock.test failed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack_3}, + "error.type" : "org.spockframework.runtime.SpockComparisonFailure", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "spock", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.is_retry" : "true", + "test.module" : "spock-junit-5", + "test.name" : "test failed", + "test.retry_reason" : "efd", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test failed()V", + "test.status" : "fail", + "test.suite" : "org.example.TestFailedSpock", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailedSpock.test failed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_3}, + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_3} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "spock-junit-5", + "test.early_flake.enabled" : "true", + "test.framework" : "spock", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test_session", + "resource" : "spock-junit-5", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_5}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_6}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.early_flake.enabled" : "true", + "test.framework" : "spock", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "spock-junit-5", + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_6} + }, + "name" : "junit.test_module", + "resource" : "spock-junit-5", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_6}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-failed-parameterized/coverages.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-failed-parameterized/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-failed-parameterized/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-failed-parameterized/events.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-failed-parameterized/events.ftl new file mode 100644 index 00000000000..f680cdb5dbe --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-failed-parameterized/events.ftl @@ -0,0 +1,204 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "spock", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "spock-junit-5", + "test.source.file" : "dummy_source_path", + "test.status" : "fail", + "test.suite" : "org.example.TestFailedParameterizedSpock", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, + "test.source.end" : 19, + "test.source.start" : 11 + }, + "name" : "junit.test_suite", + "resource" : "org.example.TestFailedParameterizedSpock", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "spock", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.module" : "spock-junit-5", + "test.name" : "test add 1 and 2", + "test.parameters" : "{\"metadata\":{\"test_name\":\"test add 1 and 2\"}}", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test add #a and #b(Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;)V", + "test.status" : "pass", + "test.suite" : "org.example.TestFailedParameterizedSpock", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailedParameterizedSpock.test add 1 and 2", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack}, + "error.type" : "org.spockframework.runtime.SpockComparisonFailure", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "spock", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.module" : "spock-junit-5", + "test.name" : "test add 4 and 4", + "test.parameters" : "{\"metadata\":{\"test_name\":\"test add 4 and 4\"}}", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test add #a and #b(Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;)V", + "test.status" : "fail", + "test.suite" : "org.example.TestFailedParameterizedSpock", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailedParameterizedSpock.test add 4 and 4", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "spock-junit-5", + "test.framework" : "spock", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test_session", + "resource" : "spock-junit-5", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_4}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "spock", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "spock-junit-5", + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5} + }, + "name" : "junit.test_module", + "resource" : "spock-junit-5", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_5}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-failed/events.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-failed/events.ftl index 33b2d8f38df..139ab206207 100644 --- a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-failed/events.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-failed/events.ftl @@ -54,7 +54,6 @@ "test.codeowners" : "[\"owner1\",\"owner2\"]", "test.framework" : "spock", "test.framework_version" : ${content_meta_test_framework_version}, - "test.is_new" : "true", "test.module" : "spock-junit-5", "test.name" : "test failed", "test.source.file" : "dummy_source_path", @@ -151,4 +150,4 @@ }, "type" : "test_module_end", "version" : 1 -} ] \ No newline at end of file +} ] diff --git a/dd-java-agent/instrumentation/junit-5.3/src/test/groovy/JUnit5Test.groovy b/dd-java-agent/instrumentation/junit-5.3/src/test/groovy/JUnit5Test.groovy index b51a0e11658..c4257650728 100644 --- a/dd-java-agent/instrumentation/junit-5.3/src/test/groovy/JUnit5Test.groovy +++ b/dd-java-agent/instrumentation/junit-5.3/src/test/groovy/JUnit5Test.groovy @@ -164,6 +164,55 @@ class JUnit5Test extends CiVisibilityInstrumentationTest { "test-succeed-impacted" | [TestSucceed] | new LineDiff([(DUMMY_SOURCE_PATH): lines(DUMMY_TEST_METHOD_START)]) } + def "test quarantined #testcaseName"() { + givenQuarantineEnabled(true) + givenQuarantinedTests(quarantined) + + runTests(tests, true) + + assertSpansData(testcaseName) + + where: + testcaseName | tests | quarantined + "test-failed" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] + "test-failed-parameterized" | [TestFailedParameterized] | [new TestIdentifier("org.example.TestFailedParameterized", "test_failed_parameterized", null)] + } + + def "test quarantined auto-retries #testcaseName"() { + givenQuarantineEnabled(true) + givenQuarantinedTests(quarantined) + + givenFlakyRetryEnabled(true) + givenFlakyTests(retried) + + // every test retry fails, but the build status is successful + runTests(tests, true) + + assertSpansData(testcaseName) + + where: + testcaseName | tests | quarantined | retried + "test-retry-failed" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] + } + + def "test quarantined early flakiness detection #testcaseName"() { + givenQuarantineEnabled(true) + givenQuarantinedTests(quarantined) + + givenEarlyFlakinessDetectionEnabled(true) + givenKnownTests(known) + + // every retry fails, but the build status is successful + runTests(tests, true) + + assertSpansData(testcaseName) + + where: + testcaseName | tests | quarantined | known + "test-failed" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] + "test-failed-efd" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] | [] + } + protected void runTests(List> tests, boolean expectSuccess = true) { TestEventsHandlerHolder.startForcefully() diff --git a/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-failed-efd/coverages.ftl b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-failed-efd/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-failed-efd/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-failed-efd/events.ftl b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-failed-efd/events.ftl new file mode 100644 index 00000000000..78b17e5d2a8 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-failed-efd/events.ftl @@ -0,0 +1,262 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "junit-5.3", + "test.early_flake.enabled" : "true", + "test.framework" : "junit5", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test_session", + "resource" : "junit-5.3", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.early_flake.enabled" : "true", + "test.framework" : "junit5", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "junit-5.3", + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2} + }, + "name" : "junit.test_module", + "resource" : "junit-5.3", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "junit5", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "junit-5.3", + "test.source.file" : "dummy_source_path", + "test.status" : "fail", + "test.suite" : "org.example.TestFailed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "test.source.end" : 19, + "test.source.start" : 11 + }, + "name" : "junit.test_suite", + "resource" : "org.example.TestFailed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack}, + "error.type" : "org.opentest4j.AssertionFailedError", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "junit5", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.module" : "junit-5.3", + "test.name" : "test_failed", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test_failed()V", + "test.status" : "fail", + "test.suite" : "org.example.TestFailed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailed.test_failed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack_2}, + "error.type" : "org.opentest4j.AssertionFailedError", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "junit5", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.is_retry" : "true", + "test.module" : "junit-5.3", + "test.name" : "test_failed", + "test.retry_reason" : "efd", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test_failed()V", + "test.status" : "fail", + "test.suite" : "org.example.TestFailed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailed.test_failed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_5}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_6}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack_3}, + "error.type" : "org.opentest4j.AssertionFailedError", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "junit5", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.is_retry" : "true", + "test.module" : "junit-5.3", + "test.name" : "test_failed", + "test.retry_reason" : "efd", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test_failed()V", + "test.status" : "fail", + "test.suite" : "org.example.TestFailed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_6}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailed.test_failed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_3}, + "start" : ${content_start_6}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_3} + }, + "type" : "test", + "version" : 2 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-failed-parameterized/coverages.ftl b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-failed-parameterized/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-failed-parameterized/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-failed-parameterized/events.ftl b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-failed-parameterized/events.ftl new file mode 100644 index 00000000000..1dfdad89f47 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-failed-parameterized/events.ftl @@ -0,0 +1,205 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "junit-5.3", + "test.framework" : "junit5", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test_session", + "resource" : "junit-5.3", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "junit5", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "junit-5.3", + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2} + }, + "name" : "junit.test_module", + "resource" : "junit-5.3", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "junit5", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "junit-5.3", + "test.source.file" : "dummy_source_path", + "test.status" : "fail", + "test.suite" : "org.example.TestFailedParameterized", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "test.source.end" : 19, + "test.source.start" : 11 + }, + "name" : "junit.test_suite", + "resource" : "org.example.TestFailedParameterized", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack}, + "error.type" : "org.opentest4j.AssertionFailedError", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "junit5", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "junit-5.3", + "test.name" : "test_failed_parameterized", + "test.parameters" : "{\"metadata\":{\"test_name\":\"[1] 0, 0, 42\"}}", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test_failed_parameterized(III)V", + "test.status" : "fail", + "test.suite" : "org.example.TestFailedParameterized", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailedParameterized.test_failed_parameterized", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message_2}, + "error.stack" : ${content_meta_error_stack_2}, + "error.type" : "org.opentest4j.AssertionFailedError", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "junit5", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "junit-5.3", + "test.name" : "test_failed_parameterized", + "test.parameters" : "{\"metadata\":{\"test_name\":\"[2] 1, 1, 42\"}}", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test_failed_parameterized(III)V", + "test.status" : "fail", + "test.suite" : "org.example.TestFailedParameterized", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailedParameterized.test_failed_parameterized", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_5}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "test", + "version" : 2 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/karate/src/test/groovy/KarateTest.groovy b/dd-java-agent/instrumentation/karate/src/test/groovy/KarateTest.groovy index b818fbbf985..4f3b72c4657 100644 --- a/dd-java-agent/instrumentation/karate/src/test/groovy/KarateTest.groovy +++ b/dd-java-agent/instrumentation/karate/src/test/groovy/KarateTest.groovy @@ -3,19 +3,7 @@ import datadog.trace.api.DisableTestTrace import datadog.trace.api.civisibility.config.TestIdentifier import datadog.trace.civisibility.CiVisibilityInstrumentationTest import datadog.trace.instrumentation.karate.TestEventsHandlerHolder -import org.example.TestFailedBuiltInRetryKarate -import org.example.TestFailedKarate -import org.example.TestFailedParameterizedKarate -import org.example.TestFailedThenSucceedKarate -import org.example.TestParameterizedKarate -import org.example.TestParameterizedMoreCasesKarate -import org.example.TestSkippedFeatureKarate -import org.example.TestSucceedKarate -import org.example.TestSucceedKarateSlow -import org.example.TestSucceedOneCaseKarate -import org.example.TestSucceedParallelKarate -import org.example.TestUnskippableKarate -import org.example.TestWithSetupKarate +import org.example.* import org.junit.jupiter.api.Assumptions import org.junit.jupiter.engine.JupiterTestEngine import org.junit.platform.engine.DiscoverySelector @@ -107,6 +95,19 @@ class KarateTest extends CiVisibilityInstrumentationTest { "test-efd-faulty-session-threshold" | [TestParameterizedMoreCasesKarate] | [] } + def "test quarantined #testcaseName"() { + givenQuarantineEnabled(true) + givenQuarantinedTests(quarantined) + + runTests(tests, true) + + assertSpansData(testcaseName) + + where: + testcaseName | tests | quarantined + "test-failed-quarantine" | [TestFailedKarate] | [new TestIdentifier("[org/example/test_failed] test failed", "second scenario", null)] + } + private void runTests(List> tests, boolean expectSuccess = true) { TestEventsHandlerHolder.start() diff --git a/dd-java-agent/instrumentation/karate/src/test/resources/test-failed-quarantine/coverages.ftl b/dd-java-agent/instrumentation/karate/src/test/resources/test-failed-quarantine/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/karate/src/test/resources/test-failed-quarantine/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/karate/src/test/resources/test-failed-quarantine/events.ftl b/dd-java-agent/instrumentation/karate/src/test/resources/test-failed-quarantine/events.ftl new file mode 100644 index 00000000000..869575044b3 --- /dev/null +++ b/dd-java-agent/instrumentation/karate/src/test/resources/test-failed-quarantine/events.ftl @@ -0,0 +1,234 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* assert false" + }, + "metrics" : { + "step.endLine" : 7, + "step.startLine" : 7 + }, + "name" : "karate.step", + "parent_id" : ${content_parent_id}, + "resource" : "* assert false", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start}, + "trace_id" : ${content_trace_id} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* print 'first'" + }, + "metrics" : { + "step.endLine" : 4, + "step.startLine" : 4 + }, + "name" : "karate.step", + "parent_id" : ${content_parent_id_2}, + "resource" : "* print 'first'", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_2}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "karate", + "test.status" : "fail", + "test.suite" : "[org/example/test_failed] test failed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count} + }, + "name" : "karate.test_suite", + "resource" : "[org/example/test_failed] test failed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "karate", + "test.name" : "first scenario", + "test.status" : "pass", + "test.suite" : "[org/example/test_failed] test failed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "karate.test", + "parent_id" : ${content_parent_id_3}, + "resource" : "[org/example/test_failed] test failed.first scenario", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_parent_id_2}, + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack}, + "error.type" : "com.intuit.karate.KarateException", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "karate", + "test.name" : "second scenario", + "test.status" : "fail", + "test.suite" : "[org/example/test_failed] test failed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "karate.test", + "parent_id" : ${content_parent_id_3}, + "resource" : "[org/example/test_failed] test failed.second scenario", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_parent_id}, + "start" : ${content_start_5}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_6}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "karate", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "karate.test_session", + "resource" : "karate", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_6}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_7}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "karate", + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5} + }, + "name" : "karate.test_module", + "resource" : "karate", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_7}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/scalatest/src/test/groovy/ScalatestTest.groovy b/dd-java-agent/instrumentation/scalatest/src/test/groovy/ScalatestTest.groovy index 6b21cb3c247..43741eaf554 100644 --- a/dd-java-agent/instrumentation/scalatest/src/test/groovy/ScalatestTest.groovy +++ b/dd-java-agent/instrumentation/scalatest/src/test/groovy/ScalatestTest.groovy @@ -101,6 +101,54 @@ class ScalatestTest extends CiVisibilityInstrumentationTest { "test-succeed-impacted" | [TestSucceed] | new LineDiff([(DUMMY_SOURCE_PATH): lines(DUMMY_TEST_METHOD_START)]) } + def "test quarantined #testcaseName"() { + givenQuarantineEnabled(true) + givenQuarantinedTests(quarantined) + + runTests(tests, true) + + assertSpansData(testcaseName) + + where: + testcaseName | tests | quarantined + "test-failed" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "Example.add adds two numbers", null)] + } + + def "test quarantined auto-retries #testcaseName"() { + givenQuarantineEnabled(true) + givenQuarantinedTests(quarantined) + + givenFlakyRetryEnabled(true) + givenFlakyTests(retried) + + // every test retry fails, but the build status is successful + runTests(tests, true) + + assertSpansData(testcaseName) + + where: + testcaseName | tests | quarantined | retried + "test-retry-failed" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "Example.add adds two numbers", null)] | [new TestIdentifier("org.example.TestFailed", "Example.add adds two numbers", null)] + } + + def "test quarantined early flakiness detection #testcaseName"() { + givenQuarantineEnabled(true) + givenQuarantinedTests(quarantined) + + givenEarlyFlakinessDetectionEnabled(true) + givenKnownTests(known) + + // every retry fails, but the build status is successful + runTests(tests, true) + + assertSpansData(testcaseName) + + where: + testcaseName | tests | quarantined | known + "test-failed" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "Example.add adds two numbers", null)] | [new TestIdentifier("org.example.TestFailed", "Example.add adds two numbers", null)] + "test-failed-efd" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "Example.add adds two numbers", null)] | [] + } + @Override String instrumentedLibraryName() { return "scalatest" diff --git a/dd-java-agent/instrumentation/scalatest/src/test/resources/test-failed-efd/coverages.ftl b/dd-java-agent/instrumentation/scalatest/src/test/resources/test-failed-efd/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/scalatest/src/test/resources/test-failed-efd/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/scalatest/src/test/resources/test-failed-efd/events.ftl b/dd-java-agent/instrumentation/scalatest/src/test/resources/test-failed-efd/events.ftl new file mode 100644 index 00000000000..aba4496f09d --- /dev/null +++ b/dd-java-agent/instrumentation/scalatest/src/test/resources/test-failed-efd/events.ftl @@ -0,0 +1,253 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "scalatest", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "scalatest", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "scalatest", + "test.source.file" : "dummy_source_path", + "test.status" : "fail", + "test.suite" : "org.example.TestFailed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, + "test.source.end" : 19, + "test.source.start" : 11 + }, + "name" : "scalatest.test_suite", + "resource" : "org.example.TestFailed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "scalatest", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack}, + "error.type" : "org.scalatest.exceptions.TestFailedException", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "scalatest", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.module" : "scalatest", + "test.name" : "Example.add adds two numbers", + "test.source.file" : "dummy_source_path", + "test.status" : "fail", + "test.suite" : "org.example.TestFailed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "scalatest.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailed.Example.add adds two numbers", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "scalatest", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack_2}, + "error.type" : "org.scalatest.exceptions.TestFailedException", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "scalatest", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.is_retry" : "true", + "test.module" : "scalatest", + "test.name" : "Example.add adds two numbers", + "test.retry_reason" : "efd", + "test.source.file" : "dummy_source_path", + "test.status" : "fail", + "test.suite" : "org.example.TestFailed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "scalatest.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailed.Example.add adds two numbers", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "scalatest", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack_3}, + "error.type" : "org.scalatest.exceptions.TestFailedException", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "scalatest", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.is_retry" : "true", + "test.module" : "scalatest", + "test.name" : "Example.add adds two numbers", + "test.retry_reason" : "efd", + "test.source.file" : "dummy_source_path", + "test.status" : "fail", + "test.suite" : "org.example.TestFailed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "scalatest.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailed.Example.add adds two numbers", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_3}, + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_3} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "scalatest", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "scalatest", + "test.early_flake.enabled" : "true", + "test.framework" : "scalatest", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "scalatest.test_session", + "resource" : "scalatest", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_5}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_6}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "scalatest", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.early_flake.enabled" : "true", + "test.framework" : "scalatest", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "scalatest", + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_6} + }, + "name" : "scalatest.test_module", + "resource" : "scalatest", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_6}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/testng/src/main/java/datadog/trace/instrumentation/testng/execution/RetryAnalyzer.java b/dd-java-agent/instrumentation/testng/src/main/java/datadog/trace/instrumentation/testng/execution/RetryAnalyzer.java index 7d3b1a7d0a4..9d9604e0363 100644 --- a/dd-java-agent/instrumentation/testng/src/main/java/datadog/trace/instrumentation/testng/execution/RetryAnalyzer.java +++ b/dd-java-agent/instrumentation/testng/src/main/java/datadog/trace/instrumentation/testng/execution/RetryAnalyzer.java @@ -36,4 +36,8 @@ public boolean retry(ITestResult result) { public RetryReason currentExecutionRetryReason() { return executionPolicy != null ? executionPolicy.currentExecutionRetryReason() : null; } + + public boolean suppressFailures() { + return executionPolicy != null && executionPolicy.suppressFailures(); + } } diff --git a/dd-java-agent/instrumentation/testng/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy b/dd-java-agent/instrumentation/testng/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy index f0eb856e518..f7a4ca4f3d1 100644 --- a/dd-java-agent/instrumentation/testng/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy +++ b/dd-java-agent/instrumentation/testng/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy @@ -154,6 +154,61 @@ abstract class TestNGTest extends CiVisibilityInstrumentationTest { "test-succeed-impacted" | [TestSucceed] | new LineDiff([(DUMMY_SOURCE_PATH): lines(DUMMY_TEST_METHOD_START)]) } + def "test quarantined #testcaseName"() { + Assumptions.assumeTrue(isExceptionSuppressionSupported()) + + givenQuarantineEnabled(true) + givenQuarantinedTests(quarantined) + + runTests(tests, null, true) + + assertSpansData(testcaseName) + + where: + testcaseName | tests | quarantined + "test-failed-${version()}" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] + "test-failed-parameterized" | [TestFailedParameterized] | [new TestIdentifier("org.example.TestFailedParameterized", "parameterized_test_succeed", null)] + } + + def "test quarantined auto-retries #testcaseName"() { + Assumptions.assumeTrue(isExceptionSuppressionSupported()) + + givenQuarantineEnabled(true) + givenQuarantinedTests(quarantined) + + givenFlakyRetryEnabled(true) + givenFlakyTests(retried) + + // every test retry fails, but the build status is successful + runTests(tests, null, true) + + assertSpansData(testcaseName) + + where: + testcaseName | tests | quarantined | retried + "test-retry-failed-${version()}" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] + } + + def "test quarantined early flakiness detection #testcaseName"() { + Assumptions.assumeTrue(isExceptionSuppressionSupported()) + + givenQuarantineEnabled(true) + givenQuarantinedTests(quarantined) + + givenEarlyFlakinessDetectionEnabled(true) + givenKnownTests(known) + + // every retry fails, but the build status is successful + runTests(tests, null, true) + + assertSpansData(testcaseName) + + where: + testcaseName | tests | quarantined | known + "test-failed-${version()}" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] + "test-failed-efd" | [TestFailed] | [new TestIdentifier("org.example.TestFailed", "test_failed", null)] | [] + } + private static boolean isEFDSupported() { currentTestNGVersion >= testNGv75 } diff --git a/dd-java-agent/instrumentation/testng/testng-7/src/main/java/datadog/trace/instrumentation/testng7/TestNGExecutionInstrumentation.java b/dd-java-agent/instrumentation/testng/testng-7/src/main/java/datadog/trace/instrumentation/testng7/TestNGExecutionInstrumentation.java index 838a04cb93c..1a415f7afed 100644 --- a/dd-java-agent/instrumentation/testng/testng-7/src/main/java/datadog/trace/instrumentation/testng7/TestNGExecutionInstrumentation.java +++ b/dd-java-agent/instrumentation/testng/testng-7/src/main/java/datadog/trace/instrumentation/testng7/TestNGExecutionInstrumentation.java @@ -14,7 +14,6 @@ import java.util.Set; import net.bytebuddy.asm.Advice; import org.testng.IRetryAnalyzer; -import org.testng.ITestNGMethod; import org.testng.ITestResult; @AutoService(InstrumenterModule.class) @@ -43,6 +42,10 @@ public void methodAdvice(MethodTransformer transformer) { transformer.applyAdvice( named("shouldRetryTestMethod").and(takesArgument(1, named("org.testng.ITestResult"))), TestNGExecutionInstrumentation.class.getName() + "$ExecutionAdvice"); + + transformer.applyAdvice( + named("runTestResultListener").and(takesArgument(0, named("org.testng.ITestResult"))), + TestNGExecutionInstrumentation.class.getName() + "$SuppressFailuresAdvice"); } @Override @@ -63,12 +66,33 @@ public static void shouldRetryTestMethod( @Advice.Argument(1) final ITestResult result, @Advice.Return(readOnly = false) boolean retry) { if (!retry && result.isSuccess()) { - ITestNGMethod method = result.getMethod(); - IRetryAnalyzer retryAnalyzer = method.getRetryAnalyzer(result); + IRetryAnalyzer retryAnalyzer = TestNGUtils.getRetryAnalyzer(result); if (retryAnalyzer instanceof RetryAnalyzer) { retry = retryAnalyzer.retry(result); } } } } + + public static class SuppressFailuresAdvice { + @SuppressWarnings("bytebuddy-exception-suppression") + @Advice.OnMethodEnter + public static void suppressFailures(@Advice.Argument(0) final ITestResult result) { + if (result.getStatus() != ITestResult.FAILURE) { + // nothing to suppress + return; + } + + IRetryAnalyzer retryAnalyzer = TestNGUtils.getRetryAnalyzer(result); + if (!(retryAnalyzer instanceof RetryAnalyzer)) { + // test execution policies not injected + return; + } + RetryAnalyzer ddRetryAnalyzer = (RetryAnalyzer) retryAnalyzer; + if (ddRetryAnalyzer.suppressFailures()) { + // "failed but within success percentage" + result.setStatus(ITestResult.SUCCESS_PERCENTAGE_FAILURE); + } + } + } } diff --git a/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-failed-efd/coverages.ftl b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-failed-efd/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-failed-efd/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-failed-efd/events.ftl b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-failed-efd/events.ftl new file mode 100644 index 00000000000..993a1c17b0f --- /dev/null +++ b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-failed-efd/events.ftl @@ -0,0 +1,262 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "testng-7", + "test.source.file" : "dummy_source_path", + "test.status" : "fail", + "test.suite" : "org.example.TestFailed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, + "test.source.end" : 19, + "test.source.start" : 11 + }, + "name" : "testng.test_suite", + "resource" : "org.example.TestFailed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack}, + "error.type" : "java.lang.AssertionError", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.module" : "testng-7", + "test.name" : "test_failed", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test_failed()V", + "test.status" : "fail", + "test.suite" : "org.example.TestFailed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "testng.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailed.test_failed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack_2}, + "error.type" : "java.lang.AssertionError", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.is_retry" : "true", + "test.module" : "testng-7", + "test.name" : "test_failed", + "test.retry_reason" : "efd", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test_failed()V", + "test.status" : "fail", + "test.suite" : "org.example.TestFailed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "testng.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailed.test_failed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack_3}, + "error.type" : "java.lang.AssertionError", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.is_retry" : "true", + "test.module" : "testng-7", + "test.name" : "test_failed", + "test.retry_reason" : "efd", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test_failed()V", + "test.status" : "fail", + "test.suite" : "org.example.TestFailed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "testng.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailed.test_failed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_3}, + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_3} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "testng-7", + "test.early_flake.enabled" : "true", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "testng.test_session", + "resource" : "testng-7", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_5}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_6}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.early_flake.enabled" : "true", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "testng-7", + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_6} + }, + "name" : "testng.test_module", + "resource" : "testng-7", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_6}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-failed-parameterized/coverages.ftl b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-failed-parameterized/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-failed-parameterized/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-failed-parameterized/events.ftl b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-failed-parameterized/events.ftl new file mode 100644 index 00000000000..fd3e510422b --- /dev/null +++ b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-failed-parameterized/events.ftl @@ -0,0 +1,201 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "testng-7", + "test.source.file" : "dummy_source_path", + "test.status" : "fail", + "test.suite" : "org.example.TestFailedParameterized", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, + "test.source.end" : 19, + "test.source.start" : 11 + }, + "name" : "testng.test_suite", + "resource" : "org.example.TestFailedParameterized", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "testng-7", + "test.name" : "parameterized_test_succeed", + "test.parameters" : "{\"arguments\":{\"0\":\"hello\",\"1\":\"true\"}}", + "test.source.file" : "dummy_source_path", + "test.source.method" : "parameterized_test_succeed(Ljava/lang/String;Z)V", + "test.status" : "pass", + "test.suite" : "org.example.TestFailedParameterized", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "testng.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailedParameterized.parameterized_test_succeed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.stack" : ${content_meta_error_stack}, + "error.type" : "java.lang.AssertionError", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "testng-7", + "test.name" : "parameterized_test_succeed", + "test.parameters" : "{\"arguments\":{\"0\":\"\\\"goodbye\\\"\",\"1\":\"false\"}}", + "test.source.file" : "dummy_source_path", + "test.source.method" : "parameterized_test_succeed(Ljava/lang/String;Z)V", + "test.status" : "fail", + "test.suite" : "org.example.TestFailedParameterized", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "testng.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailedParameterized.parameterized_test_succeed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "testng-7", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "testng.test_session", + "resource" : "testng-7", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_4}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "testng-7", + "test.status" : "fail", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5} + }, + "name" : "testng.test_module", + "resource" : "testng-7", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_5}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file