From 068a1c9f2a750155b21a08ab34c54da9a86edf02 Mon Sep 17 00:00:00 2001 From: ds-hzimmer Date: Mon, 10 Feb 2025 09:19:27 +0100 Subject: [PATCH 1/7] Suggestion Application Testing TRG Suggestion for a Tractus-X Release Guideline (TRG) for Code Coverage --- docs/release/trg-10/_category_.json | 3 ++ docs/release/trg-10/trg-10-01.md | 63 +++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 docs/release/trg-10/_category_.json create mode 100644 docs/release/trg-10/trg-10-01.md diff --git a/docs/release/trg-10/_category_.json b/docs/release/trg-10/_category_.json new file mode 100644 index 000000000000..c8f469a73a4c --- /dev/null +++ b/docs/release/trg-10/_category_.json @@ -0,0 +1,3 @@ +{ + "label": "TRG 10 - Application Testing" +} diff --git a/docs/release/trg-10/trg-10-01.md b/docs/release/trg-10/trg-10-01.md new file mode 100644 index 000000000000..548d294fbbc7 --- /dev/null +++ b/docs/release/trg-10/trg-10-01.md @@ -0,0 +1,63 @@ +--- +title: TRG 10.01 - Code Coverage +sidebar_position: 1 +--- + +| Status | Created | Post-History | +|--------|-------------|-------------------------| +| Draft | 07-Feb-2025 | Initial version created | + +## Why + +Goal: To ensure that all released software components meet sufficient test coverage to guarantee quality, stability, and reliability. + +This guideline applies to all software components and projects that are part of the Eclipse Tractus-X release process. + +## Description + +### 1. Minimum Requirements + +1.1. **Code Coverage Threshold** + +- Quality Gate: The minimum threshold for code coverage (line coverage) should be **80.0%**. +- This applies to both unit tests and integration tests. +- Every project not specifically associated to a different Quality Gate will be associated to this one by default. + + +1.2. **Exceptions** + +Certain code sections may be excluded from counting towards the code coverage percentage if they for example: + - Are themselves test code (no "test of tests"). + - Autogenerated code, for example Swagger-generated API clients. + - Configuration files with no logic to test. + - Are experimental or prototype code, for example incomplete implementations hidden behind feature toggles. + - Are boilerplate code or entity classes that follow a known, predictable pattern. + - Code for logging, metrics, or monitoring. + - Depend on platform-specifics, for example hardware that cannot be simulated in a test. + +All exceptions should be documented and approved by the project's committer. + +### 2. Analysis and Reporting + +2.1. **Tools for Code Coverage Measurement** + +- Recommended tool: **SonarCloud** as provided by Elipse Tractus-X. +Reference: https://sonarcloud.io/organizations/eclipse-tractusx/projects +- Alternatives: **SonarQube, JaCoCo** +- The tool used to track code coverage should be specified and documented within the respective project. + +2.2. **Regular Review** + +- Code coverage should be measured by all Eclipse Tractus-X software products at least once before each release. +- Coverage measurement results should be integrated into CI/CD pipelines and automated as part of the release process. + +### 3. Quality Assurance + +3.1. **Code Review Requirements** + +- Reviewers must ensure that new or modified code sections are sufficiently covered by test cases. +- Code changes that lower the overall coverage below the defined Quality Gate threshold may only be accepted in exceptional cases. + +3.2. **Risk Analysis** + +- If the code coverage value falls below 80.0%, the potential risks should be documented and approved by the release management team. From 49ee43bc56f1492e2336144c3f7a2e84649b950b Mon Sep 17 00:00:00 2001 From: ds-hzimmer Date: Mon, 10 Feb 2025 12:57:59 +0100 Subject: [PATCH 2/7] Update trg-10-01.md Adapted Markdown for a bare URL external link. --- docs/release/trg-10/trg-10-01.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/release/trg-10/trg-10-01.md b/docs/release/trg-10/trg-10-01.md index 548d294fbbc7..a5f5b375397d 100644 --- a/docs/release/trg-10/trg-10-01.md +++ b/docs/release/trg-10/trg-10-01.md @@ -41,8 +41,7 @@ All exceptions should be documented and approved by the project's committer. 2.1. **Tools for Code Coverage Measurement** -- Recommended tool: **SonarCloud** as provided by Elipse Tractus-X. -Reference: https://sonarcloud.io/organizations/eclipse-tractusx/projects +- Recommended tool: **SonarCloud** as provided by Elipse Tractus-X. [Reference link](https://sonarcloud.io/organizations/eclipse-tractusx/projects) - Alternatives: **SonarQube, JaCoCo** - The tool used to track code coverage should be specified and documented within the respective project. From 42a1a84ae937ee7770ca9c8ea47b54a13005a984 Mon Sep 17 00:00:00 2001 From: ds-hzimmer Date: Mon, 10 Feb 2025 13:20:59 +0100 Subject: [PATCH 3/7] Fix mardown formatting blank lines, unordered list --- docs/release/trg-10/trg-10-01.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/release/trg-10/trg-10-01.md b/docs/release/trg-10/trg-10-01.md index a5f5b375397d..7fb9c06b0a99 100644 --- a/docs/release/trg-10/trg-10-01.md +++ b/docs/release/trg-10/trg-10-01.md @@ -23,17 +23,17 @@ This guideline applies to all software components and projects that are part of - This applies to both unit tests and integration tests. - Every project not specifically associated to a different Quality Gate will be associated to this one by default. - 1.2. **Exceptions** -Certain code sections may be excluded from counting towards the code coverage percentage if they for example: - - Are themselves test code (no "test of tests"). - - Autogenerated code, for example Swagger-generated API clients. - - Configuration files with no logic to test. - - Are experimental or prototype code, for example incomplete implementations hidden behind feature toggles. - - Are boilerplate code or entity classes that follow a known, predictable pattern. - - Code for logging, metrics, or monitoring. - - Depend on platform-specifics, for example hardware that cannot be simulated in a test. +Certain code sections may be excluded from counting towards the code coverage percentage. For example this could apply to code sections that: + +- Are themselves test code (no "test of tests"). +- Are autogenerated code, for example Swagger-generated API clients. +- Are configuration files with no logic to test. +- Are experimental or prototype code, for example incomplete implementations hidden behind feature toggles. +- Are boilerplate code or entity classes that follow a known, predictable pattern. +- Are code for logging, metrics, or monitoring. +- Depend on platform-specifics, for example hardware that cannot be simulated in a test. All exceptions should be documented and approved by the project's committer. From 640cd63b54dfa410eee550bc892f7eafa688ede6 Mon Sep 17 00:00:00 2001 From: ds-hzimmer Date: Fri, 21 Mar 2025 09:27:46 +0100 Subject: [PATCH 4/7] Update trg-10-01.md Update with an example integration for Java. Additional changes pending. --- docs/release/trg-10/trg-10-01.md | 57 ++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/docs/release/trg-10/trg-10-01.md b/docs/release/trg-10/trg-10-01.md index 7fb9c06b0a99..b03baaedf7f0 100644 --- a/docs/release/trg-10/trg-10-01.md +++ b/docs/release/trg-10/trg-10-01.md @@ -6,6 +6,7 @@ sidebar_position: 1 | Status | Created | Post-History | |--------|-------------|-------------------------| | Draft | 07-Feb-2025 | Initial version created | +| Draft | 21-Mar-2025 | Update with example integration for Java | ## Why @@ -41,8 +42,7 @@ All exceptions should be documented and approved by the project's committer. 2.1. **Tools for Code Coverage Measurement** -- Recommended tool: **SonarCloud** as provided by Elipse Tractus-X. [Reference link](https://sonarcloud.io/organizations/eclipse-tractusx/projects) -- Alternatives: **SonarQube, JaCoCo** +- Recommended: **SonarQube Cloud** as provided by Elipse Tractus-X. [Reference link](https://sonarcloud.io/organizations/eclipse-tractusx/projects) - The tool used to track code coverage should be specified and documented within the respective project. 2.2. **Regular Review** @@ -60,3 +60,56 @@ All exceptions should be documented and approved by the project's committer. 3.2. **Risk Analysis** - If the code coverage value falls below 80.0%, the potential risks should be documented and approved by the release management team. + +### 4. Examples and Resources + +#### 4.1. **Code Examples** + +An example of a GitHub workflow integration can be found below. + +```yaml +name: "Run unit and integration tests" + +on: + pull_request: + workflow_dispatch: + push: + branches: + - main + +env: + JAVA_VERSION: 17 + +jobs: + Test-and-Sonar: + permissions: + contents: read + checks: write + pull-requests: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + java-version: '${{ env.JAVA_VERSION }}' + distribution: 'temurin' + cache: 'maven' + + - name: Run unit & integration tests + run: | + mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B verify + + - name: Publish integration test results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + files: "${{ github.workspace }}/target/failsafe-reports/integration-tests.xml" + check_name: "Integration Test Results" + + - name: Publish unit test results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + files: "**/surefire-reports/unit-tests.xml" + check_name: "Unit Test Results" +``` From 80809abb36f80c5f60777534a3d7fb0db6211bd2 Mon Sep 17 00:00:00 2001 From: ds-hzimmer Date: Mon, 24 Mar 2025 16:28:44 +0100 Subject: [PATCH 5/7] Adaptations for sections 2.1 and distinction between code coverage reporting and code coverage analysis itself, and additional references and clarifications. --- docs/release/trg-10/trg-10-01.md | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/docs/release/trg-10/trg-10-01.md b/docs/release/trg-10/trg-10-01.md index b03baaedf7f0..b21f5c2b89fb 100644 --- a/docs/release/trg-10/trg-10-01.md +++ b/docs/release/trg-10/trg-10-01.md @@ -21,7 +21,7 @@ This guideline applies to all software components and projects that are part of 1.1. **Code Coverage Threshold** - Quality Gate: The minimum threshold for code coverage (line coverage) should be **80.0%**. -- This applies to both unit tests and integration tests. +- The code coverage is calculated using both unit tests and integration tests combined. - Every project not specifically associated to a different Quality Gate will be associated to this one by default. 1.2. **Exceptions** @@ -36,14 +36,16 @@ Certain code sections may be excluded from counting towards the code coverage pe - Are code for logging, metrics, or monitoring. - Depend on platform-specifics, for example hardware that cannot be simulated in a test. -All exceptions should be documented and approved by the project's committer. +All exceptions should be documented and approved by the project's lead/main committers. ### 2. Analysis and Reporting -2.1. **Tools for Code Coverage Measurement** +2.1. **Tools for Code Coverage Reporting** -- Recommended: **SonarQube Cloud** as provided by Elipse Tractus-X. [Reference link](https://sonarcloud.io/organizations/eclipse-tractusx/projects) -- The tool used to track code coverage should be specified and documented within the respective project. +- The recommended code coverage reporting tool is **SonarQube Cloud** as provided by Elipse Tractus-X. [Reference link](https://sonarcloud.io/organizations/eclipse-tractusx/projects) +- Please note that SonarQube Cloud does not calculate the code coverage itself. To include coverage results you must set up a third-party coverage analysis tool and configure SonarQube Cloud to import the analysis results produced by that tool. +- SonarQube Cloud supports the import of coverage data in formats native to a variety of tools for a variety of languages. It also provides a possibility to import a generic format from tools that are not directly supported. See the guideline in section [4. Resources and Examples](#sonarqube-implementation-guides) as reference on how to implement this. +- The tool or tools used for test coverage analysis and to produce the code coverage report should be specified and documented within the respective Eclipse Tractus-X project. 2.2. **Regular Review** @@ -59,13 +61,17 @@ All exceptions should be documented and approved by the project's committer. 3.2. **Risk Analysis** -- If the code coverage value falls below 80.0%, the potential risks should be documented and approved by the release management team. +- If the code coverage value for an Eclipse Tractus-X project falls below 80.0% for an (upcoming) release and this cannot be improved anymore in time before the release, this potential risk should be documented and approved by the project lead/main committers. -### 4. Examples and Resources +### 4. Resources and Examples + +#### 4.1 **SonarQube Cloud Implementation Guides** -#### 4.1. **Code Examples** +See the following SonarQube Cloud guideline for the importing of test coverage reports with detailed guides for several popular programming languages as [reference](https://docs.sonarsource.com/sonarqube-cloud/enriching/test-coverage/overview/). -An example of a GitHub workflow integration can be found below. +#### 4.2. **GitHub Workflow Integration Code Example** + +An example of a GitHub workflow integration using Java and Apache Maven can be found below. ```yaml name: "Run unit and integration tests" From 12d2281a4e7b3089c1a2bf91194bad590fa73009 Mon Sep 17 00:00:00 2001 From: ds-hzimmer Date: Mon, 24 Mar 2025 16:46:49 +0100 Subject: [PATCH 6/7] Corrected Markdown, removed statement on Quality Gate profiles (default setting and thus does not need to be icluded in TRG) --- docs/release/trg-10/trg-10-01.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/release/trg-10/trg-10-01.md b/docs/release/trg-10/trg-10-01.md index b21f5c2b89fb..6de589bc0954 100644 --- a/docs/release/trg-10/trg-10-01.md +++ b/docs/release/trg-10/trg-10-01.md @@ -6,7 +6,8 @@ sidebar_position: 1 | Status | Created | Post-History | |--------|-------------|-------------------------| | Draft | 07-Feb-2025 | Initial version created | -| Draft | 21-Mar-2025 | Update with example integration for Java | +| Draft | 21-Mar-2025 | Update with example integration for GitHub Workflow | +| Draft | 24-Mar-2025 | Update with SonarQube Cloud reference implementation and clarifications| ## Why @@ -22,7 +23,6 @@ This guideline applies to all software components and projects that are part of - Quality Gate: The minimum threshold for code coverage (line coverage) should be **80.0%**. - The code coverage is calculated using both unit tests and integration tests combined. -- Every project not specifically associated to a different Quality Gate will be associated to this one by default. 1.2. **Exceptions** @@ -42,7 +42,7 @@ All exceptions should be documented and approved by the project's lead/main comm 2.1. **Tools for Code Coverage Reporting** -- The recommended code coverage reporting tool is **SonarQube Cloud** as provided by Elipse Tractus-X. [Reference link](https://sonarcloud.io/organizations/eclipse-tractusx/projects) +- The recommended code coverage reporting tool is **SonarQube Cloud** as provided by Elipse Tractus-X at this [reference link](https://sonarcloud.io/organizations/eclipse-tractusx/projects). - Please note that SonarQube Cloud does not calculate the code coverage itself. To include coverage results you must set up a third-party coverage analysis tool and configure SonarQube Cloud to import the analysis results produced by that tool. - SonarQube Cloud supports the import of coverage data in formats native to a variety of tools for a variety of languages. It also provides a possibility to import a generic format from tools that are not directly supported. See the guideline in section [4. Resources and Examples](#sonarqube-implementation-guides) as reference on how to implement this. - The tool or tools used for test coverage analysis and to produce the code coverage report should be specified and documented within the respective Eclipse Tractus-X project. @@ -64,7 +64,9 @@ All exceptions should be documented and approved by the project's lead/main comm - If the code coverage value for an Eclipse Tractus-X project falls below 80.0% for an (upcoming) release and this cannot be improved anymore in time before the release, this potential risk should be documented and approved by the project lead/main committers. ### 4. Resources and Examples + + #### 4.1 **SonarQube Cloud Implementation Guides** See the following SonarQube Cloud guideline for the importing of test coverage reports with detailed guides for several popular programming languages as [reference](https://docs.sonarsource.com/sonarqube-cloud/enriching/test-coverage/overview/). From 828a1ad5e92ddb7ea769a86c74baeb4f5859dceb Mon Sep 17 00:00:00 2001 From: ds-hzimmer Date: Wed, 2 Apr 2025 20:47:35 +0200 Subject: [PATCH 7/7] Update trg-10-01.md Updated workflow integration example --- docs/release/trg-10/trg-10-01.md | 33 +++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/docs/release/trg-10/trg-10-01.md b/docs/release/trg-10/trg-10-01.md index 6de589bc0954..6a295e334d9b 100644 --- a/docs/release/trg-10/trg-10-01.md +++ b/docs/release/trg-10/trg-10-01.md @@ -8,6 +8,7 @@ sidebar_position: 1 | Draft | 07-Feb-2025 | Initial version created | | Draft | 21-Mar-2025 | Update with example integration for GitHub Workflow | | Draft | 24-Mar-2025 | Update with SonarQube Cloud reference implementation and clarifications| +| Draft | 02-Apr-2025 | Updated workflow integration example | ## Why @@ -76,7 +77,7 @@ See the following SonarQube Cloud guideline for the importing of test coverage r An example of a GitHub workflow integration using Java and Apache Maven can be found below. ```yaml -name: "Run unit and integration tests" +name: "Sonar Check" on: pull_request: @@ -105,19 +106,25 @@ jobs: - name: Run unit & integration tests run: | - mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B verify + mvn -B verify - - name: Publish integration test results - uses: EnricoMi/publish-unit-test-result-action@v2 - if: always() - with: - files: "${{ github.workspace }}/target/failsafe-reports/integration-tests.xml" - check_name: "Integration Test Results" + - name: Clean working directories + run: | + rm -rf .scannerwork + rm -rf .sonar - - name: Publish unit test results - uses: EnricoMi/publish-unit-test-result-action@v2 - if: always() + - name: Cache SonarCloud packages + uses: actions/cache@v4 with: - files: "**/surefire-reports/unit-tests.xml" - check_name: "Unit Test Results" + path: ~/.sonar/cache + key: ${{ runner.os }}-sonar + restore-keys: ${{ runner.os }}-sonar + + - name: Verify Sonar Scan + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN_BACKEND }} + SONAR_ORGANIZATION: ${{ vars.SONAR_ORGANIZATION }} + SONAR_PROJECT_KEY: ${{ vars.SONAR_PROJECT_KEY_BACKEND }} + run: mvn --batch-mode sonar:sonar -Dsonar.coverage.jacoco.xmlReportPaths=${{ github.workspace }}/target/site/jacoco-aggregate/jacoco.xml -Dsonar.projectKey=${{ vars.SONAR_PROJECT_KEY_BACKEND }} -Dsonar.organization=${{ vars.SONAR_ORGANIZATION }} ```