Skip to content

Commit 09a10ca

Browse files
yasakova-anastasiayasakova-anastasia
authored
Code coverage (#6173)
- Replaced Coveralls with Codecov - Added code coverage to full workflow - Enabled code coverage for tests
1 parent 8fae5d3 commit 09a10ca

23 files changed

+265
-160
lines changed

‎.coveragerc

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
[run]
22
branch = true
3-
# relative_files = true # does not work?
43

54
source =
65
cvat/apps/

‎.github/codecov.yml

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
comment:
2+
layout: "header, diff, components"
3+
4+
component_management:
5+
individual_components:
6+
- component_id: cvat-ui
7+
name: cvat-ui
8+
paths:
9+
- cvat-canvas/**
10+
- cvat-canvas3d/**
11+
- cvat-core/**
12+
- cvat-data/**
13+
- cvat-ui/**
14+
- component_id: cvat-server
15+
name: cvat-server
16+
paths:
17+
- cvat/**
18+
- cvat-cli/**
19+
- cvat-sdk/**
20+
- utils/**
21+

‎.github/workflows/full.yml

Lines changed: 28 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -52,12 +52,19 @@ jobs:
5252
- name: CVAT server. Build and push
5353
uses: docker/build-push-action@v3
5454
with:
55+
build-args: |
56+
"COVERAGE_PROCESS_START=.coveragerc"
5557
cache-from: type=local,src=/tmp/cvat_cache_server
5658
context: .
5759
file: Dockerfile
5860
tags: cvat/server
5961
outputs: type=docker,dest=/tmp/cvat_server/image.tar
6062

63+
- name: Instrumentation of the code then rebuilding the CVAT UI
64+
run: |
65+
yarn --frozen-lockfile
66+
yarn run coverage
67+
6168
- name: CVAT UI. Build and push
6269
uses: docker/build-push-action@v3
6370
with:
@@ -154,11 +161,20 @@ jobs:
154161

155162
- name: Running REST API and SDK tests
156163
id: run_tests
164+
env:
165+
COVERAGE_PROCESS_START: ".coveragerc"
157166
run: |
158-
pip3 install --user '/tmp/cvat_sdk/[pytorch]'
159-
pip3 install --user cvat-cli/
160-
pip3 install --user -r tests/python/requirements.txt
161-
pytest tests/python/ -s -v
167+
pip3 install -r cvat-sdk/gen/requirements.txt
168+
./cvat-sdk/gen/generate.sh
169+
170+
pip3 install -r ./tests/python/requirements.txt
171+
pip3 install -e ./cvat-sdk
172+
pip3 install -e ./cvat-cli
173+
174+
pytest tests/python/ --cov --cov-report xml
175+
176+
- name: Upload coverage reports to Codecov with GitHub Action
177+
uses: codecov/codecov-action@v3
162178

163179
- name: Creating a log file from cvat containers
164180
if: failure() && steps.run_tests.conclusion == 'failure'
@@ -218,10 +234,13 @@ jobs:
218234
while [[ $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health?bundles) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done
219235
220236
docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \
221-
-c 'python manage.py test cvat/apps -v 2'
237+
-c 'coverage run -a manage.py test cvat/apps && coverage json && mv coverage.json ${CONTAINER_COVERAGE_DATA_DIR}'
222238
223239
docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \
224-
-c 'yarn --frozen-lockfile --ignore-scripts && yarn workspace cvat-core run test'
240+
-c 'yarn --frozen-lockfile --ignore-scripts && yarn workspace cvat-core run test && mv cvat-core/reports/coverage/coverage-final.json ${CONTAINER_COVERAGE_DATA_DIR}'
241+
242+
- name: Upload coverage reports to Codecov with GitHub Action
243+
uses: codecov/codecov-action@v3
225244

226245
- name: Creating a log file from cvat containers
227246
if: failure()
@@ -317,16 +336,17 @@ jobs:
317336
npx cypress run \
318337
--headed \
319338
--browser chrome \
320-
--env coverage=false \
321339
--config-file cypress_canvas3d.config.js \
322340
--spec 'cypress/e2e/${{ matrix.specs }}/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js'
323341
else
324342
npx cypress run \
325343
--browser chrome \
326-
--env coverage=false \
327344
--spec 'cypress/e2e/${{ matrix.specs }}/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js'
328345
fi
329346
347+
- name: Upload coverage reports to Codecov with GitHub Action
348+
uses: codecov/codecov-action@v3
349+
330350
- name: Creating a log file from "cvat" container logs
331351
if: failure()
332352
run: |

‎.github/workflows/main.yml

Lines changed: 28 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -52,12 +52,19 @@ jobs:
5252
- name: CVAT server. Build and push
5353
uses: docker/build-push-action@v3
5454
with:
55+
build-args: |
56+
"COVERAGE_PROCESS_START=.coveragerc"
5557
cache-from: type=local,src=/tmp/cvat_cache_server
5658
context: .
5759
file: Dockerfile
5860
tags: cvat/server
5961
outputs: type=docker,dest=/tmp/cvat_server/image.tar
6062

63+
- name: Instrumentation of the code then rebuilding the CVAT UI
64+
run: |
65+
yarn --frozen-lockfile
66+
yarn run coverage
67+
6168
- name: CVAT UI. Build and push
6269
uses: docker/build-push-action@v3
6370
with:
@@ -130,12 +137,6 @@ jobs:
130137
name: cvat_ui
131138
path: /tmp/cvat_ui/
132139

133-
- name: Download CVAT SDK package
134-
uses: actions/download-artifact@v3
135-
with:
136-
name: cvat_sdk
137-
path: /tmp/cvat_sdk/
138-
139140
- name: Load Docker images
140141
run: |
141142
docker load --input /tmp/cvat_server/image.tar
@@ -144,7 +145,6 @@ jobs:
144145
docker tag cvat/ui:latest cvat/ui:${CVAT_VERSION}
145146
docker image ls -a
146147
147-
148148
- name: Upload expected schema as an artifact
149149
if: failure() && steps.verify_schema.conclusion == 'failure'
150150
uses: actions/[email protected]
@@ -154,11 +154,20 @@ jobs:
154154

155155
- name: Running REST API and SDK tests
156156
id: run_tests
157+
env:
158+
COVERAGE_PROCESS_START: ".coveragerc"
157159
run: |
158-
pip3 install --user '/tmp/cvat_sdk/[pytorch]'
159-
pip3 install --user cvat-cli/
160-
pip3 install --user -r tests/python/requirements.txt
161-
pytest tests/python/ -s -v
160+
pip3 install -r cvat-sdk/gen/requirements.txt
161+
./cvat-sdk/gen/generate.sh
162+
163+
pip3 install -r ./tests/python/requirements.txt
164+
pip3 install -e ./cvat-sdk
165+
pip3 install -e ./cvat-cli
166+
167+
pytest tests/python/ --cov --cov-report xml
168+
169+
- name: Upload coverage reports to Codecov with GitHub Action
170+
uses: codecov/codecov-action@v3
162171

163172
- name: Creating a log file from cvat containers
164173
if: failure() && steps.run_tests.conclusion == 'failure'
@@ -216,10 +225,13 @@ jobs:
216225
while [[ $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health?bundles) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done
217226
218227
docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \
219-
-c 'python manage.py test cvat/apps -v 2'
228+
-c 'coverage run -a manage.py test cvat/apps && coverage json && mv coverage.json ${CONTAINER_COVERAGE_DATA_DIR}'
220229
221230
docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \
222-
-c 'yarn --frozen-lockfile --ignore-scripts && yarn workspace cvat-core run test'
231+
-c 'yarn --frozen-lockfile --ignore-scripts && yarn workspace cvat-core run test && mv cvat-core/reports/coverage/coverage-final.json ${CONTAINER_COVERAGE_DATA_DIR}'
232+
233+
- name: Upload coverage reports to Codecov with GitHub Action
234+
uses: codecov/codecov-action@v3
223235

224236
- name: Creating a log file from cvat containers
225237
if: failure()
@@ -313,16 +325,17 @@ jobs:
313325
npx cypress run \
314326
--headed \
315327
--browser chrome \
316-
--env coverage=false \
317328
--config-file cypress_canvas3d.config.js \
318329
--spec 'cypress/e2e/${{ matrix.specs }}/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js'
319330
else
320331
npx cypress run \
321332
--browser chrome \
322-
--env coverage=false \
323333
--spec 'cypress/e2e/${{ matrix.specs }}/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js'
324334
fi
325335
336+
- name: Upload coverage reports to Codecov with GitHub Action
337+
uses: codecov/codecov-action@v3
338+
326339
- name: Creating a log file from "cvat" container logs
327340
if: failure()
328341
run: |

‎.github/workflows/schedule.yml

Lines changed: 17 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,8 @@ jobs:
8888
- name: CVAT server. Build and push
8989
uses: docker/build-push-action@v3
9090
with:
91+
build-args: |
92+
"COVERAGE_PROCESS_START=.coveragerc"
9193
cache-from: type=local,src=/tmp/cvat_cache_server
9294
context: .
9395
file: Dockerfile
@@ -163,14 +165,17 @@ jobs:
163165
./opa test cvat/apps/iam/rules
164166
165167
- name: REST API and SDK tests
168+
env:
169+
COVERAGE_PROCESS_START: ".coveragerc"
166170
run: |
167-
pip3 install --user -r cvat-sdk/gen/requirements.txt
171+
pip3 install -r cvat-sdk/gen/requirements.txt
168172
./cvat-sdk/gen/generate.sh
169173
170-
pip3 install --user 'cvat-sdk/[pytorch]'
171-
pip3 install --user cvat-cli/
172-
pip3 install --user -r tests/python/requirements.txt
173-
pytest tests/python/
174+
pip3 install -r ./tests/python/requirements.txt
175+
pip3 install -e ./cvat-sdk
176+
pip3 install -e ./cvat-cli
177+
178+
pytest tests/python/ --cov --cov-report xml
174179
pytest tests/python/ --stop-services
175180
176181
- name: Unit tests
@@ -183,20 +188,15 @@ jobs:
183188
while [[ $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health?bundles) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done
184189
185190
docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \
186-
-c 'coverage run -a manage.py test cvat/apps && mv .coverage ${CONTAINER_COVERAGE_DATA_DIR}'
191+
-c 'coverage run -a manage.py test cvat/apps && coverage json && mv coverage.json ${CONTAINER_COVERAGE_DATA_DIR}'
187192
188193
docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \
189-
-c 'yarn --frozen-lockfile --ignore-scripts && yarn workspace cvat-core run test'
194+
-c 'yarn --frozen-lockfile --ignore-scripts && yarn workspace cvat-core run test && mv cvat-core/reports/coverage/coverage-final.json ${CONTAINER_COVERAGE_DATA_DIR}'
190195
191196
docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml down -v
192197
193-
- name: Uploading code coverage results as an artifact
194-
uses: actions/[email protected]
195-
with:
196-
name: coverage_results
197-
path: |
198-
${{ github.workspace }}/lcov.info
199-
${{ github.workspace }}/.coverage
198+
- name: Upload coverage reports to Codecov with GitHub Action
199+
uses: codecov/codecov-action@v3
200200

201201
e2e_testing:
202202
needs: build
@@ -304,14 +304,15 @@ jobs:
304304
--browser chrome \
305305
--config-file cypress_canvas3d.config.js \
306306
--spec 'cypress/e2e/${{ matrix.specs }}/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js'
307-
mv ./.nyc_output/out.json ./.nyc_output/out_${{ matrix.specs }}.json
308307
else
309308
npx cypress run \
310309
--browser chrome \
311310
--spec 'cypress/e2e/${{ matrix.specs }}/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js'
312-
mv ./.nyc_output/out.json ./.nyc_output/out_${{ matrix.specs }}.json
313311
fi
314312
313+
- name: Upload coverage reports to Codecov with GitHub Action
314+
uses: codecov/codecov-action@v3
315+
315316
- name: Creating a log file from "cvat" container logs
316317
if: failure()
317318
run: |
@@ -330,72 +331,3 @@ jobs:
330331
with:
331332
name: cvat_container_logs
332333
path: ${{ github.workspace }}/tests/cvat.log
333-
334-
- name: Uploading code coverage results as an artifact
335-
uses: actions/[email protected]
336-
with:
337-
name: coverage_results
338-
path: ${{ github.workspace }}/tests/.nyc_output
339-
340-
coveralls:
341-
runs-on: ubuntu-latest
342-
needs: [unit_testing, e2e_testing]
343-
steps:
344-
- uses: actions/checkout@v3
345-
346-
- name: CVAT server. Extract metadata (tags, labels) for Docker
347-
id: meta-server
348-
uses: docker/metadata-action@master
349-
with:
350-
images: ${{ secrets.DOCKERHUB_CI_WORKSPACE }}/${{ env.SERVER_IMAGE_TEST_REPO }}
351-
tags:
352-
type=raw,value=nightly
353-
354-
- name: Login to Docker Hub
355-
uses: docker/login-action@v2
356-
with:
357-
username: ${{ secrets.DOCKERHUB_CI_USERNAME }}
358-
password: ${{ secrets.DOCKERHUB_CI_TOKEN }}
359-
360-
- name: Pull CVAT server image
361-
run: |
362-
docker pull ${{ steps.meta-server.outputs.tags }}
363-
docker tag ${{ steps.meta-server.outputs.tags }} cvat/server:local
364-
365-
- name: Downloading coverage results
366-
uses: actions/download-artifact@v2
367-
with:
368-
name: coverage_results
369-
370-
- name: Combining coverage results
371-
run: |
372-
mkdir -p ./nyc_output_tmp
373-
mv ./out_*.json ./nyc_output_tmp
374-
mkdir -p ./.nyc_output
375-
yarn --frozen-lockfile
376-
npx nyc merge ./nyc_output_tmp ./.nyc_output/out.json
377-
378-
- name: Sending results to Coveralls
379-
env:
380-
HOST_COVERAGE_DATA_DIR: ${{ github.workspace }}
381-
CONTAINER_COVERAGE_DATA_DIR: "/coverage_data"
382-
COVERALLS_SERVICE_NAME: github
383-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
384-
run: |
385-
npx nyc report --reporter=text-lcov >> ${HOST_COVERAGE_DATA_DIR}/lcov.info
386-
387-
docker compose \
388-
-f docker-compose.yml \
389-
-f docker-compose.dev.yml \
390-
-f docker-compose.ci.yml \
391-
run cvat_ci /bin/bash -c 'cd ${CONTAINER_COVERAGE_DATA_DIR} && coveralls-lcov -v -n lcov.info > ${CONTAINER_COVERAGE_DATA_DIR}/coverage.json'
392-
393-
docker compose \
394-
-f docker-compose.yml \
395-
-f docker-compose.dev.yml \
396-
-f docker-compose.ci.yml \
397-
run cvat_ci /bin/bash -c '\
398-
ln -s ${CONTAINER_COVERAGE_DATA_DIR}/.git . \
399-
&& ln -s ${CONTAINER_COVERAGE_DATA_DIR}/.coverage . \
400-
&& ln -s ${CONTAINER_COVERAGE_DATA_DIR}/coverage.json . \
401-
&& coveralls --merge=coverage.json'

‎Dockerfile

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,11 @@ COPY --chown=${USER} wait-for-it.sh manage.py backend_entrypoint.sh ${HOME}/
178178
COPY --chown=${USER} utils/ ${HOME}/utils
179179
COPY --chown=${USER} cvat/ ${HOME}/cvat
180180

181+
ARG COVERAGE_PROCESS_START
182+
RUN if [ "${COVERAGE_PROCESS_START}" ]; then \
183+
echo "import coverage; coverage.process_startup()" > /opt/venv/lib/python3.10/site-packages/coverage_subprocess.pth; \
184+
fi
185+
181186
# RUN all commands below as 'django' user
182187
USER ${USER}
183188
WORKDIR ${HOME}

‎Dockerfile.ci

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,7 @@ RUN apt-get update && \
2525
COPY cvat/requirements/ /tmp/cvat/requirements/
2626
COPY utils/dataset_manifest/requirements.txt /tmp/utils/dataset_manifest/requirements.txt
2727

28-
RUN DATUMARO_HEADLESS=1 python3 -m pip install --no-cache-dir -r /tmp/cvat/requirements/${DJANGO_CONFIGURATION}.txt && \
29-
python3 -m pip install --no-cache-dir coveralls
30-
RUN gem install coveralls-lcov
28+
RUN DATUMARO_HEADLESS=1 python3 -m pip install --no-cache-dir -r /tmp/cvat/requirements/${DJANGO_CONFIGURATION}.txt
3129

3230
COPY cvat-core ${HOME}/cvat-core
3331
COPY cvat-data ${HOME}/cvat-data

‎README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -256,8 +256,8 @@ questions and get our support.
256256
[ci-url]: https://github.com/opencv/cvat/actions
257257
[gitter-img]: https://img.shields.io/gitter/room/opencv-cvat/public?style=flat
258258
[gitter-url]: https://gitter.im/opencv-cvat
259-
[coverage-img]: https://coveralls.io/repos/github/cvat-ai/cvat/badge.svg?branch=develop
260-
[coverage-url]: https://coveralls.io/github/cvat-ai/cvat?branch=develop
259+
[coverage-img]: https://codecov.io/github/opencv/cvat/branch/develop/graph/badge.svg
260+
[coverage-url]: https://codecov.io/github/opencv/cvat
261261
[doi-img]: https://zenodo.org/badge/139156354.svg
262262
[doi-url]: https://zenodo.org/badge/latestdoi/139156354
263263
[discord-img]: https://img.shields.io/discord/1000789942802337834?label=discord

0 commit comments

Comments
 (0)