Skip to content

Commit b6b6daf

Browse files
ydshiehydshieh
authored
Refactor doctest (#30210)
* fix * update * fix * update * fix --------- Co-authored-by: ydshieh <[email protected]>
1 parent b3595cf commit b6b6daf

File tree

5 files changed

+295
-132
lines changed

5 files changed

+295
-132
lines changed

‎.github/workflows/doctest_job.yml

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
name: Doctest job
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
job_splits:
7+
required: true
8+
type: string
9+
split_keys:
10+
required: true
11+
type: string
12+
13+
env:
14+
HF_HOME: /mnt/cache
15+
TRANSFORMERS_IS_CI: yes
16+
RUN_SLOW: yes
17+
OMP_NUM_THREADS: 16
18+
MKL_NUM_THREADS: 16
19+
SIGOPT_API_TOKEN: ${{ secrets.SIGOPT_API_TOKEN }}
20+
TF_FORCE_GPU_ALLOW_GROWTH: true
21+
22+
jobs:
23+
run_doctests:
24+
name: " "
25+
strategy:
26+
fail-fast: false
27+
matrix:
28+
split_keys: ${{ fromJson(inputs.split_keys) }}
29+
runs-on: [single-gpu, nvidia-gpu, t4, ci]
30+
container:
31+
image: huggingface/transformers-all-latest-gpu
32+
options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
33+
steps:
34+
- name: Update clone
35+
working-directory: /transformers
36+
run: git fetch && git checkout ${{ github.sha }}
37+
38+
- name: Reinstall transformers in edit mode (remove the one installed during docker image build)
39+
working-directory: /transformers
40+
run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .[flax]
41+
42+
- name: GPU visibility
43+
working-directory: /transformers
44+
run: |
45+
python3 utils/print_env.py
46+
47+
- name: Show installed libraries and their versions
48+
run: pip freeze
49+
50+
- name: Get doctest files
51+
working-directory: /transformers
52+
run: |
53+
echo "${{ toJson(fromJson(inputs.job_splits)[matrix.split_keys]) }}" > doc_tests.txt
54+
cat doc_tests.txt
55+
56+
- name: Set `split_keys`
57+
shell: bash
58+
run: |
59+
echo "${{ matrix.split_keys }}"
60+
split_keys=${{ matrix.split_keys }}
61+
split_keys=${split_keys//'/'/'_'}
62+
echo "split_keys"
63+
echo "split_keys=$split_keys" >> $GITHUB_ENV
64+
65+
- name: Run doctests
66+
working-directory: /transformers
67+
run: |
68+
cat doc_tests.txt
69+
python3 -m pytest -v --make-reports doc_tests_gpu_${{ env.split_keys }} --doctest-modules $(cat doc_tests.txt) -sv --doctest-continue-on-failure --doctest-glob="*.md"
70+
71+
- name: Failure short reports
72+
if: ${{ failure() }}
73+
continue-on-error: true
74+
run: cat /transformers/reports/doc_tests_gpu_${{ env.split_keys }}/failures_short.txt
75+
76+
- name: "Test suite reports artifacts: doc_tests_gpu_test_reports_${{ env.split_keys }}"
77+
if: ${{ always() }}
78+
uses: actions/upload-artifact@v3
79+
with:
80+
name: doc_tests_gpu_test_reports_${{ env.split_keys }}
81+
path: /transformers/reports/doc_tests_gpu_${{ env.split_keys }}

‎.github/workflows/doctests.yml

Lines changed: 46 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -3,81 +3,85 @@ name: Doctests
33
on:
44
push:
55
branches:
6-
- doctest*
6+
- run_doctest*
77
repository_dispatch:
88
schedule:
99
- cron: "17 2 * * *"
1010

11-
1211
env:
13-
HF_HOME: /mnt/cache
14-
TRANSFORMERS_IS_CI: yes
15-
RUN_SLOW: yes
16-
OMP_NUM_THREADS: 16
17-
MKL_NUM_THREADS: 16
18-
SIGOPT_API_TOKEN: ${{ secrets.SIGOPT_API_TOKEN }}
19-
TF_FORCE_GPU_ALLOW_GROWTH: true
12+
NUM_SLICES: 3
2013

2114
jobs:
22-
run_doctests:
15+
setup:
16+
name: Setup
2317
runs-on: [single-gpu, nvidia-gpu, t4, ci]
2418
container:
2519
image: huggingface/transformers-all-latest-gpu
2620
options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
21+
outputs:
22+
job_splits: ${{ steps.set-matrix.outputs.job_splits }}
23+
split_keys: ${{ steps.set-matrix.outputs.split_keys }}
2724
steps:
28-
- name: uninstall transformers (installed during docker image build)
29-
run: python3 -m pip uninstall -y transformers
30-
31-
- uses: actions/checkout@v3
32-
- name: NVIDIA-SMI
25+
- name: Update clone
26+
working-directory: /transformers
3327
run: |
34-
nvidia-smi
28+
git fetch && git checkout ${{ github.sha }}
3529
36-
- name: Install transformers in edit mode
37-
run: python3 -m pip install -e .[flax]
38-
39-
- name: GPU visibility
40-
run: |
41-
python3 utils/print_env.py
30+
- name: Reinstall transformers in edit mode (remove the one installed during docker image build)
31+
working-directory: /transformers
32+
run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .
4233

4334
- name: Show installed libraries and their versions
35+
working-directory: /transformers
4436
run: pip freeze
4537

46-
- name: Get doctest files
38+
- name: Check values for matrix
39+
working-directory: /transformers
4740
run: |
48-
$(python3 -c 'from utils.tests_fetcher import get_all_doctest_files; to_test = get_all_doctest_files(); to_test = " ".join(to_test); fp = open("doc_tests.txt", "w"); fp.write(to_test); fp.close()')
41+
python3 utils/split_doctest_jobs.py
42+
python3 utils/split_doctest_jobs.py --only_return_keys --num_splits ${{ env.NUM_SLICES }}
4943
50-
- name: Run doctests
44+
- id: set-matrix
45+
working-directory: /transformers
46+
name: Set values for matrix
5147
run: |
52-
python3 -m pytest -v --make-reports doc_tests_gpu --doctest-modules $(cat doc_tests.txt) -sv --doctest-continue-on-failure --doctest-glob="*.md"
53-
54-
- name: Failure short reports
55-
if: ${{ failure() }}
56-
continue-on-error: true
57-
run: cat reports/doc_tests_gpu/failures_short.txt
58-
59-
- name: Test suite reports artifacts
60-
if: ${{ always() }}
61-
uses: actions/upload-artifact@v3
62-
with:
63-
name: doc_tests_gpu_test_reports
64-
path: reports/doc_tests_gpu
48+
echo "job_splits=$(python3 utils/split_doctest_jobs.py)" >> $GITHUB_OUTPUT
49+
echo "split_keys=$(python3 utils/split_doctest_jobs.py --only_return_keys --num_splits ${{ env.NUM_SLICES }})" >> $GITHUB_OUTPUT
6550
51+
call_doctest_job:
52+
name: "Call doctest jobs"
53+
needs: setup
54+
strategy:
55+
fail-fast: false
56+
matrix:
57+
split_keys: ${{ fromJson(needs.setup.outputs.split_keys) }}
58+
uses: ./.github/workflows/doctest_job.yml
59+
with:
60+
job_splits: ${{ needs.setup.outputs.job_splits }}
61+
split_keys: ${{ toJson(matrix.split_keys) }}
62+
secrets: inherit
6663

6764
send_results:
6865
name: Send results to webhook
6966
runs-on: ubuntu-22.04
7067
if: always()
71-
needs: [run_doctests]
68+
needs: [call_doctest_job]
7269
steps:
7370
- uses: actions/checkout@v3
7471
- uses: actions/download-artifact@v3
7572
- name: Send message to Slack
7673
env:
7774
CI_SLACK_BOT_TOKEN: ${{ secrets.CI_SLACK_BOT_TOKEN }}
78-
CI_SLACK_CHANNEL_ID: ${{ secrets.CI_SLACK_CHANNEL_ID_DAILY_DOCS }}
79-
CI_SLACK_CHANNEL_ID_DAILY: ${{ secrets.CI_SLACK_CHANNEL_ID_DAILY_DOCS }}
80-
CI_SLACK_CHANNEL_DUMMY_TESTS: ${{ secrets.CI_SLACK_CHANNEL_DUMMY_TESTS }}
75+
ACCESS_REPO_INFO_TOKEN: ${{ secrets.ACCESS_REPO_INFO_TOKEN }}
76+
# Use `CI_SLACK_CHANNEL_DUMMY_TESTS` when doing experimentation
77+
SLACK_REPORT_CHANNEL: ${{ secrets.CI_SLACK_CHANNEL_ID_DAILY_DOCS }}
8178
run: |
8279
pip install slack_sdk
8380
python utils/notification_service_doc_tests.py
81+
82+
- name: "Upload results"
83+
if: ${{ always() }}
84+
uses: actions/upload-artifact@v3
85+
with:
86+
name: doc_test_results
87+
path: doc_test_results

0 commit comments

Comments
 (0)