Skip to content

Commit e536f6a

Browse files
molbapmolbap
authored
Merge pull request #9 from huggingface/update
Update
2 parents 3f69f41 + ed74d97 commit e536f6a

File tree

1,347 files changed

+47152
-13386
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,347 files changed

+47152
-13386
lines changed

‎.circleci/create_circleci_config.py

+30-22
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
"RUN_PT_FLAX_CROSS_TESTS": False,
3333
}
3434
# Disable the use of {"s": None} as the output is way too long, causing the navigation on CircleCI impractical
35-
COMMON_PYTEST_OPTIONS = {"max-worker-restart": 0, "dist": "loadfile"}
35+
COMMON_PYTEST_OPTIONS = {"max-worker-restart": 0, "dist": "loadfile", "v": None}
3636
DEFAULT_DOCKER_IMAGE = [{"image": "cimg/python:3.8.12"}]
3737

3838

@@ -52,7 +52,7 @@ class CircleCIJob:
5252
name: str
5353
additional_env: Dict[str, Any] = None
5454
cache_name: str = None
55-
cache_version: str = "0.7"
55+
cache_version: str = "0.8.2"
5656
docker_image: List[Dict[str, str]] = None
5757
install_steps: List[str] = None
5858
marker: Optional[str] = None
@@ -128,22 +128,6 @@ def to_dict(self):
128128
steps.extend([{"run": l} for l in self.install_steps])
129129
steps.extend([{"run": 'pip install "fsspec>=2023.5.0,<2023.10.0"'}])
130130
steps.extend([{"run": "pip install pytest-subtests"}])
131-
steps.append(
132-
{
133-
"save_cache": {
134-
"key": f"v{self.cache_version}-{self.cache_name}-{cache_branch_prefix}-pip-" + '{{ checksum "setup.py" }}',
135-
"paths": ["~/.cache/pip"],
136-
}
137-
}
138-
)
139-
steps.append(
140-
{
141-
"save_cache": {
142-
"key": f"v{self.cache_version}-{self.cache_name}-{cache_branch_prefix}-site-packages-" + '{{ checksum "setup.py" }}',
143-
"paths": ["~/.pyenv/versions/"],
144-
}
145-
}
146-
)
147131
steps.append({"run": {"name": "Show installed libraries and their versions", "command": "pip freeze | tee installed.txt"}})
148132
steps.append({"store_artifacts": {"path": "~/transformers/installed.txt"}})
149133

@@ -227,7 +211,7 @@ def to_dict(self):
227211
# failure.
228212
test_command = f"({test_command}) || true"
229213
else:
230-
test_command += " || true"
214+
test_command = f"({test_command} | tee tests_output.txt) || true"
231215
steps.append({"run": {"name": "Run tests", "command": test_command}})
232216

233217
# Deal with errors
@@ -264,6 +248,25 @@ def to_dict(self):
264248

265249
steps.append({"store_artifacts": {"path": "~/transformers/tests_output.txt"}})
266250
steps.append({"store_artifacts": {"path": "~/transformers/reports"}})
251+
252+
# save cache at the end: so pytest step runs before cache saving and we can see results earlier
253+
steps.append(
254+
{
255+
"save_cache": {
256+
"key": f"v{self.cache_version}-{self.cache_name}-{cache_branch_prefix}-pip-" + '{{ checksum "setup.py" }}',
257+
"paths": ["~/.cache/pip"],
258+
}
259+
}
260+
)
261+
steps.append(
262+
{
263+
"save_cache": {
264+
"key": f"v{self.cache_version}-{self.cache_name}-{cache_branch_prefix}-site-packages-" + '{{ checksum "setup.py" }}',
265+
"paths": ["~/.pyenv/versions/"],
266+
}
267+
}
268+
)
269+
267270
job["steps"] = steps
268271
return job
269272

@@ -283,6 +286,8 @@ def job_name(self):
283286
"pip install -U --upgrade-strategy eager .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision]",
284287
"pip install -U --upgrade-strategy eager tensorflow_probability",
285288
"pip install -U --upgrade-strategy eager -e git+https://github.com/huggingface/accelerate@main#egg=accelerate",
289+
# TODO: remove this one after fixing the dependency issue(s) above
290+
"pip install -U --upgrade-strategy eager torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu",
286291
],
287292
marker="is_pt_tf_cross_test",
288293
pytest_options={"rA": None, "durations": 0},
@@ -470,10 +475,11 @@ def job_name(self):
470475
"pip install -U --upgrade-strategy eager 'git+https://github.com/facebookresearch/detectron2.git'",
471476
"sudo apt install tesseract-ocr",
472477
"pip install -U --upgrade-strategy eager pytesseract",
473-
"pip install -U --upgrade-strategy eager 'natten<0.15.0'",
478+
"pip install -U --upgrade-strategy eager natten==0.15.1+torch210cpu -f https://shi-labs.com/natten/wheels",
474479
"pip install -U --upgrade-strategy eager python-Levenshtein",
475480
"pip install -U --upgrade-strategy eager opencv-python",
476481
"pip install -U --upgrade-strategy eager nltk",
482+
"pip uninstall -y torch torchvision torchaudio && pip install -U --upgrade-strategy eager 'torch<2.2.0' 'torchvision<0.17' 'torchaudio<2.2.0'"
477483
],
478484
tests_to_run=[
479485
"tests/models/*layoutlmv*",
@@ -513,9 +519,11 @@ def job_name(self):
513519
"pip install --upgrade --upgrade-strategy eager pip",
514520
"pip install -U --upgrade-strategy eager -e .[dev]",
515521
"pip install -U --upgrade-strategy eager -e git+https://github.com/huggingface/accelerate@main#egg=accelerate",
516-
"pip install --upgrade --upgrade-strategy eager pytest pytest-sugar",
517-
"pip install -U --upgrade-strategy eager 'natten<0.15.0'",
522+
"pip install --upgrade --upgrade-strategy eager 'pytest<8.0.0' pytest-sugar",
523+
"pip install -U --upgrade-strategy eager natten==0.15.1+torch210cpu -f https://shi-labs.com/natten/wheels",
518524
"pip install -U --upgrade-strategy eager g2p-en",
525+
# TODO: remove this one after fixing the dependency issue(s) above
526+
"pip install -U --upgrade-strategy eager torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu",
519527
"find -name __pycache__ -delete",
520528
"find . -name \*.pyc -delete",
521529
# Add an empty file to keep the test step running correctly even no file is selected to be tested.

‎.github/PULL_REQUEST_TEMPLATE.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ Fixes # (issue)
1717

1818
## Before submitting
1919
- [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case).
20-
- [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests),
20+
- [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#create-a-pull-request),
2121
Pull Request section?
2222
- [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link
2323
to it if that's the case.

‎.github/workflows/build-docker-images.yml

+29-11
Original file line numberDiff line numberDiff line change
@@ -20,18 +20,8 @@ concurrency:
2020
jobs:
2121
latest-docker:
2222
name: "Latest PyTorch + TensorFlow [dev]"
23-
runs-on: ubuntu-22.04
23+
runs-on: [intel-cpu, 8-cpu, ci]
2424
steps:
25-
- name: Cleanup disk
26-
run: |
27-
sudo ls -l /usr/local/lib/
28-
sudo ls -l /usr/share/
29-
sudo du -sh /usr/local/lib/
30-
sudo du -sh /usr/share/
31-
sudo rm -rf /usr/local/lib/android
32-
sudo rm -rf /usr/share/dotnet
33-
sudo du -sh /usr/local/lib/
34-
sudo du -sh /usr/share/
3525
-
3626
name: Set up Docker Buildx
3727
uses: docker/setup-buildx-action@v3
@@ -307,3 +297,31 @@ jobs:
307297
# REF=main
308298
# push: true
309299
# tags: huggingface/transformers-pytorch-deepspeed-amd-gpu-push-ci
300+
301+
latest-quantization-torch-docker:
302+
name: "Latest Pytorch + Quantization [dev]"
303+
# Push CI doesn't need this image
304+
if: inputs.image_postfix != '-push-ci'
305+
runs-on: [intel-cpu, 8-cpu, ci]
306+
steps:
307+
-
308+
name: Set up Docker Buildx
309+
uses: docker/setup-buildx-action@v3
310+
-
311+
name: Check out code
312+
uses: actions/checkout@v3
313+
-
314+
name: Login to DockerHub
315+
uses: docker/login-action@v3
316+
with:
317+
username: ${{ secrets.DOCKERHUB_USERNAME }}
318+
password: ${{ secrets.DOCKERHUB_PASSWORD }}
319+
-
320+
name: Build and push
321+
uses: docker/build-push-action@v5
322+
with:
323+
context: ./docker/transformers-quantization-latest-gpu
324+
build-args: |
325+
REF=main
326+
push: true
327+
tags: huggingface/transformers-quantization-latest-gpu${{ inputs.image_postfix }}

‎.github/workflows/model_jobs.yml

+102
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
name: model jobs
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
folder_slices:
7+
required: true
8+
type: string
9+
machine_type:
10+
required: true
11+
type: string
12+
slice_id:
13+
required: true
14+
type: number
15+
16+
env:
17+
HF_HOME: /mnt/cache
18+
TRANSFORMERS_IS_CI: yes
19+
OMP_NUM_THREADS: 8
20+
MKL_NUM_THREADS: 8
21+
RUN_SLOW: yes
22+
# For gated repositories, we still need to agree to share information on the Hub repo. page in order to get access.
23+
# This token is created under the bot `hf-transformers-bot`.
24+
HF_HUB_READ_TOKEN: ${{ secrets.HF_HUB_READ_TOKEN }}
25+
SIGOPT_API_TOKEN: ${{ secrets.SIGOPT_API_TOKEN }}
26+
TF_FORCE_GPU_ALLOW_GROWTH: true
27+
RUN_PT_TF_CROSS_TESTS: 1
28+
CUDA_VISIBLE_DEVICES: 0,1
29+
30+
jobs:
31+
model_job:
32+
name: " "
33+
strategy:
34+
fail-fast: false
35+
matrix:
36+
folders: ${{ fromJson(inputs.folder_slices)[inputs.slice_id] }}
37+
runs-on: ['${{ inputs.machine_type }}', nvidia-gpu, t4, daily-ci]
38+
container:
39+
image: huggingface/transformers-all-latest-gpu
40+
options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
41+
steps:
42+
- name: Echo input and matrix info
43+
shell: bash
44+
run: |
45+
echo "${{ inputs.folder_slices }}"
46+
echo "${{ matrix.folders }}"
47+
echo "${{ toJson(fromJson(inputs.folder_slices)[inputs.slice_id]) }}"
48+
49+
- name: Echo folder ${{ matrix.folders }}
50+
shell: bash
51+
# For folders like `models/bert`, set an env. var. (`matrix_folders`) to `models_bert`, which will be used to
52+
# set the artifact folder names (because the character `/` is not allowed).
53+
run: |
54+
echo "${{ matrix.folders }}"
55+
matrix_folders=${{ matrix.folders }}
56+
matrix_folders=${matrix_folders/'models/'/'models_'}
57+
echo "$matrix_folders"
58+
echo "matrix_folders=$matrix_folders" >> $GITHUB_ENV
59+
60+
- name: Update clone
61+
working-directory: /transformers
62+
run: git fetch && git checkout ${{ github.sha }}
63+
64+
- name: Reinstall transformers in edit mode (remove the one installed during docker image build)
65+
working-directory: /transformers
66+
run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .
67+
68+
- name: NVIDIA-SMI
69+
run: |
70+
nvidia-smi
71+
72+
- name: Environment
73+
working-directory: /transformers
74+
run: |
75+
python3 utils/print_env.py
76+
77+
- name: Show installed libraries and their versions
78+
working-directory: /transformers
79+
run: pip freeze
80+
81+
- name: Run all tests on GPU
82+
working-directory: /transformers
83+
run: python3 -m pytest -v --make-reports=${{ inputs.machine_type }}_tests_gpu_${{ matrix.folders }} tests/${{ matrix.folders }}
84+
85+
- name: Failure short reports
86+
if: ${{ failure() }}
87+
continue-on-error: true
88+
run: cat /transformers/reports/${{ inputs.machine_type }}_tests_gpu_${{ matrix.folders }}/failures_short.txt
89+
90+
- name: Run test
91+
shell: bash
92+
run: |
93+
mkdir -p /transformers/reports/${{ inputs.machine_type }}_tests_gpu_${{ matrix.folders }}
94+
echo "hello" > /transformers/reports/${{ inputs.machine_type }}_tests_gpu_${{ matrix.folders }}/hello.txt
95+
echo "${{ inputs.machine_type }}_tests_gpu_${{ matrix.folders }}"
96+
97+
- name: "Test suite reports artifacts: ${{ inputs.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports"
98+
if: ${{ always() }}
99+
uses: actions/upload-artifact@v3
100+
with:
101+
name: ${{ inputs.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports
102+
path: /transformers/reports/${{ inputs.machine_type }}_tests_gpu_${{ matrix.folders }}

‎.github/workflows/self-nightly-scheduled.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ jobs:
115115
continue-on-error: true
116116
run: cat /transformers/reports/${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }}/failures_short.txt
117117

118-
- name: Test suite reports artifacts
118+
- name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports_postfix_nightly"
119119
if: ${{ always() }}
120120
uses: actions/upload-artifact@v3
121121
with:
@@ -176,7 +176,7 @@ jobs:
176176
continue-on-error: true
177177
run: cat /transformers/reports/${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }}/failures_short.txt
178178

179-
- name: Test suite reports artifacts
179+
- name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports_postfix_nightly"
180180
if: ${{ always() }}
181181
uses: actions/upload-artifact@v3
182182
with:
@@ -238,7 +238,7 @@ jobs:
238238
continue-on-error: true
239239
run: cat /workspace/transformers/reports/${{ matrix.machine_type }}_tests_torch_cuda_extensions_gpu/failures_short.txt
240240

241-
- name: Test suite reports artifacts
241+
- name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_tests_torch_cuda_extensions_gpu_test_reports_postfix_nightly"
242242
if: ${{ always() }}
243243
uses: actions/upload-artifact@v3
244244
with:

‎.github/workflows/self-past.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,7 @@ jobs:
141141
echo "$job_name"
142142
echo "$job_name" > /transformers/reports/${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }}/job_name.txt
143143
144-
- name: Test suite reports artifacts
144+
- name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports_postfix_${{ inputs.framework }}-${{ inputs.version }}"
145145
if: ${{ always() }}
146146
uses: actions/upload-artifact@v3
147147
with:
@@ -221,7 +221,7 @@ jobs:
221221
echo "$job_name"
222222
echo "$job_name" > /transformers/reports/${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }}/job_name.txt
223223
224-
- name: Test suite reports artifacts
224+
- name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports_postfix_${{ inputs.framework }}-${{ inputs.version }}"
225225
if: ${{ always() }}
226226
uses: actions/upload-artifact@v3
227227
with:
@@ -293,7 +293,7 @@ jobs:
293293
continue-on-error: true
294294
run: cat /transformers/reports/${{ matrix.machine_type }}_tests_torch_cuda_extensions_gpu/failures_short.txt
295295

296-
- name: Test suite reports artifacts
296+
- name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_tests_torch_cuda_extensions_gpu_test_reports_postfix_${{ inputs.framework }}-${{ inputs.version }}"
297297
if: ${{ always() }}
298298
uses: actions/upload-artifact@v3
299299
with:

‎.github/workflows/self-push-amd.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -237,7 +237,7 @@ jobs:
237237
continue-on-error: true
238238
run: cat /transformers/reports/${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }}/failures_short.txt
239239

240-
- name: Test suite reports artifacts
240+
- name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports"
241241
if: ${{ always() }}
242242
uses: actions/upload-artifact@v3
243243
with:

‎.github/workflows/self-push.yml

+4-4
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,7 @@ jobs:
207207
continue-on-error: true
208208
run: cat /transformers/reports/${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }}/failures_short.txt
209209

210-
- name: Test suite reports artifacts
210+
- name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports"
211211
if: ${{ always() }}
212212
uses: actions/upload-artifact@v3
213213
with:
@@ -302,7 +302,7 @@ jobs:
302302
continue-on-error: true
303303
run: cat /transformers/reports/${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }}/failures_short.txt
304304

305-
- name: Test suite reports artifacts
305+
- name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports"
306306
if: ${{ always() }}
307307
uses: actions/upload-artifact@v3
308308
with:
@@ -392,7 +392,7 @@ jobs:
392392
continue-on-error: true
393393
run: cat /workspace/transformers/reports/${{ matrix.machine_type }}_tests_torch_cuda_extensions_gpu/failures_short.txt
394394

395-
- name: Test suite reports artifacts
395+
- name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_tests_torch_cuda_extensions_gpu_test_reports"
396396
if: ${{ always() }}
397397
uses: actions/upload-artifact@v3
398398
with:
@@ -482,7 +482,7 @@ jobs:
482482
continue-on-error: true
483483
run: cat /workspace/transformers/reports/${{ matrix.machine_type }}_tests_torch_cuda_extensions_gpu/failures_short.txt
484484

485-
- name: Test suite reports artifacts
485+
- name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_tests_torch_cuda_extensions_gpu_test_reports"
486486
if: ${{ always() }}
487487
uses: actions/upload-artifact@v3
488488
with:

0 commit comments

Comments
 (0)