Skip to content

Commit 6d10046

Browse files
jamesthesnakejamesthesnake
authored
Merge pull request huggingface#13 from huggingface/main
ad
2 parents 9272a10 + c612628 commit 6d10046

34 files changed

+1632
-852
lines changed

‎.circleci/config.yml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,6 @@ jobs:
176176
- run: python utils/check_config_attributes.py
177177
- run: python utils/check_doctest_list.py
178178
- run: make deps_table_check_updated
179-
- run: python utils/tests_fetcher.py --sanity_check
180179
- run: python utils/update_metadata.py --check-only
181180
- run: python utils/check_task_guides.py
182181

‎.github/workflows/build-docker-images.yml

Lines changed: 1 addition & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ name: Build docker images (scheduled)
33
on:
44
push:
55
branches:
6-
- docker-image*
6+
- build_ci_docker_image*
77
repository_dispatch:
88
workflow_call:
99
inputs:
@@ -67,35 +67,6 @@ jobs:
6767
push: true
6868
tags: huggingface/transformers-all-latest-gpu-push-ci
6969

70-
latest-with-torch-nightly-docker:
71-
name: "Nightly PyTorch + Stable TensorFlow"
72-
# Push CI doesn't need this image
73-
if: inputs.image_postfix != '-push-ci'
74-
runs-on: ubuntu-latest
75-
steps:
76-
-
77-
name: Set up Docker Buildx
78-
uses: docker/setup-buildx-action@v2
79-
-
80-
name: Check out code
81-
uses: actions/checkout@v3
82-
-
83-
name: Login to DockerHub
84-
uses: docker/login-action@v2
85-
with:
86-
username: ${{ secrets.DOCKERHUB_USERNAME }}
87-
password: ${{ secrets.DOCKERHUB_PASSWORD }}
88-
-
89-
name: Build and push
90-
uses: docker/build-push-action@v3
91-
with:
92-
context: ./docker/transformers-all-latest-gpu
93-
build-args: |
94-
REF=main
95-
PYTORCH=pre
96-
push: true
97-
tags: huggingface/transformers-all-latest-torch-nightly-gpu
98-
9970
latest-torch-deepspeed-docker:
10071
name: "Latest PyTorch + DeepSpeed"
10172
runs-on: ubuntu-latest
@@ -153,34 +124,6 @@ jobs:
153124
push: true
154125
tags: huggingface/transformers-pytorch-deepspeed-latest-gpu-push-ci
155126

156-
nightly-torch-deepspeed-docker:
157-
name: "Nightly PyTorch + DeepSpeed"
158-
# Push CI doesn't need this image
159-
if: inputs.image_postfix != '-push-ci'
160-
runs-on: ubuntu-latest
161-
steps:
162-
-
163-
name: Set up Docker Buildx
164-
uses: docker/setup-buildx-action@v2
165-
-
166-
name: Check out code
167-
uses: actions/checkout@v3
168-
-
169-
name: Login to DockerHub
170-
uses: docker/login-action@v2
171-
with:
172-
username: ${{ secrets.DOCKERHUB_USERNAME }}
173-
password: ${{ secrets.DOCKERHUB_PASSWORD }}
174-
-
175-
name: Build and push
176-
uses: docker/build-push-action@v3
177-
with:
178-
context: ./docker/transformers-pytorch-deepspeed-nightly-gpu
179-
build-args: |
180-
REF=main
181-
push: true
182-
tags: huggingface/transformers-pytorch-deepspeed-nightly-gpu
183-
184127
doc-builder:
185128
name: "Doc builder"
186129
# Push CI doesn't need this image

‎.github/workflows/build-nightly-ci-docker-images.yml

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
name: Build docker images (Nightly CI)
2+
3+
on:
4+
workflow_call:
5+
push:
6+
branches:
7+
- build_nightly_ci_docker_image*
8+
9+
concurrency:
10+
group: docker-images-builds
11+
cancel-in-progress: false
12+
13+
jobs:
14+
latest-with-torch-nightly-docker:
15+
name: "Nightly PyTorch + Stable TensorFlow"
16+
runs-on: ubuntu-latest
17+
steps:
18+
- name: Cleanup disk
19+
run: |
20+
sudo ls -l /usr/local/lib/
21+
sudo ls -l /usr/share/
22+
sudo du -sh /usr/local/lib/
23+
sudo du -sh /usr/share/
24+
sudo rm -rf /usr/local/lib/android
25+
sudo rm -rf /usr/share/dotnet
26+
sudo du -sh /usr/local/lib/
27+
sudo du -sh /usr/share/
28+
-
29+
name: Set up Docker Buildx
30+
uses: docker/setup-buildx-action@v2
31+
-
32+
name: Check out code
33+
uses: actions/checkout@v3
34+
-
35+
name: Login to DockerHub
36+
uses: docker/login-action@v2
37+
with:
38+
username: ${{ secrets.DOCKERHUB_USERNAME }}
39+
password: ${{ secrets.DOCKERHUB_PASSWORD }}
40+
-
41+
name: Build and push
42+
uses: docker/build-push-action@v3
43+
with:
44+
context: ./docker/transformers-all-latest-gpu
45+
build-args: |
46+
REF=main
47+
PYTORCH=pre
48+
push: true
49+
tags: huggingface/transformers-all-latest-torch-nightly-gpu
50+
51+
nightly-torch-deepspeed-docker:
52+
name: "Nightly PyTorch + DeepSpeed"
53+
runs-on: ubuntu-latest
54+
steps:
55+
-
56+
name: Set up Docker Buildx
57+
uses: docker/setup-buildx-action@v2
58+
-
59+
name: Check out code
60+
uses: actions/checkout@v3
61+
-
62+
name: Login to DockerHub
63+
uses: docker/login-action@v2
64+
with:
65+
username: ${{ secrets.DOCKERHUB_USERNAME }}
66+
password: ${{ secrets.DOCKERHUB_PASSWORD }}
67+
-
68+
name: Build and push
69+
uses: docker/build-push-action@v3
70+
with:
71+
context: ./docker/transformers-pytorch-deepspeed-nightly-gpu
72+
build-args: |
73+
REF=main
74+
push: true
75+
tags: huggingface/transformers-pytorch-deepspeed-nightly-gpu

‎.github/workflows/build-past-ci-docker-images.yml

Lines changed: 26 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ name: Build docker images (Past CI)
33
on:
44
push:
55
branches:
6-
- past-ci-docker-image*
6+
- build_past_ci_docker_image*
77

88
concurrency:
99
group: docker-images-builds
@@ -15,7 +15,7 @@ jobs:
1515
strategy:
1616
fail-fast: false
1717
matrix:
18-
version: ["1.11", "1.10", "1.9", "1.8", "1.7", "1.6", "1.5", "1.4"]
18+
version: ["1.13", "1.12", "1.11", "1.10", "1.9"]
1919
runs-on: ubuntu-latest
2020
steps:
2121
-
@@ -24,6 +24,17 @@ jobs:
2424
-
2525
name: Check out code
2626
uses: actions/checkout@v3
27+
-
28+
id: get-base-image
29+
name: Get Base Image
30+
env:
31+
framework_version: ${{ matrix.version }}
32+
run: |
33+
echo "base_image=$(python3 -c 'import os; from utils.past_ci_versions import past_versions_testing; base_image = past_versions_testing["pytorch"][os.environ["framework_version"]]["base_image"]; print(base_image)')" >> $GITHUB_OUTPUT
34+
-
35+
name: Print Base Image
36+
run: |
37+
echo ${{ steps.get-base-image.outputs.base_image }}
2738
-
2839
name: Login to DockerHub
2940
uses: docker/login-action@v2
@@ -37,6 +48,7 @@ jobs:
3748
context: ./docker/transformers-past-gpu
3849
build-args: |
3950
REF=main
51+
BASE_DOCKER_IMAGE=${{ steps.get-base-image.outputs.base_image }}
4052
FRAMEWORK=pytorch
4153
VERSION=${{ matrix.version }}
4254
push: true
@@ -47,7 +59,7 @@ jobs:
4759
strategy:
4860
fail-fast: false
4961
matrix:
50-
version: ["2.8", "2.7", "2.6", "2.5"]
62+
version: ["2.11", "2.10", "2.9", "2.8", "2.7", "2.6", "2.5"]
5163
runs-on: ubuntu-latest
5264
steps:
5365
-
@@ -57,37 +69,16 @@ jobs:
5769
name: Check out code
5870
uses: actions/checkout@v3
5971
-
60-
name: Login to DockerHub
61-
uses: docker/login-action@v2
62-
with:
63-
username: ${{ secrets.DOCKERHUB_USERNAME }}
64-
password: ${{ secrets.DOCKERHUB_PASSWORD }}
72+
id: get-base-image
73+
name: Get Base Image
74+
env:
75+
framework_version: ${{ matrix.version }}
76+
run: |
77+
echo "base_image=$(python3 -c 'import os; from utils.past_ci_versions import past_versions_testing; base_image = past_versions_testing["tensorflow"][os.environ["framework_version"]]["base_image"]; print(base_image)')" >> $GITHUB_OUTPUT
6578
-
66-
name: Build and push
67-
uses: docker/build-push-action@v3
68-
with:
69-
context: ./docker/transformers-past-gpu
70-
build-args: |
71-
REF=main
72-
FRAMEWORK=tensorflow
73-
VERSION=${{ matrix.version }}
74-
push: true
75-
tags: huggingface/transformers-tensorflow-past-${{ matrix.version }}-gpu
76-
77-
past-tensorflow-docker-2-4:
78-
name: "Past TensorFlow Docker"
79-
strategy:
80-
fail-fast: false
81-
matrix:
82-
version: ["2.4"]
83-
runs-on: ubuntu-latest
84-
steps:
85-
-
86-
name: Set up Docker Buildx
87-
uses: docker/setup-buildx-action@v2
88-
-
89-
name: Check out code
90-
uses: actions/checkout@v3
79+
name: Print Base Image
80+
run: |
81+
echo ${{ steps.get-base-image.outputs.base_image }}
9182
-
9283
name: Login to DockerHub
9384
uses: docker/login-action@v2
@@ -101,8 +92,8 @@ jobs:
10192
context: ./docker/transformers-past-gpu
10293
build-args: |
10394
REF=main
104-
BASE_DOCKER_IMAGE=nvidia/cuda:11.0.3-cudnn8-devel-ubuntu20.04
95+
BASE_DOCKER_IMAGE=${{ steps.get-base-image.outputs.base_image }}
10596
FRAMEWORK=tensorflow
10697
VERSION=${{ matrix.version }}
10798
push: true
108-
tags: huggingface/transformers-tensorflow-past-${{ matrix.version }}-gpu
99+
tags: huggingface/transformers-tensorflow-past-${{ matrix.version }}-gpu

0 commit comments

Comments
 (0)