Skip to content

Commit 64c4a79

Browse files
scottsscotts
committed
Merge branch 'main' of github.com:pytorch/torchcodec into benchmark_test
2 parents 6cc503c + bd9d5cb commit 64c4a79

15 files changed

+452
-103
lines changed

Diff for: ‎.github/workflows/build_ffmpeg.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ on:
1414
pull_request:
1515
paths:
1616
- packaging/build_ffmpeg.sh
17+
- .github/workflows/build_ffmpeg.yaml # self reference
1718
schedule:
1819
- cron: '0 0 * * 0' # on sunday
1920

Diff for: ‎.github/workflows/cpp_tests.yaml

+4
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@ on:
44
push:
55
branches: [ main ]
66
pull_request:
7+
paths:
8+
- src/torchcodec/*
9+
- test/*
10+
- .github/workflows/cpp_tests.yaml # self reference
711

812
concurrency:
913
group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }}

Diff for: ‎.github/workflows/docs.yaml

+90-20
Original file line numberDiff line numberDiff line change
@@ -4,46 +4,116 @@ on:
44
push:
55
branches: [ main ]
66
pull_request:
7+
paths:
8+
- src/torchcodec/*
9+
- docs/*
10+
- .github/workflows/docs.yaml # self reference
11+
12+
permissions:
13+
id-token: write
14+
contents: write
715

816
defaults:
917
run:
1018
shell: bash -l -eo pipefail {0}
1119

1220
jobs:
21+
generate-matrix:
22+
uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
23+
with:
24+
package-type: wheel
25+
os: linux
26+
test-infra-repository: pytorch/test-infra
27+
test-infra-ref: main
28+
with-cpu: disable
29+
with-xpu: disable
30+
with-rocm: disable
31+
with-cuda: enable
32+
build-python-only: "disable"
1333
build:
14-
runs-on: ubuntu-latest
34+
needs: generate-matrix
35+
strategy:
36+
fail-fast: false
37+
name: Build and Upload wheel
38+
uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main
39+
with:
40+
repository: pytorch/torchcodec
41+
ref: ""
42+
test-infra-repository: pytorch/test-infra
43+
test-infra-ref: main
44+
build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
45+
post-script: packaging/post_build_script.sh
46+
smoke-test-script: packaging/fake_smoke_test.py
47+
package-name: torchcodec
48+
trigger-event: ${{ github.event_name }}
49+
build-platform: "python-build-package"
50+
build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 ENABLE_CUDA=1 python -m build --wheel -vvv --no-isolation"
51+
52+
build-docs:
53+
runs-on: linux.4xlarge.nvidia.gpu
1554
strategy:
1655
fail-fast: false
56+
matrix:
57+
# 3.9 corresponds to the minimum python version for which we build
58+
# the wheel unless the label cliflow/binaries/all is present in the
59+
# PR.
60+
python-version: ['3.9']
61+
cuda-version: ['12.4']
62+
ffmpeg-version-for-tests: ['7']
63+
container:
64+
image: "pytorch/manylinux-builder:cuda${{ matrix.cuda-version }}"
65+
options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility"
66+
needs: build
1767
steps:
18-
- name: Check out repo
19-
uses: actions/checkout@v3
20-
- name: Setup conda env
21-
uses: conda-incubator/setup-miniconda@v2
68+
- name: Setup env vars
69+
run: |
70+
cuda_version_without_periods=$(echo "${{ matrix.cuda-version }}" | sed 's/\.//g')
71+
echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV
72+
- uses: actions/download-artifact@v3
2273
with:
23-
auto-update-conda: true
24-
miniconda-version: "latest"
25-
activate-environment: test
26-
python-version: '3.12'
74+
name: pytorch_torchcodec__3.9_cu${{ env.cuda_version_without_periods }}_x86_64
75+
path: pytorch/torchcodec/dist/
76+
- name: Setup miniconda using test-infra
77+
uses: pytorch/test-infra/.github/actions/setup-miniconda@main
78+
with:
79+
python-version: ${{ matrix.python-version }}
80+
#
81+
# For some reason nvidia::libnpp=12.4 doesn't install but nvidia/label/cuda-12.4.0::libnpp does.
82+
# So we use the latter convention for libnpp.
83+
# We install conda packages at the start because otherwise conda may have conflicts with dependencies.
84+
default-packages: "nvidia/label/cuda-${{ matrix.cuda-version }}.0::libnpp nvidia::cuda-nvrtc=${{ matrix.cuda-version }} nvidia::cuda-toolkit=${{ matrix.cuda-version }} nvidia::cuda-cudart=${{ matrix.cuda-version }} nvidia::cuda-driver-dev=${{ matrix.cuda-version }} conda-forge::ffmpeg=${{ matrix.ffmpeg-version-for-tests }}"
85+
- name: Check env
86+
run: |
87+
${CONDA_RUN} env
88+
${CONDA_RUN} conda info
89+
${CONDA_RUN} nvidia-smi
90+
${CONDA_RUN} conda list
91+
- name: Assert ffmpeg exists
92+
run: |
93+
${CONDA_RUN} ffmpeg -buildconf
2794
- name: Update pip
28-
run: python -m pip install --upgrade pip
29-
- name: Install dependencies and FFmpeg
95+
run: ${CONDA_RUN} python -m pip install --upgrade pip
96+
- name: Install PyTorch
3097
run: |
31-
# TODO: torchvision and torchaudio shouldn't be needed. They were only added
32-
# to silence an error as seen in https://github.com/pytorch/torchcodec/issues/203
33-
python -m pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cpu
34-
conda install "ffmpeg=7.0.1" pkg-config -c conda-forge
35-
ffmpeg -version
36-
- name: Build and install torchcodec
98+
${CONDA_RUN} python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu${{ env.cuda_version_without_periods }}
99+
${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")'
100+
- name: Install torchcodec from the wheel
37101
run: |
38-
python -m pip install -e ".[dev]" --no-build-isolation -vvv
102+
wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"`
103+
echo Installing $wheel_path
104+
${CONDA_RUN} python -m pip install $wheel_path -vvv
105+
106+
- name: Check out repo
107+
uses: actions/checkout@v3
108+
39109
- name: Install doc dependencies
40110
run: |
41111
cd docs
42-
python -m pip install -r requirements.txt
112+
${CONDA_RUN} python -m pip install -r requirements.txt
43113
- name: Build docs
44114
run: |
45115
cd docs
46-
make html
116+
${CONDA_RUN} make html
47117
- uses: actions/upload-artifact@v3
48118
with:
49119
name: Built-Docs

Diff for: ‎.github/workflows/linux_cuda_wheel.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,12 @@ name: Build and test Linux CUDA wheels
22

33
on:
44
pull_request:
5+
paths:
6+
- src/torchcodec/*
7+
- benchmarks/*
8+
- packaging/*
9+
- test/*
10+
- .github/workflows/linux_cuda_wheel.yaml # self reference
511
push:
612
branches:
713
- nightly

Diff for: ‎.github/workflows/linux_wheel.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,11 @@ name: Build and test Linux wheel
22

33
on:
44
pull_request:
5+
paths:
6+
- src/torchcodec/*
7+
- packaging/*
8+
- test/*
9+
- .github/workflows/linux_wheel.yaml # self reference
510
push:
611
branches:
712
- nightly

Diff for: ‎.github/workflows/macos_wheel.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,11 @@ name: Build and test MacOS wheel
22

33
on:
44
pull_request:
5+
paths:
6+
- src/torchcodec/*
7+
- packaging/*
8+
- test/*
9+
- .github/workflows/macos_wheel.yaml # self reference
510
push:
611
branches:
712
- nightly

Diff for: ‎.github/workflows/reference_resources.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ on:
55
pull_request:
66
paths:
77
- test/generate_reference_resources.sh
8+
- .github/workflows/reference_resources.yaml # self reference
89
schedule:
910
- cron: '0 0 * * 0' # on sunday
1011

Diff for: ‎README.md

+27-22
Original file line numberDiff line numberDiff line change
@@ -94,29 +94,38 @@ ffmpeg -f lavfi -i \
9494

9595
## Installing TorchCodec
9696

97-
Note: if you're on MacOS, you'll need to [build from source](./CONTRIBUTING.md).
98-
The instructions below assume you're on Linux.
97+
1. Install the latest stable version of PyTorch following the
98+
[official instructions](https://pytorch.org/get-started/locally/). For other
99+
versions, refer to the table below for compatibility between versions of
100+
`torch` and `torchcodec`.
99101

100-
1. Install the latest stable version of PyTorch following the
101-
[official instructions](https://pytorch.org/get-started/locally/). TorchCodec
102-
requires [PyTorch 2.4](https://pytorch.org/docs/2.4/).
102+
2. Install FFmpeg, if it's not already installed. Linux distributions usually
103+
come with FFmpeg pre-installed. TorchCodec supports all major FFmpeg versions
104+
in [4, 7].
103105

104-
2. Install FFmpeg, if it's not already installed. Your Linux distribution probably
105-
comes with FFmpeg pre-installed. TorchCodec supports all major FFmpeg versions
106-
in [4, 7].
106+
If FFmpeg is not already installed, or you need a more recent version, an
107+
easy way to install it is to use `conda`:
107108

108-
If FFmpeg is not already installed, or you need a later version, install it with:
109+
```bash
110+
conda install ffmpeg
111+
# or
112+
conda install ffmpeg -c conda-forge
113+
```
109114

110-
```bash
111-
conda install ffmpeg
112-
# or
113-
conda install ffmpeg -c conda-forge
114-
```
115-
3. Install TorchCodec:
115+
3. Install TorchCodec:
116116

117-
```bash
118-
pip install torchcodec
119-
```
117+
```bash
118+
pip install torchcodec
119+
```
120+
121+
The following table indicates the compatibility between versions of
122+
`torchcodec`, `torch` and Python.
123+
124+
| `torchcodec` | `torch` | Python |
125+
| ------------------ | ------------------ | ------------------- |
126+
| `main` / `nightly` | `main` / `nightly` | `>=3.9`, `<=3.12` |
127+
| not yet supported | `2.5` | `>=3.9`, `<=3.12` |
128+
| `0.0.3` | `2.4` | `>=3.8`, `<=3.12` |
120129

121130
## Benchmark Results
122131

@@ -134,10 +143,6 @@ encoded with libx264 and yuv420p pixel format.
134143

135144
We are actively working on the following features:
136145

137-
- [Ship wheels for MacOS](https://github.com/pytorch/torchcodec/issues/111), so
138-
that MacOS users can `pip install torchcodec`. For now this is only supported
139-
on Linux, but MacOS users can [build from source](./CONTRIBUTING.md).
140-
- [GPU decoding](https://github.com/pytorch/torchcodec/pull/58)
141146
- [Audio decoding](https://github.com/pytorch/torchcodec/issues/85)
142147

143148
Let us know if you have any feature requests by [opening an

Diff for: ‎benchmarks/decoders/benchmark_decoders.py

+2
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313

1414
from benchmark_decoders_library import (
1515
AbstractDecoder,
16+
BatchParameters,
1617
DecordAccurate,
1718
DecordAccurateBatch,
1819
plot_data,
@@ -173,6 +174,7 @@ def main() -> None:
173174
num_sequential_frames_from_start=[1, 10, 100],
174175
min_runtime_seconds=args.bm_video_speed_min_run_seconds,
175176
benchmark_video_creation=args.bm_video_creation,
177+
batch_parameters=BatchParameters(num_threads=8, batch_size=40),
176178
)
177179
plot_data(df_data, args.plot_path)
178180

0 commit comments

Comments
 (0)