From f86f2ccd80e816ad1ca643d265bf96c7f94d35c7 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Mon, 8 Apr 2024 11:36:54 -0700 Subject: [PATCH 1/8] update --- .jenkins/build.sh | 5 ++--- .jenkins/sphinx_files.py | 15 +++++++++++++++ .../super_resolution_with_onnxruntime.py | 3 ++- 3 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 .jenkins/sphinx_files.py diff --git a/.jenkins/build.sh b/.jenkins/build.sh index c7bbb5c2503..14a26b9c3e6 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -53,12 +53,11 @@ if [[ "${JOB_TYPE}" == "worker" ]]; then # Step 2: Keep certain tutorials based on file count, and remove runnable code in all other tutorials # IMPORTANT NOTE: We assume that each tutorial has a UNIQUE filename. FILES_TO_RUN=$(python .jenkins/get_files_to_run.py) - echo "FILES_TO_RUN: " ${FILES_TO_RUN} # Files to run must be accessible to subprocessed (at least to `download_data.py`) export FILES_TO_RUN - # Step 3: Run `make docs` to generate HTML files and static files for these tutorials - make docs + make download + python .jenkins/sphinx_files.py # Step 3.1: Run the post-processing script: python .jenkins/post_process_notebooks.py diff --git a/.jenkins/sphinx_files.py b/.jenkins/sphinx_files.py new file mode 100644 index 00000000000..eb41aee73b5 --- /dev/null +++ b/.jenkins/sphinx_files.py @@ -0,0 +1,15 @@ +import subprocess +import os + + +def main() -> None: + files_to_run = os.environ["FILES_TO_RUN"] + env = os.environ.copy() + for file in files_to_run.split(" "): + print(f"Running {file}") + env["GALLERY_PATTERN"] = file + subprocess.check_output(["make", "html"], env=env) + + +if __name__ == "__main__": + main() diff --git a/advanced_source/super_resolution_with_onnxruntime.py b/advanced_source/super_resolution_with_onnxruntime.py index 3f4dd439690..b409fccd2b9 100644 --- a/advanced_source/super_resolution_with_onnxruntime.py +++ b/advanced_source/super_resolution_with_onnxruntime.py @@ -254,8 +254,9 @@ def to_numpy(tensor): from PIL import Image import torchvision.transforms as transforms +import os -img = Image.open("./_static/img/cat.jpg") +img = Image.open(os.cwd() + "../_static/img/cat.jpg") resize = transforms.Resize([224, 224]) img = resize(img) From 0e49cb876a5dd38f2f9ccf59aae4b7fa1aa916dc Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Mon, 8 Apr 2024 13:31:26 -0700 Subject: [PATCH 2/8] update --- .jenkins/build.sh | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 14a26b9c3e6..bfc31b30c2a 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -58,6 +58,12 @@ if [[ "${JOB_TYPE}" == "worker" ]]; then make download python .jenkins/sphinx_files.py + mkdir docs + cp -r beginner docs + cp -r intermediate docs + cp -r prototype docs + cp -r recipes docs + cp -r advanced docs # Step 3.1: Run the post-processing script: python .jenkins/post_process_notebooks.py @@ -109,6 +115,12 @@ if [[ "${JOB_TYPE}" == "worker" ]]; then done set -x + mv docs/prototype docs/prototype_source + mv docs/beginner docs/beginner_source + mv docs/advanced docs/advanced_source + mv docs/intermediate docs/intermediate_source + mv docs/recipes docs/recipes_source + # Step 5: Remove INVISIBLE_CODE_BLOCK from .html/.rst.txt/.ipynb/.py files bash $DIR/remove_invisible_code_block_batch.sh docs python .jenkins/validate_tutorials_built.py @@ -117,10 +129,6 @@ if [[ "${JOB_TYPE}" == "worker" ]]; then 7z a worker_${WORKER_ID}.7z docs awsv2 s3 cp worker_${WORKER_ID}.7z s3://${BUCKET_NAME}/${COMMIT_ID}/worker_${WORKER_ID}.7z elif [[ "${JOB_TYPE}" == "manager" ]]; then - # Step 1: Generate no-plot HTML pages for all tutorials - make html-noplot - cp -r _build/html docs - # Step 2: Wait for all workers to finish # Don't actually need to do this because gha will wait @@ -129,9 +137,13 @@ elif [[ "${JOB_TYPE}" == "manager" ]]; then for ((worker_id=1;worker_id Date: Mon, 8 Apr 2024 15:32:32 -0700 Subject: [PATCH 3/8] update --- .jenkins/build.sh | 10 ++-------- .jenkins/sphinx_files.py | 2 +- conf.py | 2 +- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index bfc31b30c2a..7babcb47d15 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -56,14 +56,8 @@ if [[ "${JOB_TYPE}" == "worker" ]]; then # Files to run must be accessible to subprocessed (at least to `download_data.py`) export FILES_TO_RUN - make download - python .jenkins/sphinx_files.py - mkdir docs - cp -r beginner docs - cp -r intermediate docs - cp -r prototype docs - cp -r recipes docs - cp -r advanced docs + # Step 3: Run `make docs` to generate HTML files and static files for these tutorials + make docs # Step 3.1: Run the post-processing script: python .jenkins/post_process_notebooks.py diff --git a/.jenkins/sphinx_files.py b/.jenkins/sphinx_files.py index eb41aee73b5..4e2a008360d 100644 --- a/.jenkins/sphinx_files.py +++ b/.jenkins/sphinx_files.py @@ -7,7 +7,7 @@ def main() -> None: env = os.environ.copy() for file in files_to_run.split(" "): print(f"Running {file}") - env["GALLERY_PATTERN"] = file + env["RUNTHIS"] = file subprocess.check_output(["make", "html"], env=env) diff --git a/conf.py b/conf.py index 36bf506f864..faba20016f7 100644 --- a/conf.py +++ b/conf.py @@ -102,7 +102,7 @@ def reset_seeds(gallery_conf, fname): 'examples_dirs': ['beginner_source', 'intermediate_source', 'advanced_source', 'recipes_source', 'prototype_source'], 'gallery_dirs': ['beginner', 'intermediate', 'advanced', 'recipes', 'prototype'], - 'filename_pattern': re.compile(SPHINX_SHOULD_RUN), + 'filename_pattern': os.getenv("RUNTHIS"), 'promote_jupyter_magic': True, 'backreferences_dir': None, 'first_notebook_cell': ("# For tips on running notebooks in Google Colab, see\n" From f2c3cd7bf13e890392e37a2648d168919b28dfe6 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Mon, 8 Apr 2024 15:32:37 -0700 Subject: [PATCH 4/8] update --- .jenkins/build.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 7babcb47d15..bfc31b30c2a 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -56,8 +56,14 @@ if [[ "${JOB_TYPE}" == "worker" ]]; then # Files to run must be accessible to subprocessed (at least to `download_data.py`) export FILES_TO_RUN - # Step 3: Run `make docs` to generate HTML files and static files for these tutorials - make docs + make download + python .jenkins/sphinx_files.py + mkdir docs + cp -r beginner docs + cp -r intermediate docs + cp -r prototype docs + cp -r recipes docs + cp -r advanced docs # Step 3.1: Run the post-processing script: python .jenkins/post_process_notebooks.py From 9eec84cdc7d54e0e2a2adcd656d392227f404668 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Wed, 10 Apr 2024 11:16:56 -0700 Subject: [PATCH 5/8] update --- .jenkins/get_files_to_run.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.jenkins/get_files_to_run.py b/.jenkins/get_files_to_run.py index bdf4562a827..9a75b933a49 100644 --- a/.jenkins/get_files_to_run.py +++ b/.jenkins/get_files_to_run.py @@ -3,6 +3,7 @@ import os from pathlib import Path from remove_runnable_code import remove_runnable_code +from validate_tutorials_built import NOT_RUN # Calculate repo base dir @@ -96,6 +97,7 @@ def main() -> None: all_files = get_all_files() files_to_run = calculate_shards(all_files, num_shards=args.num_shards)[args.shard_num - 1] + files_to_run = [x for x in files_to_run if x not in NOT_RUN] if not args.dry_run: remove_other_files(all_files, compute_files_to_keep(files_to_run)) stripped_file_names = [Path(x).stem for x in files_to_run] From 31f56771397cf35682419e3c817d3ee7115d1ba9 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Fri, 12 Apr 2024 16:32:31 -0700 Subject: [PATCH 6/8] update --- .jenkins/sphinx_files.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.jenkins/sphinx_files.py b/.jenkins/sphinx_files.py index 4e2a008360d..79cfe7f1a65 100644 --- a/.jenkins/sphinx_files.py +++ b/.jenkins/sphinx_files.py @@ -1,6 +1,7 @@ +import shutil import subprocess import os - +import glob def main() -> None: files_to_run = os.environ["FILES_TO_RUN"] @@ -8,8 +9,12 @@ def main() -> None: for file in files_to_run.split(" "): print(f"Running {file}") env["RUNTHIS"] = file - subprocess.check_output(["make", "html"], env=env) - + subprocess.check_output(["make", "html", f"BUILDDIR=_build/{file}"], env=env) + files = glob.glob(f"_build/{file}/**", recursive=True) + for gen_file in files: + if file in str(gen_file): + rel_path = os.path.relpath(gen_file, f"_build/{file}") + shutil.copy(gen_file, f"_build/{rel_path}") if __name__ == "__main__": main() From a4be24ae78cddd3999a5be865a4b7a8709516d8d Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Wed, 17 Apr 2024 15:56:45 -0700 Subject: [PATCH 7/8] update --- .jenkins/sphinx_files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/sphinx_files.py b/.jenkins/sphinx_files.py index 79cfe7f1a65..972affd4035 100644 --- a/.jenkins/sphinx_files.py +++ b/.jenkins/sphinx_files.py @@ -10,7 +10,7 @@ def main() -> None: print(f"Running {file}") env["RUNTHIS"] = file subprocess.check_output(["make", "html", f"BUILDDIR=_build/{file}"], env=env) - files = glob.glob(f"_build/{file}/**", recursive=True) + files = glob.glob(f"_build/{file}/**/*", recursive=True) for gen_file in files: if file in str(gen_file): rel_path = os.path.relpath(gen_file, f"_build/{file}") From f89c82f83e1ce33b27b35f613b0112390c173343 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Wed, 17 Apr 2024 16:28:29 -0700 Subject: [PATCH 8/8] update --- .jenkins/sphinx_files.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.jenkins/sphinx_files.py b/.jenkins/sphinx_files.py index 972affd4035..13fb2357b38 100644 --- a/.jenkins/sphinx_files.py +++ b/.jenkins/sphinx_files.py @@ -14,6 +14,7 @@ def main() -> None: for gen_file in files: if file in str(gen_file): rel_path = os.path.relpath(gen_file, f"_build/{file}") + print(gen_file, f"_build/{rel_path}") shutil.copy(gen_file, f"_build/{rel_path}") if __name__ == "__main__":