[experiment] mucking around with sphinx gallery

.jenkins/build.sh

@@ -52,10 +52,17 @@ if [[ "${JOB_TYPE}" == "worker" ]]; then
   # Step 2: Keep certain tutorials based on file count, and remove runnable code in all other tutorials
   # IMPORTANT NOTE: We assume that each tutorial has a UNIQUE filename.
   FILES_TO_RUN=$(python .jenkins/get_files_to_run.py)
-  echo "FILES_TO_RUN: " ${FILES_TO_RUN}
   # Files to run must be accessible to subprocessed (at least to `download_data.py`)
   export FILES_TO_RUN
 
+  make download
+  python .jenkins/sphinx_files.py
+  mkdir docs
+  cp -r beginner docs
+  cp -r intermediate docs
+  cp -r prototype docs
+  cp -r recipes docs
+  cp -r advanced docs
   # Step 3: Run `make docs` to generate HTML files and static files for these tutorialis
   pip3 install -e git+https://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme
   make docs
@@ -110,6 +117,12 @@ if [[ "${JOB_TYPE}" == "worker" ]]; then
   done
   set -x
 
+  mv docs/prototype docs/prototype_source
+  mv docs/beginner docs/beginner_source
+  mv docs/advanced docs/advanced_source
+  mv docs/intermediate docs/intermediate_source
+  mv docs/recipes docs/recipes_source
+
   # Step 5: Remove INVISIBLE_CODE_BLOCK from .html/.rst.txt/.ipynb/.py files
   bash $DIR/remove_invisible_code_block_batch.sh docs
   python .jenkins/validate_tutorials_built.py
@@ -131,9 +144,13 @@ elif [[ "${JOB_TYPE}" == "manager" ]]; then
   for ((worker_id=1;worker_id<NUM_WORKERS+1;worker_id++)); do
     awsv2 s3 cp s3://${BUCKET_NAME}/${COMMIT_ID}/worker_$worker_id.7z worker_$worker_id.7z
     7z x worker_$worker_id.7z -oworker_$worker_id
-    yes | cp -R worker_$worker_id/docs/* docs_with_plot/docs
+    yes | cp -R worker_$worker_id/docs/* .
   done
 
+  # Step 1: Generate no-plot HTML pages for all tutorials
+  make html-noplot
+  cp -r _build/html docs
+
   # Step 4: Copy all generated files into docs
   rsync -av docs_with_plot/docs/ docs --exclude='**aws_distributed_training_tutorial*'
 

.jenkins/get_files_to_run.py

@@ -3,6 +3,7 @@
 import os
 from pathlib import Path
 from remove_runnable_code import remove_runnable_code
+from validate_tutorials_built import NOT_RUN
 
 
 # Calculate repo base dir
@@ -96,6 +97,7 @@ def main() -> None:
 
     all_files = get_all_files()
     files_to_run = calculate_shards(all_files, num_shards=args.num_shards)[args.shard_num - 1]
+    files_to_run = [x for x in files_to_run if x not in NOT_RUN]
     if not args.dry_run:
         remove_other_files(all_files, compute_files_to_keep(files_to_run))
     stripped_file_names = [Path(x).stem for x in files_to_run]

.jenkins/sphinx_files.py

@@ -0,0 +1,21 @@
+import shutil
+import subprocess
+import os
+import glob
+
+def main() -> None:
+    files_to_run = os.environ["FILES_TO_RUN"]
+    env = os.environ.copy()
+    for file in files_to_run.split(" "):
+        print(f"Running {file}")
+        env["RUNTHIS"] = file
+        subprocess.check_output(["make", "html", f"BUILDDIR=_build/{file}"], env=env)
+        files = glob.glob(f"_build/{file}/**/*", recursive=True)
+        for gen_file in files:
+            if file in str(gen_file):
+                rel_path = os.path.relpath(gen_file, f"_build/{file}")
+                print(gen_file, f"_build/{rel_path}")
+                shutil.copy(gen_file, f"_build/{rel_path}")
+
+if __name__ == "__main__":
+    main()

advanced_source/super_resolution_with_onnxruntime.py

@@ -254,8 +254,9 @@ def to_numpy(tensor):
 
 from PIL import Image
 import torchvision.transforms as transforms
+import os
 
-img = Image.open("./_static/img/cat.jpg")
+img = Image.open(os.cwd() + "../_static/img/cat.jpg")
 
 resize = transforms.Resize([224, 224])
 img = resize(img)

conf.py

@@ -101,7 +101,7 @@ def reset_seeds(gallery_conf, fname):
     'examples_dirs': ['beginner_source', 'intermediate_source',
                       'advanced_source', 'recipes_source', 'prototype_source'],
     'gallery_dirs': ['beginner', 'intermediate', 'advanced', 'recipes', 'prototype'],
-    'filename_pattern': re.compile(SPHINX_SHOULD_RUN),
+    'filename_pattern': os.getenv("RUNTHIS"),
     'promote_jupyter_magic': True,
     'backreferences_dir': None,
     'first_notebook_cell': ("# For tips on running notebooks in Google Colab, see\n"