Merge branch 'main' into logs-doc-fix

Author: svekars

Diff for: .ci/docker/Dockerfile

@@ -15,15 +15,11 @@ RUN bash ./install_user.sh && rm install_user.sh
 COPY ./common/install_docs_reqs.sh install_docs_reqs.sh
 RUN bash ./install_docs_reqs.sh && rm install_docs_reqs.sh
 
-# Install conda and other packages
-ENV ANACONDA_PYTHON_VERSION=3.10
-ENV CONDA_CMAKE yes
-ENV DOCS yes
-ENV PATH /opt/conda/envs/py_$ANACONDA_PYTHON_VERSION/bin:/opt/conda/bin:$PATH
-COPY ./requirements.txt /opt/conda/
-COPY ./common/install_conda.sh install_conda.sh
-COPY ./common/common_utils.sh common_utils.sh
-RUN bash ./install_conda.sh && rm install_conda.sh common_utils.sh /opt/conda/requirements.txt
+COPY ./common/install_pip_requirements.sh install_pip_requirements.sh
+COPY ./requirements.txt requirements.txt
+RUN bash ./install_pip_requirements.sh && rm install_pip_requirements.sh
+
+RUN ln -s /usr/bin/python3 /usr/bin/python
 
 USER ci-user
 CMD ["bash"]

Diff for: .ci/docker/build.sh

@@ -10,7 +10,7 @@ set -exu
 IMAGE_NAME="$1"
 shift
 
-export UBUNTU_VERSION="20.04"
+export UBUNTU_VERSION="22.04"
 export CUDA_VERSION="12.4.1"
 
 export BASE_IMAGE="nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION}"

Diff for: .ci/docker/common/common_utils.sh

@@ -10,7 +10,7 @@ install_ubuntu() {
   apt-get install -y --no-install-recommends \
     build-essential \
     ca-certificates \
-    cmake=3.16* \
+    cmake=3.22* \
     curl \
     git \
     wget \
@@ -27,7 +27,9 @@ install_ubuntu() {
     libglfw3-dev \
     sox \
     libsox-dev \
-    libsox-fmt-all
+    libsox-fmt-all \
+    python3-pip \
+    python3-dev
 
   # Cleanup package manager
   apt-get autoclean && apt-get clean

Diff for: .ci/docker/common/install_base.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -ex
+
+# Install pip packages
+pip install --upgrade pip
+pip install -r ./requirements.txt

Diff for: .ci/docker/common/install_conda.sh

@@ -44,6 +44,8 @@ jobs:
 
       - name: Checkout Tutorials
         uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
 
       - name: Setup Linux
         uses: pytorch/pytorch/.github/actions/setup-linux@main
@@ -115,6 +117,8 @@ jobs:
 
       - name: Checkout Tutorials
         uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
 
       - name: Setup Linux
         uses: pytorch/pytorch/.github/actions/setup-linux@main

Diff for: .ci/docker/common/install_pip_requirements.sh

@@ -0,0 +1,160 @@
+import json
+import os
+import subprocess
+import sys
+from datetime import datetime
+
+from bs4 import BeautifulSoup
+
+
+json_file_path = "tutorials-review-data.json"
+
+# paths to skip from the post-processing script
+paths_to_skip = [
+    "beginner/examples_autograd/two_layer_net_custom_function",  # not present in the repo
+    "beginner/examples_nn/two_layer_net_module",  # not present in the repo
+    "beginner/examples_tensor/two_layer_net_numpy",  # not present in the repo
+    "beginner/examples_tensor/two_layer_net_tensor",  # not present in the repo
+    "beginner/examples_autograd/two_layer_net_autograd",  # not present in the repo
+    "beginner/examples_nn/two_layer_net_optim",  # not present in the repo
+    "beginner/examples_nn/two_layer_net_nn",  # not present in the repo
+    "intermediate/coding_ddpg",  # not present in the repo - will delete the carryover
+]
+# Mapping of source directories to build directories
+source_to_build_mapping = {
+    "beginner": "beginner_source",
+    "recipes": "recipes_source",
+    "distributed": "distributed",
+    "intermediate": "intermediate_source",
+    "prototype": "prototype_source",
+    "advanced": "advanced_source",
+    "": "",  # root dir for index.rst
+}
+
+def get_git_log_date(file_path, git_log_args):
+    try:
+        result = subprocess.run(
+            ["git", "log"] + git_log_args + ["--", file_path],
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        if result.stdout:
+            date_str = result.stdout.splitlines()[0]
+            return datetime.strptime(date_str, "%a, %d %b %Y %H:%M:%S %z")
+    except subprocess.CalledProcessError:
+        pass
+    raise ValueError(f"Could not find date for {file_path}")
+
+def get_creation_date(file_path):
+    return get_git_log_date(file_path, ["--diff-filter=A", "--format=%aD"]).strftime("%b %d, %Y")
+
+
+def get_last_updated_date(file_path):
+    return get_git_log_date(file_path, ["-1", "--format=%aD"]).strftime("%b %d, %Y")
+
+# Try to find the source file with the given base path and the extensions .rst and .py
+def find_source_file(base_path):
+    for ext in [".rst", ".py"]:
+        source_file_path = base_path + ext
+        if os.path.exists(source_file_path):
+            return source_file_path
+    return None
+
+
+# Function to process a JSON file and insert the "Last Verified" information into the HTML files
+def process_json_file(build_dir , json_file_path):
+    with open(json_file_path, "r", encoding="utf-8") as json_file:
+        json_data = json.load(json_file)
+
+    for entry in json_data:
+        path = entry["Path"]
+        last_verified = entry["Last Verified"]
+        status = entry.get("Status", "")
+        if path in paths_to_skip:
+            print(f"Skipping path: {path}")
+            continue
+        if status in ["needs update", "not verified"]:
+            formatted_last_verified = "Not Verified"
+        elif last_verified:
+            try:
+                last_verified_date = datetime.strptime(last_verified, "%Y-%m-%d")
+                formatted_last_verified = last_verified_date.strftime("%b %d, %Y")
+            except ValueError:
+                formatted_last_verified = "Unknown"
+        else:
+            formatted_last_verified = "Not Verified"
+        if status == "deprecated":
+            formatted_last_verified += "Deprecated"
+
+        for build_subdir, source_subdir in source_to_build_mapping.items():
+            if path.startswith(build_subdir):
+                html_file_path = os.path.join(build_dir, path + ".html")
+                base_source_path = os.path.join(
+                    source_subdir, path[len(build_subdir) + 1 :]
+                )
+                source_file_path = find_source_file(base_source_path)
+                break
+        else:
+            print(f"Warning: No mapping found for path {path}")
+            continue
+
+        if not os.path.exists(html_file_path):
+            print(
+                f"Warning: HTML file not found for path {html_file_path}."
+                "If this is a new tutorial, please add it to the audit JSON file and set the Verified status and todays's date."
+            )
+            continue
+
+        if not source_file_path:
+            print(f"Warning: Source file not found for path {base_source_path}.")
+            continue
+
+        created_on = get_creation_date(source_file_path)
+        last_updated = get_last_updated_date(source_file_path)
+
+        with open(html_file_path, "r", encoding="utf-8") as file:
+            soup = BeautifulSoup(file, "html.parser")
+        # Check if the <p> tag with class "date-info-last-verified" already exists
+        existing_date_info = soup.find("p", {"class": "date-info-last-verified"})
+        if existing_date_info:
+            print(
+                f"Warning: <p> tag with class 'date-info-last-verified' already exists in {html_file_path}"
+            )
+            continue
+
+        h1_tag = soup.find("h1")  # Find the h1 tag to insert the dates
+        if h1_tag:
+            date_info_tag = soup.new_tag("p", **{"class": "date-info-last-verified"})
+            date_info_tag["style"] = "color: #6c6c6d; font-size: small;"
+            # Add the "Created On", "Last Updated", and "Last Verified" information
+            date_info_tag.string = (
+                f"Created On: {created_on} | "
+                f"Last Updated: {last_updated} | "
+                f"Last Verified: {formatted_last_verified}"
+            )
+            # Insert the new tag after the <h1> tag
+            h1_tag.insert_after(date_info_tag)
+            # Save back to the HTML.
+            with open(html_file_path, "w", encoding="utf-8") as file:
+                file.write(str(soup))
+        else:
+            print(f"Warning: <h1> tag not found in {html_file_path}")
+
+
+def main():
+    if len(sys.argv) < 2:
+        print("Error: Build directory not provided. Exiting.")
+        exit(1)
+    build_dir = sys.argv[1]
+    print(f"Build directory: {build_dir}")
+    process_json_file(build_dir , json_file_path)
+    print(
+        "Finished processing JSON file. Please check the output for any warnings. "
+        "Pages like `nlp/index.html` are generated only during the full `make docs` "
+        "or `make html` build. Warnings about these files when you run `make html-noplot` "
+        "can be ignored."
+    )
+
+if __name__ == "__main__":
+    main()

Diff for: .github/workflows/build-tutorials.yml

@@ -33,7 +33,7 @@
   },
   "recipes_source/torch_export_aoti_python.py": {
     "needs": "linux.g5.4xlarge.nvidia.gpu"
-  }, 
+  },
   "advanced_source/pendulum.py": {
     "needs": "linux.g5.4xlarge.nvidia.gpu",
     "_comment": "need to be here for the compiling_optimizer_lr_scheduler.py to run."
@@ -58,6 +58,9 @@
   "intermediate_source/scaled_dot_product_attention_tutorial.py": {
     "needs": "linux.g5.4xlarge.nvidia.gpu"
   },
+  "intermediate_source/transformer_building_blocks.py": {
+    "needs": "linux.g5.4xlarge.nvidia.gpu"
+  },
   "recipes_source/torch_compile_user_defined_triton_kernel_tutorial.py": {
     "needs": "linux.g5.4xlarge.nvidia.gpu"
   },

Diff for: .jenkins/insert_last_verified.py

@@ -25,6 +25,7 @@
     "intermediate_source/mnist_train_nas",  # used by ax_multiobjective_nas_tutorial.py
     "intermediate_source/fx_conv_bn_fuser",
     "intermediate_source/_torch_export_nightly_tutorial",  # does not work on release
+    "intermediate_source/transformer_building_blocks",  # does not work on release
     "advanced_source/super_resolution_with_onnxruntime",
     "advanced_source/usb_semisup_learn", # fails with CUDA OOM error, should try on a different worker
     "prototype_source/fx_graph_mode_ptq_dynamic",

Diff for: .jenkins/metadata.json

@@ -9,3 +9,6 @@ file:\/\/\/.*
 
 # Ignore colab link in the setting of conf.py
 https://pytorch.org/tutorials/beginner/colab/n
+
+# Ignore local host link from intermediate_source/tensorboard_tutorial.rst
+http://localhost:6006

Diff for: .jenkins/validate_tutorials_built.py

@@ -86,18 +86,29 @@ download:
 	wget https://www.cis.upenn.edu/~jshi/ped_html/PennFudanPed.zip -P $(DATADIR)
 	unzip -o $(DATADIR)/PennFudanPed.zip -d intermediate_source/data/
 
+download-last-reviewed-json:
+	@echo "Downloading tutorials-review-data.json..."
+	curl -o tutorials-review-data.json https://raw.githubusercontent.com/pytorch/tutorials/refs/heads/last-reviewed-data-json/tutorials-review-data.json
+	@echo "Finished downloading tutorials-review-data.json."
 docs:
 	make download
+	make download-last-reviewed-json
 	make html
+	@python .jenkins/insert_last_verified.py $(BUILDDIR)/html
 	rm -rf docs
 	cp -r $(BUILDDIR)/html docs
 	touch docs/.nojekyll
+	rm -rf tutorials-review-data.json
 
 html-noplot:
 	$(SPHINXBUILD) -D plot_gallery=0 -b html $(SPHINXOPTS) "$(SOURCEDIR)" "$(BUILDDIR)/html"
 	# bash .jenkins/remove_invisible_code_block_batch.sh "$(BUILDDIR)/html"
 	@echo
+	make download-last-reviewed-json
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+	@echo "Running post-processing script to insert 'Last Verified' dates..."
+	@python .jenkins/insert_last_verified.py $(BUILDDIR)/html
+	rm -rf tutorials-review-data.json
 
 clean-cache:
 	make clean