Allow building CPython from a local source directory (#3)

* Allow building CPython from a local source directory

* Allow `version` override in `ContainerContext.install_toolchain_archive` as well

* Remove schedule?

Author: zanieb

.github/workflows/linux.yml

@@ -3,8 +3,6 @@ name: Linux Python build
 on:
   push:
   pull_request:
-  schedule:
-    - cron: '13 11 * * *'
 jobs:
   pythonbuild:
     runs-on: ubuntu-22.04

cpython-unix/Makefile

@@ -17,6 +17,10 @@ ifndef PYBUILD_HOST_PLATFORM
     $(error PYBUILD_HOST_PLATFORM not defined)
 endif
 
+ifndef PYBUILD_PYTHON_SOURCE
+    $(error PYBUILD_PYTHON_SOURCE not defined)
+endif
+
 ifndef PYBUILD_PYTHON_VERSION
     $(error PYBUILD_PYTHON_VERSION not defined)
 endif
@@ -33,6 +37,7 @@ RUN_BUILD = $(BUILD) \
     --host-platform $(HOST_PLATFORM) \
     --target-triple $(TARGET_TRIPLE) \
     --optimizations $(PYBUILD_OPTIMIZATIONS) \
+    --python-source $(PYBUILD_PYTHON_SOURCE) \
     --dest-archive $@ \
     $(NULL)
 

cpython-unix/build-main.py

@@ -71,6 +71,11 @@ def main():
         default="cpython-3.11",
         help="Python distribution to build",
     )
+    parser.add_argument(
+        "--python-source",
+        default=None,
+        help="A custom path to CPython source files to use",
+    )
     parser.add_argument(
         "--break-on-failure",
         action="store_true",
@@ -118,31 +123,45 @@ def main():
         )
         return 1
 
+    python_source = (
+        (str(pathlib.Path(args.python_source).resolve()))
+        if args.python_source
+        else "null"
+    )
+
     musl = "musl" in target_triple
 
     env = dict(os.environ)
 
     env["PYBUILD_HOST_PLATFORM"] = host_platform
     env["PYBUILD_TARGET_TRIPLE"] = target_triple
     env["PYBUILD_OPTIMIZATIONS"] = args.optimizations
+    env["PYBUILD_PYTHON_SOURCE"] = python_source
     if musl:
         env["PYBUILD_MUSL"] = "1"
     if args.break_on_failure:
         env["PYBUILD_BREAK_ON_FAILURE"] = "1"
     if args.no_docker:
         env["PYBUILD_NO_DOCKER"] = "1"
 
-    entry = DOWNLOADS[args.python]
-    env["PYBUILD_PYTHON_VERSION"] = entry["version"]
-    env["PYBUILD_PYTHON_MAJOR_VERSION"] = ".".join(entry["version"].split(".")[0:2])
+    if not args.python_source:
+        entry = DOWNLOADS[args.python]
+        env["PYBUILD_PYTHON_VERSION"] = cpython_version = entry["version"]
+    else:
+        if "PYBUILD_PYTHON_VERSION" not in env:
+            print("PYBUILD_PYTHON_VERSION must be set when using `--python-source`")
+            return 1
+        cpython_version = env["PYBUILD_PYTHON_VERSION"]
+
+    env["PYBUILD_PYTHON_MAJOR_VERSION"] = ".".join(cpython_version.split(".")[0:2])
 
     if "PYBUILD_RELEASE_TAG" in os.environ:
         release_tag = os.environ["PYBUILD_RELEASE_TAG"]
     else:
         release_tag = release_tag_from_git()
 
     archive_components = [
-        "cpython-%s" % entry["version"],
+        "cpython-%s" % cpython_version,
         target_triple,
         args.optimizations,
     ]

cpython-unix/build.py

@@ -32,12 +32,14 @@
     add_env_common,
     add_licenses_to_extension_entry,
     clang_toolchain,
+    create_tar_from_directory,
     download_entry,
     get_targets,
     get_target_settings,
     target_needs,
     validate_python_json,
     write_package_versions,
+    write_cpython_version,
     write_target_settings,
     write_triples_makefiles,
 )
@@ -62,8 +64,7 @@ def install_sccache(build_env):
     """
     candidates = [
         # Prefer a binary in the project itself.
-        ROOT
-        / "sccache",
+        ROOT / "sccache",
     ]
 
     # Look for sccache in $PATH, but only if the build environment
@@ -255,10 +256,9 @@ def simple_build(
         build_env.copy_file(SUPPORT / ("build-%s.sh" % entry))
 
         env = {
-            "%s_VERSION"
-            % entry.upper()
-            .replace("-", "_")
-            .replace(".", "_"): DOWNLOADS[entry]["version"],
+            "%s_VERSION" % entry.upper().replace("-", "_").replace(".", "_"): DOWNLOADS[
+                entry
+            ]["version"],
         }
 
         add_target_env(env, host_platform, target_triple, build_env)
@@ -684,13 +684,23 @@ def build_cpython(
     optimizations,
     dest_archive,
     version=None,
+    python_source=None,
 ):
     """Build CPython in a Docker image'"""
     entry_name = "cpython-%s" % version
     entry = DOWNLOADS[entry_name]
-    python_version = entry["version"]
+    if not python_source:
+        python_version = entry["version"]
+        python_archive = download_entry(entry_name, DOWNLOADS_PATH)
+    else:
+        python_version = os.environ["PYBUILD_PYTHON_VERSION"]
+        python_archive = DOWNLOADS_PATH / ("Python-%s.tar.xz" % python_version)
+        print("Compressing %s to %s" % (python_source, python_archive))
+        with python_archive.open("wb") as fh:
+            create_tar_from_directory(
+                fh, python_source, path_prefix="Python-%s" % python_version
+            )
 
-    python_archive = download_entry(entry_name, DOWNLOADS_PATH)
     setuptools_archive = download_entry("setuptools", DOWNLOADS_PATH)
     pip_archive = download_entry("pip", DOWNLOADS_PATH)
 
@@ -726,7 +736,9 @@ def build_cpython(
         for p in sorted(packages):
             build_env.install_artifact_archive(BUILD, p, target_triple, optimizations)
 
-        build_env.install_toolchain_archive(BUILD, entry_name, host_platform)
+        build_env.install_toolchain_archive(
+            BUILD, entry_name, host_platform, version=python_version
+        )
 
         for p in (
             python_archive,
@@ -762,8 +774,8 @@ def build_cpython(
 
         env = {
             "PIP_VERSION": DOWNLOADS["pip"]["version"],
-            "PYTHON_VERSION": entry["version"],
-            "PYTHON_MAJMIN_VERSION": ".".join(entry["version"].split(".")[0:2]),
+            "PYTHON_VERSION": python_version,
+            "PYTHON_MAJMIN_VERSION": ".".join(python_version.split(".")[0:2]),
             "SETUPTOOLS_VERSION": DOWNLOADS["setuptools"]["version"],
             "TOOLCHAIN": "clang-%s" % host_platform,
         }
@@ -824,7 +836,7 @@ def build_cpython(
             "target_triple": target_triple,
             "optimizations": optimizations,
             "python_tag": entry["python_tag"],
-            "python_version": entry["version"],
+            "python_version": python_version,
             "python_stdlib_test_packages": sorted(STDLIB_TEST_PACKAGES),
             "python_symbol_visibility": python_symbol_visibility,
             "python_extension_module_loading": extension_module_loading,
@@ -924,6 +936,11 @@ def main():
         "--dest-archive", required=True, help="Path to archive that we are producing"
     )
     parser.add_argument("--docker-image", help="Docker image to use for building")
+    parser.add_argument(
+        "--python-source",
+        default=None,
+        help="A custom path to CPython source files to use",
+    )
     parser.add_argument("action")
 
     args = parser.parse_args()
@@ -933,6 +950,9 @@ def main():
     target_triple = args.target_triple
     host_platform = args.host_platform
     optimizations = args.optimizations
+    python_source = (
+        pathlib.Path(args.python_source) if args.python_source != "null" else None
+    )
     dest_archive = pathlib.Path(args.dest_archive)
     docker_image = args.docker_image
 
@@ -969,6 +989,12 @@ def main():
             write_target_settings(targets, BUILD / "targets")
             write_package_versions(BUILD / "versions")
 
+            # Override the DOWNLOADS package entry for CPython for the local build
+            if python_source:
+                write_cpython_version(
+                    BUILD / "versions", os.environ["PYBUILD_PYTHON_VERSION"]
+                )
+
         elif action.startswith("image-"):
             image_name = action[6:]
             image_path = BUILD / ("%s.Dockerfile" % image_name)
@@ -1179,6 +1205,7 @@ def main():
                 optimizations=optimizations,
                 dest_archive=dest_archive,
                 version=action.split("-")[1],
+                python_source=python_source,
             )
 
         else:

cpython-windows/build.py

@@ -2656,6 +2656,7 @@ def main():
             "cpython-3.10",
             "cpython-3.11",
             "cpython-3.12",
+            "cpython-3.13",
         },
         default="cpython-3.11",
         help="Python distribution to build",

pythonbuild/buildenv.py

@@ -38,9 +38,15 @@ def copy_file(self, source: pathlib.Path, dest_path=None, dest_name=None):
         dest_path = dest_path or "/build"
         copy_file_to_container(source, self.container, dest_path, dest_name)
 
-    def install_toolchain_archive(self, build_dir, package_name, host_platform):
+    def install_toolchain_archive(
+        self, build_dir, package_name, host_platform, version=None
+    ):
         entry = DOWNLOADS[package_name]
-        basename = "%s-%s-%s.tar" % (package_name, entry["version"], host_platform)
+        basename = "%s-%s-%s.tar" % (
+            package_name,
+            version or entry["version"],
+            host_platform,
+        )
 
         p = build_dir / basename
         self.copy_file(p)
@@ -152,9 +158,15 @@ def copy_file(self, source: pathlib.Path, dest_path=None, dest_name=None):
         log("copying %s to %s/%s" % (source, dest_dir, dest_name))
         shutil.copy(source, dest_dir / dest_name)
 
-    def install_toolchain_archive(self, build_dir, package_name, host_platform):
+    def install_toolchain_archive(
+        self, build_dir, package_name, host_platform, version=None
+    ):
         entry = DOWNLOADS[package_name]
-        basename = "%s-%s-%s.tar" % (package_name, entry["version"], host_platform)
+        basename = "%s-%s-%s.tar" % (
+            package_name,
+            version or entry["version"],
+            host_platform,
+        )
 
         p = build_dir / basename
         dest_path = self.td / "tools"

pythonbuild/utils.py

@@ -182,6 +182,17 @@ def write_package_versions(dest_path: pathlib.Path):
         write_if_different(p, content.encode("ascii"))
 
 
+def write_cpython_version(dest_path: pathlib.Path, version: str):
+    """Write a CPython version in a directory."""
+    dest_path.mkdir(parents=True, exist_ok=True)
+
+    major_minor = ".".join(version.split(".")[:2])
+    k = "cpython-%s" % major_minor
+    p = dest_path / ("VERSION.%s" % k)
+    content = "%s_VERSION := %s\n" % (k.upper().replace("-", "_"), version)
+    write_if_different(p, content.encode("ascii"))
+
+
 def write_target_settings(targets, dest_path: pathlib.Path):
     dest_path.mkdir(parents=True, exist_ok=True)
 
@@ -621,7 +632,7 @@ def release_download_statistics(mode="by_asset"):
             print("%d\t%s" % (count, build))
     elif mode == "by_tag":
         for tag, count in sorted(by_tag.items()):
-            print("%d\t%s"% (count, tag))
+            print("%d\t%s" % (count, tag))
     elif mode == "total":
         print("%d" % by_tag.total())
     else: