Skip to content

GH-131556: calculate PYBUILDDIR in the makefile instead of sysconfig #131761

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
+79 −80
Open

GH-131556: calculate PYBUILDDIR in the makefile instead of sysconfig #131761

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6d86d82
GH-131556: calculate PYBUILDDIR in configure instead of sysconfig
FFY00 Mar 26, 2025
72af97f
Merge branch 'main' into gh-131556
FFY00 Apr 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 3 additions & 27 deletions Lib/sysconfig/__main__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
import json
import os
import sys
import types
from sysconfig import (
_ALWAYS_STR,
_PROJECT_BASE,
_PYTHON_BUILD,
_get_sysconfigdata_name,
get_config_h_filename,
get_config_var,
get_config_vars,
get_default_scheme,
get_makefile_filename,
Expand Down Expand Up @@ -161,10 +160,8 @@ def _print_config_dict(d, stream):


def _get_pybuilddir():
pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}'
if get_config_var('Py_DEBUG') == '1':
pybuilddir += '-pydebug'
return pybuilddir
with open(os.path.join(_PROJECT_BASE, 'pybuilddir.txt')) as f:
return f.read()


def _get_json_data_name():
Expand Down Expand Up @@ -203,23 +200,6 @@ def _generate_posix_vars():

name = _get_sysconfigdata_name()

# There's a chicken-and-egg situation on OS X with regards to the
# _sysconfigdata module after the changes introduced by #15298:
# get_config_vars() is called by get_platform() as part of the
# `make pybuilddir.txt` target -- which is a precursor to the
# _sysconfigdata.py module being constructed. Unfortunately,
# get_config_vars() eventually calls _init_posix(), which attempts
# to import _sysconfigdata, which we won't have built yet. In order
# for _init_posix() to work, if we're on Darwin, just mock up the
# _sysconfigdata module manually and populate it with the build vars.
# This is more than sufficient for ensuring the subsequent call to
# get_platform() succeeds.
# GH-127178: Since we started generating a .json file, we also need this to
# be able to run sysconfig.get_config_vars().
module = types.ModuleType(name)
module.build_time_vars = vars
sys.modules[name] = module

pybuilddir = _get_pybuilddir()
os.makedirs(pybuilddir, exist_ok=True)
destfile = os.path.join(pybuilddir, name + '.py')
Expand All @@ -243,10 +223,6 @@ def _generate_posix_vars():

print(f'Written {jsonfile}')

# Create file used for sys.path fixup -- see Modules/getpath.c
with open('pybuilddir.txt', 'w', encoding='utf8') as f:
f.write(pybuilddir)


def _print_dict(title, data):
for index, (key, value) in enumerate(sorted(data.items())):
Expand Down
76 changes: 41 additions & 35 deletions Makefile.pre.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ srcdir= @srcdir@
VPATH= @srcdir@
abs_srcdir= @abs_srcdir@
abs_builddir= @abs_builddir@
PYBUILDDIR= build/lib.@MACHDEP@-@MULTIARCH@-@VERSION@-@ABIFLAGS@


CC= @CC@
Expand Down Expand Up @@ -135,6 +136,10 @@ MACHDEP= @MACHDEP@
MULTIARCH= @MULTIARCH@
MULTIARCH_CPPFLAGS = @MULTIARCH_CPPFLAGS@

# Sysconfig data
SYSCONFIGDATA_NAME= @SYSCONFIGDATA_NAME@
SYSCONFIGVARS_JSON_NAME= @SYSCONFIGVARS_JSON_NAME@

# Install prefix for architecture-independent files
prefix= @prefix@

Expand Down Expand Up @@ -731,11 +736,12 @@ list-targets:

.PHONY: build_all
build_all: check-clean-src check-app-store-compliance $(BUILDPYTHON) platform sharedmods \
gdbhooks Programs/_testembed scripts checksharedmods rundsymutil build-details.json
gdbhooks Programs/_testembed scripts checksharedmods rundsymutil pybuilddir.txt \
$(PYBUILDDIR)/build-details.json $(PYBUILDDIR)/$(SYSCONFIGDATA_NAME).py

.PHONY: build_wasm
build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \
python-config checksharedmods
$(PYBUILDDIR)/$(SYSCONFIGDATA_NAME).py python-config checksharedmods

.PHONY: build_emscripten
build_emscripten: build_wasm web_example
Expand Down Expand Up @@ -918,27 +924,37 @@ clinic-tests: check-clean-src $(srcdir)/Lib/test/clinic.test.c
$(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS)
$(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS)

platform: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt
platform: $(PYTHON_FOR_BUILD_DEPS) $(PYBUILDDIR)/$(SYSCONFIGDATA_NAME).py
$(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform

# Create build directory and generate the sysconfig build-time data there.
# pybuilddir.txt contains the name of the build dir and is used for
# sys.path fixup -- see Modules/getpath.c.
# Since this step runs before shared modules are built, try to avoid bootstrap
# problems by creating a dummy pybuilddir.txt just to allow interpreter
# initialization to succeed. It will be overwritten by generate-posix-vars
# or removed in case of failure.
pybuilddir.txt: $(PYTHON_FOR_BUILD_DEPS)
@echo "none" > ./pybuilddir.txt
$(RUNSHARED) $(PYTHON_FOR_BUILD) -S -m sysconfig --generate-posix-vars ;\
if test $$? -ne 0 ; then \
echo "generate-posix-vars failed" ; \
rm -f ./pybuilddir.txt ; \
exit 1 ; \
fi
# Create build directory and pybuilddir.txt.
# pybuilddir.txt which is used by Modules/getpath.c to detect the build directory.
pybuilddir.txt:
mkdir -p $(PYBUILDDIR)
printf $(PYBUILDDIR) >pybuilddir.txt

SYSCONFIG_SRC= \
$(srcdir)/Lib/sysconfig/__init__.py \
$(srcdir)/Lib/sysconfig/__main__.py
SYSCONFIGVARS_DEPS= \
$(SYSCONFIG_SRC) \
Makefile \
Python/sysmodule.o \
Modules/posixmodule.o

# Generate the sysconfig data module.
$(PYBUILDDIR)/$(SYSCONFIGDATA_NAME).py $(PYBUILDDIR)/$(SYSCONFIGVARS_JSON_NAME): $(SYSCONFIGVARS_DEPS) $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt
$(RUNSHARED) $(PYTHON_FOR_BUILD) -S -m sysconfig --generate-posix-vars
@# Sanity check, to make sure sysconfig._get_sysconfigdata_name() returns the same value.
@for file in $@; do \
if test ! -f $<; then \
echo "generate-posix-vars didn't generate '$(PYBUILDDIR)/$(SYSCONFIGDATA_NAME).py'" \
exit 1; \
fi; \
done

build-details.json: pybuilddir.txt
$(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/generate-build-details.py `cat pybuilddir.txt`/build-details.json
$(PYBUILDDIR)/build-details.json: $(PYTHON_FOR_BUILD_DEPS) $(PYBUILDDIR)/$(SYSCONFIGDATA_NAME).py Makefile pybuilddir.txt
$(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/generate-build-details.py $@

# Build static library
$(LIBRARY): $(LIBRARY_OBJS)
Expand Down Expand Up @@ -1444,18 +1460,8 @@ $(LIBHACL_BLAKE2_A): $(LIBHACL_BLAKE2_OBJS)
-rm -f $@
$(AR) $(ARFLAGS) $@ $(LIBHACL_BLAKE2_OBJS)

# create relative links from build/lib.platform/egg.so to Modules/egg.so
# pybuilddir.txt is created too late. We cannot use it in Makefile
# targets. ln --relative is not portable.
.PHONY: sharedmods
sharedmods: $(SHAREDMODS) pybuilddir.txt
@target=`cat pybuilddir.txt`; \
$(MKDIR_P) $$target; \
for mod in X $(SHAREDMODS); do \
if test $$mod != X; then \
$(LN) -sf ../../$$mod $$target/`basename $$mod`; \
fi; \
done
sharedmods: $(SHAREDMODS)

# dependency on BUILDPYTHON ensures that the target is run last
.PHONY: checksharedmods
Expand Down Expand Up @@ -2657,9 +2663,9 @@ libinstall: all $(srcdir)/Modules/xxmodule.c
esac; \
done; \
done
$(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py $(DESTDIR)$(LIBDEST); \
$(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfig_vars_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).json $(DESTDIR)$(LIBDEST); \
$(INSTALL_DATA) `cat pybuilddir.txt`/build-details.json $(DESTDIR)$(LIBDEST); \
$(INSTALL_DATA) $(PYBUILDDIR)/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py $(DESTDIR)$(LIBDEST); \
$(INSTALL_DATA) $(PYBUILDDIR)/_sysconfig_vars_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).json $(DESTDIR)$(LIBDEST); \
$(INSTALL_DATA) $(PYBUILDDIR)/build-details.json $(DESTDIR)$(LIBDEST); \
$(INSTALL_DATA) $(srcdir)/LICENSE $(DESTDIR)$(LIBDEST)/LICENSE.txt
@ # If app store compliance has been configured, apply the patch to the
@ # installed library code. The patch has been previously validated against
Expand Down Expand Up @@ -3064,7 +3070,7 @@ clean-retain-profile: pycremoval
find build -name 'fficonfig.h' -exec rm -f {} ';' || true
find build -name '*.py' -exec rm -f {} ';' || true
find build -name '*.py[co]' -exec rm -f {} ';' || true
-rm -f pybuilddir.txt
-rm -rf pybuilddir.txt $(PYBUILDDIR)
-rm -f _bootstrap_python
-rm -rf web_example python.mjs python.wasm python*.symbols python*.map
-rm -f Programs/_testembed Programs/_freeze_module
Expand Down
4 changes: 2 additions & 2 deletions Modules/makesetup
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -266,15 +266,15 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' |
esac
for mod in $mods
do
file="$srcdir/$mod\$(EXT_SUFFIX)"
file="\$(PYBUILDDIR)/$mod\$(EXT_SUFFIX)"
case $doconfig in
no)
SHAREDMODS="$SHAREDMODS $file"
BUILT_SHARED="$BUILT_SHARED $mod"
;;
esac
rule="$file: $objs"
rule="$rule; \$(BLDSHARED) $objs $libs \$(LIBPYTHON) -o $file"
rule="$rule; @mkdir -p \$(PYBUILDDIR); \$(BLDSHARED) $objs $libs \$(LIBPYTHON) -o $file"
echo "$rule" >>$rulesf
done
done
Expand Down
16 changes: 10 additions & 6 deletions Tools/wasm/emscripten/__main__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,11 +195,10 @@ def configure_emscripten_python(context, working_dir):
assert (
len(lib_dirs) == 1
), f"Expected a single lib.* directory in {python_build_dir}"
lib_dir = os.fsdecode(lib_dirs[0])
pydebug = lib_dir.endswith("-pydebug")
python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1]
_, python_version, abiflags = os.fsdecode(lib_dirs[0]).rsplit('-', maxsplit=2)
sysconfig_data = (
f"{emscripten_build_dir}/build/lib.emscripten-wasm32-{python_version}"
f"{emscripten_build_dir}/build/"
f"lib.emscripten-wasm32-emscripten-{python_version}-{abiflags}"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This won't quite work - pydebug is used in a couple of places (e.g., L203) that isn't covered by this change. I agree using abiflags is a better way to handle it - but we should probably use abiflags to set pydebug.

Copy link
Member Author

@FFY00 FFY00 Mar 27, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Uh, I missed that line, I meant to do the exact same change I did to the WASI script. Since I changed the format of pybuilddir.txt, which is where sysconfig_data points to, we shouldn't be adding -pydebug. My change to the sysconfig_data variable already sets it to the correct value, I just forgot to remove the line below in the commit 😅.

I don't remember seeing pydebug being used elsewhere, but I will check when I am back at the computer.

That said, the script is broken as-is, meaning it must not be covered by the CI. We should probably fix that.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That said, the script is broken as-is, meaning must not be covered by the CI. We should probably fix that.

Correct - is isn't, and yes, we should.

The machine that will be used for the Emscripten buildbot has been commissioned. Right now, Emscripten hard-crashes on a couple of tests when -uall is enabled, but once those are resolved, we should be able to add the Emscripten buildbot to the pool.

)
if pydebug:
sysconfig_data += "-pydebug"
Expand Down Expand Up @@ -227,8 +226,13 @@ def configure_emscripten_python(context, working_dir):
"--enable-wasm-dynamic-linking",
f"--prefix={PREFIX_DIR}",
]
if pydebug:
configure.append("--with-pydebug")
for flag in abiflags:
if flag == 'd':
configure.append("--with-pydebug")
elif flag == 't':
configure.append("--disable-gil")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure what the state of Emscripten on free threading will be... I'm torn on whether it's better to leave the option enabled, and just be aware that it will quite possible break, or to error out if a free threaded build is used. Maybe the former with a warning message?

Either way, we should probably handle it similar to pydebug - set a flag (freethreaded?) earlier in the process and then use that flag.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Considering the build-python installation built by this script is specifically meant to be used in the cross-compilation process for the Emscripten build, I don't think we need remove it. Having it here means the user specifically asked for it.

Regarding the warning, I think it makes sense, but I think the build system is probably a more suitable place for it. What do you think?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes - I think the build system is a better place for that warning - assuming it's even needed. Its possible free threaded builds might work... especially given that there's no thread support in Emscripten.

else:
raise ValueError(f"Unknown ABI flag: {flag}")
if context.args:
configure.extend(context.args)
call(
Expand Down
17 changes: 9 additions & 8 deletions Tools/wasm/wasi.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,12 +212,8 @@ def configure_wasi_python(context, working_dir):
python_build_dir = BUILD_DIR / "build"
lib_dirs = list(python_build_dir.glob("lib.*"))
assert len(lib_dirs) == 1, f"Expected a single lib.* directory in {python_build_dir}"
lib_dir = os.fsdecode(lib_dirs[0])
pydebug = lib_dir.endswith("-pydebug")
python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1]
sysconfig_data = f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}"
if pydebug:
sysconfig_data += "-pydebug"
_, python_version, abiflags = os.fsdecode(lib_dirs[0]).rsplit('-', maxsplit=2)
sysconfig_data = f"{wasi_build_dir}/build/lib.wasi-wasm32-wasi-{python_version}-{abiflags}"

# Use PYTHONPATH to include sysconfig data which must be anchored to the
# WASI guest's `/` directory.
Expand All @@ -244,8 +240,13 @@ def configure_wasi_python(context, working_dir):
f"--host={context.host_triple}",
f"--build={build_platform()}",
f"--with-build-python={build_python}"]
if pydebug:
configure.append("--with-pydebug")
for flag in abiflags:
if flag == 'd':
configure.append("--with-pydebug")
elif flag == 't':
configure.append("--disable-gil")
else:
raise ValueError(f"Unknown ABI flag: {flag}")
if context.args:
configure.extend(context.args)
call(configure,
Expand Down
9 changes: 8 additions & 1 deletion configure
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 6 additions & 1 deletion configure.ac
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ AC_ARG_WITH([build-python],
dnl Build Python interpreter is used for regeneration and freezing.
ac_cv_prog_PYTHON_FOR_REGEN=$with_build_python
PYTHON_FOR_FREEZE="$with_build_python"
PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) _PYTHON_SYSCONFIGDATA_PATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`) '$with_build_python
PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=$(SYSCONFIGDATA_NAME) _PYTHON_SYSCONFIGDATA_PATH=$(abs_builddir)/$(PYBUILDDIR) '$with_build_python
AC_MSG_RESULT([$with_build_python])
], [
AS_VAR_IF([cross_compiling], [yes],
Expand All @@ -176,6 +176,11 @@ AC_ARG_WITH([build-python],
)
AC_SUBST([PYTHON_FOR_BUILD])

SYSCONFIGDATA_NAME='_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH)'
SYSCONFIGVARS_JSON_NAME='_sysconfigvars_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).json'
AC_SUBST([SYSCONFIGDATA_NAME])
AC_SUBST([SYSCONFIGVARS_JSON_NAME])

AC_MSG_CHECKING([for Python interpreter freezing])
AC_MSG_RESULT([$PYTHON_FOR_FREEZE])
AC_SUBST([PYTHON_FOR_FREEZE])
Expand Down