Skip to content

Commit eaede0d

Browse files
vstinnervstinner
authoredMay 17, 2021
bpo-44131: Test Py_FrozenMain() (pythonGH-26126)
* Add test_frozenmain to test_embed * Add Programs/test_frozenmain.py * Add Programs/freeze_test_frozenmain.py * Add Programs/test_frozenmain.h * Add make regen-test-frozenmain * Add test_frozenmain command to Programs/_testembed * _testembed.c: add error(msg) function
1 parent f32c795 commit eaede0d

10 files changed

+203
-32
lines changed
 

Diff for: ‎.gitattributes

+1
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ Doc/library/token-list.inc linguist-generated=true
5757
Include/token.h linguist-generated=true
5858
Lib/token.py linguist-generated=true
5959
Parser/token.c linguist-generated=true
60+
Programs/test_frozenmain.h linguist-generated=true
6061

6162
# Language aware diff headers
6263
# https://tekin.co.uk/2020/10/better-git-diff-output-for-ruby-python-elixir-and-more

Diff for: ‎Lib/test/test_embed.py

+15
Original file line numberDiff line numberDiff line change
@@ -1480,6 +1480,21 @@ def test_unicode_id_init(self):
14801480
# when Python is initialized multiples times.
14811481
self.run_embedded_interpreter("test_unicode_id_init")
14821482

1483+
# See bpo-44133
1484+
@unittest.skipIf(os.name == 'nt',
1485+
'Py_FrozenMain is not exported on Windows')
1486+
def test_frozenmain(self):
1487+
out, err = self.run_embedded_interpreter("test_frozenmain")
1488+
exe = os.path.realpath('./argv0')
1489+
expected = textwrap.dedent(f"""
1490+
Frozen Hello World
1491+
sys.argv ['./argv0', '-E', 'arg1', 'arg2']
1492+
config program_name: ./argv0
1493+
config executable: {exe}
1494+
config use_environment: 1
1495+
""").lstrip()
1496+
self.assertEqual(out, expected)
1497+
14831498

14841499
class StdPrinterTests(EmbeddingTestsMixin, unittest.TestCase):
14851500
# Test PyStdPrinter_Type which is used by _PySys_SetPreliminaryStderr():

Diff for: ‎Makefile.pre.in

+10-2
Original file line numberDiff line numberDiff line change
@@ -720,6 +720,14 @@ Makefile Modules/config.c: Makefile.pre \
720720
@mv config.c Modules
721721
@echo "The Makefile was updated, you may need to re-run make."
722722

723+
regen-test-frozenmain: $(BUILDPYTHON)
724+
# Regenerate Programs/test_frozenmain.h
725+
# from Programs/test_frozenmain.py
726+
# using Programs/freeze_test_frozenmain.py
727+
$(RUNSHARED) ./$(BUILDPYTHON) Programs/freeze_test_frozenmain.py Programs/test_frozenmain.h
728+
729+
Programs/test_frozenmain.h: Programs/freeze_test_frozenmain.py Programs/test_frozenmain.py
730+
$(MAKE) regen-test-frozenmain
723731

724732
Programs/_testembed: Programs/_testembed.o $(LIBRARY_DEPS)
725733
$(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS)
@@ -763,7 +771,7 @@ regen-limited-abi: all
763771

764772
regen-all: regen-opcode regen-opcode-targets regen-typeslots \
765773
regen-token regen-ast regen-keyword regen-importlib clinic \
766-
regen-pegen-metaparser regen-pegen regen-frozen
774+
regen-pegen-metaparser regen-pegen regen-frozen regen-test-frozenmain
767775
@echo
768776
@echo "Note: make regen-stdlib-module-names and autoconf should be run manually"
769777

@@ -794,7 +802,7 @@ Modules/getpath.o: $(srcdir)/Modules/getpath.c Makefile
794802
Programs/python.o: $(srcdir)/Programs/python.c
795803
$(MAINCC) -c $(PY_CORE_CFLAGS) -o $@ $(srcdir)/Programs/python.c
796804

797-
Programs/_testembed.o: $(srcdir)/Programs/_testembed.c
805+
Programs/_testembed.o: $(srcdir)/Programs/_testembed.c Programs/test_frozenmain.h
798806
$(MAINCC) -c $(PY_CORE_CFLAGS) -o $@ $(srcdir)/Programs/_testembed.c
799807

800808
Modules/_sre.o: $(srcdir)/Modules/_sre.c $(srcdir)/Modules/sre.h $(srcdir)/Modules/sre_constants.h $(srcdir)/Modules/sre_lib.h

Diff for: ‎Misc/NEWS.d/next/Tests/2021-05-14-14-13-44.bpo-44131.YPizoC.rst

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Add test_frozenmain to test_embed to test the :c:func:`Py_FrozenMain` C
2+
function. Patch by Victor Stinner.

Diff for: ‎Programs/_testembed.c

+63-9
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,14 @@
2727
_Py_COMP_DIAG_PUSH
2828
_Py_COMP_DIAG_IGNORE_DEPR_DECLS
2929

30+
31+
static void error(const char *msg)
32+
{
33+
fprintf(stderr, "ERROR: %s\n", msg);
34+
fflush(stderr);
35+
}
36+
37+
3038
static void _testembed_Py_Initialize(void)
3139
{
3240
Py_SetProgramName(PROGRAM_NAME);
@@ -239,7 +247,7 @@ static void bpo20891_thread(void *lockp)
239247

240248
PyGILState_STATE state = PyGILState_Ensure();
241249
if (!PyGILState_Check()) {
242-
fprintf(stderr, "PyGILState_Check failed!");
250+
error("PyGILState_Check failed!");
243251
abort();
244252
}
245253

@@ -259,15 +267,15 @@ static int test_bpo20891(void)
259267
crash. */
260268
PyThread_type_lock lock = PyThread_allocate_lock();
261269
if (!lock) {
262-
fprintf(stderr, "PyThread_allocate_lock failed!");
270+
error("PyThread_allocate_lock failed!");
263271
return 1;
264272
}
265273

266274
_testembed_Py_Initialize();
267275

268276
unsigned long thrd = PyThread_start_new_thread(bpo20891_thread, &lock);
269277
if (thrd == PYTHREAD_INVALID_THREAD_ID) {
270-
fprintf(stderr, "PyThread_start_new_thread failed!");
278+
error("PyThread_start_new_thread failed!");
271279
return 1;
272280
}
273281
PyThread_acquire_lock(lock, WAIT_LOCK);
@@ -1397,12 +1405,12 @@ static int test_init_setpath(void)
13971405
{
13981406
char *env = getenv("TESTPATH");
13991407
if (!env) {
1400-
fprintf(stderr, "missing TESTPATH env var\n");
1408+
error("missing TESTPATH env var");
14011409
return 1;
14021410
}
14031411
wchar_t *path = Py_DecodeLocale(env, NULL);
14041412
if (path == NULL) {
1405-
fprintf(stderr, "failed to decode TESTPATH\n");
1413+
error("failed to decode TESTPATH");
14061414
return 1;
14071415
}
14081416
Py_SetPath(path);
@@ -1430,12 +1438,12 @@ static int test_init_setpath_config(void)
14301438

14311439
char *env = getenv("TESTPATH");
14321440
if (!env) {
1433-
fprintf(stderr, "missing TESTPATH env var\n");
1441+
error("missing TESTPATH env var");
14341442
return 1;
14351443
}
14361444
wchar_t *path = Py_DecodeLocale(env, NULL);
14371445
if (path == NULL) {
1438-
fprintf(stderr, "failed to decode TESTPATH\n");
1446+
error("failed to decode TESTPATH");
14391447
return 1;
14401448
}
14411449
Py_SetPath(path);
@@ -1459,12 +1467,12 @@ static int test_init_setpythonhome(void)
14591467
{
14601468
char *env = getenv("TESTHOME");
14611469
if (!env) {
1462-
fprintf(stderr, "missing TESTHOME env var\n");
1470+
error("missing TESTHOME env var");
14631471
return 1;
14641472
}
14651473
wchar_t *home = Py_DecodeLocale(env, NULL);
14661474
if (home == NULL) {
1467-
fprintf(stderr, "failed to decode TESTHOME\n");
1475+
error("failed to decode TESTHOME");
14681476
return 1;
14691477
}
14701478
Py_SetPythonHome(home);
@@ -1726,6 +1734,48 @@ static int test_unicode_id_init(void)
17261734
}
17271735

17281736

1737+
#ifndef MS_WINDOWS
1738+
#include "test_frozenmain.h" // M_test_frozenmain
1739+
1740+
static int test_frozenmain(void)
1741+
{
1742+
// Get "_frozen_importlib" and "_frozen_importlib_external"
1743+
// from PyImport_FrozenModules
1744+
const struct _frozen *importlib = NULL, *importlib_external = NULL;
1745+
for (const struct _frozen *mod = PyImport_FrozenModules; mod->name != NULL; mod++) {
1746+
if (strcmp(mod->name, "_frozen_importlib") == 0) {
1747+
importlib = mod;
1748+
}
1749+
else if (strcmp(mod->name, "_frozen_importlib_external") == 0) {
1750+
importlib_external = mod;
1751+
}
1752+
}
1753+
if (importlib == NULL || importlib_external == NULL) {
1754+
error("cannot find frozen importlib and importlib_external");
1755+
return 1;
1756+
}
1757+
1758+
static struct _frozen frozen_modules[4] = {
1759+
{0, 0, 0}, // importlib
1760+
{0, 0, 0}, // importlib_external
1761+
{"__main__", M_test_frozenmain, sizeof(M_test_frozenmain)},
1762+
{0, 0, 0} // sentinel
1763+
};
1764+
frozen_modules[0] = *importlib;
1765+
frozen_modules[1] = *importlib_external;
1766+
1767+
char* argv[] = {
1768+
"./argv0",
1769+
"-E",
1770+
"arg1",
1771+
"arg2",
1772+
};
1773+
PyImport_FrozenModules = frozen_modules;
1774+
return Py_FrozenMain(Py_ARRAY_LENGTH(argv), argv);
1775+
}
1776+
#endif // !MS_WINDOWS
1777+
1778+
17291779
// List frozen modules.
17301780
// Command used by Tools/scripts/generate_stdlib_module_names.py script.
17311781
static int list_frozen(void)
@@ -1811,11 +1861,15 @@ static struct TestCase TestCases[] = {
18111861
{"test_audit_run_stdin", test_audit_run_stdin},
18121862

18131863
{"test_unicode_id_init", test_unicode_id_init},
1864+
#ifndef MS_WINDOWS
1865+
{"test_frozenmain", test_frozenmain},
1866+
#endif
18141867

18151868
{"list_frozen", list_frozen},
18161869
{NULL, NULL}
18171870
};
18181871

1872+
18191873
int main(int argc, char *argv[])
18201874
{
18211875
if (argc > 1) {

Diff for: ‎Programs/freeze_test_frozenmain.py

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
import marshal
2+
import tokenize
3+
import os.path
4+
import sys
5+
6+
PROGRAM_DIR = os.path.dirname(__file__)
7+
SRC_DIR = os.path.dirname(PROGRAM_DIR)
8+
9+
10+
def writecode(fp, mod, data):
11+
print('unsigned char M_%s[] = {' % mod, file=fp)
12+
indent = ' ' * 4
13+
for i in range(0, len(data), 16):
14+
print(indent, file=fp, end='')
15+
for c in bytes(data[i:i+16]):
16+
print('%d,' % c, file=fp, end='')
17+
print('', file=fp)
18+
print('};', file=fp)
19+
20+
21+
def dump(fp, filename, name):
22+
# Strip the directory to get reproducible marshal dump
23+
code_filename = os.path.basename(filename)
24+
25+
with tokenize.open(filename) as source_fp:
26+
source = source_fp.read()
27+
code = compile(source, code_filename, 'exec')
28+
29+
data = marshal.dumps(code)
30+
writecode(fp, name, data)
31+
32+
33+
def main():
34+
if len(sys.argv) < 2:
35+
print(f"usage: {sys.argv[0]} filename")
36+
sys.exit(1)
37+
filename = sys.argv[1]
38+
39+
with open(filename, "w") as fp:
40+
print("// Auto-generated by Programs/freeze_test_frozenmain.py", file=fp)
41+
frozenmain = os.path.join(PROGRAM_DIR, 'test_frozenmain.py')
42+
dump(fp, frozenmain, 'test_frozenmain')
43+
44+
print(f"{filename} written")
45+
46+
47+
if __name__ == "__main__":
48+
main()

Diff for: ‎Programs/test_frozenmain.h

+30
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Diff for: ‎Programs/test_frozenmain.py

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
import sys
2+
import _testinternalcapi
3+
4+
print("Frozen Hello World")
5+
print("sys.argv", sys.argv)
6+
config = _testinternalcapi.get_configs()['config']
7+
print(f"config program_name: {config['program_name']}")
8+
print(f"config executable: {config['executable']}")
9+
print(f"config use_environment: {config['use_environment']}")

Diff for: ‎Python/frozenmain.c

+16-10
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
21
/* Python interpreter main program for frozen scripts */
32

43
#include "Python.h"
@@ -43,10 +42,12 @@ Py_FrozenMain(int argc, char **argv)
4342
PyConfig_InitPythonConfig(&config);
4443
config.pathconfig_warnings = 0; /* Suppress errors from getpath.c */
4544

46-
if ((p = Py_GETENV("PYTHONINSPECT")) && *p != '\0')
45+
if ((p = Py_GETENV("PYTHONINSPECT")) && *p != '\0') {
4746
inspect = 1;
48-
if ((p = Py_GETENV("PYTHONUNBUFFERED")) && *p != '\0')
47+
}
48+
if ((p = Py_GETENV("PYTHONUNBUFFERED")) && *p != '\0') {
4949
unbuffered = 1;
50+
}
5051

5152
if (unbuffered) {
5253
setbuf(stdin, (char *)NULL);
@@ -65,8 +66,9 @@ Py_FrozenMain(int argc, char **argv)
6566
argv_copy[i] = Py_DecodeLocale(argv[i], NULL);
6667
argv_copy2[i] = argv_copy[i];
6768
if (!argv_copy[i]) {
68-
fprintf(stderr, "Unable to decode the command line argument #%i\n",
69-
i + 1);
69+
fprintf(stderr,
70+
"Unable to decode the command line argument #%i\n",
71+
i + 1);
7072
argc = i;
7173
goto error;
7274
}
@@ -97,24 +99,28 @@ Py_FrozenMain(int argc, char **argv)
9799
PyWinFreeze_ExeInit();
98100
#endif
99101

100-
if (Py_VerboseFlag)
102+
if (Py_VerboseFlag) {
101103
fprintf(stderr, "Python %s\n%s\n",
102-
Py_GetVersion(), Py_GetCopyright());
104+
Py_GetVersion(), Py_GetCopyright());
105+
}
103106

104107
PySys_SetArgv(argc, argv_copy);
105108

106109
n = PyImport_ImportFrozenModule("__main__");
107-
if (n == 0)
110+
if (n == 0) {
108111
Py_FatalError("the __main__ module is not frozen");
112+
}
109113
if (n < 0) {
110114
PyErr_Print();
111115
sts = 1;
112116
}
113-
else
117+
else {
114118
sts = 0;
119+
}
115120

116-
if (inspect && isatty((int)fileno(stdin)))
121+
if (inspect && isatty((int)fileno(stdin))) {
117122
sts = PyRun_AnyFile(stdin, "<stdin>") != 0;
123+
}
118124

119125
#ifdef MS_WINDOWS
120126
PyWinFreeze_ExeTerm();

Diff for: ‎Tools/freeze/makefreeze.py

+9-11
Original file line numberDiff line numberDiff line change
@@ -74,14 +74,12 @@ def makefreeze(base, dict, debug=0, entry_point=None, fail_import=()):
7474
# Write a C initializer for a module containing the frozen python code.
7575
# The array is called M_<mod>.
7676

77-
def writecode(outfp, mod, str):
78-
outfp.write('unsigned char M_%s[] = {' % mod)
79-
for i in range(0, len(str), 16):
80-
outfp.write('\n\t')
81-
for c in bytes(str[i:i+16]):
82-
outfp.write('%d,' % c)
83-
outfp.write('\n};\n')
84-
85-
## def writecode(outfp, mod, str):
86-
## outfp.write('unsigned char M_%s[%d] = "%s";\n' % (mod, len(str),
87-
## '\\"'.join(map(lambda s: repr(s)[1:-1], str.split('"')))))
77+
def writecode(fp, mod, data):
78+
print('unsigned char M_%s[] = {' % mod, file=fp)
79+
indent = ' ' * 4
80+
for i in range(0, len(data), 16):
81+
print(indent, file=fp, end='')
82+
for c in bytes(data[i:i+16]):
83+
print('%d,' % c, file=fp, end='')
84+
print('', file=fp)
85+
print('};', file=fp)

0 commit comments

Comments
 (0)
Please sign in to comment.