Skip to content

Commit 1037ca5

Browse files
gvanrossumgvanrossum
authored
bpo-45850: Implement deep-freeze on Windows (#29648)
Implement changes to build with deep-frozen modules on Windows. Note that we now require Python 3.10 as the "bootstrap" or "host" Python. This causes a modest startup speed (around 7%) on Windows.
1 parent 4d6c0c0 commit 1037ca5

File tree

9 files changed

+500
-24
lines changed

9 files changed

+500
-24
lines changed

‎Misc/NEWS.d/next/Windows/2021-11-20-00-06-59.bpo-45850.q9lofz.rst

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Implement changes to build with deep-frozen modules on Windows.
2+
Note that we now require Python 3.10 as the "bootstrap" or "host" Python.

‎PCbuild/_freeze_module.vcxproj

+55-3
Original file line numberDiff line numberDiff line change
@@ -236,119 +236,171 @@
236236
<ModName>importlib._bootstrap</ModName>
237237
<IntFile>$(IntDir)importlib._bootstrap.g.h</IntFile>
238238
<OutFile>$(PySourcePath)Python\frozen_modules\importlib._bootstrap.h</OutFile>
239+
<DeepIntFile>$(IntDir)importlib._bootstrap.g.c</DeepIntFile>
240+
<DeepOutFile>$(PySourcePath)Python\deepfreeze\df.importlib._bootstrap.c</DeepOutFile>
239241
</None>
240242
<None Include="..\Lib\importlib\_bootstrap_external.py">
241243
<ModName>importlib._bootstrap_external</ModName>
242244
<IntFile>$(IntDir)importlib._bootstrap_external.g.h</IntFile>
243245
<OutFile>$(PySourcePath)Python\frozen_modules\importlib._bootstrap_external.h</OutFile>
246+
<DeepIntFile>$(IntDir)importlib._bootstrap_external.g.c</DeepIntFile>
247+
<DeepOutFile>$(PySourcePath)Python\deepfreeze\df.importlib._bootstrap_external.c</DeepOutFile>
244248
</None>
245249
<None Include="..\Lib\zipimport.py">
246250
<ModName>zipimport</ModName>
247251
<IntFile>$(IntDir)zipimport.g.h</IntFile>
248252
<OutFile>$(PySourcePath)Python\frozen_modules\zipimport.h</OutFile>
253+
<DeepIntFile>$(IntDir)zipimport.g.c</DeepIntFile>
254+
<DeepOutFile>$(PySourcePath)Python\deepfreeze\df.zipimport.c</DeepOutFile>
249255
</None>
250256
<None Include="..\Lib\abc.py">
251257
<ModName>abc</ModName>
252258
<IntFile>$(IntDir)abc.g.h</IntFile>
253259
<OutFile>$(PySourcePath)Python\frozen_modules\abc.h</OutFile>
260+
<DeepIntFile>$(IntDir)abc.g.c</DeepIntFile>
261+
<DeepOutFile>$(PySourcePath)Python\deepfreeze\df.abc.c</DeepOutFile>
254262
</None>
255263
<None Include="..\Lib\codecs.py">
256264
<ModName>codecs</ModName>
257265
<IntFile>$(IntDir)codecs.g.h</IntFile>
258266
<OutFile>$(PySourcePath)Python\frozen_modules\codecs.h</OutFile>
267+
<DeepIntFile>$(IntDir)codecs.g.c</DeepIntFile>
268+
<DeepOutFile>$(PySourcePath)Python\deepfreeze\df.codecs.c</DeepOutFile>
259269
</None>
260270
<None Include="..\Lib\io.py">
261271
<ModName>io</ModName>
262272
<IntFile>$(IntDir)io.g.h</IntFile>
263273
<OutFile>$(PySourcePath)Python\frozen_modules\io.h</OutFile>
274+
<DeepIntFile>$(IntDir)io.g.c</DeepIntFile>
275+
<DeepOutFile>$(PySourcePath)Python\deepfreeze\df.io.c</DeepOutFile>
264276
</None>
265277
<None Include="..\Lib\_collections_abc.py">
266278
<ModName>_collections_abc</ModName>
267279
<IntFile>$(IntDir)_collections_abc.g.h</IntFile>
268280
<OutFile>$(PySourcePath)Python\frozen_modules\_collections_abc.h</OutFile>
281+
<DeepIntFile>$(IntDir)_collections_abc.g.c</DeepIntFile>
282+
<DeepOutFile>$(PySourcePath)Python\deepfreeze\df._collections_abc.c</DeepOutFile>
269283
</None>
270284
<None Include="..\Lib\_sitebuiltins.py">
271285
<ModName>_sitebuiltins</ModName>
272286
<IntFile>$(IntDir)_sitebuiltins.g.h</IntFile>
273287
<OutFile>$(PySourcePath)Python\frozen_modules\_sitebuiltins.h</OutFile>
288+
<DeepIntFile>$(IntDir)_sitebuiltins.g.c</DeepIntFile>
289+
<DeepOutFile>$(PySourcePath)Python\deepfreeze\df._sitebuiltins.c</DeepOutFile>
274290
</None>
275291
<None Include="..\Lib\genericpath.py">
276292
<ModName>genericpath</ModName>
277293
<IntFile>$(IntDir)genericpath.g.h</IntFile>
278294
<OutFile>$(PySourcePath)Python\frozen_modules\genericpath.h</OutFile>
295+
<DeepIntFile>$(IntDir)genericpath.g.c</DeepIntFile>
296+
<DeepOutFile>$(PySourcePath)Python\deepfreeze\df.genericpath.c</DeepOutFile>
279297
</None>
280298
<None Include="..\Lib\ntpath.py">
281299
<ModName>ntpath</ModName>
282300
<IntFile>$(IntDir)ntpath.g.h</IntFile>
283301
<OutFile>$(PySourcePath)Python\frozen_modules\ntpath.h</OutFile>
302+
<DeepIntFile>$(IntDir)ntpath.g.c</DeepIntFile>
303+
<DeepOutFile>$(PySourcePath)Python\deepfreeze\df.ntpath.c</DeepOutFile>
284304
</None>
285305
<None Include="..\Lib\posixpath.py">
286306
<ModName>posixpath</ModName>
287307
<IntFile>$(IntDir)posixpath.g.h</IntFile>
288308
<OutFile>$(PySourcePath)Python\frozen_modules\posixpath.h</OutFile>
309+
<DeepIntFile>$(IntDir)posixpath.g.c</DeepIntFile>
310+
<DeepOutFile>$(PySourcePath)Python\deepfreeze\df.posixpath.c</DeepOutFile>
289311
</None>
290312
<None Include="..\Lib\os.py">
291313
<ModName>os</ModName>
292314
<IntFile>$(IntDir)os.g.h</IntFile>
293315
<OutFile>$(PySourcePath)Python\frozen_modules\os.h</OutFile>
316+
<DeepIntFile>$(IntDir)os.g.c</DeepIntFile>
317+
<DeepOutFile>$(PySourcePath)Python\deepfreeze\df.os.c</DeepOutFile>
294318
</None>
295319
<None Include="..\Lib\site.py">
296320
<ModName>site</ModName>
297321
<IntFile>$(IntDir)site.g.h</IntFile>
298322
<OutFile>$(PySourcePath)Python\frozen_modules\site.h</OutFile>
323+
<DeepIntFile>$(IntDir)site.g.c</DeepIntFile>
324+
<DeepOutFile>$(PySourcePath)Python\deepfreeze\df.site.c</DeepOutFile>
299325
</None>
300326
<None Include="..\Lib\stat.py">
301327
<ModName>stat</ModName>
302328
<IntFile>$(IntDir)stat.g.h</IntFile>
303329
<OutFile>$(PySourcePath)Python\frozen_modules\stat.h</OutFile>
330+
<DeepIntFile>$(IntDir)stat.g.c</DeepIntFile>
331+
<DeepOutFile>$(PySourcePath)Python\deepfreeze\df.stat.c</DeepOutFile>
304332
</None>
305333
<None Include="..\Lib\__hello__.py">
306334
<ModName>__hello__</ModName>
307335
<IntFile>$(IntDir)__hello__.g.h</IntFile>
308336
<OutFile>$(PySourcePath)Python\frozen_modules\__hello__.h</OutFile>
337+
<DeepIntFile>$(IntDir)__hello__.g.c</DeepIntFile>
338+
<DeepOutFile>$(PySourcePath)Python\deepfreeze\df.__hello__.c</DeepOutFile>
309339
</None>
310340
<None Include="..\Lib\__phello__\__init__.py">
311341
<ModName>__phello__</ModName>
312342
<IntFile>$(IntDir)__phello__.g.h</IntFile>
313343
<OutFile>$(PySourcePath)Python\frozen_modules\__phello__.h</OutFile>
344+
<DeepIntFile>$(IntDir)__phello__.g.c</DeepIntFile>
345+
<DeepOutFile>$(PySourcePath)Python\deepfreeze\df.__phello__.c</DeepOutFile>
314346
</None>
315347
<None Include="..\Lib\__phello__\ham\__init__.py">
316348
<ModName>__phello__.ham</ModName>
317349
<IntFile>$(IntDir)__phello__.ham.g.h</IntFile>
318350
<OutFile>$(PySourcePath)Python\frozen_modules\__phello__.ham.h</OutFile>
351+
<DeepIntFile>$(IntDir)__phello__.ham.g.c</DeepIntFile>
352+
<DeepOutFile>$(PySourcePath)Python\deepfreeze\df.__phello__.ham.c</DeepOutFile>
319353
</None>
320354
<None Include="..\Lib\__phello__\ham\eggs.py">
321355
<ModName>__phello__.ham.eggs</ModName>
322356
<IntFile>$(IntDir)__phello__.ham.eggs.g.h</IntFile>
323357
<OutFile>$(PySourcePath)Python\frozen_modules\__phello__.ham.eggs.h</OutFile>
358+
<DeepIntFile>$(IntDir)__phello__.ham.eggs.g.c</DeepIntFile>
359+
<DeepOutFile>$(PySourcePath)Python\deepfreeze\df.__phello__.ham.eggs.c</DeepOutFile>
324360
</None>
325361
<None Include="..\Lib\__phello__\spam.py">
326362
<ModName>__phello__.spam</ModName>
327363
<IntFile>$(IntDir)__phello__.spam.g.h</IntFile>
328364
<OutFile>$(PySourcePath)Python\frozen_modules\__phello__.spam.h</OutFile>
365+
<DeepIntFile>$(IntDir)__phello__.spam.g.c</DeepIntFile>
366+
<DeepOutFile>$(PySourcePath)Python\deepfreeze\df.__phello__.spam.c</DeepOutFile>
329367
</None>
330368
<None Include="..\Tools\freeze\flag.py">
331369
<ModName>frozen_only</ModName>
332370
<IntFile>$(IntDir)frozen_only.g.h</IntFile>
333371
<OutFile>$(PySourcePath)Python\frozen_modules\frozen_only.h</OutFile>
372+
<DeepIntFile>$(IntDir)frozen_only.g.c</DeepIntFile>
373+
<DeepOutFile>$(PySourcePath)Python\deepfreeze\df.frozen_only.c</DeepOutFile>
334374
</None>
335375
<!-- END frozen modules -->
336376
</ItemGroup>
337377
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
338378
<ImportGroup Label="ExtensionTargets">
339379
</ImportGroup>
340380
<Target Name="_RebuildFrozen" AfterTargets="AfterBuild" Condition="$(Configuration) != 'PGUpdate'">
341-
<Exec Command='"$(TargetPath)" "%(None.ModName)" "%(None.FullPath)" "%(None.IntFile)"' />
381+
<Exec Command='"$(TargetPath)" "%(None.ModName)" "%(None.FullPath)" "%(None.DeepIntFile)"' />
342382

343-
<Copy SourceFiles="%(None.IntFile)"
383+
<Copy SourceFiles="%(None.DeepIntFile)"
344384
DestinationFiles="%(None.OutFile)"
345-
Condition="!Exists(%(None.OutFile)) or (Exists(%(None.IntFile)) and '$([System.IO.File]::ReadAllText(%(None.OutFile)).Replace(`&#x0D;&#x0A;`, `&#x0A;`))' != '$([System.IO.File]::ReadAllText(%(None.IntFile)).Replace(`&#x0D;&#x0A;`, `&#x0A;`))')">
385+
Condition="!Exists(%(None.OutFile)) or (Exists(%(None.DeepIntFile)) and '$([System.IO.File]::ReadAllText(%(None.OutFile)).Replace(`&#x0D;&#x0A;`, `&#x0A;`))' != '$([System.IO.File]::ReadAllText(%(None.DeepIntFile)).Replace(`&#x0D;&#x0A;`, `&#x0A;`))')">
346386
<Output TaskParameter="CopiedFiles" ItemName="_Updated" />
347387
</Copy>
348388

349389
<Message Text="Updated files: @(_Updated->'%(Filename)%(Extension)',', ')"
350390
Condition="'@(_Updated)' != ''" Importance="high" />
351391
</Target>
392+
<Target Name="_RebuildDeepFrozen" AfterTargets="_RebuildFrozen" Condition="$(Configuration) != 'PGUpdate'">
393+
<Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\scripts\deepfreeze.py" "%(None.OutFile)" "-m" "%(None.ModName)" -o "%(None.IntFile)"' />
394+
395+
<Copy SourceFiles="%(None.IntFile)"
396+
DestinationFiles="%(None.DeepOutFile)"
397+
Condition="!Exists(%(None.DeepOutFile)) or (Exists(%(None.IntFile)) and '$([System.IO.File]::ReadAllText(%(None.DeepOutFile)).Replace(`&#x0D;&#x0A;`, `&#x0A;`))' != '$([System.IO.File]::ReadAllText(%(None.IntFile)).Replace(`&#x0D;&#x0A;`, `&#x0A;`))')">
398+
<Output TaskParameter="CopiedFiles" ItemName="_DeepUpdated" />
399+
</Copy>
400+
401+
<Message Text="Updated files: @(_DeepUpdated->'%(Filename)%(Extension)',', ')"
402+
Condition="'@(_DeepUpdated)' != ''" Importance="high" />
403+
</Target>
352404
<Target Name="_CleanFrozen" BeforeTargets="CoreClean" Condition="$(Configuration) != 'PGUpdate'">
353405
<ItemGroup>
354406
<Clean Include="%(None.IntFile)" />

‎PCbuild/find_python.bat

+3-3
Original file line numberDiff line numberDiff line change
@@ -31,13 +31,13 @@
3131
@if "%_Py_EXTERNALS_DIR%"=="" (set _Py_EXTERNALS_DIR=%~dp0\..\externals)
3232

3333
@rem If we have Python in externals, use that one
34-
@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found in externals directory) & goto :found
34+
@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" ("%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" -Ec "import sys; assert sys.version_info[:2] >= (3, 10)" >nul 2>nul) && (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") && (set _Py_Python_Source=found in externals directory) && goto :found || rmdir /Q /S "%_Py_EXTERNALS_DIR%\pythonx86"
3535

3636
@rem If HOST_PYTHON is recent enough, use that
37-
@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -Ec "import sys; assert sys.version_info[:2] >= (3, 8)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found
37+
@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -Ec "import sys; assert sys.version_info[:2] >= (3, 10)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found
3838

3939
@rem If py.exe finds a recent enough version, use that one
40-
@for %%p in (3.9 3.8) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found
40+
@for %%p in (3.10) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found
4141

4242
@if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%"
4343
@set _Py_NUGET=%NUGET%

‎PCbuild/pythoncore.vcxproj

+24
Original file line numberDiff line numberDiff line change
@@ -502,6 +502,30 @@
502502
<ClCompile Include="..\Python\thread.c" />
503503
<ClCompile Include="..\Python\traceback.c" />
504504
</ItemGroup>
505+
<ItemGroup>
506+
<!-- BEGIN deepfreeze -->
507+
<ClCompile Include="..\Python\deepfreeze\df.importlib._bootstrap.c" />
508+
<ClCompile Include="..\Python\deepfreeze\df.importlib._bootstrap_external.c" />
509+
<ClCompile Include="..\Python\deepfreeze\df.zipimport.c" />
510+
<ClCompile Include="..\Python\deepfreeze\df.abc.c" />
511+
<ClCompile Include="..\Python\deepfreeze\df.codecs.c" />
512+
<ClCompile Include="..\Python\deepfreeze\df.io.c" />
513+
<ClCompile Include="..\Python\deepfreeze\df._collections_abc.c" />
514+
<ClCompile Include="..\Python\deepfreeze\df._sitebuiltins.c" />
515+
<ClCompile Include="..\Python\deepfreeze\df.genericpath.c" />
516+
<ClCompile Include="..\Python\deepfreeze\df.ntpath.c" />
517+
<ClCompile Include="..\Python\deepfreeze\df.posixpath.c" />
518+
<ClCompile Include="..\Python\deepfreeze\df.os.c" />
519+
<ClCompile Include="..\Python\deepfreeze\df.site.c" />
520+
<ClCompile Include="..\Python\deepfreeze\df.stat.c" />
521+
<ClCompile Include="..\Python\deepfreeze\df.__hello__.c" />
522+
<ClCompile Include="..\Python\deepfreeze\df.__phello__.c" />
523+
<ClCompile Include="..\Python\deepfreeze\df.__phello__.ham.c" />
524+
<ClCompile Include="..\Python\deepfreeze\df.__phello__.ham.eggs.c" />
525+
<ClCompile Include="..\Python\deepfreeze\df.__phello__.spam.c" />
526+
<ClCompile Include="..\Python\deepfreeze\df.frozen_only.c" />
527+
<!-- END deepfreeze -->
528+
</ItemGroup>
505529
<ItemGroup Condition="$(IncludeExternals)">
506530
<ClCompile Include="..\Modules\zlibmodule.c" />
507531
<ClCompile Include="$(zlibDir)\adler32.c" />

‎Python/frozen.c

-5
Original file line numberDiff line numberDiff line change
@@ -61,12 +61,7 @@
6161
#include "frozen_modules/frozen_only.h"
6262
/* End includes */
6363

64-
#ifdef MS_WINDOWS
65-
/* Deepfreeze isn't supported on Windows yet. */
66-
#define GET_CODE(name) NULL
67-
#else
6864
#define GET_CODE(name) _Py_get_##name##_toplevel
69-
#endif
7065

7166
/* Start extern declarations */
7267
extern PyObject *_Py_get_importlib__bootstrap_toplevel(void);

‎Tools/scripts/deepfreeze.py

+46-12
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,16 @@
11
import argparse
2+
import ast
23
import builtins
34
import collections
45
import contextlib
56
import os
6-
import sys
7+
import re
78
import time
89
import types
910
import typing
1011

12+
import umarshal
13+
1114
verbose = False
1215

1316

@@ -55,7 +58,8 @@ def get_localsplus_counts(code: types.CodeType,
5558
nplaincellvars += 1
5659
elif kind & CO_FAST_FREE:
5760
nfreevars += 1
58-
assert nlocals == len(code.co_varnames) == code.co_nlocals
61+
assert nlocals == len(code.co_varnames) == code.co_nlocals, \
62+
(nlocals, len(code.co_varnames), code.co_nlocals)
5963
assert ncellvars == len(code.co_cellvars)
6064
assert nfreevars == len(code.co_freevars)
6165
assert len(names) == nlocals + nplaincellvars + nfreevars
@@ -274,14 +278,7 @@ def generate_tuple(self, name: str, t: tuple[object, ...]) -> str:
274278
self.write(item + ",")
275279
return f"& {name}._object.ob_base.ob_base"
276280

277-
def generate_int(self, name: str, i: int) -> str:
278-
maxint = sys.maxsize
279-
if maxint == 2**31 - 1:
280-
digit = 2**15
281-
elif maxint == 2**63 - 1:
282-
digit = 2**30
283-
else:
284-
assert False, f"What int size is this system?!? {maxint=}"
281+
def _generate_int_for_bits(self, name: str, i: int, digit: int) -> None:
285282
sign = -1 if i < 0 else 0 if i == 0 else +1
286283
i = abs(i)
287284
digits: list[int] = []
@@ -298,6 +295,20 @@ def generate_int(self, name: str, i: int) -> str:
298295
if digits:
299296
ds = ", ".join(map(str, digits))
300297
self.write(f".ob_digit = {{ {ds} }},")
298+
299+
def generate_int(self, name: str, i: int) -> str:
300+
if abs(i) < 2**15:
301+
self._generate_int_for_bits(name, i, 2**15)
302+
else:
303+
connective = "if"
304+
for bits_in_digit in 15, 30:
305+
self.write(f"#{connective} PYLONG_BITS_IN_DIGIT == {bits_in_digit}")
306+
self._generate_int_for_bits(name, i, 2**bits_in_digit)
307+
connective = "elif"
308+
self.write("#else")
309+
self.write('#error "PYLONG_BITS_IN_DIGIT should be 15 or 30"')
310+
self.write("#endif")
311+
# If neither clause applies, it won't compile
301312
return f"& {name}.ob_base.ob_base"
302313

303314
def generate_float(self, name: str, x: float) -> str:
@@ -326,7 +337,7 @@ def generate(self, name: str, obj: object) -> str:
326337
return self.cache[key]
327338
self.misses += 1
328339
match obj:
329-
case types.CodeType() as code:
340+
case types.CodeType() | umarshal.Code() as code:
330341
val = self.generate_code(name, code)
331342
case tuple(t):
332343
val = self.generate_tuple(name, t)
@@ -367,8 +378,31 @@ def generate(self, name: str, obj: object) -> str:
367378
}
368379
"""
369380

381+
FROZEN_COMMENT = "/* Auto-generated by Programs/_freeze_module.c */"
382+
383+
FROZEN_DATA_LINE = r"\s*(\d+,\s*)+\s*"
384+
385+
386+
def is_frozen_header(source: str) -> bool:
387+
return source.startswith(FROZEN_COMMENT)
388+
389+
390+
def decode_frozen_data(source: str) -> types.CodeType:
391+
lines = source.splitlines()
392+
while lines and re.match(FROZEN_DATA_LINE, lines[0]) is None:
393+
del lines[0]
394+
while lines and re.match(FROZEN_DATA_LINE, lines[-1]) is None:
395+
del lines[-1]
396+
values: tuple[int, ...] = ast.literal_eval("".join(lines))
397+
data = bytes(values)
398+
return umarshal.loads(data)
399+
400+
370401
def generate(source: str, filename: str, modname: str, file: typing.TextIO) -> None:
371-
code = compile(source, filename, "exec")
402+
if is_frozen_header(source):
403+
code = decode_frozen_data(source)
404+
else:
405+
code = compile(source, filename, "exec")
372406
printer = Printer(file)
373407
printer.generate("toplevel", code)
374408
printer.write("")

0 commit comments

Comments
 (0)