Skip to content

gh-109515: Allow a large number of frozen modules on Windows #109516

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 21 commits into from
Oct 30, 2023
Merged

gh-109515: Allow a large number of frozen modules on Windows #109516

Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
97c9b8c
fix windows command line limitations when freezing many modules
rghe Sep 9, 2023
5d640cb
Merge branch 'python:main' into allow-large-number-of-frozen-modules-…
rghe Sep 17, 2023
ddf9082
fix spacing
rghe Sep 17, 2023
9058b8d
📜🤖 Added by blurb_it.
blurb-it[bot] Sep 17, 2023
e98c3d0
missing regen
rghe Sep 17, 2023
86d1818
add generated file
rghe Sep 17, 2023
ca98aa1
add buildpath-dependent generated files to .gitignore
rghe Sep 17, 2023
5463e79
try relative paths
rghe Sep 17, 2023
7456fe5
fill mappings file with relative paths from vcxproj
rghe Sep 21, 2023
031ea3e
Implement S.Dower review suggestions
rghe Sep 23, 2023
e6fb7f5
Merge branch 'main' into allow-large-number-of-frozen-modules-vstudio
rghe Sep 23, 2023
b78bb97
clean gitignore
rghe Sep 23, 2023
93750fa
Merge branch 'main' into allow-large-number-of-frozen-modules-vstudio
rghe Sep 24, 2023
d584d2d
Update Tools/build/freeze_modules.py
rghe Sep 26, 2023
48112ab
Update Tools/build/deepfreeze.py
rghe Sep 26, 2023
74a255a
Update Tools/build/deepfreeze.py
rghe Sep 26, 2023
ecc334f
Update Tools/build/deepfreeze.py
rghe Sep 26, 2023
bb42f32
expand \t and align vcxfile by AA-Turner suggestion
rghe Sep 26, 2023
4cc3458
typo
rghe Sep 26, 2023
1c20fcd
reformat vcxproj
rghe Sep 27, 2023
5620f94
Update Tools/build/deepfreeze.py
rghe Sep 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ PCbuild/*-pgi
PCbuild/*-pgo
PCbuild/*.VC.db
PCbuild/*.VC.opendb
PCbuild/deepfreeze_mappings.txt
PCbuild/amd64/
PCbuild/arm32/
PCbuild/arm64/
Expand Down
66 changes: 40 additions & 26 deletions PCbuild/_freeze_module.vcxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -374,6 +374,33 @@
<OutFile>$(PySourcePath)Python\frozen_modules\getpath.h</OutFile>
</GetPath>
</ItemGroup>
<ItemGroup>
<!-- BEGIN freeze mappings -->
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It would be nice to fix the indentation on these lines. I don't see where it is though.

Otherwise anyone who edits this file manually is likely to fix it and trigger regeneration.

<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\importlib._bootstrap.h" FrozenId="importlib._bootstrap" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\importlib._bootstrap_external.h" FrozenId="importlib._bootstrap_external" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\zipimport.h" FrozenId="zipimport" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\abc.h" FrozenId="abc" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\codecs.h" FrozenId="codecs" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\io.h" FrozenId="io" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\_collections_abc.h" FrozenId="_collections_abc" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\_sitebuiltins.h" FrozenId="_sitebuiltins" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\genericpath.h" FrozenId="genericpath" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\ntpath.h" FrozenId="ntpath" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\posixpath.h" FrozenId="posixpath" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\os.h" FrozenId="os" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\site.h" FrozenId="site" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\stat.h" FrozenId="stat" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\importlib.util.h" FrozenId="importlib.util" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\importlib.machinery.h" FrozenId="importlib.machinery" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\runpy.h" FrozenId="runpy" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\__hello__.h" FrozenId="__hello__" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\__phello__.h" FrozenId="__phello__" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\__phello__.ham.h" FrozenId="__phello__.ham" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\__phello__.ham.eggs.h" FrozenId="__phello__.ham.eggs" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\__phello__.spam.h" FrozenId="__phello__.spam" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\frozen_only.h" FrozenId="frozen_only" />
<!-- END freeze mappings -->
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
Expand Down Expand Up @@ -405,32 +432,19 @@
AfterTargets="_RebuildFrozen"
DependsOnTargets="FindPythonForBuild"
Condition="$(Configuration) != 'PGUpdate'">
<!-- BEGIN deepfreeze rule -->
<Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\build\deepfreeze.py" ^
"$(PySourcePath)Python\frozen_modules\importlib._bootstrap.h:importlib._bootstrap" ^
"$(PySourcePath)Python\frozen_modules\importlib._bootstrap_external.h:importlib._bootstrap_external" ^
"$(PySourcePath)Python\frozen_modules\zipimport.h:zipimport" ^
"$(PySourcePath)Python\frozen_modules\abc.h:abc" ^
"$(PySourcePath)Python\frozen_modules\codecs.h:codecs" ^
"$(PySourcePath)Python\frozen_modules\io.h:io" ^
"$(PySourcePath)Python\frozen_modules\_collections_abc.h:_collections_abc" ^
"$(PySourcePath)Python\frozen_modules\_sitebuiltins.h:_sitebuiltins" ^
"$(PySourcePath)Python\frozen_modules\genericpath.h:genericpath" ^
"$(PySourcePath)Python\frozen_modules\ntpath.h:ntpath" ^
"$(PySourcePath)Python\frozen_modules\posixpath.h:posixpath" ^
"$(PySourcePath)Python\frozen_modules\os.h:os" ^
"$(PySourcePath)Python\frozen_modules\site.h:site" ^
"$(PySourcePath)Python\frozen_modules\stat.h:stat" ^
"$(PySourcePath)Python\frozen_modules\importlib.util.h:importlib.util" ^
"$(PySourcePath)Python\frozen_modules\importlib.machinery.h:importlib.machinery" ^
"$(PySourcePath)Python\frozen_modules\runpy.h:runpy" ^
"$(PySourcePath)Python\frozen_modules\__hello__.h:__hello__" ^
"$(PySourcePath)Python\frozen_modules\__phello__.h:__phello__" ^
"$(PySourcePath)Python\frozen_modules\__phello__.ham.h:__phello__.ham" ^
"$(PySourcePath)Python\frozen_modules\__phello__.ham.eggs.h:__phello__.ham.eggs" ^
"$(PySourcePath)Python\frozen_modules\__phello__.spam.h:__phello__.spam" ^
"$(PySourcePath)Python\frozen_modules\frozen_only.h:frozen_only" ^
"-o" "$(PySourcePath)Python\deepfreeze\deepfreeze.c"'/>
<PropertyGroup>
<FreezeMappingsFile>
<!-- BEGIN freeze mapping file -->
$(IntDir)\deepfreeze_mappings.txt
<!-- END freeze mapping file -->
</FreezeMappingsFile>
</PropertyGroup>
<WriteLinesToFile
File="$(FreezeMappingsFile)"
Overwrite="true"
Lines="@(FrozenModule->'%(FullPath):%(FrozenId)')" />
<!-- BEGIN deepfreeze rule -->
<Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\build\deepfreeze.py" -f "$(IntDir)\deepfreeze_mappings.txt" -o "$(PySourcePath)Python\deepfreeze\deepfreeze.c"' />
<!-- END deepfreeze rule -->
</Target>
<Target Name="_CleanFrozen" BeforeTargets="CoreClean" Condition="$(Configuration) != 'PGUpdate'">
Expand Down
16 changes: 14 additions & 2 deletions Tools/build/deepfreeze.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -491,7 +491,8 @@ def generate(args: list[str], output: TextIO) -> None:
parser = argparse.ArgumentParser()
parser.add_argument("-o", "--output", help="Defaults to deepfreeze.c", default="deepfreeze.c")
parser.add_argument("-v", "--verbose", action="store_true", help="Print diagnostics")
parser.add_argument('args', nargs="+", help="Input file and module name (required) in file:modname format")
parser.add_argument("-f", "--file", help="reads rule lines from a file")
parser.add_argument('args', nargs="*", help="Input file and module name (required) in file:modname format")

@contextlib.contextmanager
def report_time(label: str):
Expand All @@ -509,9 +510,20 @@ def main() -> None:
args = parser.parse_args()
verbose = args.verbose
output = args.output

if args.file:
if verbose:
print(f"reading rules from {args.file}")
with open(args.file, "rt", encoding="utf-8-sig") as fin:
rules = [x.strip() for x in fin]
elif args.args:
rules = args.args
else:
raise argparse.ArgumentError(None, "no rule(s) specified")

with open(output, "w", encoding="utf-8") as file:
with report_time("generate"):
generate(args.args, file)
generate(rules, file)
if verbose:
print(f"Wrote {os.path.getsize(output)} bytes to {output}")

Expand Down
28 changes: 25 additions & 3 deletions Tools/build/freeze_modules.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
# .gitattributes and .gitignore files needs to be updated.
FROZEN_MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'frozen_modules')
DEEPFROZEN_MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'deepfreeze')
DEEPFREEZE_MAPPING_FNAME = 'deepfreeze_mappings.txt'

FROZEN_FILE = os.path.join(ROOT_DIR, 'Python', 'frozen.c')
MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in')
Expand Down Expand Up @@ -645,7 +646,9 @@ def regen_pcbuild(modules):
projlines = []
filterlines = []
corelines = []
deepfreezerules = ['\t<Exec Command=\'$(PythonForBuild) "$(PySourcePath)Tools\\build\\deepfreeze.py" ^']
deepfreezemappingsfile = f'$(IntDir)\\{DEEPFREEZE_MAPPING_FNAME}'
deepfreezerules = [f'\t<Exec Command=\'$(PythonForBuild) "$(PySourcePath)Tools\\build\\deepfreeze.py" -f "{deepfreezemappingsfile}" -o "$(PySourcePath)Python\\deepfreeze\\deepfreeze.c"\' />']
deepfreezemappings = []
for src in _iter_sources(modules):
pyfile = relpath_for_windows_display(src.pyfile, ROOT_DIR)
header = relpath_for_windows_display(src.frozenfile, ROOT_DIR)
Expand All @@ -659,8 +662,7 @@ def regen_pcbuild(modules):
filterlines.append(f' <None Include="..\\{pyfile}">')
filterlines.append(' <Filter>Python Files</Filter>')
filterlines.append(' </None>')
deepfreezerules.append(f'\t\t "$(PySourcePath){header}:{src.frozenid}" ^')
deepfreezerules.append('\t\t "-o" "$(PySourcePath)Python\\deepfreeze\\deepfreeze.c"\'/>' )
deepfreezemappings.append(f'\t<FrozenModule Include="$(PySourcePath)\\{header}" FrozenId="{src.frozenid}" />\n')

corelines.append(f' <ClCompile Include="..\\Python\\deepfreeze\\deepfreeze.c" />')

Expand All @@ -675,6 +677,26 @@ def regen_pcbuild(modules):
PCBUILD_PROJECT,
)
outfile.writelines(lines)
with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile):
lines = infile.readlines()
lines = replace_block(
lines,
'<!-- BEGIN freeze mappings -->',
'<!-- END freeze mappings -->',
deepfreezemappings,
PCBUILD_PROJECT,
)
outfile.writelines(lines)
with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile):
lines = infile.readlines()
lines = replace_block(
lines,
'<!-- BEGIN freeze mapping file -->',
'<!-- END freeze mapping file -->',
[deepfreezemappingsfile, ],
PCBUILD_PROJECT,
)
outfile.writelines(lines)
with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile):
lines = infile.readlines()
lines = replace_block(
Expand Down