Skip to content

Commit 7482838

Browse files
zoobazooba
authored
bpo-43567: Improved generated code refresh on Windows (GH-25120)
Generated files are now refreshed automatically on regular build, or may be forcibly regenerated by calling `build.bat --regen`.
1 parent 0fdf11e commit 7482838

File tree

6 files changed

+109
-228
lines changed

6 files changed

+109
-228
lines changed

‎Misc/NEWS.d/next/Build/2021-03-31-19-50-01.bpo-43567.vd0a-p.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Improved generated code refresh (AST/tokens/opcodes/keywords) on Windows.

‎PCbuild/_freeze_importlib.vcxproj

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -140,10 +140,12 @@
140140
Condition="!Exists(%(None.OutFile)) or (Exists(%(None.IntFile)) and '$([System.IO.File]::ReadAllText(%(None.OutFile)).Replace(`
`, `
`))' != '$([System.IO.File]::ReadAllText(%(None.IntFile)).Replace(`
`, `
`))')">
141141
<Output TaskParameter="CopiedFiles" ItemName="_Updated" />
142142
</Copy>
143-
144-
<Warning Text="@(_Updated->'%(Filename)%(Extension)',', ') updated. You will need to rebuild pythoncore to see the changes.%0D%0A%0D%0AIf you are not developing on Windows but you see this error on a continuous integration build, you need to run 'make regen-all' and commit any changes."
143+
144+
<Message Text="Updated files: @(_Updated->'%(Filename)%(Extension)',', ')"
145+
Condition="'@(_Updated)' != ''" Importance="high" />
146+
<Warning Text="Frozen importlib files were updated. Please rebuild to pick up the changes.%0D%0A%0D%0AIf you are not developing on Windows but you see this error on a continuous integration build, please run 'make regen-all' and commit anything that changes."
145147
Condition="'@(_Updated)' != '' and $(Configuration) == 'Debug'" />
146-
<Error Text="@(_Updated->'%(Filename)%(Extension)',', ') updated. You will need to rebuild pythoncore to see the changes.%0D%0A%0D%0AIf you are not developing on Windows but you see this error on a continuous integration build, you need to run 'make regen-all' and commit any changes."
148+
<Error Text="Frozen importlib files were updated. Please rebuild to pick up the changes.%0D%0A%0D%0AIf you are not developing on Windows but you see this error on a continuous integration build, please run 'make regen-all' and commit anything that changes."
147149
Condition="'@(_Updated)' != '' and $(Configuration) == 'Release'" />
148150
</Target>
149151
<Target Name="RebuildImportLib" AfterTargets="AfterBuild" Condition="$(Configuration) == 'Debug' or $(Configuration) == 'Release'"

‎PCbuild/build.bat

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ echo. automatically by the pythoncore project)
3434
echo. --pgo Build with Profile-Guided Optimization. This flag
3535
echo. overrides -c and -d
3636
echo. --test-marker Enable the test marker within the build.
37-
echo. --regen Regenerate all opcodes, grammar and tokens
37+
echo. --regen Regenerate all opcodes, grammar and tokens.
3838
echo.
3939
echo.Available flags to avoid building certain modules.
4040
echo.These flags have no effect if '-e' is not given:
@@ -116,8 +116,14 @@ rem Setup the environment
116116
call "%dir%find_msbuild.bat" %MSBUILD%
117117
if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2)
118118

119+
call "%dir%find_python.bat" %PYTHON%
120+
if ERRORLEVEL 1 (echo Cannot locate python.exe on PATH or as PYTHON variable & exit /b 3)
121+
set PythonForBuild=%PYTHON%
122+
119123
if "%kill%"=="true" call :Kill
120-
if ERRORLEVEL 1 exit /B 3
124+
if ERRORLEVEL 1 exit /B %ERRORLEVEL%
125+
126+
if "%regen%"=="true" goto :Regen
121127

122128
if "%do_pgo%"=="true" (
123129
set conf=PGInstrument
@@ -147,6 +153,15 @@ echo on
147153
@echo off
148154
exit /B %ERRORLEVEL%
149155

156+
:Regen
157+
echo on
158+
%MSBUILD% "%dir%\pythoncore.vcxproj" /t:Regen %verbose%^
159+
/p:Configuration=%conf% /p:Platform=%platf%^
160+
/p:ForceRegen=true
161+
162+
@echo off
163+
exit /B %ERRORLEVEL%
164+
150165
:Build
151166
rem Call on MSBuild to do the work, echo the command.
152167
rem Passing %1-9 is not the preferred option, but argument parsing in
@@ -160,14 +175,6 @@ echo on
160175
/p:UseTestMarker=%UseTestMarker% %GITProperty%^
161176
%1 %2 %3 %4 %5 %6 %7 %8 %9
162177

163-
@if not ERRORLEVEL 1 @if "%Regen%"=="true" (
164-
%MSBUILD% "%dir%regen.vcxproj" /t:%target% %parallel% %verbose%^
165-
/p:IncludeExternals=%IncludeExternals%^
166-
/p:Configuration=%conf% /p:Platform=%platf%^
167-
/p:UseTestMarker=%UseTestMarker% %GITProperty%^
168-
%1 %2 %3 %4 %5 %6 %7 %8 %9
169-
)
170-
171178
@echo off
172179
exit /b %ERRORLEVEL%
173180

‎PCbuild/pythoncore.vcxproj

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -513,7 +513,9 @@
513513
</ItemGroup>
514514
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
515515
<ImportGroup Label="ExtensionTargets">
516+
<Import Project="regen.targets" />
516517
</ImportGroup>
518+
<Target Name="_TriggerRegen" BeforeTargets="PrepareForBuild" DependsOnTargets="Regen" />
517519
<Target Name="_GetBuildInfo" BeforeTargets="PrepareForBuild">
518520
<PropertyGroup>
519521
<GIT Condition="$(GIT) == ''">git</GIT>

‎PCbuild/regen.targets

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
3+
<Target Name="_SetPythonForBuild">
4+
<PropertyGroup>
5+
<PythonForBuild>$(PYTHON)</PythonForBuild>
6+
</PropertyGroup>
7+
</Target>
8+
9+
<ItemGroup>
10+
<_PegenSources Include="$(PySourcePath)Grammar\python.gram;$(PySourcePath)Grammar\Tokens" />
11+
<_PegenOutputs Include="$(PySourcePath)Parser\parser.c" />
12+
<_ASTSources Include="$(PySourcePath)Parser\Python.asdl" />
13+
<_ASTOutputs Include="$(PySourcePath)Include\internal\pycore_ast.h">
14+
<Argument>-H</Argument>
15+
</_ASTOutputs>
16+
<_ASTOutputs Include="$(PySourcePath)Include\internal\pycore_ast_state.h">
17+
<Argument>-I</Argument>
18+
</_ASTOutputs>
19+
<_ASTOutputs Include="$(PySourcePath)Python\Python-ast.c">
20+
<Argument>-C</Argument>
21+
</_ASTOutputs>
22+
<_OpcodeSources Include="$(PySourcePath)Tools\scripts\generate_opcode_h.py;$(PySourcePath)Lib\opcode.py" />
23+
<_OpcodeOutputs Include="$(PySourcePath)Include\opcode.h;$(PySourcePath)Python\opcode_targets.h" />
24+
<_TokenSources Include="$(PySourcePath)Grammar\Tokens" />
25+
<_TokenOutputs Include="$(PySourcePath)Doc\library\token-list.inc">
26+
<Format>rst</Format>
27+
</_TokenOutputs>
28+
<_TokenOutputs Include="$(PySourcePath)Include\token.h">
29+
<Format>h</Format>
30+
</_TokenOutputs>
31+
<_TokenOutputs Include="$(PySourcePath)Parser\token.c">
32+
<Format>c</Format>
33+
</_TokenOutputs>
34+
<_TokenOutputs Include="$(PySourcePath)Lib\token.py">
35+
<Format>py</Format>
36+
</_TokenOutputs>
37+
<_KeywordSources Include="$(PySourcePath)Grammar\python.gram;$(PySourcePath)Grammar\Tokens" />
38+
<_KeywordOutputs Include="$(PySourcePath)Lib\keyword.py" />
39+
</ItemGroup>
40+
41+
<Target Name="_TouchRegenSources" Condition="$(ForceRegen) == 'true'">
42+
<Message Text="Touching source files to force regeneration" Importance="high" />
43+
<Touch Files="@(_PegenSources);@(_ASTSources);@(_OpcodeSources);@(_TokenSources);@(_KeywordOutputs)"
44+
AlwaysCreate="False" />
45+
</Target>
46+
47+
<Target Name="_RegenPegen" Inputs="@(_PegenSources)" Outputs="@(_PegenOutputs)">
48+
<Message Text="Regenerate @(_PegenOutputs->'%(Filename)%(Extension)',' ')" Importance="high" />
49+
<!-- Specify python.gram with POSIX-like path because the argument gets written into the file verbatim -->
50+
<Exec Command="set PYTHONPATH=Tools\peg_generator%0D%0A$(PythonForBuild) -m pegen -q c ./Grammar/python.gram Grammar\Tokens -o Parser\parser.c"
51+
WorkingDirectory="$(PySourcePath)" />
52+
</Target>
53+
54+
<Target Name="_RegenAST_H" Inputs="@(_ASTSources)" Outputs="@(_ASTOutputs)">
55+
<Message Text="Regenerate @(_ASTOutputs->'%(Filename)%(Extension)',' ')" Importance="high" />
56+
<Exec Command="$(PythonForBuild) Parser\asdl_c.py Parser\Python.asdl @(_ASTOutputs->'%(Argument) &quot;%(Identity)&quot;',' ')"
57+
WorkingDirectory="$(PySourcePath)" />
58+
</Target>
59+
60+
<Target Name="_RegenOpcodes" Inputs="@(_OpcodeSources)" Outputs="@(_OpcodeOutputs)">
61+
<Message Text="Regenerate @(_OpcodeOutputs->'%(Filename)%(Extension)',' ')" Importance="high" />
62+
<Exec Command="$(PythonForBuild) Tools\scripts\generate_opcode_h.py Lib\opcode.py Include\opcode.h"
63+
WorkingDirectory="$(PySourcePath)" />
64+
<Exec Command="$(PythonForBuild) Python\makeopcodetargets.py Python\opcode_targets.h"
65+
WorkingDirectory="$(PySourcePath)" />
66+
</Target>
67+
68+
<Target Name="_RegenTokens" Inputs="@(_TokenSources)" Outputs="@(_TokenOutputs)">
69+
<Message Text="Regenerate @(_TokenOutputs->'%(Filename)%(Extension)',' ')" Importance="high" />
70+
<Exec Command="$(PythonForBuild) Tools\scripts\generate_token.py %(_TokenOutputs.Format) Grammar\Tokens &quot;%(_TokenOutputs.Identity)&quot;"
71+
WorkingDirectory="$(PySourcePath)" />
72+
<Touch Files="@(_TokenOutputs)" />
73+
</Target>
74+
75+
<Target Name="_RegenKeywords" Inputs="@(_KeywordSources)" Outputs="@(_KeywordOutputs)">
76+
<Message Text="Regenerate @(KeywordOutputs->'%(Filename)%(Extension)',' ')" Importance="high" />
77+
<Exec Command="set PYTHONPATH=Tools\peg_generator%0D%0A$(PythonForBuild) -m pegen.keywordgen Grammar\python.gram Grammar\Tokens Lib\keyword.py"
78+
WorkingDirectory="$(PySourcePath)" />
79+
</Target>
80+
81+
<Target Name="Regen" DependsOnTargets="_TouchRegenSources;_RegenPegen;_RegenAST_H;_RegenOpcodes;_RegenTokens;_RegenKeywords">
82+
<Message Text="Generated sources are up to date" Importance="high" />
83+
</Target>
84+
</Project>

0 commit comments

Comments
 (0)