bpo-43567: Improved generated code refresh on Windows (pythonGH-25120)

Generated files are now refreshed automatically on regular build, or may be forcibly regenerated by calling `build.bat --regen`.

Misc/NEWS.d/next/Build/2021-03-31-19-50-01.bpo-43567.vd0a-p.rst

@@ -0,0 +1 @@
+Improved generated code refresh (AST/tokens/opcodes/keywords) on Windows.

PCbuild/_freeze_importlib.vcxproj

@@ -140,10 +140,12 @@
           Condition="!Exists(%(None.OutFile)) or (Exists(%(None.IntFile)) and '$([System.IO.File]::ReadAllText(%(None.OutFile)).Replace(`
`, `
`))' != '$([System.IO.File]::ReadAllText(%(None.IntFile)).Replace(`
`, `
`))')">
       <Output TaskParameter="CopiedFiles" ItemName="_Updated" />
     </Copy>
-
-    <Warning Text="@(_Updated->'%(Filename)%(Extension)',', ') updated. You will need to rebuild pythoncore to see the changes.%0D%0A%0D%0AIf you are not developing on Windows but you see this error on a continuous integration build, you need to run 'make regen-all' and commit any changes."
+
+    <Message Text="Updated files: @(_Updated->'%(Filename)%(Extension)',', ')"
+             Condition="'@(_Updated)' != ''" Importance="high" />
+    <Warning Text="Frozen importlib files were updated. Please rebuild to pick up the changes.%0D%0A%0D%0AIf you are not developing on Windows but you see this error on a continuous integration build, please run 'make regen-all' and commit anything that changes."
              Condition="'@(_Updated)' != '' and $(Configuration) == 'Debug'" />
-    <Error Text="@(_Updated->'%(Filename)%(Extension)',', ') updated. You will need to rebuild pythoncore to see the changes.%0D%0A%0D%0AIf you are not developing on Windows but you see this error on a continuous integration build, you need to run 'make regen-all' and commit any changes."
+    <Error Text="Frozen importlib files were updated. Please rebuild to pick up the changes.%0D%0A%0D%0AIf you are not developing on Windows but you see this error on a continuous integration build, please run 'make regen-all' and commit anything that changes."
            Condition="'@(_Updated)' != '' and $(Configuration) == 'Release'" />
   </Target>
   <Target Name="RebuildImportLib" AfterTargets="AfterBuild" Condition="$(Configuration) == 'Debug' or $(Configuration) == 'Release'"

PCbuild/build.bat

@@ -34,7 +34,7 @@ echo.      automatically by the pythoncore project)
 echo.  --pgo          Build with Profile-Guided Optimization.  This flag
 echo.                 overrides -c and -d
 echo.  --test-marker  Enable the test marker within the build.
-echo.  --regen        Regenerate all opcodes, grammar and tokens
+echo.  --regen        Regenerate all opcodes, grammar and tokens.
 echo.
 echo.Available flags to avoid building certain modules.
 echo.These flags have no effect if '-e' is not given:
@@ -116,8 +116,14 @@ rem Setup the environment
 call "%dir%find_msbuild.bat" %MSBUILD%
 if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2)
 
+call "%dir%find_python.bat" %PYTHON%
+if ERRORLEVEL 1 (echo Cannot locate python.exe on PATH or as PYTHON variable & exit /b 3)
+set PythonForBuild=%PYTHON%
+
 if "%kill%"=="true" call :Kill
-if ERRORLEVEL 1 exit /B 3
+if ERRORLEVEL 1 exit /B %ERRORLEVEL%
+
+if "%regen%"=="true" goto :Regen
 
 if "%do_pgo%"=="true" (
     set conf=PGInstrument
@@ -147,6 +153,15 @@ echo on
 @echo off
 exit /B %ERRORLEVEL%
 
+:Regen
+echo on
+%MSBUILD% "%dir%\pythoncore.vcxproj" /t:Regen %verbose%^
+ /p:Configuration=%conf% /p:Platform=%platf%^
+ /p:ForceRegen=true
+
+@echo off
+exit /B %ERRORLEVEL%
+
 :Build
 rem Call on MSBuild to do the work, echo the command.
 rem Passing %1-9 is not the preferred option, but argument parsing in
@@ -160,14 +175,6 @@ echo on
  /p:UseTestMarker=%UseTestMarker% %GITProperty%^
  %1 %2 %3 %4 %5 %6 %7 %8 %9
 
-@if not ERRORLEVEL 1 @if "%Regen%"=="true" (
-    %MSBUILD% "%dir%regen.vcxproj" /t:%target% %parallel% %verbose%^
-     /p:IncludeExternals=%IncludeExternals%^
-     /p:Configuration=%conf% /p:Platform=%platf%^
-     /p:UseTestMarker=%UseTestMarker% %GITProperty%^
-     %1 %2 %3 %4 %5 %6 %7 %8 %9
-)
-
 @echo off
 exit /b %ERRORLEVEL%
 

PCbuild/pythoncore.vcxproj

@@ -513,7 +513,9 @@
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
+    <Import Project="regen.targets" />
   </ImportGroup>
+  <Target Name="_TriggerRegen" BeforeTargets="PrepareForBuild" DependsOnTargets="Regen" />
   <Target Name="_GetBuildInfo" BeforeTargets="PrepareForBuild">
     <PropertyGroup>
       <GIT Condition="$(GIT) == ''">git</GIT>

PCbuild/regen.targets

@@ -0,0 +1,84 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Target Name="_SetPythonForBuild">
+    <PropertyGroup>
+      <PythonForBuild>$(PYTHON)</PythonForBuild>
+    </PropertyGroup>
+  </Target>
+
+  <ItemGroup>
+    <_PegenSources Include="$(PySourcePath)Grammar\python.gram;$(PySourcePath)Grammar\Tokens" />
+    <_PegenOutputs Include="$(PySourcePath)Parser\parser.c" />
+    <_ASTSources Include="$(PySourcePath)Parser\Python.asdl" />
+    <_ASTOutputs Include="$(PySourcePath)Include\internal\pycore_ast.h">
+      <Argument>-H</Argument>
+    </_ASTOutputs>
+    <_ASTOutputs Include="$(PySourcePath)Include\internal\pycore_ast_state.h">
+      <Argument>-I</Argument>
+    </_ASTOutputs>
+    <_ASTOutputs Include="$(PySourcePath)Python\Python-ast.c">
+      <Argument>-C</Argument>
+    </_ASTOutputs>
+    <_OpcodeSources Include="$(PySourcePath)Tools\scripts\generate_opcode_h.py;$(PySourcePath)Lib\opcode.py" />
+    <_OpcodeOutputs Include="$(PySourcePath)Include\opcode.h;$(PySourcePath)Python\opcode_targets.h" />
+    <_TokenSources Include="$(PySourcePath)Grammar\Tokens" />
+    <_TokenOutputs Include="$(PySourcePath)Doc\library\token-list.inc">
+      <Format>rst</Format>
+    </_TokenOutputs>
+    <_TokenOutputs Include="$(PySourcePath)Include\token.h">
+      <Format>h</Format>
+    </_TokenOutputs>
+    <_TokenOutputs Include="$(PySourcePath)Parser\token.c">
+      <Format>c</Format>
+    </_TokenOutputs>
+    <_TokenOutputs Include="$(PySourcePath)Lib\token.py">
+      <Format>py</Format>
+    </_TokenOutputs>
+    <_KeywordSources Include="$(PySourcePath)Grammar\python.gram;$(PySourcePath)Grammar\Tokens" />
+    <_KeywordOutputs Include="$(PySourcePath)Lib\keyword.py" />
+  </ItemGroup>
+
+  <Target Name="_TouchRegenSources" Condition="$(ForceRegen) == 'true'">
+    <Message Text="Touching source files to force regeneration" Importance="high" />
+    <Touch Files="@(_PegenSources);@(_ASTSources);@(_OpcodeSources);@(_TokenSources);@(_KeywordOutputs)"
+           AlwaysCreate="False" />
+  </Target>
+
+  <Target Name="_RegenPegen" Inputs="@(_PegenSources)" Outputs="@(_PegenOutputs)">
+    <Message Text="Regenerate @(_PegenOutputs->'%(Filename)%(Extension)',' ')" Importance="high" />
+    <!-- Specify python.gram with POSIX-like path because the argument gets written into the file verbatim -->
+    <Exec Command="set PYTHONPATH=Tools\peg_generator%0D%0A$(PythonForBuild) -m pegen -q c ./Grammar/python.gram Grammar\Tokens -o Parser\parser.c"
+          WorkingDirectory="$(PySourcePath)" />
+  </Target>
+
+  <Target Name="_RegenAST_H" Inputs="@(_ASTSources)" Outputs="@(_ASTOutputs)">
+    <Message Text="Regenerate @(_ASTOutputs->'%(Filename)%(Extension)',' ')" Importance="high" />
+    <Exec Command="$(PythonForBuild) Parser\asdl_c.py Parser\Python.asdl @(_ASTOutputs->'%(Argument) &quot;%(Identity)&quot;',' ')"
+          WorkingDirectory="$(PySourcePath)" />
+  </Target>
+
+  <Target Name="_RegenOpcodes" Inputs="@(_OpcodeSources)" Outputs="@(_OpcodeOutputs)">
+    <Message Text="Regenerate @(_OpcodeOutputs->'%(Filename)%(Extension)',' ')" Importance="high" />
+    <Exec Command="$(PythonForBuild) Tools\scripts\generate_opcode_h.py Lib\opcode.py Include\opcode.h"
+          WorkingDirectory="$(PySourcePath)" />
+    <Exec Command="$(PythonForBuild) Python\makeopcodetargets.py Python\opcode_targets.h"
+          WorkingDirectory="$(PySourcePath)" />
+  </Target>
+
+  <Target Name="_RegenTokens" Inputs="@(_TokenSources)" Outputs="@(_TokenOutputs)">
+    <Message Text="Regenerate @(_TokenOutputs->'%(Filename)%(Extension)',' ')" Importance="high" />
+    <Exec Command="$(PythonForBuild) Tools\scripts\generate_token.py %(_TokenOutputs.Format) Grammar\Tokens &quot;%(_TokenOutputs.Identity)&quot;"
+          WorkingDirectory="$(PySourcePath)" />
+    <Touch Files="@(_TokenOutputs)" />
+  </Target>
+
+  <Target Name="_RegenKeywords" Inputs="@(_KeywordSources)" Outputs="@(_KeywordOutputs)">
+    <Message Text="Regenerate @(KeywordOutputs->'%(Filename)%(Extension)',' ')" Importance="high" />
+    <Exec Command="set PYTHONPATH=Tools\peg_generator%0D%0A$(PythonForBuild) -m pegen.keywordgen Grammar\python.gram Grammar\Tokens Lib\keyword.py"
+          WorkingDirectory="$(PySourcePath)" />
+  </Target>
+
+  <Target Name="Regen" DependsOnTargets="_TouchRegenSources;_RegenPegen;_RegenAST_H;_RegenOpcodes;_RegenTokens;_RegenKeywords">
+    <Message Text="Generated sources are up to date" Importance="high" />
+  </Target>
+</Project>