Clean up/refactor the batch scripts used for building on Windows.

This is mostly a backport of issue python#21907, but also includes a few
extras necessary to make the bulidbot scripts as thin as possible.

PCbuild/build.bat

@@ -1,19 +1,107 @@
 @echo off
-rem A batch program to build or rebuild a particular configuration.
+rem A batch program to build or rebuild a particular configuration,
 rem just for convenience.
 
+rem Arguments:
+rem  -c  Set the configuration (default: Release)
+rem  -p  Set the platform (x64 or Win32, default: Win32)
+rem  -r  Target Rebuild instead of Build
+rem  -t  Set the target manually (Build, Rebuild, or Clean)
+rem  -d  Set the configuration to Debug
+rem  -e  Pull in external libraries using get_externals.bat
+rem  -m  Enable parallel build
+rem  -M  Disable parallel build (disabled by default)
+rem  -v  Increased output messages
+rem  -k  Attempt to kill any running Pythons before building
+
 setlocal
 set platf=Win32
+set vs_platf=x86
 set conf=Release
-set target=build
+set target=Build
 set dir=%~dp0
+set parallel=
+set verbose=/nologo /v:m
+set kill=
+set build_tkinter=
 
 :CheckOpts
-if "%1"=="-c" (set conf=%2) & shift & shift & goto CheckOpts
-if "%1"=="-p" (set platf=%2) & shift & shift & goto CheckOpts
-if "%1"=="-r" (set target=rebuild) & shift & goto CheckOpts
-if "%1"=="-d" (set conf=Debug) & shift & goto CheckOpts
-
-set cmd=msbuild /p:useenv=true %dir%pcbuild.sln /t:%target% /p:Configuration=%conf% /p:Platform=%platf%
-echo %cmd%
-%cmd%
+if '%1'=='-c' (set conf=%2) & shift & shift & goto CheckOpts
+if '%1'=='-p' (set platf=%2) & shift & shift & goto CheckOpts
+if '%1'=='-r' (set target=Rebuild) & shift & goto CheckOpts
+if '%1'=='-t' (set target=%2) & shift & shift & goto CheckOpts
+if '%1'=='-d' (set conf=Debug) & shift & goto CheckOpts
+if '%1'=='-e' call "%dir%get_externals.bat" & (set build_tkinter=true) & shift & goto CheckOpts
+if '%1'=='-m' (set parallel=/m) & shift & goto CheckOpts
+if '%1'=='-M' (set parallel=) & shift & goto CheckOpts
+if '%1'=='-v' (set verbose=/v:n) & shift & goto CheckOpts
+if '%1'=='-k' (set kill=true) & shift & goto CheckOpts
+
+if '%conf%'=='Debug' (set dbg_ext=_d) else (set dbg_ext=)
+if '%platf%'=='x64' (
+    set vs_platf=x86_amd64
+    set builddir=%dir%amd64\
+) else (
+    set builddir=%dir%
+)
+
+rem Setup the environment
+call "%dir%env.bat" %vs_platf%
+
+if '%kill%'=='true' (
+    msbuild "%dir%kill_python.vcxproj" %verbose% /p:Configuration=%conf% /p:Platform=%platf% && "%builddir%kill_python%dbg_ext%.exe"
+)
+
+set externals_dir=%dir%..\externals
+if '%build_tkinter%'=='true' (
+    if '%platf%'=='x64' (
+        set tcltkdir=%externals_dir%\tcltk64
+        set machine=AMD64
+    ) else (
+        set tcltkdir=%externals_dir%\tcltk
+        set machine=IX86
+    )
+    if '%conf%'=='Debug' (
+        set tcl_dbg_ext=g
+        set debug_flag=1
+        set options=symbols
+    ) else (
+        set tcl_dbg_ext=
+        set debug_flag=0
+        set options=
+    )
+    set tcldir=%externals_dir%\tcl-8.6.1.0
+    set tkdir=%externals_dir%\tk-8.6.1.0
+    set tixdir=%externals_dir%\tix-8.4.3.4
+)
+if '%build_tkinter%'=='true' (
+    if not exist "%tcltkdir%\bin\tcl86t%tcl_dbg_ext%.dll" (
+        @rem all and install need to be separate invocations, otherwise nmakehlp is not found on install
+        pushd "%tcldir%\win"
+        nmake -f makefile.vc MACHINE=%machine% OPTS=%options% INSTALLDIR="%tcltkdir%" clean all
+        nmake -f makefile.vc MACHINE=%machine% OPTS=%options% INSTALLDIR="%tcltkdir%" install-binaries install-libraries
+        popd
+    )
+
+    if not exist "%tcltkdir%\bin\tk86t%tcl_dbg_ext%.dll" (
+        pushd "%tkdir%\win"
+        nmake -f makefile.vc MACHINE=%machine% OPTS=%options% INSTALLDIR="%tcltkdir%" TCLDIR="%tcldir%" clean
+        nmake -f makefile.vc MACHINE=%machine% OPTS=%options% INSTALLDIR="%tcltkdir%" TCLDIR="%tcldir%" all
+        nmake -f makefile.vc MACHINE=%machine% OPTS=%options% INSTALLDIR="%tcltkdir%" TCLDIR="%tcldir%" install-binaries install-libraries
+        popd
+    )
+
+    if not exist "%tcltkdir%\lib\tix8.4.3\tix84%tcl_dbg_ext%.dll" (
+        pushd "%tixdir%\win"
+        nmake -f python.mak DEBUG=%debug_flag% MACHINE=%machine% TCL_DIR="%tcldir%" TK_DIR="%tkdir%" INSTALL_DIR="%tcltkdir%" clean
+        nmake -f python.mak DEBUG=%debug_flag% MACHINE=%machine% TCL_DIR="%tcldir%" TK_DIR="%tkdir%" INSTALL_DIR="%tcltkdir%" all
+        nmake -f python.mak DEBUG=%debug_flag% MACHINE=%machine% TCL_DIR="%tcldir%" TK_DIR="%tkdir%" INSTALL_DIR="%tcltkdir%" install
+        popd
+    )
+)
+
+rem Call on MSBuild to do the work, echo the command.
+rem Passing %1-9 is not the preferred option, but argument parsing in
+rem batch is, shall we say, "lackluster"
+echo on
+msbuild "%dir%pcbuild.sln" /t:%target% %parallel% %verbose% /p:Configuration=%conf% /p:Platform=%platf% %1 %2 %3 %4 %5 %6 %7 %8 %9

PCbuild/env.bat

@@ -1,9 +1,4 @@
 @echo off
-set VS10=%ProgramFiles(x86)%\Microsoft Visual Studio 10.0
-IF EXIST "%VS10%" GOTO ok
-set VS10=%ProgramFiles%\Microsoft Visual Studio 10.0
-:ok
-
 echo Build environments: x86, ia64, amd64, x86_amd64, x86_ia64
 echo.
-call "%VS10%\VC\vcvarsall.bat" %1
+call "%VS100COMNTOOLS%..\..\VC\vcvarsall.bat" %*

PCbuild/get_externals.bat

@@ -0,0 +1,103 @@
+@echo off
+setlocal
+rem Simple script to fetch source for external libraries
+
+if not exist "%~dp0..\externals" mkdir "%~dp0..\externals"
+pushd "%~dp0..\externals"
+
+if "%SVNROOT%"=="" set SVNROOT=http://svn.python.org/projects/external/
+
+rem Optionally clean up first.  Be warned that this can be very destructive!
+if not "%1"=="" (
+    for %%c in (-c --clean --clean-only) do (
+        if "%1"=="%%c" goto clean
+    )
+    goto usage
+)
+goto fetch
+
+:clean
+echo.Cleaning up external libraries.
+for /D %%d in (
+               bzip2-*
+               db-*
+               nasm-*
+               openssl-*
+               tcl-*
+               tcltk*
+               tk-*
+               tix-*
+               sqlite-*
+               xz-*
+               ) do (
+    echo.Removing %%d
+    rmdir /s /q %%d
+)
+if "%1"=="--clean-only" (
+    goto end
+)
+
+:fetch
+rem Fetch current versions
+
+svn --version > nul 2>&1
+if ERRORLEVEL 9009 (
+    echo.svn.exe must be on your PATH.
+    echo.Try TortoiseSVN (http://tortoisesvn.net/^) and be sure to check the
+    echo.command line tools option.
+    popd
+    exit /b 1
+)
+
+echo.Fetching external libraries...
+
+for %%e in (
+            bzip2-1.0.6
+            nasm-2.11.06
+            openssl-1.0.2a
+            tcl-8.6.1.0
+            tk-8.6.1.0
+            tix-8.4.3.4
+            sqlite-3.8.3.1
+            xz-5.0.5
+            ) do (
+    if exist %%e (
+        echo.%%e already exists, skipping.
+    ) else (
+        echo.Fetching %%e...
+        svn export %SVNROOT%%%e
+    )
+)
+
+goto end
+
+:usage
+echo.invalid argument: %1
+echo.usage: %~n0 [[ -c ^| --clean ] ^| --clean-only ]
+echo.
+echo.Pull all sources necessary for compiling optional extension modules
+echo.that rely on external libraries.  Requires svn.exe to be on your PATH
+echo.and pulls sources from %SVNROOT%.
+echo.
+echo.Use the -c or --clean option to clean up all external library sources
+echo.before pulling in the current versions.
+echo.
+echo.Use the --clean-only option to do the same cleaning, without pulling in
+echo.anything new.
+echo.
+echo.Only the first argument is checked, all others are ignored.
+echo.
+echo.**WARNING**: the cleaning options unconditionally remove any directory
+echo.that is a child of
+echo.   %CD%
+echo.and matches wildcard patterns beginning with bzip2-, db-, nasm-, openssl-,
+echo.tcl-, tcltk, tk-, tix-, sqlite-, or xz-, and as such has the potential
+echo.to be very destructive if you are not aware of what it is doing.  Use with
+echo.caution!
+popd
+exit /b -1
+
+
+:end
+echo Finished.
+popd

PCbuild/rt.bat

@@ -9,7 +9,8 @@ rem      -q runs the tests just once, and without deleting .py[co] files.
 rem -x64 Run the 64-bit build of python (or python_d if -d was specified)
 rem      from the 'amd64' dir instead of the 32-bit build in this dir.
 rem All leading instances of these switches are shifted off, and
-rem whatever remains is passed to regrtest.py.  For example,
+rem whatever remains (up to 9 arguments) is passed to regrtest.py.
+rem For example,
 rem     rt -O -d -x test_thread
 rem runs
 rem     python_d -O ../lib/test/regrtest.py -x test_thread
@@ -26,7 +27,8 @@ rem     rt -u "network,largefile"
 
 setlocal
 
-set prefix=.\
+set pcbuild=%~dp0
+set prefix=%pcbuild%
 set suffix=
 set qmode=
 set dashO=
@@ -36,15 +38,15 @@ set tcltk=tcltk
 if "%1"=="-O" (set dashO=-O)     & shift & goto CheckOpts
 if "%1"=="-q" (set qmode=yes)    & shift & goto CheckOpts
 if "%1"=="-d" (set suffix=_d)    & shift & goto CheckOpts
-if "%1"=="-x64" (set prefix=amd64) & (set tcltk=tcltk64) & shift & goto CheckOpts
+if "%1"=="-x64" (set prefix=%prefix%amd64) & (set tcltk=tcltk64) & shift & goto CheckOpts
 
-PATH %PATH%;%~dp0..\externals\%tcltk%\bin
-set exe=%prefix%\python%suffix%
-set cmd=%exe% %dashO% -Wd -E -bb ../lib/test/regrtest.py %1 %2 %3 %4 %5 %6 %7 %8 %9
+PATH %PATH%;%pcbuild%..\externals\%tcltk%\bin
+set exe="%prefix%\python%suffix%"
+set cmd=%exe% %dashO% -Wd -E -bb "%pcbuild%..\lib\test\regrtest.py" %1 %2 %3 %4 %5 %6 %7 %8 %9
 if defined qmode goto Qmode
 
 echo Deleting .pyc/.pyo files ...
-%exe% rmpyc.py
+%exe% "%pcbuild%\rmpyc.py"
 
 echo on
 %cmd%

Tools/buildbot/build-amd64.bat

@@ -1,6 +1,2 @@
 @rem Used by the buildbot "compile" step.
-cmd /c Tools\buildbot\external-amd64.bat
-call "%VS100COMNTOOLS%\..\..\VC\vcvarsall.bat" x86_amd64
-cmd /c Tools\buildbot\clean-amd64.bat
-
-msbuild PCbuild\pcbuild.sln /p:Configuration=Debug /p:Platform=x64
+call "%~dp0build.bat" -p x64 %*

Tools/buildbot/build.bat

@@ -1,7 +1,17 @@
 @rem Used by the buildbot "compile" step.
-cmd /c Tools\buildbot\external.bat
-call "%VS100COMNTOOLS%vsvars32.bat"
-cmd /c Tools\buildbot\clean.bat
 
-msbuild PCbuild\pcbuild.sln /p:Configuration=Debug /p:Platform=Win32
+@rem Clean up
+call "%~dp0clean.bat" %*
 
+@rem If you need the buildbots to start fresh (such as when upgrading to
+@rem a new version of an external library, especially Tcl/Tk):
+@rem 1) uncomment the following line:
+
+@rem    call "%~dp0..\..\PCbuild\get_externals.bat" --clean-only
+
+@rem 2) commit and push
+@rem 3) wait for all Windows bots to start a build with that changeset
+@rem 4) re-comment, commit and push again
+
+@rem Do the build
+call "%~dp0..\..\PCbuild\build.bat" -e -d -k -v %*

Tools/buildbot/clean-amd64.bat

@@ -1,10 +1,2 @@
 @rem Used by the buildbot "clean" step.
-call "%VS100COMNTOOLS%\..\..\VC\vcvarsall.bat" x86_amd64
-@echo Deleting .pyc/.pyo files ...
-del /s Lib\*.pyc Lib\*.pyo
-@echo Deleting test leftovers ...
-rmdir /s /q build
-cd PCbuild
-msbuild /target:clean pcbuild.sln /p:Configuration=Release /p:PlatformTarget=x64
-msbuild /target:clean pcbuild.sln /p:Configuration=Debug /p:PlatformTarget=x64
-cd ..
+call "%~dp0clean.bat" -p x64 %*

Tools/buildbot/clean.bat

@@ -1,8 +1,16 @@
-@rem Used by the buildbot "clean" step.
-call "%VS100COMNTOOLS%vsvars32.bat"
-@echo Deleting test leftovers ...
-rmdir /s /q build
-cd PCbuild
-msbuild /target:clean pcbuild.sln /p:Configuration=Release /p:PlatformTarget=x86
-msbuild /target:clean pcbuild.sln /p:Configuration=Debug /p:PlatformTarget=x86
-cd ..
+@echo off
+rem Used by the buildbot "clean" step.
+
+setlocal
+set root=%~dp0..\..
+set pcbuild=%root%\PCbuild
+
+echo Deleting build
+call "%pcbuild%\build.bat" -t Clean -k %*
+call "%pcbuild%\build.bat" -t Clean -k -d %*
+
+echo Deleting .pyc/.pyo files ...
+del /s "%root%\Lib\*.pyc" "%root%\Lib\*.pyo"
+
+echo Deleting test leftovers ...
+rmdir /s /q "%root%\build"