bpo-30687: Fixes build scripts to find msbuild.exe and stop relying on vcvarsall.bat (python#2252)

* Fixes build scripts to find msbuild.exe and stop relying on vcvarsall.bat
Also fixes bdist_wininst.vcxproj to use correct version in generated name.

(cherry-picked from parts of 40a23e8)

Author: zooba

PCbuild/build.bat

@@ -10,7 +10,12 @@ echo.given.
 echo.
 echo.After the flags recognized by this script, up to 9 arguments to be passed
 echo.directly to MSBuild may be passed.  If the argument contains an '=', the
-echo.entire argument must be quoted (e.g. `%~nx0 "/p:PlatformToolset=v100"`)
+echo.entire argument must be quoted (e.g. `%~nx0 "/p:PlatformToolset=v100"`).
+echo.Alternatively you can put extra flags for MSBuild in a file named 
+echo.`msbuild.rsp` in the `PCbuild` directory, one flag per line. This file
+echo.will be picked automatically by MSBuild. Flags put in this file does not
+echo.need to be quoted. You can still use environment variables inside the 
+echo.response file.
 echo.
 echo.Available flags:
 echo.  -h  Display this help message
@@ -47,7 +52,6 @@ exit /b 127
 :Run
 setlocal
 set platf=Win32
-set vs_platf=x86
 set conf=Release
 set target=Build
 set dir=%~dp0
@@ -56,10 +60,6 @@ set verbose=/nologo /v:m
 set kill=
 set do_pgo=
 set pgo_job=-m test.regrtest --pgo
-set on_64_bit=true
-
-rem This may not be 100% accurate, but close enough.
-if "%ProgramFiles(x86)%"=="" (set on_64_bit=false)
 
 :CheckOpts
 if "%~1"=="-h" goto Usage
@@ -89,18 +89,12 @@ if "%IncludeBsddb%"=="" set IncludeBsddb=true
 
 if "%IncludeExternals%"=="true" call "%dir%get_externals.bat"
 
-if "%platf%"=="x64" (
-    if "%on_64_bit%"=="true" (
-        rem This ought to always be correct these days...
-        set vs_platf=amd64
-    ) else (
-        if "%do_pgo%"=="true" (
-            echo.ERROR: Cannot cross-compile with PGO
-            echo.    32bit operating system detected, if this is incorrect,
-            echo.    make sure the ProgramFiles(x86^) environment variable is set
-            exit /b 1
-        )
-        set vs_platf=x86_amd64
+if "%do_pgo%" EQU "true" if "%platf%" EQU "x64" (
+    if "%PROCESSOR_ARCHITEW6432%" NEQ "AMD64" if "%PROCESSOR_ARCHITECTURE%" NEQ "AMD64" (
+        echo.ERROR: Cannot cross-compile with PGO 
+        echo.       32bit operating system detected. Ensure your PROCESSOR_ARCHITECTURE
+        echo.       and PROCESSOR_ARCHITEW6432 environment variables are correct.
+        exit /b 1 
     )
 )
 
@@ -109,26 +103,28 @@ if exist "%GIT%" set GITProperty=/p:GIT="%GIT%"
 if not exist "%GIT%" echo Cannot find Git on PATH & set GITProperty=
 
 rem Setup the environment
-call "%dir%env.bat" %vs_platf% >nul
+call "%dir%find_msbuild.bat" %MSBUILD%
+if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2)
 
 if "%kill%"=="true" call :Kill
 
 if "%do_pgo%"=="true" (
     set conf=PGInstrument
-    call :Build
+    call :Build %1 %2 %3 %4 %5 %6 %7 %8 %9
     del /s "%dir%\*.pgc"
     del /s "%dir%\..\Lib\*.pyc"
     echo on
     call "%dir%\..\python.bat" %pgo_job%
     @echo off
     call :Kill
     set conf=PGUpdate
+    set target=Build
 )
 goto Build
 
 :Kill
 echo on
-msbuild "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^
+%MSBUILD% "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^
  /p:Configuration=%conf% /p:Platform=%platf%^
  /p:KillPython=true
 
@@ -140,7 +136,7 @@ rem Call on MSBuild to do the work, echo the command.
 rem Passing %1-9 is not the preferred option, but argument parsing in
 rem batch is, shall we say, "lackluster"
 echo on
-msbuild "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^
+%MSBUILD% "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^
  /p:Configuration=%conf% /p:Platform=%platf%^
  /p:IncludeExternals=%IncludeExternals%^
  /p:IncludeSSL=%IncludeSSL% /p:IncludeTkinter=%IncludeTkinter%^

PCbuild/find_msbuild.bat

@@ -0,0 +1,51 @@
+@rem
+@rem Searches for MSBuild.exe. This is the only tool we need to initiate
+@rem a build, so we no longer search for the full VC toolset.
+@rem
+@rem This file is supposed to modify the state of the caller (specifically
+@rem the MSBUILD variable), so we do not use setlocal or echo, and avoid
+@rem changing any other persistent state.
+@rem
+
+@rem No arguments provided means do full search
+@if '%1' EQU '' goto :begin_search
+
+@rem One argument may be the full path. Use a goto so we don't try to
+@rem parse the next if statement - incorrect quoting in the multi-arg
+@rem case can cause us to break immediately.
+@if '%2' EQU '' goto :one_arg
+
+@rem Entire command line may represent the full path if quoting failed.
+@if exist "%*" (set MSBUILD="%*") & (set _Py_MSBuild_Source=environment) & goto :found
+@goto :begin_search
+
+:one_arg
+@if exist "%~1" (set MSBUILD="%~1") & (set _Py_MSBuild_Source=environment) & goto :found
+
+:begin_search
+@set MSBUILD=
+
+@rem If msbuild.exe is on the PATH, assume that the user wants that one.
+@where msbuild > "%TEMP%\msbuild.loc" 2> nul && set /P MSBUILD= < "%TEMP%\msbuild.loc" & del "%TEMP%\msbuild.loc"
+@if exist "%MSBUILD%" set MSBUILD="%MSBUILD%" & (set _Py_MSBuild_Source=PATH) & goto :found
+
+@rem VS 2017 sets exactly one install as the "main" install, so we may find MSBuild in there.
+@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32 >nul 2>nul
+@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32') DO @(
+    @if "%%i"=="15.0" @if exist "%%k\MSBuild\15.0\Bin\msbuild.exe" @(set MSBUILD="%%k\MSBuild\15.0\Bin\msbuild.exe")
+)
+@if exist %MSBUILD% (set _Py_MSBuild_Source=Visual Studio 2017 registry) & goto :found
+
+@rem VS 2015 and earlier register MSBuild separately, so we can find it.
+@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32 >nul 2>nul
+@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32') DO @(
+    @if "%%i"=="MSBuildToolsPath" @if exist "%%k\msbuild.exe" @(set MSBUILD="%%k\msbuild.exe")
+)
+@if exist %MSBUILD% (set _Py_MSBuild_Source=registry) & goto :found
+
+
+@exit /b 1
+
+:found
+@echo Using %MSBUILD% (found in the %_Py_MSBuild_Source%)
+@set _Py_MSBuild_Source=

PCbuild/python.vcxproj

@@ -93,13 +93,18 @@
   </ImportGroup>
   <Target Name="GeneratePythonBat" AfterTargets="AfterBuild">
     <PropertyGroup>
+      <_PGOPath Condition="$(Configuration) == 'PGInstrument' and $(Platform) == 'Win32'">@set PATH=%PATH%%3B$(VCInstallDir)bin</_PGOPath>
+      <_PGOPath Condition="$(Configuration) == 'PGInstrument' and $(Platform) == 'x64'">@set PATH=%PATH%%3B$(VCInstallDir)bin\amd64</_PGOPath>
       <_Content>@rem This script invokes the most recently built Python with all arguments
 @rem passed through to the interpreter.  This file is generated by the
 @rem build process and any changes *will* be thrown away by the next
 @rem rebuild.
 @rem This is only meant as a convenience for developing CPython
 @rem and using it outside of that context is ill-advised.
 @echo Running $(Configuration)^|$(Platform) interpreter...
+@setlocal
+@set PYTHONHOME=$(PySourcePath)
+$(_PGOPath)
 @"$(OutDir)python$(PyDebugExt).exe" %*
 </_Content>
       <_ExistingContent Condition="Exists('$(PySourcePath)python.bat')">$([System.IO.File]::ReadAllText('$(PySourcePath)python.bat'))</_ExistingContent>

Tools/nuget/build.bat

@@ -21,6 +21,8 @@ if "%~1" EQU "-p" (set PACKAGES=%PACKAGES% %~2) && shift && shift && goto CheckO
 if not defined BUILDX86 if not defined BUILDX64 (set BUILDX86=1) && (set BUILDX64=1)
 
 if not defined NUGET where nuget -q || echo Cannot find nuget.exe on PATH and NUGET is not set. && exit /B 1
+call "%PCBUILD%find_msbuild.bat" %MSBUILD%
+if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2)
 if not defined PYTHON set PYTHON=py -3
 
 @%PYTHON% -c "" >nul 2>nul
@@ -29,7 +31,6 @@ if not defined PYTHON set PYTHON=py -3
     set PYTHON="%D%obj\python\tools\python.exe"
 )
 
-call "%PCBUILD%env.bat" x86
 
 if defined PACKAGES set PACKAGES="/p:Packages=%PACKAGES%"
 
@@ -38,7 +39,7 @@ if defined BUILDX86 (
     ) else if not exist "%PCBUILD%python.exe" call "%PCBUILD%build.bat" -e
     if errorlevel 1 goto :eof
 
-    msbuild "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x86 %OUTPUT% %PACKAGES%
+    %MSBUILD% "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x86 %OUTPUT% %PACKAGES%
     if errorlevel 1 goto :eof
 )
 
@@ -47,7 +48,7 @@ if defined BUILDX64 (
     ) else if not exist "%PCBUILD%amd64\python.exe" call "%PCBUILD%build.bat" -p x64 -e
     if errorlevel 1 goto :eof
 
-    msbuild "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x64 %OUTPUT% %PACKAGES%
+    %MSBUILD% "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x64 %OUTPUT% %PACKAGES%
     if errorlevel 1 goto :eof
 )