Skip to content

Commit 68d663c

Browse files
zoobazooba
authored
[bpo-30916] Pre-build OpenSSL and Tcl/Tk for Windows (#2688)
Updates ssl and tkinter projects to use pre-built externals
1 parent 49f6449 commit 68d663c

34 files changed

+495
-1400
lines changed

‎Doc/make.bat

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,19 @@ pushd %~dp0
55

66
set this=%~n0
77

8-
if "%SPHINXBUILD%" EQU "" set SPHINXBUILD=sphinx-build
8+
call ..\PCBuild\find_python.bat %PYTHON%
9+
if "%SPHINXBUILD%" EQU "" if "%PYTHON%" NEQ "" (
10+
set SPHINXBUILD=%PYTHON%\..\Scripts\sphinx-build.exe
11+
rem Cannot use %SPHINXBUILD% in the same block where we set it
12+
if not exist "%PYTHON%\..\Scripts\sphinx-build.exe" (
13+
echo Installing sphinx with %PYTHON%
14+
"%PYTHON%" -m pip install sphinx
15+
if errorlevel 1 exit /B
16+
)
17+
)
18+
919
if "%PYTHON%" EQU "" set PYTHON=py
20+
if "%SPHINXBUILD%" EQU "" set SPHINXBUILD=sphinx-build
1021

1122
if "%1" NEQ "htmlhelp" goto :skiphhcsearch
1223
if exist "%HTMLHELP%" goto :skiphhcsearch

‎Lib/test/test_ssl.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1244,6 +1244,7 @@ def test_load_default_certs_env(self):
12441244
self.assertEqual(ctx.cert_store_stats(), {"crl": 0, "x509": 1, "x509_ca": 0})
12451245

12461246
@unittest.skipUnless(sys.platform == "win32", "Windows specific")
1247+
@unittest.skipIf(hasattr(sys, "gettotalrefcount"), "Debug build does not share environment between CRTs")
12471248
def test_load_default_certs_env_windows(self):
12481249
ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
12491250
ctx.load_default_certs()

‎Misc/NEWS.d/next/Windows/2017-07-15-00-40-12.bpo-30916.BpCrro.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Pre-build OpenSSL, Tcl and Tk and include the binaries in the build.

‎Modules/_ssl.c

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,13 @@
2121
#ifdef WITH_THREAD
2222
#include "pythread.h"
2323

24+
/* Redefined below for Windows debug builds after important #includes */
25+
#define _PySSL_FIX_ERRNO
2426

2527
#define PySSL_BEGIN_ALLOW_THREADS_S(save) \
2628
do { if (_ssl_locks_count>0) { (save) = PyEval_SaveThread(); } } while (0)
2729
#define PySSL_END_ALLOW_THREADS_S(save) \
28-
do { if (_ssl_locks_count>0) { PyEval_RestoreThread(save); } } while (0)
30+
do { if (_ssl_locks_count>0) { PyEval_RestoreThread(save); } _PySSL_FIX_ERRNO; } while (0)
2931
#define PySSL_BEGIN_ALLOW_THREADS { \
3032
PyThreadState *_save = NULL; \
3133
PySSL_BEGIN_ALLOW_THREADS_S(_save);
@@ -96,6 +98,40 @@ struct py_ssl_library_code {
9698
int code;
9799
};
98100

101+
#if defined(MS_WINDOWS) && defined(Py_DEBUG)
102+
/* Debug builds on Windows rely on getting errno directly from OpenSSL.
103+
* However, because it uses a different CRT, we need to transfer the
104+
* value of errno from OpenSSL into our debug CRT.
105+
*
106+
* Don't be fooled - this is horribly ugly code. The only reasonable
107+
* alternative is to do both debug and release builds of OpenSSL, which
108+
* requires much uglier code to transform their automatically generated
109+
* makefile. This is the lesser of all the evils.
110+
*/
111+
112+
static void _PySSLFixErrno(void) {
113+
HMODULE ucrtbase = GetModuleHandleW(L"ucrtbase.dll");
114+
if (!ucrtbase) {
115+
/* If ucrtbase.dll is not loaded but the SSL DLLs are, we likely
116+
* have a catastrophic failure, but this function is not the
117+
* place to raise it. */
118+
return;
119+
}
120+
121+
typedef int *(__stdcall *errno_func)(void);
122+
errno_func ssl_errno = (errno_func)GetProcAddress(ucrtbase, "_errno");
123+
if (ssl_errno) {
124+
errno = *ssl_errno();
125+
*ssl_errno() = 0;
126+
} else {
127+
errno = ENOTRECOVERABLE;
128+
}
129+
}
130+
131+
#undef _PySSL_FIX_ERRNO
132+
#define _PySSL_FIX_ERRNO _PySSLFixErrno()
133+
#endif
134+
99135
/* Include generated data (error codes) */
100136
#include "_ssl_data.h"
101137

‎PCbuild/_hashlib.vcxproj

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -54,17 +54,15 @@
5454
<ImportGroup Label="PropertySheets">
5555
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
5656
<Import Project="pyproject.props" />
57+
<Import Project="openssl.props" />
5758
</ImportGroup>
5859
<PropertyGroup Label="UserMacros" />
5960
<PropertyGroup>
6061
<_ProjectFileVersion>10.0.30319.1</_ProjectFileVersion>
6162
</PropertyGroup>
6263
<ItemDefinitionGroup>
63-
<ClCompile>
64-
<AdditionalIncludeDirectories>$(opensslIncludeDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
65-
</ClCompile>
6664
<Link>
67-
<AdditionalDependencies>ws2_32.lib;$(OutDir)libeay$(PyDebugExt).lib;$(OutDir)ssleay$(PyDebugExt).lib;%(AdditionalDependencies)</AdditionalDependencies>
65+
<AdditionalDependencies>ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>
6866
</Link>
6967
</ItemDefinitionGroup>
7068
<ItemGroup>
@@ -78,14 +76,6 @@
7876
<Project>{cf7ac3d1-e2df-41d2-bea6-1e2556cdea26}</Project>
7977
<ReferenceOutputAssembly>false</ReferenceOutputAssembly>
8078
</ProjectReference>
81-
<ProjectReference Include="ssleay.vcxproj">
82-
<Project>{10615b24-73bf-4efa-93aa-236916321317}</Project>
83-
<ReferenceOutputAssembly>false</ReferenceOutputAssembly>
84-
</ProjectReference>
85-
<ProjectReference Include="libeay.vcxproj">
86-
<Project>{e5b04cc0-eb4c-42ab-b4dc-18ef95f864b0}</Project>
87-
<ReferenceOutputAssembly>false</ReferenceOutputAssembly>
88-
</ProjectReference>
8979
</ItemGroup>
9080
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
9181
<ImportGroup Label="ExtensionTargets">

‎PCbuild/_ssl.vcxproj

Lines changed: 6 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -54,21 +54,22 @@
5454
<ImportGroup Label="PropertySheets">
5555
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
5656
<Import Project="pyproject.props" />
57+
<Import Project="openssl.props" />
5758
</ImportGroup>
5859
<PropertyGroup Label="UserMacros" />
5960
<PropertyGroup>
6061
<_ProjectFileVersion>10.0.30319.1</_ProjectFileVersion>
6162
</PropertyGroup>
6263
<ItemDefinitionGroup>
63-
<ClCompile>
64-
<AdditionalIncludeDirectories>$(opensslIncludeDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
65-
</ClCompile>
6664
<Link>
67-
<AdditionalDependencies>ws2_32.lib;crypt32.lib;$(OutDir)libeay$(PyDebugExt).lib;$(OutDir)ssleay$(PyDebugExt).lib;%(AdditionalDependencies)</AdditionalDependencies>
65+
<AdditionalDependencies>ws2_32.lib;crypt32.lib;%(AdditionalDependencies)</AdditionalDependencies>
6866
</Link>
6967
</ItemDefinitionGroup>
7068
<ItemGroup>
7169
<ClCompile Include="..\Modules\_ssl.c" />
70+
<ClCompile Include="$(opensslIncludeDir)\applink.c">
71+
<PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;$(PreprocessorDefinitions)</PreprocessorDefinitions>
72+
</ClCompile>
7273
</ItemGroup>
7374
<ItemGroup>
7475
<ResourceCompile Include="..\PC\python_nt.rc" />
@@ -78,14 +79,6 @@
7879
<Project>{cf7ac3d1-e2df-41d2-bea6-1e2556cdea26}</Project>
7980
<ReferenceOutputAssembly>false</ReferenceOutputAssembly>
8081
</ProjectReference>
81-
<ProjectReference Include="libeay.vcxproj">
82-
<Project>{e5b04cc0-eb4c-42ab-b4dc-18ef95f864b0}</Project>
83-
<ReferenceOutputAssembly>false</ReferenceOutputAssembly>
84-
</ProjectReference>
85-
<ProjectReference Include="ssleay.vcxproj">
86-
<Project>{10615b24-73bf-4efa-93aa-236916321317}</Project>
87-
<ReferenceOutputAssembly>false</ReferenceOutputAssembly>
88-
</ProjectReference>
8982
<ProjectReference Include="_socket.vcxproj">
9083
<Project>{86937f53-c189-40ef-8ce8-8759d8e7d480}</Project>
9184
<ReferenceOutputAssembly>false</ReferenceOutputAssembly>
@@ -94,4 +87,4 @@
9487
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
9588
<ImportGroup Label="ExtensionTargets">
9689
</ImportGroup>
97-
</Project>
90+
</Project>

‎PCbuild/_ssl.vcxproj.filters

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,5 +9,11 @@
99
<ClCompile Include="..\Modules\_ssl.c">
1010
<Filter>Source Files</Filter>
1111
</ClCompile>
12+
<ClCompile Include="$(opensslIncludeDir)\applink.c">
13+
<Filter>Source Files</Filter>
14+
</ClCompile>
15+
</ItemGroup>
16+
<ItemGroup>
17+
<ResourceCompile Include="..\PC\python_nt.rc" />
1218
</ItemGroup>
1319
</Project>

‎PCbuild/_tkinter.vcxproj

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -76,19 +76,23 @@
7676
<ItemGroup>
7777
<ResourceCompile Include="..\PC\python_nt.rc" />
7878
</ItemGroup>
79+
<ItemGroup>
80+
<_TclTkDLL Include="$(tcltkdir)\bin\$(tclDllName)" />
81+
<_TclTkDLL Include="$(tcltkdir)\bin\$(tkDllName)" />
82+
</ItemGroup>
7983
<ItemGroup>
8084
<ProjectReference Include="pythoncore.vcxproj">
8185
<Project>{cf7ac3d1-e2df-41d2-bea6-1e2556cdea26}</Project>
8286
<ReferenceOutputAssembly>false</ReferenceOutputAssembly>
8387
</ProjectReference>
84-
<ProjectReference Include="tcl.vcxproj">
85-
<Project>{b5fd6f1d-129e-4bff-9340-03606fac7283}</Project>
86-
</ProjectReference>
87-
<ProjectReference Include="tk.vcxproj">
88-
<Project>{7e85eccf-a72c-4da4-9e52-884508e80ba1}</Project>
89-
</ProjectReference>
9088
</ItemGroup>
9189
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
9290
<ImportGroup Label="ExtensionTargets">
9391
</ImportGroup>
92+
<Target Name="_CopyTclTkDLL" Inputs="@(_TclTkDLL)" Outputs="@(_TclTkDLL->'$(OutDir)%(Filename)%(Extension)')" AfterTargets="Build">
93+
<Copy SourceFiles="@(_TclTkDLL)" DestinationFolder="$(OutDir)" />
94+
</Target>
95+
<Target Name="_CleanTclTkDLL" BeforeTargets="Clean">
96+
<Delete Files="@(_TclTkDLL->'$(OutDir)%(Filename)%(Extension)')" />
97+
</Target>
9498
</Project>

‎PCbuild/build.bat

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,6 @@ echo.%~nx0 [flags and arguments] [quoted MSBuild options]
55
echo.
66
echo.Build CPython from the command line. Requires the appropriate
77
echo.version(s) of Microsoft Visual Studio to be installed (see readme.txt).
8-
echo.Also requires Subversion (svn.exe) to be on PATH if the '-e' flag is
9-
echo.given.
108
echo.
119
echo.After the flags recognized by this script, up to 9 arguments to be passed
1210
echo.directly to MSBuild may be passed. If the argument contains an '=', the

‎PCbuild/find_msbuild.bat

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -29,20 +29,21 @@
2929
@where msbuild > "%TEMP%\msbuild.loc" 2> nul && set /P MSBUILD= < "%TEMP%\msbuild.loc" & del "%TEMP%\msbuild.loc"
3030
@if exist "%MSBUILD%" set MSBUILD="%MSBUILD%" & (set _Py_MSBuild_Source=PATH) & goto :found
3131

32-
@rem VS 2017 sets exactly one install as the "main" install, so we may find MSBuild in there.
33-
@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32 >nul 2>nul
34-
@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32') DO @(
35-
@if "%%i"=="15.0" @if exist "%%k\MSBuild\15.0\Bin\msbuild.exe" @(set MSBUILD="%%k\MSBuild\15.0\Bin\msbuild.exe")
36-
)
37-
@if exist %MSBUILD% (set _Py_MSBuild_Source=Visual Studio 2017 registry) & goto :found
38-
3932
@rem VS 2015 and earlier register MSBuild separately, so we can find it.
33+
@rem Prefer MSBuild 14.0 over MSBuild 15.0, since the latter may not be able to find a VC14 install.
4034
@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32 >nul 2>nul
4135
@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32') DO @(
4236
@if "%%i"=="MSBuildToolsPath" @if exist "%%k\msbuild.exe" @(set MSBUILD="%%k\msbuild.exe")
4337
)
4438
@if exist %MSBUILD% (set _Py_MSBuild_Source=registry) & goto :found
4539

40+
@rem VS 2017 sets exactly one install as the "main" install, so we may find MSBuild in there.
41+
@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32 >nul 2>nul
42+
@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32') DO @(
43+
@if "%%i"=="15.0" @if exist "%%k\MSBuild\15.0\Bin\msbuild.exe" @(set MSBUILD="%%k\MSBuild\15.0\Bin\msbuild.exe")
44+
)
45+
@if exist %MSBUILD% (set _Py_MSBuild_Source=Visual Studio 2017 registry) & goto :found
46+
4647

4748
@exit /b 1
4849

‎PCbuild/find_python.bat

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
@rem
2+
@rem Searches for python.exe and may download a private copy from nuget.
3+
@rem
4+
@rem This file is supposed to modify the state of the caller (specifically
5+
@rem the MSBUILD variable), so we do not use setlocal or echo, and avoid
6+
@rem changing any other persistent state.
7+
@rem
8+
9+
@rem No arguments provided means do full search
10+
@if '%1' EQU '' goto :begin_search
11+
12+
@rem One argument may be the full path. Use a goto so we don't try to
13+
@rem parse the next if statement - incorrect quoting in the multi-arg
14+
@rem case can cause us to break immediately.
15+
@if '%2' EQU '' goto :one_arg
16+
17+
@rem Entire command line may represent the full path if quoting failed.
18+
@if exist "%*" (set PYTHON="%*") & (set _Py_Python_Source=from environment) & goto :found
19+
@goto :begin_search
20+
21+
:one_arg
22+
@if exist "%~1" (set PYTHON="%~1") & (set _Py_Python_Source=from environment) & goto :found
23+
24+
:begin_search
25+
@set PYTHON=
26+
27+
@set _Py_EXTERNALS_DIR=%EXTERNAL_DIR%
28+
@if "%_Py_EXTERNALS_DIR%"=="" (set _Py_EXTERNALS_DIR=%~dp0\..\externals)
29+
30+
@rem If we have Python in externals, use that one
31+
@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found in externals directory) & goto :found
32+
33+
@rem If py.exe finds a recent enough version, use that one
34+
@py -3.6 -V >nul 2>&1 && (set PYTHON=py -3.6) && (set _Py_Python_Source=found with py.exe) && goto :found
35+
36+
@if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%"
37+
@set _Py_NUGET=%NUGET%
38+
@set _Py_NUGET_URL=%NUGET_URL%
39+
@if "%_Py_NUGET%"=="" (set _Py_NUGET=%EXTERNALS_DIR%\nuget.exe)
40+
@if "%_Py_NUGET_URL%"=="" (set _Py_NUGET_URL=https://aka.ms/nugetclidl)
41+
@if NOT exist "%_Py_NUGET%" (
42+
@echo Downloading nuget...
43+
@rem NB: Must use single quotes around NUGET here, NOT double!
44+
@rem Otherwise, a space in the path would break things
45+
@powershell.exe -Command Invoke-WebRequest %_Py_NUGET_URL% -OutFile '%_Py_NUGET%'
46+
)
47+
@echo Installing Python via nuget...
48+
@"%_Py_NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%_Py_EXTERNALS_DIR%"
49+
@rem Quote it here; it's not quoted later because "py -3.6" wouldn't work
50+
@if not errorlevel 1 (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found on nuget.org) & goto :found
51+
52+
53+
@exit /b 1
54+
55+
:found
56+
@echo Using %PYTHON% (%_Py_Python_Source%)
57+
@set _Py_Python_Source=

0 commit comments

Comments
 (0)