bpo-31392: Update SSL build for 1.1.0 (#3448)

Misc/NEWS.d/next/Windows/2017-09-07-20-09-04.bpo-31392.h92bWF.rst

@@ -0,0 +1 @@
+Update Windows build to use OpenSSL 1.1.0f

PCbuild/get_externals.bat

@@ -49,7 +49,7 @@ echo.Fetching external libraries...
 
 set libraries=
 set libraries=%libraries%                                       bzip2-1.0.6
-if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries%     openssl-1.0.2k
+if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries%     openssl-1.1.0f
 set libraries=%libraries%                                       sqlite-3.14.2.0
 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.6.0
 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.6.0
@@ -72,7 +72,7 @@ for %%e in (%libraries%) do (
 echo.Fetching external binaries...
 
 set binaries=
-if NOT "%IncludeSSL%"=="false"     set binaries=%binaries% openssl-bin-1.0.2k
+if NOT "%IncludeSSL%"=="false"     set binaries=%binaries% openssl-bin-1.1.0f
 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.6.0
 if NOT "%IncludeSSLSrc%"=="false"  set binaries=%binaries% nasm-2.11.06
 

PCbuild/openssl.props

@@ -6,14 +6,18 @@
     </ClCompile>
     <Link>
       <AdditionalLibraryDirectories>$(opensslOutDir);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
-      <AdditionalDependencies>ws2_32.lib;libeay32.lib;ssleay32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>ws2_32.lib;libcrypto.lib;libssl.lib;%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
   </ItemDefinitionGroup>
+  <PropertyGroup>
+    <_DLLSuffix>-1_1</_DLLSuffix>
+    <_DLLSuffix Condition="$(Platform) == 'x64'">$(_DLLSuffix)-x64</_DLLSuffix>
+  </PropertyGroup>
   <ItemGroup>
-    <_SSLDLL Include="$(opensslOutDir)\libeay32.dll" />
+    <_SSLDLL Include="$(opensslOutDir)\libcrypto$(_DLLSuffix).dll" />
-    <_SSLDLL Include="$(opensslOutDir)\libeay32.pdb" />
+    <_SSLDLL Include="$(opensslOutDir)\libcrypto$(_DLLSuffix).pdb" />
-    <_SSLDLL Include="$(opensslOutDir)\ssleay32.dll" />
+    <_SSLDLL Include="$(opensslOutDir)\libssl$(_DLLSuffix).dll" />
-    <_SSLDLL Include="$(opensslOutDir)\ssleay32.pdb" />
+    <_SSLDLL Include="$(opensslOutDir)\libssl$(_DLLSuffix).pdb" />
   </ItemGroup>
   <Target Name="_CopySSLDLL" Inputs="@(_SSLDLL)" Outputs="@(_SSLDLL->'$(OutDir)%(Filename)%(Extension)')" AfterTargets="Build">
     <Copy SourceFiles="@(_SSLDLL)" DestinationFolder="$(OutDir)" />

PCbuild/openssl.vcxproj

@@ -47,52 +47,68 @@
     <Bitness Condition="$(Platform) == 'x64'">64</Bitness>
     <ArchName>x86</ArchName>
     <ArchName Condition="$(Platform) == 'x64'">amd64</ArchName>
+    <OpenSSLPlatform>VC-WIN32</OpenSSLPlatform>
+    <OpenSSLPlatform Condition="$(Platform) == 'x64'">VC-WIN64A</OpenSSLPlatform>
     <SupportSigning>true</SupportSigning>
   </PropertyGroup>
 
-  <ItemGroup>
-    <PrepareSSL Include="prepare_ssl.py" />
-    <Perl Include="$(Perl)" />
-  </ItemGroup>
-
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <Import Project="pyproject.props" />
 
   <PropertyGroup>
     <IntDir>$(opensslDir)\tmp$(Bitness)dll</IntDir>
-    <OutDir>$(opensslDir)\out$(Bitness)dll</OutDir>
+    <OutDir>$(opensslOutDir)</OutDir>
-    <MakeFile>ms\ntdll$(Bitness).mak</MakeFile>
-    <NMakeOptions>LIB_D="$(opensslOutDir.TrimEnd(`\`))" OUT_D=out$(Bitness)dll TMP_D=tmp$(Bitness)dll INC_D=inc$(Bitness) INCO_D=inc$(Bitness)\openssl</NMakeOptions>
     <NMakeBuildCommandLine>setlocal
-set PATH=%PATH%;$(nasmDir);@(Perl->'%(RootDir)%(Directory)',';')
 set VCINSTALLDIR=$(VCInstallDir)
-cd /D "$(opensslDir.TrimEnd(`\`))"
 if not exist "$(IntDir.TrimEnd('\'))" mkdir "$(IntDir.TrimEnd('\'))"
-if not exist "$(OutDir.TrimEnd('\'))" mkdir "$(OutDir.TrimEnd('\'))"
+cd /D "$(IntDir.TrimEnd('\'))"
-if not exist "$(opensslOutDir.TrimEnd(`\`))" mkdir "$(opensslOutDir.TrimEnd(`\`))"
+$(Perl) "$(opensslDir)\configure" $(OpenSSLPlatform) no-asm
-$(PYTHON) "@(PrepareSSL->'%(FullPath)')" "$(opensslDir.TrimEnd(`\`))" $(ArchName)
+nmake
-nmake -f $(MakeFile) $(NMakeOptions) headers lib
-copy /y LICENSE "$(opensslOutDir)\LICENSE"
 </NMakeBuildCommandLine>
   </PropertyGroup>
 
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
 
-  <Target Name="SuppressOriginalBuildinfH" Condition="Exists('$(opensslDir)crypto\buildinf.h')" BeforeTargets="CreateBuildinfH">
+  <Target Name="_PatchUplink" BeforeTargets="Build">
-    <Move SourceFiles="$(opensslDir)crypto\buildinf.h" DestinationFiles="$(opensslDir)crypto\buildinf.h.orig" />
+    <PropertyGroup>
+      <Uplink>$(opensslDir)\ms\uplink.c</Uplink>
+      <BeforePatch>((h = GetModuleHandle(NULL)) == NULL)</BeforePatch>
+      <AfterPatch>((h = GetModuleHandleA("_ssl.pyd")) == NULL) if ((h = GetModuleHandleA("_ssl_d.pyd")) == NULL) if ((h = GetModuleHandle(NULL)) == NULL /*patched*/)</AfterPatch>
+    </PropertyGroup>
+    <Error Text="Cannot find $(Uplink)" Condition="!Exists($(Uplink))" />
+    <PropertyGroup>
+      <_Original>$([System.IO.File]::ReadAllText($(Uplink)))</_Original>
+      <_Patched>$(_Original.Replace($(BeforePatch), $(AfterPatch)))</_Patched>
+      <IsPatched>false</IsPatched>
+      <IsPatched Condition="$(_Patched) == $(_Original)">true</IsPatched>
+    </PropertyGroup>
+    <Message Text="$(Uplink) is already patched" Importance="normal" Condition="$(IsPatched)" />
+    <Message Text="Patching $(Uplink)" Importance="high" Condition="!$(IsPatched)" />
+    <WriteLinesToFile File="$(Uplink)"
+                      Lines="$(_Patched)"
+                      Overwrite="true"
+                      Encoding="ASCII"
+                      Condition="!$(IsPatched)" />
   </Target>
 
-  <Target Name="_CopyIncludes" AfterTargets="Build">
+  <Target Name="_CopyToOutput" AfterTargets="Build">
     <ItemGroup>
-      <Sources Include="$(opensslDir)inc$(Bitness)\**\*.h" />
+      <_Built Include="$(opensslDir)\LICENSE" />
-      <Sources Include="$(opensslDir)ms\applink.c" />
+      <_Built Include="$(IntDir)\libcrypto.lib;$(IntDir)\libcrypto-*.dll;$(IntDir)\libcrypto-*.pdb" />
+      <_Built Include="$(IntDir)\libssl.lib;$(IntDir)\libssl-*.dll;$(IntDir)\libssl-*.pdb" />
+      <_AppLink Include="$(opensslDir)\ms\applink.c" />
+      <_Include Include="$(opensslDir)\Include\openssl\*.h" />
+      <_Include Include="$(IntDir)\include\openssl\*.h" />
     </ItemGroup>
-    <Copy SourceFiles="%(Sources.FullPath)" DestinationFiles="$(opensslOutDir)\include\%(Sources.RecursiveDir)\%(Sources.Filename)%(Sources.Extension)" />
+    <MakeDir Directories="$(opensslOutDir)\include\openssl" />
+    <Copy SourceFiles="@(_Built)" DestinationFolder="$(opensslOutDir)" />
+    <Copy SourceFiles="@(_AppLink)" DestinationFolder="$(opensslOutDir)\include" />
+    <Copy SourceFiles="@(_Include)" DestinationFolder="$(opensslOutDir)\include\openssl" />
   </Target>
 
   <Target Name="SignFiles" AfterTargets="Build" Condition="$(_SignCommand) != ''">
     <ItemGroup>
-      <FilesToSign Include="$(opensslOutDir)\libeay32.dll;$(opensslOutDir)\ssleay32.dll" />
+      <FilesToSign Include="$(opensslOutDir)\lib*.dll" />
     </ItemGroup>
     <Exec Command="$(_SignCommand) %(FilesToSign.FullPath)" ContinueOnError="true" />
   </Target>

PCbuild/prepare_ssl.bat

@@ -23,6 +23,8 @@ setlocal
 if "%PCBUILD%"=="" (set PCBUILD=%~dp0)
 if "%EXTERNALS_DIR%"=="" (set EXTERNALS_DIR=%PCBUILD%\..\externals)
 
+set OUT=
+set SRC=
 set ORG_SETTING=
 
 :CheckOpts
@@ -30,12 +32,19 @@ if "%~1"=="-h" shift & goto Usage
 if "%~1"=="--certificate" (set SigningCertificate=%~2) && shift && shift & goto CheckOpts
 if "%~1"=="-c" (set SigningCertificate=%~2) && shift && shift & goto CheckOpts
 if "%~1"=="--organization" (set ORG_SETTING=--organization "%~2") && shift && shift && goto CheckOpts
+if "%~1"=="-i" (SET SRC=$~2) && shift && shift && goto CheckOpts
+if "%~1"=="--in" (SET SRC=$~2) && shift && shift && goto CheckOpts
+if "%~1"=="-o" (set OUT=$~2) && shift && shift && goto CheckOpts
+if "%~1"=="--out" (set OUT=$~2) && shift && shift && goto CheckOpts
 
 if "%~1"=="" goto Build
 echo Unrecognized option: %1
 goto Usage
 
 :Build
+if not defined SRC (echo --in directory is required & exit /b 1)
+if not defined OUT (echo --out directory is required & exit /b 1)
+
 call "%PCBUILD%find_msbuild.bat" %MSBUILD%
 if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2)
 
@@ -51,3 +60,4 @@ if "%PERL%" == "" (echo Cannot locate perl.exe on PATH or as PERL variable & exi
 if errorlevel 1 exit /b
 %MSBUILD% "%PCBUILD%openssl.vcxproj" /p:Configuration=Release /p:Platform=x64
 if errorlevel 1 exit /b
+

PCbuild/python.props

@@ -47,8 +47,8 @@
     <sqlite3Dir>$(ExternalsDir)sqlite-3.14.2.0\</sqlite3Dir>
     <bz2Dir>$(ExternalsDir)bzip2-1.0.6\</bz2Dir>
     <lzmaDir>$(ExternalsDir)xz-5.2.2\</lzmaDir>
-    <opensslDir>$(ExternalsDir)openssl-1.0.2k\</opensslDir>
+    <opensslDir>$(ExternalsDir)openssl-1.1.0f\</opensslDir>
-    <opensslOutDir>$(ExternalsDir)openssl-bin-1.0.2k\$(ArchName)\</opensslOutDir>
+    <opensslOutDir>$(ExternalsDir)openssl-bin-1.1.0f\$(ArchName)\</opensslOutDir>
     <opensslIncludeDir>$(opensslOutDir)include</opensslIncludeDir>
     <nasmDir>$(ExternalsDir)\nasm-2.11.06\</nasmDir>
     <zlibDir>$(ExternalsDir)\zlib-1.2.11\</zlibDir>

Tools/msi/lib/lib_files.wxs

@@ -14,11 +14,11 @@
             <Component Id="sqlite3.dll" Directory="DLLs" Guid="*">
                 <File Name="sqlite3.dll" KeyPath="yes" />
             </Component>
-            <Component Id="libeay32.dll" Directory="DLLs" Guid="*">
+            <Component Id="libcrypto.dll" Directory="DLLs" Guid="*">
-                <File Name="libeay32.dll" KeyPath="yes" />
+                <File Name="libcrypto$(var.ssltag).dll" KeyPath="yes" />
             </Component>
-            <Component Id="ssleay32.dll" Directory="DLLs" Guid="*">
+            <Component Id="libssl.dll" Directory="DLLs" Guid="*">
-                <File Name="ssleay32.dll" KeyPath="yes" />
+                <File Name="libssl$(var.ssltag).dll" KeyPath="yes" />
             </Component>
         </ComponentGroup>
     </Fragment>
@@ -36,11 +36,11 @@
             <Component Id="sqlite3.pdb" Directory="DLLs" Guid="*">
                 <File Name="sqlite3.pdb" />
             </Component>
-            <Component Id="libeay32.pdb" Directory="DLLs" Guid="*">
+            <Component Id="libcrypto.pdb" Directory="DLLs" Guid="*">
-                <File Name="libeay32.pdb" KeyPath="yes" />
+                <File Name="libcrypto$(var.ssltag).pdb" KeyPath="yes" />
             </Component>
-            <Component Id="ssleay32.pdb" Directory="DLLs" Guid="*">
+            <Component Id="libssl.pdb" Directory="DLLs" Guid="*">
-                <File Name="ssleay32.pdb" KeyPath="yes" />
+                <File Name="libssl$(var.ssltag).pdb" KeyPath="yes" />
             </Component>
         </ComponentGroup>
     </Fragment>

Tools/msi/make_zip.py

@@ -47,8 +47,6 @@
 
 EXCLUDE_FILE_FROM_LIBS = {
     'liblzma',
-    'ssleay',
-    'libeay',
     'python3stub',
 }
 

Tools/msi/msi.props

@@ -92,10 +92,10 @@
             $(DefineConstants);CRTRedist=$(CRTRedist);
         </DefineConstants>
         <DefineConstants Condition="$(Platform) != 'x64'">
-            $(DefineConstants);Suffix32=-32;
+            $(DefineConstants);Suffix32=-32;ssltag=-1_1;
         </DefineConstants>
         <DefineConstants Condition="$(Platform) == 'x64'">
-            $(DefineConstants);Suffix32=;
+            $(DefineConstants);Suffix32=;ssltag=-1_1-x64;
         </DefineConstants>
     </PropertyGroup>
 

Tools/nuget/pythondaily.symbols.nuspec

@@ -18,7 +18,7 @@
     <file src="python*.pdb" target="tools" exclude="*_d*" />
     <file src="*.pdb" target="tools\DLLs" exclude="python*;*_test*;xx*;_freeze*;*_d*;py.*;pyw.*;pyshellext.*" />
     <file src="*.pyd" target="tools\DLLs" exclude="python*;*_test*;xx*;_freeze*;*_d*;py.*;pyw.*;pyshellext.*" />
-    <file src="libeay32.dll;ssleay32.dll;sqlite3.dll" target="tools\DLLs" />
+    <file src="libcrypto*.dll;libssl*.dll;sqlite3.dll" target="tools\DLLs" />
-    <file src="libeay32.pdb;ssleay32.pdb;sqlite3.pdb" target="tools\DLLs" />
+    <file src="libcrypto*.pdb;libssl*.pdb;sqlite3.pdb" target="tools\DLLs" />
   </files>
 </package>

Tools/ssl/sslspeed.vcxproj

@@ -1,70 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
-  <ItemGroup Label="ProjectConfigurations">
-    <ProjectConfiguration Include="Debug|Win32">
-      <Configuration>Debug</Configuration>
-      <Platform>Win32</Platform>
-    </ProjectConfiguration>
-    <ProjectConfiguration Include="Release|Win32">
-      <Configuration>Release</Configuration>
-      <Platform>Win32</Platform>
-    </ProjectConfiguration>
-    <ProjectConfiguration Include="Debug|x64">
-      <Configuration>Debug</Configuration>
-      <Platform>x64</Platform>
-    </ProjectConfiguration>
-    <ProjectConfiguration Include="Release|x64">
-      <Configuration>Release</Configuration>
-      <Platform>x64</Platform>
-    </ProjectConfiguration>
-  </ItemGroup>
-  
-  <PropertyGroup Label="Globals">
-    <ProjectGuid>{F068BCCF-C0D6-478D-A2C5-26BA3237C992}</ProjectGuid>
-    <RootNamespace>sslspeed</RootNamespace>
-  </PropertyGroup>
-  
-  <Import Project="..\..\PCBuild\python.props" />
-  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
-  <Import Project="..\..\PCBuild\openssl.props" />
-  
-  <PropertyGroup Label="Configuration">
-    <OriginalOutDir>$(OutDir)</OriginalOutDir>
-    <OutDir>$(MSBuildProjectDirectory)\$(ArchName)\</OutDir>
-    <IntDir>$(MSBuildProjectDirectory)\$(ArchName)\obj\</IntDir>
-    <ConfigurationType>Application</ConfigurationType>
-    <CharacterSet>MultiByte</CharacterSet>
-  </PropertyGroup>
-  
-  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-  
-  <ItemGroup>
-    <ClCompile Include="$(opensslDir)apps\speed.c" />
-    <ClCompile Include="$(opensslDir)apps\apps.c" />
-  </ItemGroup>
-  
-  <ItemDefinitionGroup>
-    <ClCompile>
-      <PreprocessorDefinitions>_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <AdditionalIncludeDirectories>$(opensslIncDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-    </ClCompile>
-    <Link>
-      <AdditionalDependencies>ws2_32.lib;crypt32.lib;libeay$(PyDebugExt).lib;ssleay$(PyDebugExt).lib;%(AdditionalDependencies)</AdditionalDependencies>
-      <AdditionalLibraryDirectories>$(OriginalOutDir);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
-      <SubSystem>Console</SubSystem>
-    </Link>
-  </ItemDefinitionGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\..\PCBuild\ssleay.vcxproj">
-      <Project>{10615b24-73bf-4efa-93aa-236916321317}</Project>
-      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
-    </ProjectReference>
-    <ProjectReference Include="..\..\PCBuild\libeay.vcxproj">
-      <Project>{e5b04cc0-eb4c-42ab-b4dc-18ef95f864b0}</Project>
-      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
-    </ProjectReference>
-  </ItemGroup>
-
-  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
-</Project>