Update MSVC to 2017 Update 9 (#271)

This commit is contained in:
Chinmay Garde 2019-07-11 15:43:34 -07:00 committed by GitHub
parent 5a3c4ab16d
commit c4df4a7b61
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -11,6 +11,9 @@
# To update to a new MSVC toolchain, copy the updated script from the Chromium
# tree, and edit to make it work in the Dart tree by updating paths in the original script.
from __future__ import print_function
import collections
import glob
import json
import os
@ -22,26 +25,29 @@ import stat
import subprocess
import sys
from gn_helpers import ToGNString
script_dir = os.path.dirname(os.path.realpath(__file__))
chrome_src = os.path.abspath(os.path.join(script_dir, os.pardir))
SRC_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, os.path.join(chrome_src, 'tools'))
json_data_file = os.path.join(script_dir, 'win_toolchain.json')
sys.path.insert(0, os.path.join(script_dir, '..', 'tools'))
# Use MSVS2017 as the default toolchain.
CURRENT_DEFAULT_TOOLCHAIN_VERSION = '2017'
# VS versions are listed in descending order of priority (highest first).
MSVS_VERSIONS = collections.OrderedDict([
('2017', '15.0'),
('2019', '16.0'),
])
def SetEnvironmentAndGetRuntimeDllDirs():
"""Sets up os.environ to use the depot_tools VS toolchain with gyp, and
returns the location of the VS runtime DLLs so they can be copied into
returns the location of the VC runtime DLLs so they can be copied into
the output directory after gyp generation.
Return value is [x64path, x86path] or None
Return value is [x64path, x86path, 'Arm64Unused'] or None. arm64path is
generated separately because there are multiple folders for the arm64 VC
runtime.
"""
vs_runtime_dll_dirs = None
depot_tools_win_toolchain = \
@ -51,7 +57,10 @@ def SetEnvironmentAndGetRuntimeDllDirs():
if ((sys.platform in ('win32', 'cygwin') or os.path.exists(json_data_file))
and depot_tools_win_toolchain):
if ShouldUpdateToolchain():
update_result = Update()
if len(sys.argv) > 1 and sys.argv[1] == 'update':
update_result = Update()
else:
update_result = Update(no_download=True)
if update_result != 0:
raise Exception('Failed to update, error code %d.' % update_result)
with open(json_data_file, 'r') as tempf:
@ -97,7 +106,9 @@ def SetEnvironmentAndGetRuntimeDllDirs():
# don't build on ARM64 machines.
x64_path = 'System32' if bitness == '64bit' else 'Sysnative'
x64_path = os.path.join(os.path.expandvars('%windir%'), x64_path)
vs_runtime_dll_dirs = [x64_path, os.path.expandvars('%windir%/SysWOW64'),
vs_runtime_dll_dirs = [x64_path,
os.path.join(os.path.expandvars('%windir%'),
'SysWOW64'),
'Arm64Unused']
return vs_runtime_dll_dirs
@ -131,9 +142,36 @@ def _RegistryGetValue(key, value):
def GetVisualStudioVersion():
"""Return GYP_MSVS_VERSION of Visual Studio.
"""Return best available version of Visual Studio.
"""
return os.environ.get('GYP_MSVS_VERSION', CURRENT_DEFAULT_TOOLCHAIN_VERSION)
env_version = os.environ.get('GYP_MSVS_VERSION')
if env_version:
return env_version
supported_versions = MSVS_VERSIONS.keys()
# VS installed in depot_tools for Googlers
if bool(int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', '1'))):
return supported_versions[0]
# VS installed in system for external developers
supported_versions_str = ', '.join('{} ({})'.format(v,k)
for k,v in MSVS_VERSIONS.items())
available_versions = []
for version in supported_versions:
for path in (
os.environ.get('vs%s_install' % version),
os.path.expandvars('%ProgramFiles(x86)%' +
'/Microsoft Visual Studio/%s' % version)):
if path and os.path.exists(path):
available_versions.append(version)
break
if not available_versions:
raise Exception('No supported Visual Studio can be found.'
' Supported versions are: %s.' % supported_versions_str)
return available_versions[0]
def DetectVisualStudioPath():
@ -143,31 +181,30 @@ def DetectVisualStudioPath():
# Note that this code is used from
# build/toolchain/win/setup_toolchain.py as well.
version_as_year = GetVisualStudioVersion()
year_to_version = {
'2017': '15.0',
}
if version_as_year not in year_to_version:
raise Exception(('Visual Studio version %s (from GYP_MSVS_VERSION)'
' not supported. Supported versions are: %s') % (
version_as_year, ', '.join(year_to_version.keys())))
if version_as_year == '2017':
# The VC++ 2017 install location needs to be located using COM instead of
# the registry. For details see:
# https://blogs.msdn.microsoft.com/heaths/2016/09/15/changes-to-visual-studio-15-setup/
# For now we use a hardcoded default with an environment variable override.
for path in (
os.environ.get('vs2017_install'),
os.path.expandvars('%ProgramFiles(x86)%'
'/Microsoft Visual Studio/2017/Enterprise'),
os.path.expandvars('%ProgramFiles(x86)%'
'/Microsoft Visual Studio/2017/Professional'),
os.path.expandvars('%ProgramFiles(x86)%'
'/Microsoft Visual Studio/2017/Community')):
if path and os.path.exists(path):
return path
raise Exception(('Visual Studio Version %s (from GYP_MSVS_VERSION)'
' not found.') % (version_as_year))
# The VC++ >=2017 install location needs to be located using COM instead of
# the registry. For details see:
# https://blogs.msdn.microsoft.com/heaths/2016/09/15/changes-to-visual-studio-15-setup/
# For now we use a hardcoded default with an environment variable override.
for path in (
os.environ.get('vs%s_install' % version_as_year),
os.path.expandvars('%ProgramFiles(x86)%' +
'/Microsoft Visual Studio/%s/Enterprise' %
version_as_year),
os.path.expandvars('%ProgramFiles(x86)%' +
'/Microsoft Visual Studio/%s/Professional' %
version_as_year),
os.path.expandvars('%ProgramFiles(x86)%' +
'/Microsoft Visual Studio/%s/Community' %
version_as_year),
os.path.expandvars('%ProgramFiles(x86)%' +
'/Microsoft Visual Studio/%s/Preview' %
version_as_year)):
if path and os.path.exists(path):
return path
raise Exception('Visual Studio Version %s (from GYP_MSVS_VERSION)'
' not found.' % version_as_year)
def _CopyRuntimeImpl(target, source, verbose=True):
@ -180,7 +217,7 @@ def _CopyRuntimeImpl(target, source, verbose=True):
(not os.path.isfile(target) or
abs(os.stat(target).st_mtime - os.stat(source).st_mtime) >= 0.01)):
if verbose:
print 'Copying %s to %s...' % (source, target)
print('Copying %s to %s...' % (source, target))
if os.path.exists(target):
# Make the file writable so that we can delete it now, and keep it
# readable.
@ -191,10 +228,37 @@ def _CopyRuntimeImpl(target, source, verbose=True):
# keep it readable.
os.chmod(target, stat.S_IWRITE | stat.S_IREAD)
def _SortByHighestVersionNumberFirst(list_of_str_versions):
"""This sorts |list_of_str_versions| according to version number rules
so that version "1.12" is higher than version "1.9". Does not work
with non-numeric versions like 1.4.a8 which will be higher than
1.4.a12. It does handle the versions being embedded in file paths.
"""
def to_int_if_int(x):
try:
return int(x)
except ValueError:
return x
def to_number_sequence(x):
part_sequence = re.split(r'[\\/\.]', x)
return [to_int_if_int(x) for x in part_sequence]
list_of_str_versions.sort(key=to_number_sequence, reverse=True)
def _CopyUCRTRuntime(target_dir, source_dir, target_cpu, dll_pattern, suffix):
"""Copy both the msvcp and vccorlib runtime DLLs, only if the target doesn't
exist, but the target directory does exist."""
if target_cpu == 'arm64':
# Windows ARM64 VCRuntime is located at {toolchain_root}/VC/Redist/MSVC/
# {x.y.z}/[debug_nonredist/]arm64/Microsoft.VC141.CRT/.
vc_redist_root = FindVCRedistRoot()
if suffix.startswith('.'):
source_dir = os.path.join(vc_redist_root,
'arm64', 'Microsoft.VC141.CRT')
else:
source_dir = os.path.join(vc_redist_root, 'debug_nonredist',
'arm64', 'Microsoft.VC141.DebugCRT')
for file_part in ('msvcp', 'vccorlib', 'vcruntime'):
dll = dll_pattern % file_part
target = os.path.join(target_dir, dll)
@ -212,8 +276,14 @@ def _CopyUCRTRuntime(target_dir, source_dir, target_cpu, dll_pattern, suffix):
# ARM64 doesn't have a redist for the ucrt DLLs because they are always
# present in the OS.
if target_cpu != 'arm64':
ucrt_dll_dirs = os.path.join(win_sdk_dir, 'Redist', 'ucrt', 'DLLs',
target_cpu)
# Starting with the 10.0.17763 SDK the ucrt files are in a version-named
# directory - this handles both cases.
redist_dir = os.path.join(win_sdk_dir, 'Redist')
version_dirs = glob.glob(os.path.join(redist_dir, '10.*'))
if len(version_dirs) > 0:
_SortByHighestVersionNumberFirst(version_dirs)
redist_dir = version_dirs[0]
ucrt_dll_dirs = os.path.join(redist_dir, 'ucrt', 'DLLs', target_cpu)
ucrt_files = glob.glob(os.path.join(ucrt_dll_dirs, 'api-ms-win-*.dll'))
assert len(ucrt_files) > 0
for ucrt_src_file in ucrt_files:
@ -222,73 +292,60 @@ def _CopyUCRTRuntime(target_dir, source_dir, target_cpu, dll_pattern, suffix):
_CopyRuntimeImpl(ucrt_dst_file, ucrt_src_file, False)
# We must copy ucrtbase.dll for x64/x86, and ucrtbased.dll for all CPU types.
if target_cpu != 'arm64' or not suffix.startswith('.'):
if not suffix.startswith('.'):
# ucrtbased.dll is located at {win_sdk_dir}/bin/{a.b.c.d}/{target_cpu}/
# ucrt/.
sdk_redist_root = os.path.join(win_sdk_dir, 'bin')
sdk_bin_sub_dirs = os.listdir(sdk_redist_root)
# Select the most recent SDK if there are multiple versions installed.
_SortByHighestVersionNumberFirst(sdk_bin_sub_dirs)
for directory in sdk_bin_sub_dirs:
sdk_redist_root_version = os.path.join(sdk_redist_root, directory)
if not os.path.isdir(sdk_redist_root_version):
continue
if re.match(r'10\.\d+\.\d+\.\d+', directory):
source_dir = os.path.join(sdk_redist_root_version, target_cpu, 'ucrt')
break
_CopyRuntimeImpl(os.path.join(target_dir, 'ucrtbase' + suffix),
os.path.join(source_dir, 'ucrtbase' + suffix))
def FindVCToolsRoot():
"""In VS2017 the PGO runtime dependencies are located in
{toolchain_root}/VC/Tools/MSVC/{x.y.z}/bin/Host{target_cpu}/{target_cpu}/, the
{version_number} part is likely to change in case of a minor update of the
toolchain so we don't hardcode this value here (except for the major number).
This returns the '{toolchain_root}/VC/Tools/MSVC/{x.y.z}/bin/' path.
This function should only be called when using VS2017.
def FindVCComponentRoot(component):
"""Find the most recent Tools or Redist or other directory in an MSVC install.
Typical results are {toolchain_root}/VC/{component}/MSVC/{x.y.z}. The {x.y.z}
version number part changes frequently so the highest version number found is
used.
"""
assert GetVisualStudioVersion() == '2017'
SetEnvironmentAndGetRuntimeDllDirs()
assert ('GYP_MSVS_OVERRIDE_PATH' in os.environ)
vc_tools_msvc_root = os.path.join(os.environ['GYP_MSVS_OVERRIDE_PATH'],
'VC', 'Tools', 'MSVC')
for directory in os.listdir(vc_tools_msvc_root):
if not os.path.isdir(os.path.join(vc_tools_msvc_root, directory)):
vc_component_msvc_root = os.path.join(os.environ['GYP_MSVS_OVERRIDE_PATH'],
'VC', component, 'MSVC')
vc_component_msvc_contents = os.listdir(vc_component_msvc_root)
# Select the most recent toolchain if there are several.
_SortByHighestVersionNumberFirst(vc_component_msvc_contents)
for directory in vc_component_msvc_contents:
if not os.path.isdir(os.path.join(vc_component_msvc_root, directory)):
continue
if re.match('14\.\d+\.\d+', directory):
return os.path.join(vc_tools_msvc_root, directory, 'bin')
raise Exception('Unable to find the VC tools directory.')
if re.match(r'14\.\d+\.\d+', directory):
return os.path.join(vc_component_msvc_root, directory)
raise Exception('Unable to find the VC %s directory.' % component)
def _CopyPGORuntime(target_dir, target_cpu):
"""Copy the runtime dependencies required during a PGO build.
def FindVCRedistRoot():
"""In >=VS2017, Redist binaries are located in
{toolchain_root}/VC/Redist/MSVC/{x.y.z}/{target_cpu}/.
This returns the '{toolchain_root}/VC/Redist/MSVC/{x.y.z}/' path.
"""
env_version = GetVisualStudioVersion()
# These dependencies will be in a different location depending on the version
# of the toolchain.
if env_version == '2017':
pgo_runtime_root = FindVCToolsRoot()
assert pgo_runtime_root
# There's no version of pgosweep.exe in HostX64/x86, so we use the copy
# from HostX86/x86.
pgo_x86_runtime_dir = os.path.join(pgo_runtime_root, 'HostX86', 'x86')
pgo_x64_runtime_dir = os.path.join(pgo_runtime_root, 'HostX64', 'x64')
pgo_arm64_runtime_dir = os.path.join(pgo_runtime_root, 'arm64')
else:
raise Exception('Unexpected toolchain version: %s.' % env_version)
# We need to copy 2 runtime dependencies used during the profiling step:
# - pgort140.dll: runtime library required to run the instrumented image.
# - pgosweep.exe: executable used to collect the profiling data
pgo_runtimes = ['pgort140.dll', 'pgosweep.exe']
for runtime in pgo_runtimes:
if target_cpu == 'x86':
source = os.path.join(pgo_x86_runtime_dir, runtime)
elif target_cpu == 'x64':
source = os.path.join(pgo_x64_runtime_dir, runtime)
elif target_cpu == 'arm64':
source = os.path.join(pgo_arm64_runtime_dir, runtime)
else:
raise NotImplementedError('Unexpected target_cpu value: ' + target_cpu)
if not os.path.exists(source):
raise Exception('Unable to find %s.' % source)
_CopyRuntimeImpl(os.path.join(target_dir, runtime), source)
return FindVCComponentRoot('Redist')
def _CopyRuntime(target_dir, source_dir, target_cpu, debug):
"""Copy the VS runtime DLLs, only if the target doesn't exist, but the target
directory does exist. Handles VS 2015 and VS 2017."""
directory does exist. Handles VS 2015, 2017 and 2019."""
suffix = 'd.dll' if debug else '.dll'
# VS 2017 uses the same CRT DLLs as VS 2015.
# VS 2015, 2017 and 2019 use the same CRT DLLs.
_CopyUCRTRuntime(target_dir, source_dir, target_cpu, '%s140' + suffix,
suffix)
@ -297,7 +354,7 @@ def CopyDlls(target_dir, configuration, target_cpu):
"""Copy the VS runtime DLLs into the requested directory as needed.
configuration is one of 'Debug' or 'Release'.
target_cpu is one of 'x86' or 'x64'.
target_cpu is one of 'x86', 'x64' or 'arm64'.
The debug configuration gets both the debug and release DLLs; the
release config only the latter.
@ -318,16 +375,13 @@ def CopyDlls(target_dir, configuration, target_cpu):
_CopyRuntime(target_dir, runtime_dir, target_cpu, debug=False)
if configuration == 'Debug':
_CopyRuntime(target_dir, runtime_dir, target_cpu, debug=True)
else:
_CopyPGORuntime(target_dir, target_cpu)
_CopyDebugger(target_dir, target_cpu)
def _CopyDebugger(target_dir, target_cpu):
"""Copy dbghelp.dll and dbgcore.dll into the requested directory as needed.
target_cpu is one of 'x86' or 'x64'.
target_cpu is one of 'x86', 'x64' or 'arm64'.
dbghelp.dll is used when Chrome needs to symbolize stacks. Copying this file
from the SDK directory avoids using the system copy of dbghelp.dll which then
@ -353,7 +407,7 @@ def _CopyDebugger(target_dir, target_cpu):
# TODO(crbug.com/773476): remove version requirement.
raise Exception('%s not found in "%s"\r\nYou must install the '
'"Debugging Tools for Windows" feature from the Windows'
' 10 SDK. You must use v10.0.17134.0. of the SDK'
' 10 SDK.'
% (debug_file, full_path))
target_path = os.path.join(target_dir, debug_file)
_CopyRuntimeImpl(target_path, full_path)
@ -364,9 +418,10 @@ def _GetDesiredVsToolchainHashes():
to build with."""
env_version = GetVisualStudioVersion()
if env_version == '2017':
# VS 2017 Update 7.1 (15.7.1) with 10.0.17134.12 SDK, rebuilt with
# dbghelp.dll fix.
toolchain_hash = '3bc0ec615cf20ee342f3bc29bc991b5ad66d8d2c'
# VS 2017 Update 9 (15.9.12) with 10.0.18362 SDK, 10.0.17763 version of
# Debuggers, and 10.0.17134 version of d3dcompiler_47.dll, with ARM64
# libraries.
toolchain_hash = '418b3076791776573a815eb298c8aa590307af63'
# Third parties that do not have access to the canonical toolchain can map
# canonical toolchain version to their own toolchain versions.
toolchain_hash_mapping_key = 'GYP_MSVS_HASH_%s' % toolchain_hash
@ -387,13 +442,15 @@ def ShouldUpdateToolchain():
return version != env_version
def Update(force=False):
def Update(force=False, no_download=False):
"""Requests an update of the toolchain to the specific hashes we have at
this revision. The update outputs a .json of the various configuration
information required to pass to gyp which we use in |GetToolchainDir()|.
If no_download is true then the toolchain will be configured if present but
will not be downloaded.
"""
if force != False and force != '--force':
print >>sys.stderr, 'Unknown parameter "%s"' % force
print('Unknown parameter "%s"' % force, file=sys.stderr)
return 1
if force == '--force' or os.path.exists(json_data_file):
force = True
@ -441,6 +498,8 @@ def Update(force=False):
] + _GetDesiredVsToolchainHashes()
if force:
get_toolchain_args.append('--force')
if no_download:
get_toolchain_args.append('--no-download')
subprocess.check_call(get_toolchain_args)
return 0
@ -473,17 +532,15 @@ def GetToolchainDir():
runtime_dll_dirs = SetEnvironmentAndGetRuntimeDllDirs()
win_sdk_dir = SetEnvironmentAndGetSDKDir()
print '''vs_path = %s
print('''vs_path = %s
sdk_path = %s
vs_version = %s
wdk_dir = %s
runtime_dirs = %s
''' % (
ToGNString(NormalizePath(os.environ['GYP_MSVS_OVERRIDE_PATH'])),
ToGNString(win_sdk_dir),
ToGNString(GetVisualStudioVersion()),
ToGNString(NormalizePath(os.environ.get('WDK_DIR', ''))),
ToGNString(os.path.pathsep.join(runtime_dll_dirs or ['None'])))
''' % (ToGNString(NormalizePath(os.environ['GYP_MSVS_OVERRIDE_PATH'])),
ToGNString(win_sdk_dir), ToGNString(GetVisualStudioVersion()),
ToGNString(NormalizePath(os.environ.get('WDK_DIR', ''))),
ToGNString(os.path.pathsep.join(runtime_dll_dirs or ['None']))))
def main():
@ -493,7 +550,7 @@ def main():
'copy_dlls': CopyDlls,
}
if len(sys.argv) < 2 or sys.argv[1] not in commands:
print >>sys.stderr, 'Expected one of: %s' % ', '.join(commands)
print('Expected one of: %s' % ', '.join(commands), file=sys.stderr)
return 1
return commands[sys.argv[1]](*sys.argv[2:])