From 346b59e4a3296b92f2f00c2a697819bc56b7df71 Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 13 Jan 2014 16:50:37 +0000 Subject: [PATCH] deps: update gyp to 1eae492b --- tools/gyp/AUTHORS | 1 + tools/gyp/pylib/gyp/MSVSVersion.py | 6 ++- tools/gyp/pylib/gyp/generator/make.py | 9 ++-- tools/gyp/pylib/gyp/generator/msvs.py | 37 ++++++++++++---- tools/gyp/pylib/gyp/generator/ninja.py | 58 +++++++++++++++++--------- tools/gyp/pylib/gyp/msvs_emulation.py | 23 ++++++---- tools/gyp/pylib/gyp/ordered_dict.py | 2 + tools/gyp/pylib/gyp/win_tool.py | 33 ++++++++------- 8 files changed, 113 insertions(+), 56 deletions(-) diff --git a/tools/gyp/AUTHORS b/tools/gyp/AUTHORS index 234e1483fed356..9389ca0a23e48f 100644 --- a/tools/gyp/AUTHORS +++ b/tools/gyp/AUTHORS @@ -7,4 +7,5 @@ Yandex LLC Steven Knight Ryan Norton +David J. Sankel Eric N. Vander Weele diff --git a/tools/gyp/pylib/gyp/MSVSVersion.py b/tools/gyp/pylib/gyp/MSVSVersion.py index bb30a7ba048938..03b6d8ad42cc7f 100644 --- a/tools/gyp/pylib/gyp/MSVSVersion.py +++ b/tools/gyp/pylib/gyp/MSVSVersion.py @@ -96,9 +96,11 @@ def SetupScript(self, target_arch): else: assert target_arch == 'x64' arg = 'x86_amd64' - if (os.environ.get('PROCESSOR_ARCHITECTURE') == 'AMD64' or + # Use the 64-on-64 compiler if we're not using an express + # edition and we're running on a 64bit OS. + if self.short_name[-1] != 'e' and ( + os.environ.get('PROCESSOR_ARCHITECTURE') == 'AMD64' or os.environ.get('PROCESSOR_ARCHITEW6432') == 'AMD64'): - # Use the 64-on-64 compiler if we can. arg = 'amd64' return [os.path.normpath( os.path.join(self.path, 'VC/vcvarsall.bat')), arg] diff --git a/tools/gyp/pylib/gyp/generator/make.py b/tools/gyp/pylib/gyp/generator/make.py index 8fb856fad24052..b88a433d3d92de 100644 --- a/tools/gyp/pylib/gyp/generator/make.py +++ b/tools/gyp/pylib/gyp/generator/make.py @@ -1951,7 +1951,8 @@ def CalculateMakefilePath(build_file, base_name): # We write the file in the base_path directory. output_file = os.path.join(options.depth, base_path, base_name) if options.generator_output: - output_file = os.path.join(options.generator_output, output_file) + output_file = os.path.join( + options.depth, options.generator_output, base_path, base_name) base_path = gyp.common.RelativePath(os.path.dirname(build_file), options.toplevel_dir) return base_path, output_file @@ -1974,7 +1975,8 @@ def CalculateMakefilePath(build_file, base_name): makefile_path = os.path.join(options.toplevel_dir, makefile_name) if options.generator_output: global srcdir_prefix - makefile_path = os.path.join(options.generator_output, makefile_path) + makefile_path = os.path.join( + options.toplevel_dir, options.generator_output, makefile_name) srcdir = gyp.common.RelativePath(srcdir, options.generator_output) srcdir_prefix = '$(srcdir)/' @@ -2094,7 +2096,8 @@ def CalculateMakefilePath(build_file, base_name): this_make_global_settings = data[build_file].get('make_global_settings', []) assert make_global_settings_array == this_make_global_settings, ( - "make_global_settings needs to be the same for all targets.") + "make_global_settings needs to be the same for all targets. %s vs. %s" % + (this_make_global_settings, make_global_settings)) build_files.add(gyp.common.RelativePath(build_file, options.toplevel_dir)) included_files = data[build_file]['included_files'] diff --git a/tools/gyp/pylib/gyp/generator/msvs.py b/tools/gyp/pylib/gyp/generator/msvs.py index 4ca57169718943..d8e0872c347e97 100644 --- a/tools/gyp/pylib/gyp/generator/msvs.py +++ b/tools/gyp/pylib/gyp/generator/msvs.py @@ -209,13 +209,14 @@ def _FixPaths(paths): def _ConvertSourcesToFilterHierarchy(sources, prefix=None, excluded=None, - list_excluded=True): + list_excluded=True, msvs_version=None): """Converts a list split source file paths into a vcproj folder hierarchy. Arguments: sources: A list of source file paths split. prefix: A list of source file path layers meant to apply to each of sources. excluded: A set of excluded files. + msvs_version: A MSVSVersion object. Returns: A hierarchy of filenames and MSVSProject.Filter objects that matches the @@ -230,6 +231,7 @@ def _ConvertSourcesToFilterHierarchy(sources, prefix=None, excluded=None, if not prefix: prefix = [] result = [] excluded_result = [] + folders = OrderedDict() # Gather files into the final result, excluded, or folders. for s in sources: if len(s) == 1: @@ -238,10 +240,17 @@ def _ConvertSourcesToFilterHierarchy(sources, prefix=None, excluded=None, excluded_result.append(filename) else: result.append(filename) + elif msvs_version and not msvs_version.UsesVcxproj(): + # For MSVS 2008 and earlier, we need to process all files before walking + # the sub folders. + if not folders.get(s[0]): + folders[s[0]] = [] + folders[s[0]].append(s[1:]) else: contents = _ConvertSourcesToFilterHierarchy([s[1:]], prefix + [s[0]], excluded=excluded, - list_excluded=list_excluded) + list_excluded=list_excluded, + msvs_version=msvs_version) contents = MSVSProject.Filter(s[0], contents=contents) result.append(contents) # Add a folder for excluded files. @@ -249,6 +258,17 @@ def _ConvertSourcesToFilterHierarchy(sources, prefix=None, excluded=None, excluded_folder = MSVSProject.Filter('_excluded_files', contents=excluded_result) result.append(excluded_folder) + + if msvs_version and msvs_version.UsesVcxproj(): + return result + + # Populate all the folders. + for f in folders: + contents = _ConvertSourcesToFilterHierarchy(folders[f], prefix=prefix + [f], + excluded=excluded, + list_excluded=list_excluded) + contents = MSVSProject.Filter(f, contents=contents) + result.append(contents) return result @@ -971,8 +991,9 @@ def _GenerateMSVSProject(project, options, version, generator_flags): actions_to_add) list_excluded = generator_flags.get('msvs_list_excluded_files', True) sources, excluded_sources, excluded_idl = ( - _AdjustSourcesAndConvertToFilterHierarchy( - spec, options, project_dir, sources, excluded_sources, list_excluded)) + _AdjustSourcesAndConvertToFilterHierarchy(spec, options, project_dir, + sources, excluded_sources, + list_excluded, version)) # Add in files. missing_sources = _VerifySourcesExist(sources, project_dir) @@ -1416,7 +1437,7 @@ def _PrepareListOfSources(spec, generator_flags, gyp_file): def _AdjustSourcesAndConvertToFilterHierarchy( - spec, options, gyp_dir, sources, excluded_sources, list_excluded): + spec, options, gyp_dir, sources, excluded_sources, list_excluded, version): """Adjusts the list of sources and excluded sources. Also converts the sets to lists. @@ -1427,6 +1448,7 @@ def _AdjustSourcesAndConvertToFilterHierarchy( gyp_dir: The path to the gyp file being processed. sources: A set of sources to be included for this project. excluded_sources: A set of sources to be excluded for this project. + version: A MSVSVersion object. Returns: A trio of (list of sources, list of excluded sources, path of excluded IDL file) @@ -1451,7 +1473,8 @@ def _AdjustSourcesAndConvertToFilterHierarchy( # Convert to folders and the right slashes. sources = [i.split('\\') for i in sources] sources = _ConvertSourcesToFilterHierarchy(sources, excluded=fully_excluded, - list_excluded=list_excluded) + list_excluded=list_excluded, + msvs_version=version) # Prune filters with a single child to flatten ugly directory structures # such as ../../src/modules/module1 etc. @@ -3126,7 +3149,7 @@ def _GenerateMSBuildProject(project, options, version, generator_flags): _AdjustSourcesAndConvertToFilterHierarchy(spec, options, project_dir, sources, excluded_sources, - list_excluded)) + list_excluded, version)) # Don't add actions if we are using an external builder like ninja. if not spec.get('msvs_external_builder'): diff --git a/tools/gyp/pylib/gyp/generator/ninja.py b/tools/gyp/pylib/gyp/generator/ninja.py index d3db2c887685be..74618147452569 100644 --- a/tools/gyp/pylib/gyp/generator/ninja.py +++ b/tools/gyp/pylib/gyp/generator/ninja.py @@ -1037,12 +1037,13 @@ def WriteLinkForArch(self, ninja_file, spec, config_name, config, self.GypPathToNinja, arch) ldflags = env_ldflags + ldflags elif self.flavor == 'win': - manifest_name = self.GypPathToUniqueOutput( + manifest_base_name = self.GypPathToUniqueOutput( self.ComputeOutputFileName(spec)) ldflags, intermediate_manifest, manifest_files = \ self.msvs_settings.GetLdflags(config_name, self.GypPathToNinja, - self.ExpandSpecial, manifest_name, - is_executable, self.toplevel_build) + self.ExpandSpecial, manifest_base_name, + output, is_executable, + self.toplevel_build) ldflags = env_ldflags + ldflags self.WriteVariableList(ninja_file, 'manifests', manifest_files) implicit_deps = implicit_deps.union(manifest_files) @@ -1095,16 +1096,27 @@ def WriteLinkForArch(self, ninja_file, spec, config_name, config, extra_bindings.append(('lib', gyp.common.EncodePOSIXShellArgument(output))) if self.flavor == 'win': - extra_bindings.append(('dll', output)) + extra_bindings.append(('binary', output)) if '/NOENTRY' not in ldflags: self.target.import_lib = output + '.lib' extra_bindings.append(('implibflag', '/IMPLIB:%s' % self.target.import_lib)) + pdbname = self.msvs_settings.GetPDBName( + config_name, self.ExpandSpecial, output + '.pdb') output = [output, self.target.import_lib] + if pdbname: + output.append(pdbname) elif not self.is_mac_bundle: output = [output, output + '.TOC'] else: command = command + '_notoc' + elif self.flavor == 'win': + extra_bindings.append(('binary', output)) + pdbname = self.msvs_settings.GetPDBName( + config_name, self.ExpandSpecial, output + '.pdb') + if pdbname: + output = [output, pdbname] + if len(solibs): extra_bindings.append(('solibs', gyp.common.EncodePOSIXShellList(solibs))) @@ -1545,7 +1557,10 @@ class MEMORYSTATUSEX(ctypes.Structure): mem_limit = max(1, stat.ullTotalPhys / (4 * (2 ** 30))) # total / 4GB hard_cap = max(1, int(os.getenv('GYP_LINK_CONCURRENCY_MAX', 2**32))) - return min(mem_limit, hard_cap) + # return min(mem_limit, hard_cap) + # TODO(scottmg): Temporary speculative fix for OOM on builders + # See http://crbug.com/333000. + return 2 elif sys.platform.startswith('linux'): with open("/proc/meminfo") as meminfo: memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB') @@ -1591,33 +1606,35 @@ def FullLinkCommand(ldcmd, out, binary_type): 'resname': resource_name, 'embed': embed_manifest } rule_name_suffix = _GetWinLinkRuleNameSuffix(embed_manifest) - dlldesc = 'LINK%s(DLL) $dll' % rule_name_suffix.upper() - dllcmd = ('%s gyp-win-tool link-wrapper $arch ' - '$ld /nologo $implibflag /DLL /OUT:$dll ' - '/PDB:$dll.pdb @$dll.rsp' % sys.executable) - dllcmd = FullLinkCommand(dllcmd, '$dll', 'dll') + use_separate_mspdbsrv = ( + int(os.environ.get('GYP_USE_SEPARATE_MSPDBSRV', '0')) != 0) + dlldesc = 'LINK%s(DLL) $binary' % rule_name_suffix.upper() + dllcmd = ('%s gyp-win-tool link-wrapper $arch %s ' + '$ld /nologo $implibflag /DLL /OUT:$binary ' + '@$binary.rsp' % (sys.executable, use_separate_mspdbsrv)) + dllcmd = FullLinkCommand(dllcmd, '$binary', 'dll') master_ninja.rule('solink' + rule_name_suffix, description=dlldesc, command=dllcmd, - rspfile='$dll.rsp', + rspfile='$binary.rsp', rspfile_content='$libs $in_newline $ldflags', restat=True, pool='link_pool') master_ninja.rule('solink_module' + rule_name_suffix, description=dlldesc, command=dllcmd, - rspfile='$dll.rsp', + rspfile='$binary.rsp', rspfile_content='$libs $in_newline $ldflags', restat=True, pool='link_pool') # Note that ldflags goes at the end so that it has the option of # overriding default settings earlier in the command line. - exe_cmd = ('%s gyp-win-tool link-wrapper $arch ' - '$ld /nologo /OUT:$out /PDB:$out.pdb @$out.rsp' % - sys.executable) - exe_cmd = FullLinkCommand(exe_cmd, '$out', 'exe') + exe_cmd = ('%s gyp-win-tool link-wrapper $arch %s ' + '$ld /nologo /OUT:$binary @$binary.rsp' % + (sys.executable, use_separate_mspdbsrv)) + exe_cmd = FullLinkCommand(exe_cmd, '$binary', 'exe') master_ninja.rule('link' + rule_name_suffix, - description='LINK%s $out' % rule_name_suffix.upper(), + description='LINK%s $binary' % rule_name_suffix.upper(), command=exe_cmd, - rspfile='$out.rsp', + rspfile='$binary.rsp', rspfile_content='$in_newline $libs $ldflags', pool='link_pool') @@ -1877,7 +1894,7 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params, master_ninja.rule( 'alink', description='LIB $out', - command=('%s gyp-win-tool link-wrapper $arch ' + command=('%s gyp-win-tool link-wrapper $arch False ' '$ar /nologo /ignore:4221 /OUT:$out @$out.rsp' % sys.executable), rspfile='$out.rsp', @@ -2027,7 +2044,8 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params, this_make_global_settings = data[build_file].get('make_global_settings', []) assert make_global_settings == this_make_global_settings, ( - "make_global_settings needs to be the same for all targets.") + "make_global_settings needs to be the same for all targets. %s vs. %s" % + (this_make_global_settings, make_global_settings)) spec = target_dicts[qualified_target] if flavor == 'mac': diff --git a/tools/gyp/pylib/gyp/msvs_emulation.py b/tools/gyp/pylib/gyp/msvs_emulation.py index 3435bbc524e64a..6428fced00552e 100644 --- a/tools/gyp/pylib/gyp/msvs_emulation.py +++ b/tools/gyp/pylib/gyp/msvs_emulation.py @@ -317,15 +317,20 @@ def GetOutputName(self, config, expand_special): output_file, config=config)) return output_file - def GetPDBName(self, config, expand_special): - """Gets the explicitly overridden pdb name for a target or returns None - if it's not overridden.""" + def GetPDBName(self, config, expand_special, default): + """Gets the explicitly overridden pdb name for a target or returns + default if it's not overridden, or if no pdb will be generated.""" config = self._TargetConfig(config) output_file = self._Setting(('VCLinkerTool', 'ProgramDatabaseFile'), config) - if output_file: - output_file = expand_special(self.ConvertVSMacros( - output_file, config=config)) - return output_file + generate_debug_info = self._Setting( + ('VCLinkerTool', 'GenerateDebugInformation'), config) + if generate_debug_info: + if output_file: + return expand_special(self.ConvertVSMacros(output_file, config=config)) + else: + return default + else: + return None def GetCflags(self, config): """Returns the flags that need to be added to .c and .cc compilations.""" @@ -454,7 +459,7 @@ def GetPGDName(self, config, expand_special): return output_file def GetLdflags(self, config, gyp_to_build_path, expand_special, - manifest_base_name, is_executable, build_dir): + manifest_base_name, output_name, is_executable, build_dir): """Returns the flags that need to be added to link commands, and the manifest files.""" config = self._TargetConfig(config) @@ -472,7 +477,7 @@ def GetLdflags(self, config, gyp_to_build_path, expand_special, out = self.GetOutputName(config, expand_special) if out: ldflags.append('/OUT:' + out) - pdb = self.GetPDBName(config, expand_special) + pdb = self.GetPDBName(config, expand_special, output_name + '.pdb') if pdb: ldflags.append('/PDB:' + pdb) pgd = self.GetPGDName(config, expand_special) diff --git a/tools/gyp/pylib/gyp/ordered_dict.py b/tools/gyp/pylib/gyp/ordered_dict.py index a3609add40d405..a1e89f9199087d 100644 --- a/tools/gyp/pylib/gyp/ordered_dict.py +++ b/tools/gyp/pylib/gyp/ordered_dict.py @@ -166,6 +166,8 @@ def iteritems(self): for k in self: yield (k, self[k]) + # Suppress 'OrderedDict.update: Method has no argument': + # pylint: disable=E0211 def update(*args, **kwds): '''od.update(E, **F) -> None. Update od from dict/iterable E and F. diff --git a/tools/gyp/pylib/gyp/win_tool.py b/tools/gyp/pylib/gyp/win_tool.py index 1634ff93ddc6a9..e9d7df0bb771b5 100755 --- a/tools/gyp/pylib/gyp/win_tool.py +++ b/tools/gyp/pylib/gyp/win_tool.py @@ -18,9 +18,9 @@ BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -# A regex matching an argument corresponding to a PDB filename passed as an -# argument to link.exe. -_LINK_EXE_PDB_ARG = re.compile('/PDB:(?P.+\.exe\.pdb)$', re.IGNORECASE) +# A regex matching an argument corresponding to the output filename passed to +# link.exe. +_LINK_EXE_OUT_ARG = re.compile('/OUT:(?P.+)$', re.IGNORECASE) def main(args): executor = WinTool() @@ -33,25 +33,22 @@ class WinTool(object): """This class performs all the Windows tooling steps. The methods can either be executed directly, or dispatched from an argument list.""" - def _MaybeUseSeparateMspdbsrv(self, env, args): - """Allows to use a unique instance of mspdbsrv.exe for the linkers linking - an .exe target if GYP_USE_SEPARATE_MSPDBSRV has been set.""" - if not os.environ.get('GYP_USE_SEPARATE_MSPDBSRV'): - return - + def _UseSeparateMspdbsrv(self, env, args): + """Allows to use a unique instance of mspdbsrv.exe per linker instead of a + shared one.""" if len(args) < 1: raise Exception("Not enough arguments") if args[0] != 'link.exe': return - # Checks if this linker produces a PDB for an .exe target. If so use the - # name of this PDB to generate an endpoint name for mspdbsrv.exe. + # Use the output filename passed to the linker to generate an endpoint name + # for mspdbsrv.exe. endpoint_name = None for arg in args: - m = _LINK_EXE_PDB_ARG.match(arg) + m = _LINK_EXE_OUT_ARG.match(arg) if m: - endpoint_name = '%s_%d' % (m.group('pdb'), os.getpid()) + endpoint_name = '%s_%d' % (m.group('out'), os.getpid()) break if endpoint_name is None: @@ -99,13 +96,14 @@ def ExecRecursiveMirror(self, source, dest): else: shutil.copy2(source, dest) - def ExecLinkWrapper(self, arch, *args): + def ExecLinkWrapper(self, arch, use_separate_mspdbsrv, *args): """Filter diagnostic output from link that looks like: ' Creating library ui.dll.lib and object ui.dll.exp' This happens when there are exports from the dll or exe. """ env = self._GetEnv(arch) - self._MaybeUseSeparateMspdbsrv(env, args) + if use_separate_mspdbsrv == 'True': + self._UseSeparateMspdbsrv(env, args) link = subprocess.Popen(args, shell=True, env=env, @@ -280,6 +278,11 @@ def ExecActionWrapper(self, arch, rspfile, *dir): """Runs an action command line from a response file using the environment for |arch|. If |dir| is supplied, use that as the working directory.""" env = self._GetEnv(arch) + # TODO(scottmg): This is a temporary hack to get some specific variables + # through to actions that are set after gyp-time. http://crbug.com/333738. + for k, v in os.environ.iteritems(): + if k not in env: + env[k] = v args = open(rspfile).read() dir = dir[0] if dir else None return subprocess.call(args, shell=True, env=env, cwd=dir)