From 217ad69e40ab9d291a240c5fb6ec342c2ffb4c67 Mon Sep 17 00:00:00 2001 From: Ryan Saunders Date: Fri, 25 Feb 2022 23:12:51 -0800 Subject: [PATCH 01/22] Initial implementation of vcpkg builder --- SCons/Tool/VCPkgTests.py | 139 +++++++++++++ SCons/Tool/__init__.py | 2 + SCons/Tool/default.xml | 2 +- SCons/Tool/vcpkg.py | 437 +++++++++++++++++++++++++++++++++++++++ SCons/Tool/vcpkg.xml | 92 +++++++++ bin/files | 1 + doc/generated/tools.mod | 2 + 7 files changed, 674 insertions(+), 1 deletion(-) create mode 100644 SCons/Tool/VCPkgTests.py create mode 100644 SCons/Tool/vcpkg.py create mode 100644 SCons/Tool/vcpkg.xml diff --git a/SCons/Tool/VCPkgTests.py b/SCons/Tool/VCPkgTests.py new file mode 100644 index 0000000000..7f41a08747 --- /dev/null +++ b/SCons/Tool/VCPkgTests.py @@ -0,0 +1,139 @@ +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import os +from shutil import rmtree +from pathlib import Path +import unittest + +import TestUnit + +import SCons.Errors +import SCons.Tool.vcpkg +from SCons.Environment import Environment + +class VCPkgTestCase(unittest.TestCase): + def make_mock_vcpkg_dir(self, include_vcpkg_exe = False): + """Creates a mock vcpkg directory under the temp directory and returns its path""" + + vcpkg_root = os.environ['TEMP'] + '/scons_vcpkg_test' + + # Delete it if it already exists + rmtree(vcpkg_root, ignore_errors = True) + + os.mkdir(vcpkg_root) + + # Ensure that the .vcpkg-root sentinel file exists + Path(vcpkg_root + '/.vcpkg-root').touch() + + if include_vcpkg_exe: + if os.name == 'nt': + Path(vcpkg_root + '/vcpkg.exe').touch() + else: + Path(vcpkg_root + '/vcpkg').touch() + + return vcpkg_root + + + def test_VCPKGROOT(self): + """Test that VCPkg() fails with an exception if the VCPKGROOT environment variable is unset or invalid""" + + env = Environment(tools=['default','vcpkg']) + + # VCPKGROOT unset (should fail) + exc_caught = None + try: + if 'VCPKGROOT' in env: + del env['VCPKGROOT'] + env.VCPkg('pretend_package') + except SCons.Errors.UserError as e: + exc_caught = 1 + assert "$VCPKGROOT must be set" in str(e), e + assert exc_caught, "did not catch expected UserError" + + # VCPKGROOT pointing to a bogus path (should fail) + exc_caught = None + try: + env['VCPKGROOT'] = '/usr/bin/phony/path' + env.VCPkg('pretend_package') + except SCons.Errors.UserError as e: + exc_caught = 1 + assert "$VCPKGROOT must point to" in str(e), e + assert exc_caught, "did not catch expected UserError" + + # VCPKGROOT pointing to a valid path that is not a vcpkg instance (should fail) + exc_caught = None + try: + env['VCPKGROOT'] = '#/' + env.VCPkg('pretend_package') + except SCons.Errors.UserError as e: + exc_caught = 1 + assert "$VCPKGROOT must point to" in str(e), e + assert exc_caught, "did not catch expected UserError" + + # VCPKGROOT pointing to a mock vcpkg instance (should succeed) + env['VCPKGROOT'] = self.make_mock_vcpkg_dir(include_vcpkg_exe = True) + orig_call_vcpkg = SCons.Tool.vcpkg._call_vcpkg + orig_get_package_version = SCons.Tool.vcpkg._get_package_version + orig_read_vcpkg_file_list = SCons.Tool.vcpkg._read_vcpkg_file_list + installed_package = False + def mock_call_vcpkg_exe(env, params, check_output = False): + if 'install' in params and 'pretend_package' in params: + installed_pacakge = True + return 0 + if 'depend-info' in params and check_output: + return params[1] + ':' + + if check_output: + return '' + else: + return 0 + def mock_get_package_version(env, spec): + assert spec == 'pretend_package', "queried for unexpected package '" + spec + "'" + return '1.0.0' + def mock_read_vcpkg_file_list(env, list_file): + return [env.File('$VCPKGROOT/installed/x64-windows/lib/pretend_package.lib')] + + SCons.Tool.vcpkg._call_vcpkg = mock_call_vcpkg_exe + SCons.Tool.vcpkg._get_package_version = mock_get_package_version + SCons.Tool.vcpkg._read_vcpkg_file_list = mock_read_vcpkg_file_list + try: + env.VCPkg('pretend_package') + finally: + rmtree(env['VCPKGROOT']) + SCons.Tool.vcpkg._call_vcpkg = orig_call_vcpkg + SCons.Tool.vcpkg._get_pacakge_version = orig_get_package_version + SCons.Tool.vcpkg._get_pacakge_version = orig_read_vcpkg_file_list + + +if __name__ == "__main__": + suite = unittest.makeSuite(VCPkgTestCase, 'test_') + TestUnit.run(suite) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/Tool/__init__.py b/SCons/Tool/__init__.py index c7003f03d5..710252fd30 100644 --- a/SCons/Tool/__init__.py +++ b/SCons/Tool/__init__.py @@ -815,6 +815,8 @@ def tool_list(platform, env): 'tar', 'zip', # File builders (text) 'textfile', + # Package management + 'vcpkg', ], env) tools = [ diff --git a/SCons/Tool/default.xml b/SCons/Tool/default.xml index 15dc2f7fea..d0d0124425 100644 --- a/SCons/Tool/default.xml +++ b/SCons/Tool/default.xml @@ -55,7 +55,7 @@ are selected if their respective conditions are met: &t-link-jar;, &t-link-javac;, &t-link-javah;, &t-link-rmic;, &t-link-dvipdf;, &t-link-dvips;, &t-link-gs;, &t-link-tex;, &t-link-latex;, &t-link-pdflatex;, &t-link-pdftex;, -&t-link-tar;, &t-link-zip;, &t-link-textfile;. +&t-link-tar;, &t-link-zip;, &t-link-textfile; &t-link-vcpkg;. diff --git a/SCons/Tool/vcpkg.py b/SCons/Tool/vcpkg.py new file mode 100644 index 0000000000..f6bc88a09c --- /dev/null +++ b/SCons/Tool/vcpkg.py @@ -0,0 +1,437 @@ +# -*- coding: utf-8; -*- + +"""SCons.Tool.vcpkg + +Tool-specific initialization for vcpkg. + +There normally shouldn't be any need to import this module directly. +It will usually be imported through the generic SCons.Tool.Tool() +selection method. + + +TODO: + * ensure that upgrading to a new package version works after a repo update + * ensure Linux works + * unit tests + * verify that feature supersetting works + * print errors from vcpkg on console + * print vcpkg commands unless in silent mode + * debug libs + * install/symlink built dlls into variant dir + * flag detection + +""" + +# +# Copyright (c) 2001 - 2019 The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import re +import subprocess +import SCons.Builder +import SCons.Node.Python +import SCons.Script +from SCons.Errors import UserError, InternalError, BuildError + +def _get_built_vcpkg_full_version(vcpkg_exe): + """Runs 'vcpkg version' and parses the version string from the output""" + + if not vcpkg_exe.exists(): + raise InternalError(vcpkg_exe.get_path() + ' does not exist') + + line = subprocess.check_output([vcpkg_exe.get_abspath(), 'version'], text = True) + match_version = re.search(r' version (\S+)', line) + if (match_version): + version = match_version.group(1) + return version + raise InternalError(vcpkg_exe.get_path() + ': failed to parse version string') + + +def _get_built_vcpkg_base_version(vcpkg_exe): + """Returns just the base version from 'vcpkg version' (i.e., with any '-whatever' suffix stripped off""" + full_version = _get_built_vcpkg_full_version(vcpkg_exe) + dash_pos = full_version.find('-') + if dash_pos != -1: + return full_version[0:dash_pos] + return full_version + + +def _get_source_vcpkg_version(vcpkg_root): + """Parses the vcpkg source code version out of VERSION.txt""" + + if not vcpkg_root.exists(): + raise InternalError(vcpkg_root.get_path() + ' does not exist') + + version_txt = vcpkg_root.File('toolsrc/VERSION.txt') + if not version_txt.exists(): + raise InternalError(version_txt.get_path() + ' does not exist; did something change in the vcpkg directory structure?') + + for line in open(version_txt.get_abspath()): + match_version = re.match(r'^"(.*)"$', line) + if (match_version): + return match_version.group(1) + raise InternalError(version_txt.get_path() + ": Failed to parse vcpkg version string") + + +def _bootstrap_vcpkg(env): + """Ensures that VCPKGROOT is set, and that the vcpkg executable has been built.""" + + # If we've already done these checks, return the previous result + if '_VCPKG_EXE' in env: + return env['_VCPKG_EXE'] + + if 'VCPKGROOT' not in env: + raise UserError('$VCPKGROOT must be set in order to use the VCPkg builder') + + vcpkgroot_dir = env.Dir(env.subst('$VCPKGROOT')) + sentinel_file = vcpkgroot_dir.File('.vcpkg-root') + if not sentinel_file.exists(): + raise UserError(sentinel_file.get_path() + ' does not exist...$VCPKGROOT must point to the root directory of a VCPkg repository containing this file') + + if env['PLATFORM'] == 'win32': + vcpkg_exe = vcpkgroot_dir.File('vcpkg.exe') + bootstrap_vcpkg_script = vcpkgroot_dir.File('bootstrap-vcpkg.bat') + elif env['PLATFORM'] == 'darwin' or env['PLATFORM'] == 'posix': + vcpkg_exe = vcpkgroot_dir.File('vcpkg') + bootstrap_vcpkg_script = vcpkgroot_dir.File('bootstrap-vcpkg.sh') + else: + raise UserError('This architecture/platform (%s/%s) is currently unsupported with VCPkg' % (env['TARGET_ARCH'], env['PLATFORM'])) + + # We need to build vcpkg.exe if it doesn't exist, or if it has a different "base" version than the source code + build_vcpkg = not vcpkg_exe.exists() + if not build_vcpkg: + built_version = _get_built_vcpkg_base_version(vcpkg_exe) + source_version = _get_source_vcpkg_version(vcpkgroot_dir) + if built_version != source_version and not built_version.startswith(source_version + '-'): + print(vcpkg_exe.get_path() + ' (version ' + built_version + ') is out of date (source version: ' + source_version + '); rebuilding') + build_vcpkg = True + else: + print(vcpkg_exe.get_path() + ' (version ' + built_version + ') is up-to-date') + + # If we need to build, do it now, and ensure that it built + if build_vcpkg: + if not bootstrap_vcpkg_script.exists(): + raise InternalError(bootstrap_vcpkg_script.get_path() + ' does not exist...what gives?') + print('Building vcpkg binary') + if subprocess.call(bootstrap_vcpkg_script.get_abspath()) != 0: + raise BuildError(bootstrap_vcpkg_script.get_path() + ' failed') + vcpkg_exe.clear_memoized_values() + if not vcpkg_exe.exists(): + raise BuildError(bootstrap_vcpkg_script.get_path() + ' failed to create ' + vcpkg_exe.get_path()) + + # Remember this, so we don't run these checks again + env['_VCPKG_EXE'] = vcpkg_exe + return vcpkg_exe + + +def _call_vcpkg(env, params, check_output = False): + """Run the vcpkg executable wth the given set of parameters, optionally returning its standard output as a string. If the vcpkg executable is not yet built, or out of date, it will be rebuilt.""" + + vcpkg_exe = _bootstrap_vcpkg(env) + command_line = [vcpkg_exe.get_abspath()] + params + print(str(command_line)) + if check_output: + return str(subprocess.check_output(command_line, universal_newlines = True)) + else: + return subprocess.call(args = command_line, universal_newlines = True) + + +def _get_vcpkg_triplet(env, static): + """Computes the appropriate VCPkg 'triplet' for the current build environment""" + + platform = env['PLATFORM'] + + if 'TARGET_ARCH' in env: + arch = env['TARGET_ARCH'] + else: + arch = env['HOST_ARCH'] + + if platform == 'win32': + if arch == 'x86' or arch == 'i386': + return 'x86-windows-static' if static else 'x86-windows' + elif arch == 'x86_64' or arch == 'x64' or arch == 'amd64': + return 'x64-windows-static' if static else 'x64-windows' + elif arch == 'arm': + return 'arm-windows' + elif arch == 'arm64': + return 'arm64-windows' + elif platform == 'darwin': + if arch == 'x86_64': + return 'x64-osx' + elif platform == 'posix': +# if arch == 'x86_64': + return 'x64-linux' + + raise UserError('This architecture/platform (%s/%s) is currently unsupported with VCPkg' % (arch, platform)) + + +def _parse_build_depends(s): + """Given a string from the Build-Depends field of a CONTROL file, parse just the package name from it""" + + match_name = re.match(r'^\s*([^\(\)\[\] ]+)\s*(\[.*\])?$', s) + if not match_name: + print('Failed to parse package name from string "' + s + '"') + return s + + return match_name.group(1), match_name.group(2) + + +def _read_vcpkg_file_list(env, list_file): + """Read a .list file for a built package and return a list of File nodes for the files it contains (ignoring directories)""" + + files = [] + for line in open(list_file.get_abspath()): + if not line.rstrip().endswith('/'): + files.append(env.File('$VCPKGROOT/installed/' + line)) + return files + + +def _get_package_version(env, spec): + """Read the CONTROL file for a package, returning (version, depends[])""" + + name = spec.split('[')[0] + control_file = env.File('$VCPKGROOT/ports/' + name + '/CONTROL') + version = None + for line in open(control_file.get_abspath()): + match_version = re.match(r'^Version: (\S+)$', line) + if match_version: + version = match_version.group(1) + break + if version is None: + raise InternalError('Failed to parse package version from control file "' + control_file.get_abspath() + '"') + return version + + +def _get_package_deps(env, spec, static): + """Call 'vcpkg depend-info' to query for the dependencies of a package""" + + # TODO: compute these from the vcpkg base version + _vcpkg_supports_triplet_param = True + _vcpkg_supports_max_recurse_param = True + _vcpkg_supports_no_recurse_param = False + params = ['depend-info'] + + # Try to filter to only first-level dependencies + if _vcpkg_supports_max_recurse_param: + params.append('--max-recurse=0') + elif _vcpkg_supports_no_recurse_param: + params.append('--no-recurse') + + if _vcpkg_supports_triplet_param: + # Append the package spec + triplet + params += ['--triplet', _get_vcpkg_triplet(env, static), spec] + else: + # OK, VCPkg doesn't know about the --triplet param, which means that it also doesn't understnd package specs + # containing feature specifications. So, we'll strip these out (but possibly miss some dependencies). + params.append(spec.split('[')[0]) + + output = _call_vcpkg(env, params, check_output = True) + deps_list = output[output.index(':') + 1:].split(',') + deps_list = list(map(lambda s: s.strip(), deps_list)) + deps_list = list(filter(lambda s: s != "", deps_list)) + return deps_list + + +# Global mapping of previously-computed package-name -> list-file targets. This +# exists because we may discover additional packages that we need to build, based +# on the Build-Depends field in the CONTROL file), and these packages may or may +# not have been explicitly requested by the +# CHECKIN +_package_descriptors_map = {} +_package_targets_map = {} + +class PackageDescriptor(SCons.Node.Python.Value): + + def __init__(self, env, spec, static = False): + _bootstrap_vcpkg(env) + + triplet = _get_vcpkg_triplet(env, static) + env.AppendUnique(CPPPATH = ['$VCPKGROOT/installed/' + triplet + '/include/']) + if env.subst('$VCPKGDEBUG') == 'True': + env.AppendUnique(LIBPATH = '$VCPKGROOT/installed/' + triplet + '/debug/lib/') + env.AppendUnique(PATH = '$VCPKGROOT/installed/' + triplet + '/debug/bin/') + else: + env.AppendUnique(LIBPATH = '$VCPKGROOT/installed/' + triplet + '/lib/') + env.AppendUnique(PATH = '$VCPKGROOT/installed/' + triplet + '/bin/') + + if spec is None or spec == '': + raise ValueError('VCPkg: Package spec must not be empty') + + matches = re.match(r'^([^[]+)(?:\[([^[]+)\])?$', spec) + if not matches: + raise ValueError('VCPkg: Malformed package spec "' + spec + '"') + + name = matches[1] + features = matches[2] + version = _get_package_version(env, name) + depends = _get_package_deps(env, spec, static) + value = { + 'name': name, + 'features': features, + 'static': static, + 'version': version, + } + + super(PackageDescriptor, self).__init__(value) + self.env = env + self.package_deps = depends + + def get_package_string(self): + s = self.value['name'] + if self.value['features'] is not None: + s += '[' + self.value['features'] + ']' + s += ':' + _get_vcpkg_triplet(self.env, self.value['static']) + return s + + def __str__(self): + return self.get_package_string() + + def is_mismatched_version_installed(self): + triplet = _get_vcpkg_triplet(self.env, self.value['static']) + list_file = self.env.File('$VCPKGROOT/installed/vcpkg/info/' + self.value['name'] + '_' + self.value['version'] + '_' + triplet + '.list') + if list_file.exists(): + return False + + installed = _call_vcpkg(self.env, ['list', self.value['name'] + ':' + triplet], check_output = True) + if installed == '' or installed.startswith('No packages are installed'): + return False + + match = re.match(r'^\S+\s+(\S+)\s+\S', installed) + if match[1] == self.value['version']: + raise InternalError("VCPkg thinks " + str(self) + " is installed, but '" + list_file.get_abspath() + "' does not exist") + print("Installed is " + match[1]) + return True + + def target_from_source(self, pre, suf, splitext): + _bootstrap_vcpkg(self.env) + + target = self.env.File('$VCPKGROOT/installed/vcpkg/info/' + self.value['name'] + '_' + self.value['version'] + '_' + _get_vcpkg_triplet(self.env, self.value['static']) + suf) + target.state = SCons.Node.up_to_date + +# print("target_from_source: " + self.value['name']) + for pkg in self.package_deps: + # CHECKIN: verify features + if pkg in _package_targets_map: +# print("Reused dep: " + str(_package_targets_map[pkg])) + self.env.Depends(target, _package_targets_map[pkg]) + else: +# print("New dep: " + str(pkg)) + dep = self.env.VCPkg(pkg) +# print("Depends: " + str(dep[0])) + self.env.Depends(target, dep[0]) + + if not target.exists(): + if self.is_mismatched_version_installed(): + if _call_vcpkg(self.env, ['upgrade', '--no-dry-run', str(self)]) != 0: + print("Failed to upgrade package '" + str(self) + "'") + target.state = SCons.Node.failed + elif _call_vcpkg(self.env, ['install', str(self)]) != 0: + print("Failed to install package '" + str(self) + "'") + target.state = SCons.Node.failed + target.clear_memoized_values() + if not target.exists(): + print("What gives? vcpkg install failed to create '" + target.get_abspath() + "'") + target.state = SCons.Node.failed + + target.precious = True + target.noclean = True + + print("Caching target for package: " + self.value['name']) + _package_targets_map[self.value['name']] = target + + return target + + +def get_package_descriptor(env, spec): + if spec in _package_descriptors_map: + return _package_descriptors_map[spec] + desc = PackageDescriptor(env, spec) + _package_descriptors_map[spec] = desc + return desc + + +def vcpkg_action(target, source, env): + pass +# packages = list(map(str, source)) +# print("Running action") +# return _call_vcpkg(env, ['install'] + packages) + + +def get_vcpkg_deps(node, env, path, arg): + print("Scan!!!!") + deps = [] + if not node.package_deps is None: + for pkg in node.package_deps: + target = env.VCPkg(pkg) + deps += target[0] + print("Dep: " + str(target[0])) + return deps + + +vcpkg_source_scanner = SCons.Script.Scanner(function = get_vcpkg_deps, + argument = None) + + +def vcpkg_emitter(target, source, env): + _bootstrap_vcpkg(env) + + for t in target: + if not t.exists(): + vcpkg_action(target, source, env) + break + + built = [] + for t in target: + built += _read_vcpkg_file_list(env, t) + + for f in built: + f.precious = True + f.noclean = True + f.state = SCons.Node.up_to_date + + target += built + + return target, source + + +# TODO: static? +def generate(env): + """Add Builders and construction variables for vcpkg to an Environment.""" + VCPkgBuilder = SCons.Builder.Builder(action = vcpkg_action, + source_factory = lambda spec: get_package_descriptor(env, spec), + target_factory = SCons.Node.FS.File, + source_scanner = vcpkg_source_scanner, + suffix = '.list', + emitter = vcpkg_emitter) + + env['BUILDERS']['VCPkg'] = VCPkgBuilder + +def exists(env): + return 1 + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/Tool/vcpkg.xml b/SCons/Tool/vcpkg.xml new file mode 100644 index 0000000000..d0f7b529ee --- /dev/null +++ b/SCons/Tool/vcpkg.xml @@ -0,0 +1,92 @@ + + + + + %scons; + + %builders-mod; + + %functions-mod; + + %tools-mod; + + %variables-mod; + ]> + + + + + + + Sets construction variables for the &vcpkg; package manager + + + + VCPKGROOT + + + + + + + Downloads and builds one or more software packages via the &vcpkg; package manager (plus + any dependencies of those packages), making the built artifcats available for other builders. + + + +# Install FreeImage, plus any of its dependencies +env.VCPkg('freeimage') + + + + &vcpkg; is distributed as a Git repository, containing the &vcpkg; executable and a + "snapshot" of the current versions of all available packages. A typical usage pattern is + for your project to incorporate &vcpkg; as a Git submodule underneath your project (run + 'git submodule --help'), though system-wide installation is also supported. + + + + Packages built with &vcpkg; may produce header files, static libraries and + .dll/.so files. The &vcpkg;-internal directores + containing these built artifacts are added to &cv-link-CPPPATH;, &cv-link-LIBPATH; and + &cv-link-PATH;, respectively. + + + + + + + + + Specifies the path to the root directory of the &vcpkg; installation. + This must be set in the SConstruct/SConscript file, and is typically + specified relative to the project root. + + + +# vcpkg is a submodule located in the 'vcpkg' directory underneath the project root +env['VCPKGROOT'] = '#/vcpkg' + + + + + + + + Specifies whether &vcpkg; should build debug or optimized versions of packages. + If True, then "debug" packages will be built, with full debugging information + and most optimizations disabled. If False (or unset), then packages will be + built using optimized settings. + + + + + diff --git a/bin/files b/bin/files index 08b1caa3bd..0207edc6c9 100644 --- a/bin/files +++ b/bin/files @@ -99,6 +99,7 @@ ./SCons/Tool/tar.py ./SCons/Tool/tex.py ./SCons/Tool/tlib.py +./SCons/Tool/vcpkg.py ./SCons/Tool/yacc.py ./SCons/Util.py ./SCons/Warnings.py diff --git a/doc/generated/tools.mod b/doc/generated/tools.mod index 35eea5e2b4..22143cf148 100644 --- a/doc/generated/tools.mod +++ b/doc/generated/tools.mod @@ -104,6 +104,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. tex"> textfile"> tlib"> +vcpkg"> xgettext"> yacc"> zip"> @@ -210,6 +211,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. tex"> textfile"> tlib"> +vcpkg"> xgettext"> yacc"> zip"> From cccb361557be5a1234a8d9334ebd220c36d8c5d9 Mon Sep 17 00:00:00 2001 From: Ryan Saunders Date: Sat, 26 Feb 2022 23:07:10 -0800 Subject: [PATCH 02/22] Respect --silent in vcpkg spew and support a --vcpkg-debug flag --- SCons/Tool/vcpkg.py | 124 ++++++++++++++++++++++++++++---------------- 1 file changed, 78 insertions(+), 46 deletions(-) diff --git a/SCons/Tool/vcpkg.py b/SCons/Tool/vcpkg.py index f6bc88a09c..b74bd93b32 100644 --- a/SCons/Tool/vcpkg.py +++ b/SCons/Tool/vcpkg.py @@ -15,10 +15,11 @@ * unit tests * verify that feature supersetting works * print errors from vcpkg on console - * print vcpkg commands unless in silent mode * debug libs * install/symlink built dlls into variant dir - * flag detection + * parallel builds? + * can we ensure granular detection, and fail on undetected dependencies? + * batch depend-info calls to vcpkg for better perf? """ @@ -54,6 +55,25 @@ import SCons.Script from SCons.Errors import UserError, InternalError, BuildError + +# Named constants for verbosity levels supported by _vcpkg_print +Silent = 0 +Normal = 1 +Debug = 2 + +_max_verbosity = Normal # Can be changed with --silent or --vcpkg-debug + +# Add --vcpkg-debug command-line option +SCons.Script.AddOption('--vcpkg-debug', dest = 'vcpkg-debug', default = False, action = 'store_true', + help = 'Emit verbose debugging spew from the vcpkg builder') + +def _vcpkg_print(verbosity, *args): + """Prints *args""" + + if verbosity <= _max_verbosity: + print(*args) + + def _get_built_vcpkg_full_version(vcpkg_exe): """Runs 'vcpkg version' and parses the version string from the output""" @@ -64,6 +84,7 @@ def _get_built_vcpkg_full_version(vcpkg_exe): match_version = re.search(r' version (\S+)', line) if (match_version): version = match_version.group(1) + _vcpkg_print(Debug, 'vcpkg full version is "' + version + '"') return version raise InternalError(vcpkg_exe.get_path() + ': failed to parse version string') @@ -71,9 +92,14 @@ def _get_built_vcpkg_full_version(vcpkg_exe): def _get_built_vcpkg_base_version(vcpkg_exe): """Returns just the base version from 'vcpkg version' (i.e., with any '-whatever' suffix stripped off""" full_version = _get_built_vcpkg_full_version(vcpkg_exe) + dash_pos = full_version.find('-') if dash_pos != -1: - return full_version[0:dash_pos] + base_version = full_version[0:dash_pos] + _vcpkg_print(Debug, 'vcpkg base version is "' + base_version + '"') + return base_version + + _vcpkg_print(Debug, 'vcpkg base version is identical to full version "' + full_version + '"') return full_version @@ -124,16 +150,16 @@ def _bootstrap_vcpkg(env): built_version = _get_built_vcpkg_base_version(vcpkg_exe) source_version = _get_source_vcpkg_version(vcpkgroot_dir) if built_version != source_version and not built_version.startswith(source_version + '-'): - print(vcpkg_exe.get_path() + ' (version ' + built_version + ') is out of date (source version: ' + source_version + '); rebuilding') + _vcpkg_print(Normal, 'vcpkg executable (version ' + built_version + ') is out of date (source version: ' + source_version + '); rebuilding') build_vcpkg = True else: - print(vcpkg_exe.get_path() + ' (version ' + built_version + ') is up-to-date') + _vcpkg_print(Debug, 'vcpkg executable (version ' + built_version + ') is up-to-date') # If we need to build, do it now, and ensure that it built if build_vcpkg: if not bootstrap_vcpkg_script.exists(): raise InternalError(bootstrap_vcpkg_script.get_path() + ' does not exist...what gives?') - print('Building vcpkg binary') + _vcpkg_print(Normal, 'Building vcpkg binary') if subprocess.call(bootstrap_vcpkg_script.get_abspath()) != 0: raise BuildError(bootstrap_vcpkg_script.get_path() + ' failed') vcpkg_exe.clear_memoized_values() @@ -150,11 +176,15 @@ def _call_vcpkg(env, params, check_output = False): vcpkg_exe = _bootstrap_vcpkg(env) command_line = [vcpkg_exe.get_abspath()] + params - print(str(command_line)) + _vcpkg_print(Debug, "Running " + str(command_line)) if check_output: - return str(subprocess.check_output(command_line, universal_newlines = True)) + output = str(subprocess.check_output(command_line, text = True)) + _vcpkg_print(Debug, 'Output is: ' + output) + return output else: - return subprocess.call(args = command_line, universal_newlines = True) + result = subprocess.call(args = command_line, text = True) + _vcpkg_print(Debug, 'Result is: ' + str(result)) + return result def _get_vcpkg_triplet(env, static): @@ -186,17 +216,6 @@ def _get_vcpkg_triplet(env, static): raise UserError('This architecture/platform (%s/%s) is currently unsupported with VCPkg' % (arch, platform)) -def _parse_build_depends(s): - """Given a string from the Build-Depends field of a CONTROL file, parse just the package name from it""" - - match_name = re.match(r'^\s*([^\(\)\[\] ]+)\s*(\[.*\])?$', s) - if not match_name: - print('Failed to parse package name from string "' + s + '"') - return s - - return match_name.group(1), match_name.group(2) - - def _read_vcpkg_file_list(env, list_file): """Read a .list file for a built package and return a list of File nodes for the files it contains (ignoring directories)""" @@ -253,11 +272,9 @@ def _get_package_deps(env, spec, static): return deps_list -# Global mapping of previously-computed package-name -> list-file targets. This -# exists because we may discover additional packages that we need to build, based -# on the Build-Depends field in the CONTROL file), and these packages may or may -# not have been explicitly requested by the -# CHECKIN +# Global mapping of previously-computed package-name -> list-file targets. This exists because we may discover +# additional packages that we need to build, based on running :vcpkg depend-info"), and these packages may or +# may not have been explicitly requested by calls to VCPkg. _package_descriptors_map = {} _package_targets_map = {} @@ -309,6 +326,7 @@ def __str__(self): def is_mismatched_version_installed(self): triplet = _get_vcpkg_triplet(self.env, self.value['static']) + _vcpkg_print(Debug, 'Checking for mismatched version of "' + str(self) + '"') list_file = self.env.File('$VCPKGROOT/installed/vcpkg/info/' + self.value['name'] + '_' + self.value['version'] + '_' + triplet + '.list') if list_file.exists(): return False @@ -320,7 +338,7 @@ def is_mismatched_version_installed(self): match = re.match(r'^\S+\s+(\S+)\s+\S', installed) if match[1] == self.value['version']: raise InternalError("VCPkg thinks " + str(self) + " is installed, but '" + list_file.get_abspath() + "' does not exist") - print("Installed is " + match[1]) + _vcpkg_print(Debug, "Installed is " + match[1]) return True def target_from_source(self, pre, suf, splitext): @@ -329,35 +347,36 @@ def target_from_source(self, pre, suf, splitext): target = self.env.File('$VCPKGROOT/installed/vcpkg/info/' + self.value['name'] + '_' + self.value['version'] + '_' + _get_vcpkg_triplet(self.env, self.value['static']) + suf) target.state = SCons.Node.up_to_date -# print("target_from_source: " + self.value['name']) for pkg in self.package_deps: - # CHECKIN: verify features if pkg in _package_targets_map: -# print("Reused dep: " + str(_package_targets_map[pkg])) + _vcpkg_print(Debug, 'Reused dep: ' + str(_package_targets_map[pkg])) self.env.Depends(target, _package_targets_map[pkg]) else: -# print("New dep: " + str(pkg)) + _vcpkg_print(Debug, "New dep: " + str(pkg)) dep = self.env.VCPkg(pkg) -# print("Depends: " + str(dep[0])) self.env.Depends(target, dep[0]) - if not target.exists(): - if self.is_mismatched_version_installed(): - if _call_vcpkg(self.env, ['upgrade', '--no-dry-run', str(self)]) != 0: - print("Failed to upgrade package '" + str(self) + "'") - target.state = SCons.Node.failed - elif _call_vcpkg(self.env, ['install', str(self)]) != 0: - print("Failed to install package '" + str(self) + "'") - target.state = SCons.Node.failed - target.clear_memoized_values() + if not SCons.Script.GetOption('help'): if not target.exists(): - print("What gives? vcpkg install failed to create '" + target.get_abspath() + "'") - target.state = SCons.Node.failed + if self.is_mismatched_version_installed(): + _vcpkg_print(Debug, 'Upgrading package "' + str(self) + '"') + if _call_vcpkg(self.env, ['upgrade', '--no-dry-run', str(self)]) != 0: + _vcpkg_print(Silent, "Failed to upgrade package '" + str(self) + "'") + target.state = SCons.Node.failed + else: + _vcpkg_print(Debug, 'Installing package "' + str(self) + '"') + if _call_vcpkg(self.env, ['install', str(self)]) != 0: + _vcpkg_print(Silent, "Failed to install package '" + str(self) + "'") + target.state = SCons.Node.failed + target.clear_memoized_values() + if not target.exists(): + _vcpkg_print(Silent, "What gives? vcpkg install failed to create '" + target.get_abspath() + "'") + target.state = SCons.Node.failed target.precious = True target.noclean = True - print("Caching target for package: " + self.value['name']) + _vcpkg_print(Debug, "Caching target for package: " + self.value['name']) _package_targets_map[self.value['name']] = target return target @@ -371,21 +390,24 @@ def get_package_descriptor(env, spec): return desc +# TODO: at the moment, we can't execute vcpkg install at the "normal" point in time, because we need to know what +# files are produced by running this, and we can't do that without actually running the command. Thus, we have to +# shoe-horn the building of packages into the target_from_source function. If vcpkg supported some kind of "outputs" +# mode where it could spit out the contents of the .list file without actually doing the build, then we could defer +# the build until vcpkg_action. def vcpkg_action(target, source, env): pass # packages = list(map(str, source)) -# print("Running action") # return _call_vcpkg(env, ['install'] + packages) def get_vcpkg_deps(node, env, path, arg): - print("Scan!!!!") deps = [] if not node.package_deps is None: for pkg in node.package_deps: target = env.VCPkg(pkg) deps += target[0] - print("Dep: " + str(target[0])) + _vcpkg_print(Debug, 'Found dependency: "' + str(node) + '" -> "' + str(target[0])) return deps @@ -418,6 +440,16 @@ def vcpkg_emitter(target, source, env): # TODO: static? def generate(env): """Add Builders and construction variables for vcpkg to an Environment.""" + + # Set verbosity to the appropriate level + global _max_verbosity + if SCons.Script.GetOption('vcpkg-debug'): + _max_verbosity = Debug + elif SCons.Script.GetOption('silent'): + _max_verbosity = Silent + else: + _max_verbosity = Normal + VCPkgBuilder = SCons.Builder.Builder(action = vcpkg_action, source_factory = lambda spec: get_package_descriptor(env, spec), target_factory = SCons.Node.FS.File, From d3e983bba6f0ac380c3f9a5048284695a4bc3502 Mon Sep 17 00:00:00 2001 From: Ryan Saunders Date: Mon, 28 Feb 2022 11:00:41 -0800 Subject: [PATCH 03/22] Fix break caused by reading old-style CONTROL files. Tweak output for --silent mode. --- SCons/Tool/vcpkg.py | 87 +++++++++++++++++++++++++-------------------- 1 file changed, 48 insertions(+), 39 deletions(-) diff --git a/SCons/Tool/vcpkg.py b/SCons/Tool/vcpkg.py index b74bd93b32..b892f8c653 100644 --- a/SCons/Tool/vcpkg.py +++ b/SCons/Tool/vcpkg.py @@ -20,6 +20,8 @@ * parallel builds? * can we ensure granular detection, and fail on undetected dependencies? * batch depend-info calls to vcpkg for better perf? + * bootstrap-vcpkg now installs from github...how to detect when to do this? + * Make "vcpkg search" faster by supporting a strict match option """ @@ -68,9 +70,10 @@ help = 'Emit verbose debugging spew from the vcpkg builder') def _vcpkg_print(verbosity, *args): - """Prints *args""" + """If the user wants to see messages of 'verbosity', prints *args with a 'vcpkg' prefix""" if verbosity <= _max_verbosity: + print('vcpkg: ', end='') print(*args) @@ -171,20 +174,26 @@ def _bootstrap_vcpkg(env): return vcpkg_exe -def _call_vcpkg(env, params, check_output = False): +def _call_vcpkg(env, params, check_output = False, check = True): """Run the vcpkg executable wth the given set of parameters, optionally returning its standard output as a string. If the vcpkg executable is not yet built, or out of date, it will be rebuilt.""" vcpkg_exe = _bootstrap_vcpkg(env) command_line = [vcpkg_exe.get_abspath()] + params _vcpkg_print(Debug, "Running " + str(command_line)) - if check_output: - output = str(subprocess.check_output(command_line, text = True)) - _vcpkg_print(Debug, 'Output is: ' + output) - return output - else: - result = subprocess.call(args = command_line, text = True) - _vcpkg_print(Debug, 'Result is: ' + str(result)) - return result + try: + result = subprocess.run(command_line, text = True, capture_output = check_output or _max_verbosity == Silent, check = check) + if check_output: + _vcpkg_print(Debug, result.stdout) + return result.stdout + else: + return result.returncode + except subprocess.CalledProcessError as ex: + if check_output: + _vcpkg_print(Silent, result.stdout) + _vcpkg_print(Silent, result.stderr) + return result.stdout + else: + return ex.returncode def _get_vcpkg_triplet(env, static): @@ -227,19 +236,17 @@ def _read_vcpkg_file_list(env, list_file): def _get_package_version(env, spec): - """Read the CONTROL file for a package, returning (version, depends[])""" + """Read the available version of a package (i.e., what would be installed)""" name = spec.split('[')[0] - control_file = env.File('$VCPKGROOT/ports/' + name + '/CONTROL') - version = None - for line in open(control_file.get_abspath()): - match_version = re.match(r'^Version: (\S+)$', line) - if match_version: - version = match_version.group(1) - break - if version is None: - raise InternalError('Failed to parse package version from control file "' + control_file.get_abspath() + '"') - return version + output = _call_vcpkg(env, ['search', name], check_output = True) + for line in output.split('\n'): + match = re.match(r'^(\S+)\s*(\S+)', line) + if match and match.group(1) == name: + version = match.group(2) + _vcpkg_print(Debug, 'Available version of package "' + name + '" is ' + version) + return version + raise UserError('No package "' + name + '" found via vcpkg search') def _get_package_deps(env, spec, static): @@ -321,30 +328,32 @@ def get_package_string(self): s += ':' + _get_vcpkg_triplet(self.env, self.value['static']) return s + def get_listfile_basename(self): + # Trim off any suffix like '#3' from the version, as this doesn't appear in the listfile name + version = self.value['version'] + hash_pos = version.find('#') + if hash_pos != -1: + version = version[0:hash_pos] + return self.value['name'] + '_' + version + '_' + _get_vcpkg_triplet(self.env, self.value['static']) + def __str__(self): return self.get_package_string() def is_mismatched_version_installed(self): triplet = _get_vcpkg_triplet(self.env, self.value['static']) _vcpkg_print(Debug, 'Checking for mismatched version of "' + str(self) + '"') - list_file = self.env.File('$VCPKGROOT/installed/vcpkg/info/' + self.value['name'] + '_' + self.value['version'] + '_' + triplet + '.list') - if list_file.exists(): - return False - - installed = _call_vcpkg(self.env, ['list', self.value['name'] + ':' + triplet], check_output = True) - if installed == '' or installed.startswith('No packages are installed'): - return False - - match = re.match(r'^\S+\s+(\S+)\s+\S', installed) - if match[1] == self.value['version']: - raise InternalError("VCPkg thinks " + str(self) + " is installed, but '" + list_file.get_abspath() + "' does not exist") - _vcpkg_print(Debug, "Installed is " + match[1]) - return True + output = _call_vcpkg(self.env, ['update'], check_output = True) + for line in output.split('\n'): + match = re.match(r'^\s*(\S+)\s*(\S+) -> (\S+)', line) + if match and match.group(1) == str(self): + _vcpkg_print(Debug, 'Package "' + str(self) + '" can be updated (' + match.group(2) + ' -> ' + match.group(3)) + return True + return False def target_from_source(self, pre, suf, splitext): _bootstrap_vcpkg(self.env) - target = self.env.File('$VCPKGROOT/installed/vcpkg/info/' + self.value['name'] + '_' + self.value['version'] + '_' + _get_vcpkg_triplet(self.env, self.value['static']) + suf) + target = self.env.File('$VCPKGROOT/installed/vcpkg/info/' + self.get_listfile_basename() + suf) target.state = SCons.Node.up_to_date for pkg in self.package_deps: @@ -359,14 +368,14 @@ def target_from_source(self, pre, suf, splitext): if not SCons.Script.GetOption('help'): if not target.exists(): if self.is_mismatched_version_installed(): - _vcpkg_print(Debug, 'Upgrading package "' + str(self) + '"') + _vcpkg_print(Silent, str(self) + ' (upgrade)') if _call_vcpkg(self.env, ['upgrade', '--no-dry-run', str(self)]) != 0: - _vcpkg_print(Silent, "Failed to upgrade package '" + str(self) + "'") + _vcpkg_print(Silent, "Failed to upgrade package " + str(self)) target.state = SCons.Node.failed else: - _vcpkg_print(Debug, 'Installing package "' + str(self) + '"') + _vcpkg_print(Silent, str(self) + ' (install)') if _call_vcpkg(self.env, ['install', str(self)]) != 0: - _vcpkg_print(Silent, "Failed to install package '" + str(self) + "'") + _vcpkg_print(Silent, "Failed to install package " + str(self)) target.state = SCons.Node.failed target.clear_memoized_values() if not target.exists(): From b5f070cd6b8ad46251a9b2d47ec1e46f3e4191aa Mon Sep 17 00:00:00 2001 From: Ryan Saunders Date: Mon, 28 Feb 2022 16:02:41 -0800 Subject: [PATCH 04/22] Avoid re-downloading vcpkg.exe from GitHub when we have the correct version. Also fix parsing of 'vcpkg depend-info'. --- SCons/Tool/vcpkg.py | 65 +++++++++++++++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 20 deletions(-) diff --git a/SCons/Tool/vcpkg.py b/SCons/Tool/vcpkg.py index b892f8c653..7659d9adba 100644 --- a/SCons/Tool/vcpkg.py +++ b/SCons/Tool/vcpkg.py @@ -10,17 +10,14 @@ TODO: - * ensure that upgrading to a new package version works after a repo update * ensure Linux works * unit tests * verify that feature supersetting works - * print errors from vcpkg on console * debug libs * install/symlink built dlls into variant dir * parallel builds? * can we ensure granular detection, and fail on undetected dependencies? * batch depend-info calls to vcpkg for better perf? - * bootstrap-vcpkg now installs from github...how to detect when to do this? * Make "vcpkg search" faster by supporting a strict match option """ @@ -96,9 +93,10 @@ def _get_built_vcpkg_base_version(vcpkg_exe): """Returns just the base version from 'vcpkg version' (i.e., with any '-whatever' suffix stripped off""" full_version = _get_built_vcpkg_full_version(vcpkg_exe) - dash_pos = full_version.find('-') - if dash_pos != -1: - base_version = full_version[0:dash_pos] + # Allow either 2022-02-02 or 2022.02.02 date syntax + match = re.match(r'^(\d{4}[.-]\d{2}[.-]\d{2})', full_version) + if match: + base_version = match.group(1) _vcpkg_print(Debug, 'vcpkg base version is "' + base_version + '"') return base_version @@ -112,15 +110,36 @@ def _get_source_vcpkg_version(vcpkg_root): if not vcpkg_root.exists(): raise InternalError(vcpkg_root.get_path() + ' does not exist') + # Older vcpkg versions had the source version in VERSION.txt version_txt = vcpkg_root.File('toolsrc/VERSION.txt') - if not version_txt.exists(): - raise InternalError(version_txt.get_path() + ' does not exist; did something change in the vcpkg directory structure?') + if version_txt.exists(): + _vcpkg_print(Debug, "Looking for source version in " + version_txt.get_path()) + for line in open(version_txt.get_abspath()): + match_version = re.match(r'^"(.*)"$', line) + if (match_version): + version = match_version.group(1) + + # Newer versions of vcpkg have a hard-coded invalid date in the VERSION.txt for local builds, + # and the actual version comes from the bootstrap.ps1 script + if version != '9999.99.99': + _vcpkg_print(Debug, 'Found valid vcpkg source version "' + version + '" in VERSION.txt') + return version + else: + _vcpkg_print(Debug, 'VERSION.txt contains invalid vcpkg source version "' + version + '"') + break + + # Newer versions of bootstrap-vcpkg simply download a pre-built executable from GitHub, and the version to download + # is hard-coded in a deployment script. + bootstrap_ps1 = vcpkg_root.File('scripts/bootstrap.ps1') + if bootstrap_ps1.exists(): + for line in open(bootstrap_ps1.get_abspath()): + match_version = re.match(r"\$versionDate\s*=\s*'(.*)'", line) + if match_version: + version = match_version.group(1) + _vcpkg_print(Debug, 'Found valid vcpkg source version "' + version + '" in bootstrap.ps1') + return version - for line in open(version_txt.get_abspath()): - match_version = re.match(r'^"(.*)"$', line) - if (match_version): - return match_version.group(1) - raise InternalError(version_txt.get_path() + ": Failed to parse vcpkg version string") + raise InternalError("Failed to determine vcpkg source version") def _bootstrap_vcpkg(env): @@ -189,9 +208,9 @@ def _call_vcpkg(env, params, check_output = False, check = True): return result.returncode except subprocess.CalledProcessError as ex: if check_output: - _vcpkg_print(Silent, result.stdout) - _vcpkg_print(Silent, result.stderr) - return result.stdout + _vcpkg_print(Silent, ex.stdout) + _vcpkg_print(Silent, ex.stderr) + return ex.stdout else: return ex.returncode @@ -272,11 +291,17 @@ def _get_package_deps(env, spec, static): # containing feature specifications. So, we'll strip these out (but possibly miss some dependencies). params.append(spec.split('[')[0]) + name = spec.split('[')[0] output = _call_vcpkg(env, params, check_output = True) - deps_list = output[output.index(':') + 1:].split(',') - deps_list = list(map(lambda s: s.strip(), deps_list)) - deps_list = list(filter(lambda s: s != "", deps_list)) - return deps_list + for line in output.split('\n'): + match = re.match(r'^([^:[]+)(?:\[[^]]+\])?:\s*(.+)?', line) + if match and match.group(1) == name: + deps_list = [] + if match.group(2): + deps_list = list(filter(lambda s: s != "", map(lambda s: s.strip(), match.group(2).split(',')))) + _vcpkg_print(Debug, 'Package ' + spec + ' has dependencies [' + ','.join(deps_list) + ']') + return deps_list + raise InternalError('Failed to parse output from vcpkg ' + ' '.join(params) + '\n' + output) # Global mapping of previously-computed package-name -> list-file targets. This exists because we may discover From 282785e09c473bc7fa07539479382de35aab5b56 Mon Sep 17 00:00:00 2001 From: Ryan Saunders Date: Mon, 8 Aug 2022 23:00:40 -0700 Subject: [PATCH 05/22] Enable enumeration of package contents in SConstruct file, via PackageContents 'target' node --- SCons/Tool/vcpkg.py | 98 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 85 insertions(+), 13 deletions(-) diff --git a/SCons/Tool/vcpkg.py b/SCons/Tool/vcpkg.py index 7659d9adba..1125a78379 100644 --- a/SCons/Tool/vcpkg.py +++ b/SCons/Tool/vcpkg.py @@ -13,8 +13,6 @@ * ensure Linux works * unit tests * verify that feature supersetting works - * debug libs - * install/symlink built dlls into variant dir * parallel builds? * can we ensure granular detection, and fail on undetected dependencies? * batch depend-info calls to vcpkg for better perf? @@ -304,6 +302,7 @@ def _get_package_deps(env, spec, static): raise InternalError('Failed to parse output from vcpkg ' + ' '.join(params) + '\n' + output) +# CHECKIN: move to env? # Global mapping of previously-computed package-name -> list-file targets. This exists because we may discover # additional packages that we need to build, based on running :vcpkg depend-info"), and these packages may or # may not have been explicitly requested by calls to VCPkg. @@ -311,6 +310,8 @@ def _get_package_deps(env, spec, static): _package_targets_map = {} class PackageDescriptor(SCons.Node.Python.Value): + """PackageDescriptor is the 'source' node for the VCPkg builder. A PackageDescriptor instance includes the package + name, version and linkage (static or dynamic), and a list of other packages that it depends on.""" def __init__(self, env, spec, static = False): _bootstrap_vcpkg(env) @@ -319,10 +320,8 @@ def __init__(self, env, spec, static = False): env.AppendUnique(CPPPATH = ['$VCPKGROOT/installed/' + triplet + '/include/']) if env.subst('$VCPKGDEBUG') == 'True': env.AppendUnique(LIBPATH = '$VCPKGROOT/installed/' + triplet + '/debug/lib/') - env.AppendUnique(PATH = '$VCPKGROOT/installed/' + triplet + '/debug/bin/') else: env.AppendUnique(LIBPATH = '$VCPKGROOT/installed/' + triplet + '/lib/') - env.AppendUnique(PATH = '$VCPKGROOT/installed/' + triplet + '/bin/') if spec is None or spec == '': raise ValueError('VCPkg: Package spec must not be empty') @@ -340,17 +339,21 @@ def __init__(self, env, spec, static = False): 'features': features, 'static': static, 'version': version, + 'triplet': triplet } super(PackageDescriptor, self).__init__(value) self.env = env - self.package_deps = depends + self.package_deps = list(map(lambda p: get_package_descriptor(env, p), depends)) + + def get_triplet(self): + return self.value['triplet'] def get_package_string(self): s = self.value['name'] if self.value['features'] is not None: s += '[' + self.value['features'] + ']' - s += ':' + _get_vcpkg_triplet(self.env, self.value['static']) + s += ':' + self.value['triplet'] return s def get_listfile_basename(self): @@ -359,13 +362,12 @@ def get_listfile_basename(self): hash_pos = version.find('#') if hash_pos != -1: version = version[0:hash_pos] - return self.value['name'] + '_' + version + '_' + _get_vcpkg_triplet(self.env, self.value['static']) + return self.value['name'] + '_' + version + '_' + self.value['triplet'] def __str__(self): return self.get_package_string() def is_mismatched_version_installed(self): - triplet = _get_vcpkg_triplet(self.env, self.value['static']) _vcpkg_print(Debug, 'Checking for mismatched version of "' + str(self) + '"') output = _call_vcpkg(self.env, ['update'], check_output = True) for line in output.split('\n'): @@ -378,7 +380,7 @@ def is_mismatched_version_installed(self): def target_from_source(self, pre, suf, splitext): _bootstrap_vcpkg(self.env) - target = self.env.File('$VCPKGROOT/installed/vcpkg/info/' + self.get_listfile_basename() + suf) + target = PackageContents(self.env, self) target.state = SCons.Node.up_to_date for pkg in self.package_deps: @@ -411,11 +413,80 @@ def target_from_source(self, pre, suf, splitext): target.noclean = True _vcpkg_print(Debug, "Caching target for package: " + self.value['name']) - _package_targets_map[self.value['name']] = target + _package_targets_map[self] = target return target +class PackageContents(SCons.Node.FS.File): + """PackageContents is a File node (referring to the installed package's .list file) and is the 'target' node of + the VCPkg builder (though currently, it doesn't actually get built during SCons's normal build phase, since + vcpkg currently can't tell us what files will be installed without actually doing the work). + + It includes functionality for enumerating the different kinds of files (headers, libraries, etc.) produced by + installing the package.""" + + def __init__(self, env, descriptor): + super().__init__( descriptor.get_listfile_basename() + ".list", env.Dir('$VCPKGROOT/installed/vcpkg/info/'), env.fs) + self.descriptor = descriptor + self.loaded = False + + def Headers(self, transitive = False): + """Returns the list of C/C++ header files belonging to the package. + If `transitive` is True, then files belonging to upstream dependencies of this package are also included.""" + _vcpkg_print(Debug, str(self.descriptor) + ': headers') + files = self.FilesUnderSubPath('include/', transitive) + return self.FilesUnderSubPath('include/', transitive) + + def StaticLibraries(self, transitive = False): + """Returns the list of static libraries belonging to the package. + If `transitive` is True, then files belonging to upstream dependencies of this package are also included.""" + _vcpkg_print(Debug, str(self.descriptor) + ': static libraries') + if self.env.subst('$VCPKGDEBUG') == 'True': + return self.FilesUnderSubPath('debug/lib/', transitive) + else: + return self.FilesUnderSubPath('lib/', transitive) + + def SharedLibraries(self, transitive = False): + """Returns the list of shared libraries belonging to the package. + If `transitive` is True, then files belonging to upstream dependencies of this package are also included.""" + _vcpkg_print(Debug, str(self.descriptor) + ': shared libraries') + if self.env.subst('$VCPKGDEBUG') == 'True': + return self.FilesUnderSubPath('debug/bin/', transitive) + else: + return self.FilesUnderSubPath('bin/', transitive) + + def FilesUnderSubPath(self, subpath, transitive): + """Returns a (possibly empty) list of File nodes belonging to this package that are located under the + relative path `subpath` underneath the triplet install directory. + If `transitive` is True, then files belonging to upstream dependencies of this package are also included.""" + + # Load the listfile contents, if we haven't already. This returns a list of File nodes. + if not self.loaded: + if not self.exists(): + raise InternalError(self.get_path() + ' does not exist') + _vcpkg_print(Debug, 'Loading ' + str(self.descriptor) + ' listfile: ' + self.get_path()) + self.files = _read_vcpkg_file_list(self.env, self) + self.loaded = True + + triplet = self.descriptor.get_triplet() + prefix = self.env.Dir(self.env.subst('$VCPKGROOT/installed/' + self.descriptor.get_triplet() + "/" + subpath)) + _vcpkg_print(Debug, 'Looking for files under ' + str(prefix)) + matching_files = [] + for file in self.files: + if file.is_under(prefix): + _vcpkg_print(Debug, 'Matching file ' + file.get_abspath()) + matching_files += [file] + + if transitive: + for dep in self.descriptor.package_deps: + matching_files += _package_targets_map[dep].FilesUnderSubPath(subpath, transitive) + + return SCons.Util.NodeList(matching_files) + + def __str__(self): + return "Package: " + super(PackageContents, self).__str__() + def get_package_descriptor(env, spec): if spec in _package_descriptors_map: return _package_descriptors_map[spec] @@ -449,6 +520,7 @@ def get_vcpkg_deps(node, env, path, arg): argument = None) +# TODO: do we need the emitter at all? def vcpkg_emitter(target, source, env): _bootstrap_vcpkg(env) @@ -458,8 +530,8 @@ def vcpkg_emitter(target, source, env): break built = [] - for t in target: - built += _read_vcpkg_file_list(env, t) +# for t in target: +# built += _read_vcpkg_file_list(env, t) for f in built: f.precious = True @@ -486,7 +558,7 @@ def generate(env): VCPkgBuilder = SCons.Builder.Builder(action = vcpkg_action, source_factory = lambda spec: get_package_descriptor(env, spec), - target_factory = SCons.Node.FS.File, + target_factory = lambda desc: PackageContents(env, desc), source_scanner = vcpkg_source_scanner, suffix = '.list', emitter = vcpkg_emitter) From 12d1fdd9e530be0aec6ff975b1adad6e3057c3c9 Mon Sep 17 00:00:00 2001 From: Ryan Saunders Date: Tue, 9 Aug 2022 00:28:13 -0700 Subject: [PATCH 06/22] Fix faulty use of AppendUnique --- SCons/Tool/vcpkg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SCons/Tool/vcpkg.py b/SCons/Tool/vcpkg.py index 1125a78379..907ad9ef26 100644 --- a/SCons/Tool/vcpkg.py +++ b/SCons/Tool/vcpkg.py @@ -319,9 +319,9 @@ def __init__(self, env, spec, static = False): triplet = _get_vcpkg_triplet(env, static) env.AppendUnique(CPPPATH = ['$VCPKGROOT/installed/' + triplet + '/include/']) if env.subst('$VCPKGDEBUG') == 'True': - env.AppendUnique(LIBPATH = '$VCPKGROOT/installed/' + triplet + '/debug/lib/') + env.AppendUnique(LIBPATH = ['$VCPKGROOT/installed/' + triplet + '/debug/lib/']) else: - env.AppendUnique(LIBPATH = '$VCPKGROOT/installed/' + triplet + '/lib/') + env.AppendUnique(LIBPATH = ['$VCPKGROOT/installed/' + triplet + '/lib/']) if spec is None or spec == '': raise ValueError('VCPkg: Package spec must not be empty') From 705be7bf71b504d35eb3e5dd0f6f4a135e72555f Mon Sep 17 00:00:00 2001 From: Ryan Saunders Date: Sun, 14 Aug 2022 15:30:44 -0700 Subject: [PATCH 07/22] Updates to VCPkg documentation and tests --- SCons/Tool/VCPkgTests.py | 99 +++++++++++++++++++++++++++++++++------- SCons/Tool/vcpkg.xml | 37 ++++++++------- 2 files changed, 104 insertions(+), 32 deletions(-) diff --git a/SCons/Tool/VCPkgTests.py b/SCons/Tool/VCPkgTests.py index 7f41a08747..b76810b3f0 100644 --- a/SCons/Tool/VCPkgTests.py +++ b/SCons/Tool/VCPkgTests.py @@ -57,10 +57,42 @@ def make_mock_vcpkg_dir(self, include_vcpkg_exe = False): return vcpkg_root + def run_with_mocks(payload, + call_vcpkg = None, + get_package_version = None, + read_vcpkg_file_list = None): + """Runs `payload` with one or more low-level VCPkg functions mocked""" + + # Save original functions to restore later + orig_call_vcpkg = SCons.Tool.vcpkg._call_vcpkg + orig_get_package_version = SCons.Tool.vcpkg._get_package_version + orig_read_vcpkg_file_list = SCons.Tool.vcpkg._read_vcpkg_file_list + + try: + # Override any functions for which mocks were supplied + if call_vcpkg is not None: + SCons.Tool.vcpkg._call_vcpkg = call_vcpkg + if get_package_version is not None: + SCons.Tool.vcpkg._get_package_version = get_package_version + if read_vcpkg_file_list is not None: + SCons.Tool.vcpkg._read_vcpkg_file_list = read_vcpkg_file_list + + # Run the payload + result = payload() + + finally: + # Restore original functions + SCons.Tool.vcpkg._call_vcpkg = orig_call_vcpkg + SCons.Tool.vcpkg._get_pacakge_version = orig_get_package_version + SCons.Tool.vcpkg._get_pacakge_version = orig_read_vcpkg_file_list + + return result + + def test_VCPKGROOT(self): """Test that VCPkg() fails with an exception if the VCPKGROOT environment variable is unset or invalid""" - env = Environment(tools=['default','vcpkg']) + env = Environment(tools=['vcpkg']) # VCPKGROOT unset (should fail) exc_caught = None @@ -95,11 +127,11 @@ def test_VCPKGROOT(self): # VCPKGROOT pointing to a mock vcpkg instance (should succeed) env['VCPKGROOT'] = self.make_mock_vcpkg_dir(include_vcpkg_exe = True) - orig_call_vcpkg = SCons.Tool.vcpkg._call_vcpkg - orig_get_package_version = SCons.Tool.vcpkg._get_package_version - orig_read_vcpkg_file_list = SCons.Tool.vcpkg._read_vcpkg_file_list +# orig_call_vcpkg = SCons.Tool.vcpkg._call_vcpkg +# orig_get_package_version = SCons.Tool.vcpkg._get_package_version +# orig_read_vcpkg_file_list = SCons.Tool.vcpkg._read_vcpkg_file_list installed_package = False - def mock_call_vcpkg_exe(env, params, check_output = False): + def mock_call_vcpkg(env, params, check_output = False): if 'install' in params and 'pretend_package' in params: installed_pacakge = True return 0 @@ -115,17 +147,52 @@ def mock_get_package_version(env, spec): return '1.0.0' def mock_read_vcpkg_file_list(env, list_file): return [env.File('$VCPKGROOT/installed/x64-windows/lib/pretend_package.lib')] - - SCons.Tool.vcpkg._call_vcpkg = mock_call_vcpkg_exe - SCons.Tool.vcpkg._get_package_version = mock_get_package_version - SCons.Tool.vcpkg._read_vcpkg_file_list = mock_read_vcpkg_file_list - try: - env.VCPkg('pretend_package') - finally: - rmtree(env['VCPKGROOT']) - SCons.Tool.vcpkg._call_vcpkg = orig_call_vcpkg - SCons.Tool.vcpkg._get_pacakge_version = orig_get_package_version - SCons.Tool.vcpkg._get_pacakge_version = orig_read_vcpkg_file_list + + VCPkgTestCase.run_with_mocks(lambda: env.VCPkg('pretend_package'), + call_vcpkg = mock_call_vcpkg, + get_package_version = mock_get_package_version, + read_vcpkg_file_list = mock_read_vcpkg_file_list) +# SCons.Tool.vcpkg._call_vcpkg = mock_call_vcpkg_exe +# SCons.Tool.vcpkg._get_package_version = mock_get_package_version +# SCons.Tool.vcpkg._read_vcpkg_file_list = mock_read_vcpkg_file_list +# try: +# env.VCPkg('pretend_package') +# finally: +# rmtree(env['VCPKGROOT']) +# SCons.Tool.vcpkg._call_vcpkg = orig_call_vcpkg +# SCons.Tool.vcpkg._get_pacakge_version = orig_get_package_version +# SCons.Tool.vcpkg._get_pacakge_version = orig_read_vcpkg_file_list + + + def test_bootstrap(self): + """Test that the VCPkg builder correctly bootstraps the vcpkg executable""" + # upgrade, downgrade + pass + + + def test_depend_info(self): + """Test that the VCPkg builder correctly parses package dependencies""" + # multiple levels, conflicting feature sets, ignore extra packages + pass + + + def test_install(self): + """Test that the VCPkg builder installs missing packages""" + # Multiple installs, dependencies + pass + + + def test_upgrade(self): + """Test that the VCPkg builder correctly upgrades/downgrades existing packages to match the vcpkg configuration""" + pass + + + def test_PackageContents(self): + """Test that the VCPkg builder returns a PackageContents object that can reports the files contained in the package""" + + env = Environment(tools=['vcpkg']) + + pass if __name__ == "__main__": diff --git a/SCons/Tool/vcpkg.xml b/SCons/Tool/vcpkg.xml index d0f7b529ee..80596ba416 100644 --- a/SCons/Tool/vcpkg.xml +++ b/SCons/Tool/vcpkg.xml @@ -31,14 +31,15 @@ See its __doc__ string for a discussion of the format. VCPKGROOT + VCPKGDEBUG - Downloads and builds one or more software packages via the &vcpkg; package manager (plus - any dependencies of those packages), making the built artifcats available for other builders. + Downloads and builds one or more software packages (plus any dependencies of those packages) via the + &vcpkg; package manager, making the built artifcats available for other builders. @@ -47,16 +48,16 @@ env.VCPkg('freeimage') - &vcpkg; is distributed as a Git repository, containing the &vcpkg; executable and a - "snapshot" of the current versions of all available packages. A typical usage pattern is - for your project to incorporate &vcpkg; as a Git submodule underneath your project (run - 'git submodule --help'), though system-wide installation is also supported. + &vcpkg; is distributed as a Git repository, containing the &vcpkg; executable (or a script to build it) + and a "snapshot" of the current versions of all available packages. A typical usage pattern is for your + project to incorporate &vcpkg; as a Git submodule underneath your project (run 'git submodule --help'), + though system-wide installation is also supported. - Packages built with &vcpkg; may produce header files, static libraries and - .dll/.so files. The &vcpkg;-internal directores - containing these built artifacts are added to &cv-link-CPPPATH;, &cv-link-LIBPATH; and + Packages built with &vcpkg; may produce header files, static libraries and shared libraries + (.dll/.so files). The &vcpkg;-internal directores + containing these built artifacts are added to &cv-link-CPPPATH; and &cv-link-LIBPATH; and &cv-link-PATH;, respectively. @@ -66,9 +67,10 @@ env.VCPkg('freeimage') - Specifies the path to the root directory of the &vcpkg; installation. - This must be set in the SConstruct/SConscript file, and is typically - specified relative to the project root. + Specifies the path to the root directory of the &vcpkg; installation. This must be set in the + SConstruct/SConscript file, and must point to an existing &vcpkg; installation. Often, this directory + will be a Git sub-module of your project, in which case VCPKGROOT will be specified relative to the + project root. @@ -81,10 +83,13 @@ env['VCPKGROOT'] = '#/vcpkg' - Specifies whether &vcpkg; should build debug or optimized versions of packages. - If True, then "debug" packages will be built, with full debugging information - and most optimizations disabled. If False (or unset), then packages will be - built using optimized settings. + Specifies whether &vcpkg; should build debug or optimized versions of packages. If True, then "debug" + packages will be built, with full debugging information and most optimizations disabled. If False (or + unset), then packages will be built using optimized settings. + + + Note that, while you may choose to set this to match the optimization settings of your project's build, + this is not required: it's perfectly fine to use optimized packages with a "debug" build of your project. From a7d5fd84addb5bbc18161da3b7152b75008cc956 Mon Sep 17 00:00:00 2001 From: Ryan Saunders Date: Sat, 10 Sep 2022 22:12:43 -0700 Subject: [PATCH 08/22] Significantly expanded unit tests --- SCons/Tool/VCPkgTests.py | 530 +++++++++++++++++++++++++++++++-------- SCons/Tool/vcpkg.py | 104 +++++--- 2 files changed, 497 insertions(+), 137 deletions(-) diff --git a/SCons/Tool/VCPkgTests.py b/SCons/Tool/VCPkgTests.py index b76810b3f0..2403c8e2e1 100644 --- a/SCons/Tool/VCPkgTests.py +++ b/SCons/Tool/VCPkgTests.py @@ -24,7 +24,9 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -from shutil import rmtree +import re +import tempfile +from contextlib import contextmanager from pathlib import Path import unittest @@ -34,61 +36,348 @@ import SCons.Tool.vcpkg from SCons.Environment import Environment -class VCPkgTestCase(unittest.TestCase): - def make_mock_vcpkg_dir(self, include_vcpkg_exe = False): - """Creates a mock vcpkg directory under the temp directory and returns its path""" +# TODO: +# * Test upgrade/downgrade of vcpkg itself +# * Test parsing of real vcpkg.exe output +# * Test feature super-setting +# * Test "static" installs +# * Test "debug" installs +# * Test enumerating package contents + +class MockPackage: + def __init__(self, name, version, dependencies): + self.name = name + self.version = version + self.dependencies = dependencies + self._installedFiles = [] + + def install(self, env, static): + assert not self._installedFiles, f"Trying to install package '{self.name}' more than once!" + listfile = f"{env['VCPKGROOT']}/installed/vcpkg/info/{self.name}_{self.version}_{SCons.Tool.vcpkg._get_vcpkg_triplet(env, static)}.list" + Path(listfile).touch() + self._installedFiles = [listfile] + + def clean_up(self): + for file in self._installedFiles: + os.remove(file) + self._installedFiles = [] + + +class MockVCPkg: + """Singleton object that replaces low-level VCPkg builder functions with mocks""" + + # + # MockVCPkg lifecycle management + # + + __instance = None + + # Singleton accessor + def getInstance(): + if MockVCPkg.__instance is None: + MockVCPkg.__instance = MockVCPkg() + MockVCPkg.__instance.acquire() + return MockVCPkg.__instance + + def __init__(self): + self._availablePackages = {} + self._installedPackages = {} + self._expectations = [] + self._useCount = 0 + + def assert_empty(self): + """Asserts that all test configuration and expectations have been removed""" + assert not self._availablePackages, f"There is/are still {len(self._availablePackages)} AvailablePackage(s)" + assert not self._installedPackages, f"There is/are still {len(self._installedPackages)} InstalledPackage(s)" + assert not self._expectations, f"There is/are still {len(self._expectations)} Expectation(s)" + + def acquire(self): + """Called to acquire a ref-count on the singleton MockVCPkg object. This is needed because multiple objects can be using the MockVCPkg object simultaneously, and it needs to tear itself down when the last user releases it""" + self._useCount += 1 + if self._useCount == 1: + # There shouldn't be anything configured yet + self.assert_empty() + + # Save original functions to restore later + self._orig_bootstrap_vcpkg = SCons.Tool.vcpkg._bootstrap_vcpkg + self._orig_call_vcpkg = SCons.Tool.vcpkg._call_vcpkg + self._orig_install_packages = SCons.Tool.vcpkg._install_packages + self._orig_upgrade_packages = SCons.Tool.vcpkg._upgrade_packages + self._origis_mismatched_version_installed = SCons.Tool.vcpkg.is_mismatched_version_installed + self._orig_get_package_version = SCons.Tool.vcpkg._get_package_version + self._orig_get_package_deps = SCons.Tool.vcpkg._get_package_deps + self._orig_read_vcpkg_file_list = SCons.Tool.vcpkg._read_vcpkg_file_list + + # Replace the low-level vcpkg functions with our mocks + SCons.Tool.vcpkg._bootstrap_vcpkg = MockVCPkg._bootstrap_vcpkg + SCons.Tool.vcpkg._call_vcpkg = MockVCPkg._call_vcpkg + SCons.Tool.vcpkg._install_packages = MockVCPkg._install_packages + SCons.Tool.vcpkg._upgrade_packages = MockVCPkg._upgrade_packages + SCons.Tool.vcpkg.is_mismatched_version_installed = MockVCPkg.is_mismatched_version_installed + SCons.Tool.vcpkg._get_package_version = MockVCPkg._get_package_version + SCons.Tool.vcpkg._get_package_deps = MockVCPkg._get_package_deps + SCons.Tool.vcpkg._read_vcpkg_file_list = MockVCPkg._read_vcpkg_file_list + + def release(self): + """Called to release a ref-count on the singleton MockVCPkg object. When this hits zero, the MockVCPkg instance will tear itself down""" + assert(self._useCount > 0) + self._useCount -= 1 + if self._useCount == 0: + # There shouldn't be any configuration still remaining + self.assert_empty() - vcpkg_root = os.environ['TEMP'] + '/scons_vcpkg_test' + # Restore original functions + SCons.Tool.vcpkg._bootstrap_vcpkg = self._orig_bootstrap_vcpkg + SCons.Tool.vcpkg._call_vcpkg = self._orig_call_vcpkg + SCons.Tool.vcpkg._install_packages = self._orig_install_packages + SCons.Tool.vcpkg._upgrade_packages = self._orig_upgrade_packages + SCons.Tool.vcpkg.is_mismatched_version_installed = self._origis_mismatched_version_installed + SCons.Tool.vcpkg._get_package_version = self._orig_get_package_version + SCons.Tool.vcpkg._get_package_deps = self._orig_get_package_deps + SCons.Tool.vcpkg._read_vcpkg_file_list = self._orig_read_vcpkg_file_list + + # Finally, free the singleton + MockVCPkg.__instance = None + + + # + # State modification functions used by contextmanager functions below + # + + def addAvailablePackage(self, name, version, dependencies): + assert name not in self._availablePackages, f"Already have an AvailablePackage with name '{name}' (version {self._availablePackages[name].version})" + pkg = MockPackage(name, version, dependencies) + self._availablePackages[name] = pkg + return pkg + + def removeAvailablePackage(self, pkg): + pkg.clean_up() + assert self._availablePackages.pop(pkg.name), f"Trying to remove AvailablePackage with name '{pkg.name}' that is not currently registered" + + def addInstalledPackage(self, env, name, version, dependencies, static): + assert name not in self._installedPackages, f"Already have an InstalledPackage with name '{name}' (version {self._availablePackages[name].version})" + pkg = MockPackage(name, version, dependencies) + pkg.install(env, static) + self._installedPackages[name] = pkg + return pkg + + def removeInstalledPackage(self, pkg): + pkg.clean_up() + assert self._installedPackages.pop(pkg.name), f"Trying to remove InstalledPackage with name '{pkg.name}' that is not currently registered" + + def addExpectation(self, exp): + assert exp not in self._expectations, "Trying to add an Expectation twice?" + self._expectations.append(exp) + return exp + + def removeExpectation(self, exp): + assert exp in self._expectations, "Trying to remove Expectation that is not currently registered" + self._expectations.remove(exp) + + + # + # Mock implementations of low-level VCPkg builder functions + # + + def _bootstrap_vcpkg(env): + pass - # Delete it if it already exists - rmtree(vcpkg_root, ignore_errors = True) + def _call_vcpkg(env, params, check_output = False, check = True): + assert False, "_call_vcpkg() should never be called...did we forget to hook a function?" + + def _install_packages(env, packages): + instance = MockVCPkg.__instance + for exp in instance._expectations: + exp.onInstall(env, packages) + for p in packages: + name = p.get_name() + assert name not in instance._installedPackages, f"Trying to install package with name '{name}' that is reported as already-installed" + assert name in instance._availablePackages, f"Trying to install package with name '{name}' that is not among the available packages" + instance._availablePackages[name].install(env, p.get_static()) + + def _upgrade_packages(env, packages): + instance = MockVCPkg.__instance + for exp in MockVCPkg.__instance._expectations: + exp.onUpgrade(env, packages) + for p in packages: + name = p.get_name() + assert name in instance._installedPackages, f"Trying to upgrade package with name '{name}' that is not reported as already-installed" + assert name in instance._availablePackages, f"Trying to upgrade package with name '{name}' that is not among the available packages" + instance._installedPackages[name].clean_up() + instance._availablePackages[name].install(env, p.get_static()) + + def is_mismatched_version_installed(env, spec): + name = re.sub(r':.*$', '', spec) + instance = MockVCPkg.__instance + return name in instance._installedPackages and (name not in instance._availablePackages or instance._installedPackages[name].version != instance._availablePackages[name].version) + + def _get_package_version(env, spec): + name = re.sub(r':.*$', '', spec) + pkg = MockVCPkg.__instance._availablePackages[name] + assert pkg is not None, f"_get_package_version() for not-registered package '{spec}'" + return pkg.version + + def _get_package_deps(env, spec, static): + name = re.sub(r':.*$', '', spec) + return MockVCPkg.__instance._availablePackages[name].dependencies + + def _read_vcpkg_file_list(env, list_file): + return [] + + +class InstallExpectation: + def __init__(self, packages): + self._packageInstalled = {} + for p in packages: + self._packageInstalled[p] = False + + def onInstall(self, env, packages): + for p in packages: + name = p.get_name() + assert name in self._packageInstalled, f"Installing unexpected package '{name}'" + assert self._packageInstalled[name] == False, f"Installing package '{name}' more than once!" + self._packageInstalled[name] = True + + def onUpgrade(self, env, packages): + for p in packages: + assert p.get_name() not in self._packageInstalled, f"Expected package '{p.get_name()}' to be installed, but it was upgraded instead." + + def finalize(self): + for p in self._packageInstalled: + assert self._packageInstalled[p], f"Expected package '{p}' to be installed, but it was not." + + +class UpgradeExpectation: + def __init__(self, packages, static = False): + self._packageUpgraded = {} + for p in packages: + self._packageUpgraded[p] = False + + def onInstall(self, env, packages): + for p in packages: + assert p.get_name() not in self._packageUpgraded, f"Expected package '{p.get_name()}' to be upgraded, but it was installed instead." + + def onUpgrade(self, env, packages): + for p in packages: + name = p.get_name() + assert name in self._packageUpgraded, f"Upgrading unexpected package '{name}'" + assert self._packageUpgraded[name] == False, f"Upgrading package '{name}' more than once!" + self._packageUpgraded[name] = True + + def finalize(self): + for p in self._packageUpgraded: + assert self._packageUpgraded[p], f"Expected package '{p}' to be upgraded, but it was not." + + +class NoChangeExpectation: + def onInstall(self, env, packages): + assert False, "Expected no package changes, but this/these were installed: " + ' '.join(map(lambda p: str(p), packages)) + + def onUpgrade(self, env, packages): + assert False, "Expected no package changes, but this/these were upgraded: " + ' '.join(map(lambda p: str(p), packages)) + + def finalize(self): + pass - os.mkdir(vcpkg_root) - # Ensure that the .vcpkg-root sentinel file exists - Path(vcpkg_root + '/.vcpkg-root').touch() +@contextmanager +def MockVCPkgUser(): + """ContextManager providing scoped usage of the MockVCPkg singleton""" + instance = MockVCPkg.getInstance() + try: + yield instance + finally: + instance.release() - if include_vcpkg_exe: - if os.name == 'nt': - Path(vcpkg_root + '/vcpkg.exe').touch() - else: - Path(vcpkg_root + '/vcpkg').touch() - return vcpkg_root +@contextmanager +def AvailablePackage(name, version, dependencies = []): + """ContextManager temporarily adding an 'available' package to the MockVCPkg during its 'with' scope""" + with MockVCPkgUser() as vcpkg: + pkg = vcpkg.addAvailablePackage(name, version, dependencies) + try: + yield pkg + finally: + vcpkg.removeAvailablePackage(pkg) - def run_with_mocks(payload, - call_vcpkg = None, - get_package_version = None, - read_vcpkg_file_list = None): - """Runs `payload` with one or more low-level VCPkg functions mocked""" +@contextmanager +def InstalledPackage(env, name, version, dependencies = [], static = False): + """ContextManager temporarily adding an 'installed' package to the MockVCPkg during its 'with' scope""" + with MockVCPkgUser() as vcpkg: + pkg = vcpkg.addInstalledPackage(env, name, version, dependencies, static) + try: + yield pkg + finally: + vcpkg.removeInstalledPackage(pkg) - # Save original functions to restore later - orig_call_vcpkg = SCons.Tool.vcpkg._call_vcpkg - orig_get_package_version = SCons.Tool.vcpkg._get_package_version - orig_read_vcpkg_file_list = SCons.Tool.vcpkg._read_vcpkg_file_list +@contextmanager +def Expect(exp): + """ContextManager temporarily adding an expectation to the MockVCPkg that must be fulfilled within its 'with' scope""" + with MockVCPkgUser() as vcpkg: + exp = vcpkg.addExpectation(exp) try: - # Override any functions for which mocks were supplied - if call_vcpkg is not None: - SCons.Tool.vcpkg._call_vcpkg = call_vcpkg - if get_package_version is not None: - SCons.Tool.vcpkg._get_package_version = get_package_version - if read_vcpkg_file_list is not None: - SCons.Tool.vcpkg._read_vcpkg_file_list = read_vcpkg_file_list + yield exp + exp.finalize() + finally: + vcpkg.removeExpectation(exp) - # Run the payload - result = payload() - finally: - # Restore original functions - SCons.Tool.vcpkg._call_vcpkg = orig_call_vcpkg - SCons.Tool.vcpkg._get_pacakge_version = orig_get_package_version - SCons.Tool.vcpkg._get_pacakge_version = orig_read_vcpkg_file_list +@contextmanager +def ExpectInstall(packages): + """ContextManager adding an expectation that the specified list of packages will be installed within its 'with' scope""" + exp = InstallExpectation(packages) + with Expect(exp): + yield exp + + +@contextmanager +def ExpectNoInstall(): + """ContextManager adding an expectation that no packages will be installed within its 'with' scope""" + exp = InstallExpectation(packages = []) + with Expect(exp): + yield exp + + +@contextmanager +def ExpectUpgrade(packages): + """ContextManager adding an expectation that the specified list of packages will be upgraded within its 'with' scope""" + exp = UpgradeExpectation(packages) + with Expect(exp): + yield exp + + +@contextmanager +def ExpectNoUpgrade(): + """ContextManager adding an expectation that no packages will be upgraded within its 'with' scope""" + exp = UpgradeExpectation(packages = []) + with Expect(exp): + yield exp + + +@contextmanager +def ExpectNoChange(): + """ContextManager temporarily adding an expectation that no package installation changes will occur within its 'with' scope""" + exp = NoChangeExpectation() + with Expect(exp): + yield exp - return result +@contextmanager +def MakeVCPkgEnv(): + """Returns an Environment suitable for testing VCPkg""" + with tempfile.TemporaryDirectory() as vcpkg_root: + # Ensure that the .vcpkg-root sentinel file and directory structure exists + Path(f"{vcpkg_root}/.vcpkg-root").touch() + os.makedirs(f"{vcpkg_root}/installed/vcpkg/info") + env = Environment(tools=['default', 'vcpkg']) + env['VCPKGROOT'] = vcpkg_root + yield env + + +class VCPkgTestCase(unittest.TestCase): def test_VCPKGROOT(self): """Test that VCPkg() fails with an exception if the VCPKGROOT environment variable is unset or invalid""" @@ -125,74 +414,107 @@ def test_VCPKGROOT(self): assert "$VCPKGROOT must point to" in str(e), e assert exc_caught, "did not catch expected UserError" - # VCPKGROOT pointing to a mock vcpkg instance (should succeed) - env['VCPKGROOT'] = self.make_mock_vcpkg_dir(include_vcpkg_exe = True) -# orig_call_vcpkg = SCons.Tool.vcpkg._call_vcpkg -# orig_get_package_version = SCons.Tool.vcpkg._get_package_version -# orig_read_vcpkg_file_list = SCons.Tool.vcpkg._read_vcpkg_file_list - installed_package = False - def mock_call_vcpkg(env, params, check_output = False): - if 'install' in params and 'pretend_package' in params: - installed_pacakge = True - return 0 - if 'depend-info' in params and check_output: - return params[1] + ':' - - if check_output: - return '' - else: - return 0 - def mock_get_package_version(env, spec): - assert spec == 'pretend_package', "queried for unexpected package '" + spec + "'" - return '1.0.0' - def mock_read_vcpkg_file_list(env, list_file): - return [env.File('$VCPKGROOT/installed/x64-windows/lib/pretend_package.lib')] - - VCPkgTestCase.run_with_mocks(lambda: env.VCPkg('pretend_package'), - call_vcpkg = mock_call_vcpkg, - get_package_version = mock_get_package_version, - read_vcpkg_file_list = mock_read_vcpkg_file_list) -# SCons.Tool.vcpkg._call_vcpkg = mock_call_vcpkg_exe -# SCons.Tool.vcpkg._get_package_version = mock_get_package_version -# SCons.Tool.vcpkg._read_vcpkg_file_list = mock_read_vcpkg_file_list -# try: -# env.VCPkg('pretend_package') -# finally: -# rmtree(env['VCPKGROOT']) -# SCons.Tool.vcpkg._call_vcpkg = orig_call_vcpkg -# SCons.Tool.vcpkg._get_pacakge_version = orig_get_package_version -# SCons.Tool.vcpkg._get_pacakge_version = orig_read_vcpkg_file_list - - - def test_bootstrap(self): - """Test that the VCPkg builder correctly bootstraps the vcpkg executable""" - # upgrade, downgrade - pass - - - def test_depend_info(self): - """Test that the VCPkg builder correctly parses package dependencies""" - # multiple levels, conflicting feature sets, ignore extra packages - pass - - - def test_install(self): + def test_install_existing_with_no_dependency(self): """Test that the VCPkg builder installs missing packages""" - # Multiple installs, dependencies - pass + with MakeVCPkgEnv() as env, \ + AvailablePackage("frobotz", "1.2.3"), \ + InstalledPackage(env, "frobotz", "1.2.3"), \ + ExpectNoInstall(): + env.VCPkg("frobotz") + def test_install_with_no_dependency(self): + """Test that the VCPkg builder installs missing packages""" + with MakeVCPkgEnv() as env, \ + AvailablePackage( "frobotz", "1.2.3"), \ + ExpectInstall(["frobotz"]): + env.VCPkg("frobotz") + + def test_duplicate_install(self): + """Test that duplicate invocations of the VCPkg builder installs a package only once""" + with MakeVCPkgEnv() as env, \ + AvailablePackage("frobotz", "1.2.3"), \ + ExpectInstall(["frobotz"]): + env.VCPkg("frobotz") + env.VCPkg("frobotz") + + def test_install_with_satisfied_dependency(self): + """Test that installing a package depending on an installed package does not attempt to reinstall that package""" + with MakeVCPkgEnv() as env, \ + AvailablePackage("xyzzy", "0.1"), \ + AvailablePackage("frobotz", "1.2.1", dependencies = ["xyzzy"]), \ + InstalledPackage(env, "xyzzy", "0.1"), \ + ExpectInstall(["frobotz"]), \ + ExpectNoUpgrade(): + env.VCPkg("frobotz") + + def test_install_with_missing_dependency(self): + """Test that installing a package depending on a not-installed package also installs that package""" + with MakeVCPkgEnv() as env, \ + AvailablePackage("xyzzy", "0.1"), \ + AvailablePackage("frobotz", "1.2.1", dependencies = ["xyzzy"]), \ + ExpectInstall(["xyzzy", "frobotz"]), \ + ExpectNoUpgrade(): + env.VCPkg("frobotz") + + def test_install_with_missing_dependencies(self): + """Test that installing a package depending on multiple not-installed packages also installs those packages""" + with MakeVCPkgEnv() as env, \ + AvailablePackage("xyzzy", "0.1"), \ + AvailablePackage("battered_lantern", "0.2"), \ + AvailablePackage("frobotz", "1.2.1", dependencies = ["xyzzy", "battered_lantern"]), \ + ExpectInstall(["xyzzy", "battered_lantern", "frobotz"]), \ + ExpectNoUpgrade(): + env.VCPkg("frobotz") + + def test_install_with_mixed_dependencies(self): + """Test that installing a package depending on a not-installed package also installs that package""" + with MakeVCPkgEnv() as env, \ + AvailablePackage("xyzzy", "0.1"), \ + AvailablePackage("battered_lantern", "0.2"), \ + AvailablePackage("frobotz", "1.2.1", dependencies = ["xyzzy"]), \ + InstalledPackage(env, "battered_lantern", "0.2"), \ + ExpectInstall(["xyzzy", "frobotz"]), \ + ExpectNoUpgrade(): + env.VCPkg("frobotz") def test_upgrade(self): - """Test that the VCPkg builder correctly upgrades/downgrades existing packages to match the vcpkg configuration""" - pass - - - def test_PackageContents(self): - """Test that the VCPkg builder returns a PackageContents object that can reports the files contained in the package""" - - env = Environment(tools=['vcpkg']) - - pass + """Test that the VCPkg builder correctly upgrades existing packages to match the vcpkg configuration""" + with MakeVCPkgEnv() as env, \ + AvailablePackage("frobotz", "1.2.1"), \ + InstalledPackage(env, "frobotz", "1.2.2"), \ + ExpectNoInstall(), \ + ExpectUpgrade(["frobotz"]): + env.VCPkg("frobotz") + + def test_downgrade(self): + """Test that the VCPkg builder correctly downgrades existing packages to match the vcpkg configuration""" + with MakeVCPkgEnv() as env, \ + AvailablePackage("frobotz", "1.2.3"), \ + InstalledPackage(env, "frobotz", "1.2.2"), \ + ExpectNoInstall(), \ + ExpectUpgrade(["frobotz"]): + env.VCPkg("frobotz") + + def test_upgrade_with_satisfied_dependency(self): + """Test that the VCPkg builder correctly upgrades existing packages to match the vcpkg configuration""" + with MakeVCPkgEnv() as env, \ + AvailablePackage("glowing_sword", "0.5"), \ + AvailablePackage("frobotz", "1.2.2", dependencies = ["glowing_sword"]), \ + InstalledPackage(env, "glowing_sword", "0.5"), \ + InstalledPackage(env, "frobotz", "1.2.3"), \ + ExpectNoInstall(), \ + ExpectUpgrade(["frobotz"]): + env.VCPkg("frobotz") + + def test_upgrade_with_missing_dependency(self): + """Test that the VCPkg builder correctly upgrades existing packages to match the vcpkg configuration""" + with MakeVCPkgEnv() as env, \ + AvailablePackage("glowing_sword", "0.5"), \ + AvailablePackage("frobotz", "1.2.2", dependencies = ["glowing_sword"]), \ + InstalledPackage(env, "frobotz", "1.2.3"), \ + ExpectInstall(["glowing_sword"]), \ + ExpectUpgrade(["frobotz"]): + env.VCPkg("frobotz") if __name__ == "__main__": diff --git a/SCons/Tool/vcpkg.py b/SCons/Tool/vcpkg.py index 907ad9ef26..2da713995e 100644 --- a/SCons/Tool/vcpkg.py +++ b/SCons/Tool/vcpkg.py @@ -10,13 +10,14 @@ TODO: + * find a way to push package build into SCons's normal build phase * ensure Linux works - * unit tests - * verify that feature supersetting works + * handle complex feature supersetting scenarios * parallel builds? * can we ensure granular detection, and fail on undetected dependencies? * batch depend-info calls to vcpkg for better perf? * Make "vcpkg search" faster by supporting a strict match option + * Is there a way to make vcpkg build only dgb/rel libs? """ @@ -213,11 +214,30 @@ def _call_vcpkg(env, params, check_output = False, check = True): return ex.returncode +def _install_packages(env, packages): + packages_args = list(map(lambda p: str(p), packages)) + _vcpkg_print(Silent, ' '.join(packages_args) + ' (install)') + result = _call_vcpkg(env, ['install'] + packages_args) + if result != 0: + _vcpkg_print(Silent, "Failed to install package(s) " + ' '.join(packages_args)) + return result + + +def _upgrade_packages(env, packages): + packages_args = list(map(lambda p: str(p), packages)) + _vcpkg_print(Silent, ' '.join(packages_args) + ' (upgrade)') + result = _call_vcpkg(env, ['upgrade', '--no-dry-run'] + packages_args) + if result != 0: + _vcpkg_print(Silent, "Failed to upgrade package(s) " + ' '.join(packages_args)) + return result + + def _get_vcpkg_triplet(env, static): """Computes the appropriate VCPkg 'triplet' for the current build environment""" platform = env['PLATFORM'] + # TODO: this relies on having a C++ compiler tool enabled. Is there a better way to compute this? if 'TARGET_ARCH' in env: arch = env['TARGET_ARCH'] else: @@ -252,6 +272,17 @@ def _read_vcpkg_file_list(env, list_file): return files +def is_mismatched_version_installed(env, spec): + _vcpkg_print(Debug, 'Checking for mismatched version of "' + spec + '"') + output = _call_vcpkg(env, ['update'], check_output = True) + for line in output.split('\n'): + match = re.match(r'^\s*(\S+)\s*(\S+) -> (\S+)', line) + if match and match.group(1) == spec: + _vcpkg_print(Debug, 'Package "' + spec + '" can be updated (' + match.group(2) + ' -> ' + match.group(3) + ')') + return True + return False + + def _get_package_version(env, spec): """Read the available version of a package (i.e., what would be installed)""" @@ -302,12 +333,23 @@ def _get_package_deps(env, spec, static): raise InternalError('Failed to parse output from vcpkg ' + ' '.join(params) + '\n' + output) -# CHECKIN: move to env? -# Global mapping of previously-computed package-name -> list-file targets. This exists because we may discover -# additional packages that we need to build, based on running :vcpkg depend-info"), and these packages may or -# may not have been explicitly requested by calls to VCPkg. -_package_descriptors_map = {} -_package_targets_map = {} +def get_package_descriptors_map(env): + """Returns an Environment-global mapping of previously-seen package name -> PackageDescriptor, ensuring a 1:1 + correspondence between a package and its PackageDescriptor. This global mapping is needed because a package + may need to be built due to being a dependency of a user-requested package, and a given package may be a + dependency of multiple user-requested packages, or may be a dependency of a user-requested package via multiple + paths in the package dependency graph.""" + if not hasattr(env, '_vcpkg_package_descriptors_map'): + env._vcpkg_package_descriptors_map = {} + return env._vcpkg_package_descriptors_map + +def get_package_targets_map(env): + """Returns an Environment-global mapping of previously-seen package name -> PackageContents. This global mapping + is needed because the contents-enumeration methods of PackageContents need to be able to access the contents + of their dependencies when transitive = True is specified.""" + if not hasattr(env, '_vcpkg_package_targets_map'): + env._vcpkg_package_targets_map = {} + return env._vcpkg_package_targets_map class PackageDescriptor(SCons.Node.Python.Value): """PackageDescriptor is the 'source' node for the VCPkg builder. A PackageDescriptor instance includes the package @@ -346,6 +388,12 @@ def __init__(self, env, spec, static = False): self.env = env self.package_deps = list(map(lambda p: get_package_descriptor(env, p), depends)) + def get_name(self): + return self.value['name'] + + def get_static(self): + return self.value['static'] + def get_triplet(self): return self.value['triplet'] @@ -367,26 +415,18 @@ def get_listfile_basename(self): def __str__(self): return self.get_package_string() - def is_mismatched_version_installed(self): - _vcpkg_print(Debug, 'Checking for mismatched version of "' + str(self) + '"') - output = _call_vcpkg(self.env, ['update'], check_output = True) - for line in output.split('\n'): - match = re.match(r'^\s*(\S+)\s*(\S+) -> (\S+)', line) - if match and match.group(1) == str(self): - _vcpkg_print(Debug, 'Package "' + str(self) + '" can be updated (' + match.group(2) + ' -> ' + match.group(3)) - return True - return False - def target_from_source(self, pre, suf, splitext): _bootstrap_vcpkg(self.env) target = PackageContents(self.env, self) target.state = SCons.Node.up_to_date + package_targets_map = get_package_targets_map(self.env) + for pkg in self.package_deps: - if pkg in _package_targets_map: - _vcpkg_print(Debug, 'Reused dep: ' + str(_package_targets_map[pkg])) - self.env.Depends(target, _package_targets_map[pkg]) + if pkg in package_targets_map: + _vcpkg_print(Debug, 'Reused dep: ' + str(package_targets_map[pkg])) + self.env.Depends(target, package_targets_map[pkg]) else: _vcpkg_print(Debug, "New dep: " + str(pkg)) dep = self.env.VCPkg(pkg) @@ -394,15 +434,11 @@ def target_from_source(self, pre, suf, splitext): if not SCons.Script.GetOption('help'): if not target.exists(): - if self.is_mismatched_version_installed(): - _vcpkg_print(Silent, str(self) + ' (upgrade)') - if _call_vcpkg(self.env, ['upgrade', '--no-dry-run', str(self)]) != 0: - _vcpkg_print(Silent, "Failed to upgrade package " + str(self)) + if is_mismatched_version_installed(self.env, str(self)): + if _upgrade_packages(self.env, [self]) != 0: target.state = SCons.Node.failed else: - _vcpkg_print(Silent, str(self) + ' (install)') - if _call_vcpkg(self.env, ['install', str(self)]) != 0: - _vcpkg_print(Silent, "Failed to install package " + str(self)) + if _install_packages(self.env, [self]) != 0: target.state = SCons.Node.failed target.clear_memoized_values() if not target.exists(): @@ -413,7 +449,7 @@ def target_from_source(self, pre, suf, splitext): target.noclean = True _vcpkg_print(Debug, "Caching target for package: " + self.value['name']) - _package_targets_map[self] = target + package_targets_map[self] = target return target @@ -479,8 +515,9 @@ def FilesUnderSubPath(self, subpath, transitive): matching_files += [file] if transitive: + package_targets_map = get_package_targets_map(self.env) for dep in self.descriptor.package_deps: - matching_files += _package_targets_map[dep].FilesUnderSubPath(subpath, transitive) + matching_files += package_targets_map[dep].FilesUnderSubPath(subpath, transitive) return SCons.Util.NodeList(matching_files) @@ -488,10 +525,11 @@ def __str__(self): return "Package: " + super(PackageContents, self).__str__() def get_package_descriptor(env, spec): - if spec in _package_descriptors_map: - return _package_descriptors_map[spec] + package_descriptors_map = get_package_descriptors_map(env) + if spec in package_descriptors_map: + return package_descriptors_map[spec] desc = PackageDescriptor(env, spec) - _package_descriptors_map[spec] = desc + package_descriptors_map[spec] = desc return desc From a560ee05f3d8f97cc5681659b4f65052d828e704 Mon Sep 17 00:00:00 2001 From: Ryan Saunders Date: Sat, 10 Sep 2022 22:41:09 -0700 Subject: [PATCH 09/22] Added CHANGES.txt entry --- CHANGES.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index 8e32f3d67d..dffb53149e 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -25,6 +25,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Added -fsanitize support to ParseFlags(). This will propagate to CCFLAGS and LINKFLAGS. From Ryan Saunders: + - Added VCPkg() builder, enabling cross-platform acquisition and building of 3rd party C/C++ + libraries. - Fixed runtest.py failure on Windows caused by excessive escaping of the path to python.exe. RELEASE 4.4.0 - Sat, 30 Jul 2022 14:08:29 -0700 From 7ce70888801751997d894b3018fc8ffefb5f2bce Mon Sep 17 00:00:00 2001 From: Ryan Saunders Date: Sun, 11 Sep 2022 17:44:16 -0700 Subject: [PATCH 10/22] Updated description in CHANGES.txt and replicated in RELEASE.txt --- CHANGES.txt | 5 +++-- RELEASE.txt | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index dffb53149e..e82944e054 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -25,8 +25,9 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Added -fsanitize support to ParseFlags(). This will propagate to CCFLAGS and LINKFLAGS. From Ryan Saunders: - - Added VCPkg() builder, enabling cross-platform acquisition and building of 3rd party C/C++ - libraries. + - Added VCPkg() builder, integrating the vcpkg cross-platform package management tool for + 3rd-party C/C++ libraries (http://vcpkg.io). A project using SCons can use the VCPkg() + builder to to download and build any package known to vcpkg. Works on Windows, Linux and MacOS. - Fixed runtest.py failure on Windows caused by excessive escaping of the path to python.exe. RELEASE 4.4.0 - Sat, 30 Jul 2022 14:08:29 -0700 diff --git a/RELEASE.txt b/RELEASE.txt index 93d47f1a9f..28bbb212b8 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -19,6 +19,9 @@ NEW FUNCTIONALITY - Added ValidateOptions() which will check that all command line options are in either those specified by SCons itself, or by AddOption() in SConstruct/SConscript. It should not be called until all AddOption() calls are completed. Resolves Issue #4187 +- Added VCPkg() builder, integrating the vcpkg cross-platform package management tool for + 3rd-party C/C++ libraries (http://vcpkg.io). A project using SCons can use the VCPkg() + builder to to download and build any package known to vcpkg. Works on Windows, Linux and MacOS. DEPRECATED FUNCTIONALITY ------------------------ From 2f3c24fd61f45579f7a55a58be9e48d7cdbc2765 Mon Sep 17 00:00:00 2001 From: Ryan Saunders Date: Sun, 18 Sep 2022 08:36:49 -0700 Subject: [PATCH 11/22] Updated file headers to match current templates. --- SCons/Tool/VCPkgTests.py | 10 +++++++--- SCons/Tool/vcpkg.py | 26 +++++++++++++++++++++++--- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/SCons/Tool/VCPkgTests.py b/SCons/Tool/VCPkgTests.py index 2403c8e2e1..b64a0a4b00 100644 --- a/SCons/Tool/VCPkgTests.py +++ b/SCons/Tool/VCPkgTests.py @@ -1,5 +1,8 @@ +#!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -19,9 +22,10 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" +""" +Unit tests for the VCPkg() builder +""" import os import re diff --git a/SCons/Tool/vcpkg.py b/SCons/Tool/vcpkg.py index 2da713995e..d13cbadcc7 100644 --- a/SCons/Tool/vcpkg.py +++ b/SCons/Tool/vcpkg.py @@ -1,7 +1,27 @@ -# -*- coding: utf-8; -*- - -"""SCons.Tool.vcpkg +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" Tool-specific initialization for vcpkg. There normally shouldn't be any need to import this module directly. From 70fef432ab204530e499852ef13ac459a0f234f3 Mon Sep 17 00:00:00 2001 From: Ryan Saunders Date: Sun, 18 Sep 2022 13:05:36 -0700 Subject: [PATCH 12/22] Only init vcpkg tool on supported platforms (Win/MacOS/Linux) --- SCons/Tool/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/SCons/Tool/__init__.py b/SCons/Tool/__init__.py index 708b1420f6..20118eaf68 100644 --- a/SCons/Tool/__init__.py +++ b/SCons/Tool/__init__.py @@ -696,7 +696,7 @@ def tool_list(platform, env): assemblers = ['masm', 'nasm', 'gas', '386asm'] fortran_compilers = ['gfortran', 'g77', 'ifl', 'cvf', 'f95', 'f90', 'fortran'] ars = ['mslib', 'ar', 'tlib'] - other_plat_tools = ['msvs', 'midl', 'wix'] + other_plat_tools = ['msvs', 'midl', 'vcpkg', 'wix'] elif str(platform) == 'os2': "prefer IBM tools on OS/2" linkers = ['ilink', 'gnulink', ] # 'mslink'] @@ -746,6 +746,7 @@ def tool_list(platform, env): assemblers = ['as'] fortran_compilers = ['gfortran', 'f95', 'f90', 'g77'] ars = ['ar'] + other_plat_tools += ['vcpkg'] elif str(platform) == 'cygwin': "prefer GNU tools on Cygwin, except for a platform-specific linker" linkers = ['cyglink', 'mslink', 'ilink'] @@ -762,6 +763,9 @@ def tool_list(platform, env): assemblers = ['gas', 'nasm', 'masm'] fortran_compilers = ['gfortran', 'g77', 'ifort', 'ifl', 'f95', 'f90', 'f77'] ars = ['ar', ] + # VCPkg is supported on Linux; no official support for other *nix variants + if str(platform) == 'posix': + other_plat_tools += ['vcpkg'] if not str(platform) == 'win32': other_plat_tools += ['m4', 'rpm'] @@ -809,8 +813,6 @@ def tool_list(platform, env): 'tar', 'zip', # File builders (text) 'textfile', - # Package management - 'vcpkg', ], env) tools = [ From 20ce4788a703bc4cc9bfaafb260b3f254f22d613 Mon Sep 17 00:00:00 2001 From: Ryan Saunders Date: Sun, 18 Sep 2022 17:54:21 -0700 Subject: [PATCH 13/22] Fix code review feedback and add tests for package contents --- SCons/Tool/VCPkgTests.py | 180 ++++++++++++++++++++++++++++++++++----- SCons/Tool/vcpkg.py | 44 +++++----- 2 files changed, 186 insertions(+), 38 deletions(-) diff --git a/SCons/Tool/VCPkgTests.py b/SCons/Tool/VCPkgTests.py index b64a0a4b00..849ecc69cd 100644 --- a/SCons/Tool/VCPkgTests.py +++ b/SCons/Tool/VCPkgTests.py @@ -45,21 +45,28 @@ # * Test parsing of real vcpkg.exe output # * Test feature super-setting # * Test "static" installs -# * Test "debug" installs -# * Test enumerating package contents +# * Test transitive dependencies only listing files once class MockPackage: - def __init__(self, name, version, dependencies): + def __init__(self, name, version, dependencies, files): self.name = name self.version = version self.dependencies = dependencies self._installedFiles = [] + self._packageFiles = files + + def get_list_file(self, env, static): + version = self.version + hash_pos = version.find('#') + if hash_pos != -1: + version = version[0:hash_pos] + return env.File(f"$VCPKGROOT/installed/vcpkg/info/{self.name}_{version}_{SCons.Tool.vcpkg._get_vcpkg_triplet(env, static)}.list") def install(self, env, static): assert not self._installedFiles, f"Trying to install package '{self.name}' more than once!" - listfile = f"{env['VCPKGROOT']}/installed/vcpkg/info/{self.name}_{self.version}_{SCons.Tool.vcpkg._get_vcpkg_triplet(env, static)}.list" - Path(listfile).touch() - self._installedFiles = [listfile] + listfile = self.get_list_file(env, static) + Path(listfile.get_abspath()).touch() + self._installedFiles = [listfile.get_abspath()] def clean_up(self): for file in self._installedFiles: @@ -148,9 +155,9 @@ def release(self): # State modification functions used by contextmanager functions below # - def addAvailablePackage(self, name, version, dependencies): + def addAvailablePackage(self, name, version, dependencies, files): assert name not in self._availablePackages, f"Already have an AvailablePackage with name '{name}' (version {self._availablePackages[name].version})" - pkg = MockPackage(name, version, dependencies) + pkg = MockPackage(name, version, dependencies, files) self._availablePackages[name] = pkg return pkg @@ -158,9 +165,9 @@ def removeAvailablePackage(self, pkg): pkg.clean_up() assert self._availablePackages.pop(pkg.name), f"Trying to remove AvailablePackage with name '{pkg.name}' that is not currently registered" - def addInstalledPackage(self, env, name, version, dependencies, static): + def addInstalledPackage(self, env, name, version, dependencies, files, static): assert name not in self._installedPackages, f"Already have an InstalledPackage with name '{name}' (version {self._availablePackages[name].version})" - pkg = MockPackage(name, version, dependencies) + pkg = MockPackage(name, version, dependencies, files) pkg.install(env, static) self._installedPackages[name] = pkg return pkg @@ -226,7 +233,38 @@ def _get_package_deps(env, spec, static): return MockVCPkg.__instance._availablePackages[name].dependencies def _read_vcpkg_file_list(env, list_file): - return [] + # Find the correct package. It could be in either 'available' or 'installed' + instance = MockVCPkg.__instance + package = None + static = False + for name in instance._installedPackages: + p = instance._installedPackages[name] + for s in [False, True]: + if p.get_list_file(env, s).get_abspath() == list_file.get_abspath(): + package = p + static = s + break + if package is not None: + break + if package is None: + for name in instance._availablePackages: + p = instance._availablePackages[name] + for s in [False, True]: + if p.get_list_file(env, s).get_abspath() == list_file.get_abspath(): + package = p + static = s + break + if package is not None: + break + + if package is not None: + prefix = f"$VCPKGROOT/installed/{SCons.Tool.vcpkg._get_vcpkg_triplet(env, static)}/" + files = [] + for f in package._packageFiles: + files.append(env.File(prefix + f)) + return files + + assert False, f"Did not find a package matching list_file '{list_file}'" class InstallExpectation: @@ -239,7 +277,7 @@ def onInstall(self, env, packages): for p in packages: name = p.get_name() assert name in self._packageInstalled, f"Installing unexpected package '{name}'" - assert self._packageInstalled[name] == False, f"Installing package '{name}' more than once!" + assert self._packageInstalled[name] is False, f"Installing package '{name}' more than once!" self._packageInstalled[name] = True def onUpgrade(self, env, packages): @@ -265,7 +303,7 @@ def onUpgrade(self, env, packages): for p in packages: name = p.get_name() assert name in self._packageUpgraded, f"Upgrading unexpected package '{name}'" - assert self._packageUpgraded[name] == False, f"Upgrading package '{name}' more than once!" + assert self._packageUpgraded[name] is False, f"Upgrading package '{name}' more than once!" self._packageUpgraded[name] = True def finalize(self): @@ -295,10 +333,10 @@ def MockVCPkgUser(): @contextmanager -def AvailablePackage(name, version, dependencies = []): +def AvailablePackage(name, version, dependencies = [], files = []): """ContextManager temporarily adding an 'available' package to the MockVCPkg during its 'with' scope""" with MockVCPkgUser() as vcpkg: - pkg = vcpkg.addAvailablePackage(name, version, dependencies) + pkg = vcpkg.addAvailablePackage(name, version, dependencies, files) try: yield pkg finally: @@ -306,10 +344,10 @@ def AvailablePackage(name, version, dependencies = []): @contextmanager -def InstalledPackage(env, name, version, dependencies = [], static = False): +def InstalledPackage(env, name, version, dependencies = [], files = [], static = False): """ContextManager temporarily adding an 'installed' package to the MockVCPkg during its 'with' scope""" with MockVCPkgUser() as vcpkg: - pkg = vcpkg.addInstalledPackage(env, name, version, dependencies, static) + pkg = vcpkg.addInstalledPackage(env, name, version, dependencies, files, static) try: yield pkg finally: @@ -369,7 +407,7 @@ def ExpectNoChange(): @contextmanager -def MakeVCPkgEnv(): +def MakeVCPkgEnv(debug = False): """Returns an Environment suitable for testing VCPkg""" with tempfile.TemporaryDirectory() as vcpkg_root: # Ensure that the .vcpkg-root sentinel file and directory structure exists @@ -378,9 +416,31 @@ def MakeVCPkgEnv(): env = Environment(tools=['default', 'vcpkg']) env['VCPKGROOT'] = vcpkg_root + if debug: + env['VCPKGDEBUG'] = True yield env +def assert_package_files(env, static, actual_files, expected_subpaths): + """Verify that 'actual_files' contains exactly the list in 'expected_subpaths' (after prepending the path to the + vcpkg triplet subdirectory containing installed files)""" + prefix = env.subst(f"$VCPKGROOT/installed/{SCons.Tool.vcpkg._get_vcpkg_triplet(env, static)}/") + subpath_used = {} + for s in expected_subpaths: + subpath_used[s] = False + for f in actual_files: + path = f.get_abspath() + used_subpath = None + for s in expected_subpaths: + if path == env.File(prefix + s).get_abspath(): + assert used_subpath is None, f"Used file subpath '{s}' more than once" + used_subpath = s + subpath_used[s] = True + assert used_subpath is not None, f"File '{path}' does not match any expected subpath" + for s in expected_subpaths: + assert subpath_used[s] is True, f"Suffix '{s}' did not match any file" + + class VCPkgTestCase(unittest.TestCase): def test_VCPKGROOT(self): """Test that VCPkg() fails with an exception if the VCPKGROOT environment variable is unset or invalid""" @@ -429,7 +489,7 @@ def test_install_existing_with_no_dependency(self): def test_install_with_no_dependency(self): """Test that the VCPkg builder installs missing packages""" with MakeVCPkgEnv() as env, \ - AvailablePackage( "frobotz", "1.2.3"), \ + AvailablePackage("frobotz", "1.2.3"), \ ExpectInstall(["frobotz"]): env.VCPkg("frobotz") @@ -481,6 +541,17 @@ def test_install_with_mixed_dependencies(self): ExpectNoUpgrade(): env.VCPkg("frobotz") + def test_install_with_dependency_reached_by_multiple_paths(self): + """Test that installing a package depending on a not-installed package also installs that package""" + with MakeVCPkgEnv() as env, \ + AvailablePackage("xyzzy", "0.1"), \ + AvailablePackage("glowing_sword", "0.1", dependencies = ["xyzzy"]), \ + AvailablePackage("battered_lantern", "0.2", dependencies = ["xyzzy"]), \ + AvailablePackage("frobotz", "1.2.1", dependencies = ["glowing_sword", "battered_lantern"]), \ + ExpectInstall(["xyzzy", "glowing_sword", "battered_lantern", "frobotz"]), \ + ExpectNoUpgrade(): + env.VCPkg("frobotz") + def test_upgrade(self): """Test that the VCPkg builder correctly upgrades existing packages to match the vcpkg configuration""" with MakeVCPkgEnv() as env, \ @@ -520,6 +591,77 @@ def test_upgrade_with_missing_dependency(self): ExpectUpgrade(["frobotz"]): env.VCPkg("frobotz") + def test_empty_package_spec_is_rejected(self): + """Test that the VCPkg builder rejects package names that are the empty string""" + with MakeVCPkgEnv() as env, \ + MockVCPkgUser(), \ + self.assertRaises(ValueError): + env.VCPkg('') + + def test_package_version_with_hash_suffix_is_trimmed(self): + """Ensure that package versions with #n suffix get trimmed off""" + with MakeVCPkgEnv() as env, \ + AvailablePackage("frobotz", "1.2.3#4"): + assert str(env.VCPkg("frobotz")).__contains__("1.2.3_"), "#4 suffix should've been trimmed" + + def test_enumerating_package_contents(self): + """Ensure that we can enumerate the contents of a package (and, optionally, its transitive dependency set)""" + xyzzyFiles = ['bin/xyzzy.dll', + 'bin/xyzzy.pdb', + 'debug/bin/xyzzy.dll', + 'debug/bin/xyzzy.pdb', + 'debug/lib/xyzzy.lib', + 'debug/lib/pkgconfig/xyzzy.pc', + 'include/xyzzy.h', + 'lib/xyzzy.lib', + 'lib/pkgconfig/xyzzy.pc', + 'shared/xyzzy/copyright'] + frobotzFiles = ['bin/frobotz.dll', + 'bin/frobotz.pdb', + 'debug/bin/frobotz.dll', + 'debug/bin/frobotz.pdb', + 'debug/lib/frobotz.lib', + 'debug/lib/pkgconfig/frobotz.pc', + 'include/frobotz.h', + 'lib/frobotz.lib', + 'lib/pkgconfig/frobotz.pc', + 'shared/frobotz/copyright'] + # By default, we should get libraries under bin/ and lib/ + with MakeVCPkgEnv() as env, \ + AvailablePackage("xyzzy", "0.1", files = xyzzyFiles), \ + AvailablePackage("frobotz", "1.2.3", files = frobotzFiles, dependencies = ["xyzzy"]): + # For simplicity, override the platform-specific naming for static and shared libraries so that we don't + # have to modify the file lists to make the test pass on all platforms + env['SHLIBPREFIX'] = '' + env['LIBPREFIX'] = '' + env['SHLIBSUFFIX'] = '.dll' + env['LIBSUFFIX'] = '.lib' + for pkg in env.VCPkg("frobotz"): + assert_package_files(env, False, pkg.Headers(), ['include/frobotz.h']) + assert_package_files(env, False, pkg.Headers(transitive = True), ['include/frobotz.h', 'include/xyzzy.h']) + assert_package_files(env, False, pkg.SharedLibraries(), ['bin/frobotz.dll']) + assert_package_files(env, False, pkg.SharedLibraries(transitive = True), ['bin/frobotz.dll', 'bin/xyzzy.dll']) + assert_package_files(env, False, pkg.StaticLibraries(), ['lib/frobotz.lib']) + assert_package_files(env, False, pkg.StaticLibraries(transitive = True), ['lib/frobotz.lib', 'lib/xyzzy.lib']) + + # with $VCPKGDEBUG = True, we should get libraries under debug/bin/ and debug/lib/ + with MakeVCPkgEnv(debug = True) as env, \ + AvailablePackage("xyzzy", "0.1", files = xyzzyFiles), \ + AvailablePackage("frobotz", "1.2.3", files = frobotzFiles, dependencies = ["xyzzy"]): + # For simplicity, override the platform-specific naming for static and shared libraries so that we don't + # have to modify the file lists to make the test pass on all platforms + env['SHLIBPREFIX'] = '' + env['LIBPREFIX'] = '' + env['SHLIBSUFFIX'] = '.dll' + env['LIBSUFFIX'] = '.lib' + for pkg in env.VCPkg("frobotz"): + assert_package_files(env, False, pkg.Headers(), ['include/frobotz.h']) + assert_package_files(env, False, pkg.Headers(transitive = True), ['include/frobotz.h', 'include/xyzzy.h']) + assert_package_files(env, False, pkg.SharedLibraries(), ['debug/bin/frobotz.dll']) + assert_package_files(env, False, pkg.SharedLibraries(transitive = True), ['debug/bin/frobotz.dll', 'debug/bin/xyzzy.dll']) + assert_package_files(env, False, pkg.StaticLibraries(), ['debug/lib/frobotz.lib']) + assert_package_files(env, False, pkg.StaticLibraries(transitive = True), ['debug/lib/frobotz.lib', 'debug/lib/xyzzy.lib']) + if __name__ == "__main__": suite = unittest.makeSuite(VCPkgTestCase, 'test_') diff --git a/SCons/Tool/vcpkg.py b/SCons/Tool/vcpkg.py index d13cbadcc7..50fdc708c5 100644 --- a/SCons/Tool/vcpkg.py +++ b/SCons/Tool/vcpkg.py @@ -77,7 +77,7 @@ # Named constants for verbosity levels supported by _vcpkg_print Silent = 0 Normal = 1 -Debug = 2 +Debug = 2 _max_verbosity = Normal # Can be changed with --silent or --vcpkg-debug @@ -258,7 +258,7 @@ def _get_vcpkg_triplet(env, static): platform = env['PLATFORM'] # TODO: this relies on having a C++ compiler tool enabled. Is there a better way to compute this? - if 'TARGET_ARCH' in env: + if 'TARGET_ARCH' in env and env['TARGET_ARCH'] is not None: arch = env['TARGET_ARCH'] else: arch = env['HOST_ARCH'] @@ -276,8 +276,8 @@ def _get_vcpkg_triplet(env, static): if arch == 'x86_64': return 'x64-osx' elif platform == 'posix': -# if arch == 'x86_64': - return 'x64-linux' + if arch == 'x86_64': + return 'x64-linux' raise UserError('This architecture/platform (%s/%s) is currently unsupported with VCPkg' % (arch, platform)) @@ -483,39 +483,37 @@ class PackageContents(SCons.Node.FS.File): installing the package.""" def __init__(self, env, descriptor): - super().__init__( descriptor.get_listfile_basename() + ".list", env.Dir('$VCPKGROOT/installed/vcpkg/info/'), env.fs) + super().__init__(descriptor.get_listfile_basename() + ".list", env.Dir('$VCPKGROOT/installed/vcpkg/info/'), env.fs) self.descriptor = descriptor self.loaded = False def Headers(self, transitive = False): """Returns the list of C/C++ header files belonging to the package. If `transitive` is True, then files belonging to upstream dependencies of this package are also included.""" - _vcpkg_print(Debug, str(self.descriptor) + ': headers') - files = self.FilesUnderSubPath('include/', transitive) return self.FilesUnderSubPath('include/', transitive) def StaticLibraries(self, transitive = False): """Returns the list of static libraries belonging to the package. If `transitive` is True, then files belonging to upstream dependencies of this package are also included.""" - _vcpkg_print(Debug, str(self.descriptor) + ': static libraries') if self.env.subst('$VCPKGDEBUG') == 'True': - return self.FilesUnderSubPath('debug/lib/', transitive) + return self.FilesUnderSubPath('debug/lib/', transitive, self.env['LIBSUFFIX']) else: - return self.FilesUnderSubPath('lib/', transitive) + return self.FilesUnderSubPath('lib/', transitive, self.env['LIBSUFFIX']) def SharedLibraries(self, transitive = False): """Returns the list of shared libraries belonging to the package. If `transitive` is True, then files belonging to upstream dependencies of this package are also included.""" - _vcpkg_print(Debug, str(self.descriptor) + ': shared libraries') if self.env.subst('$VCPKGDEBUG') == 'True': - return self.FilesUnderSubPath('debug/bin/', transitive) + return self.FilesUnderSubPath('debug/bin/', transitive, self.env['SHLIBSUFFIX']) else: - return self.FilesUnderSubPath('bin/', transitive) + return self.FilesUnderSubPath('bin/', transitive, self.env['SHLIBSUFFIX']) - def FilesUnderSubPath(self, subpath, transitive): + def FilesUnderSubPath(self, subpath, transitive, suffix_filters = None): """Returns a (possibly empty) list of File nodes belonging to this package that are located under the relative path `subpath` underneath the triplet install directory. - If `transitive` is True, then files belonging to upstream dependencies of this package are also included.""" + If `transitive` is True, then files belonging to upstream dependencies of this package are also included. + If 'suffix_filters is not None, but instead a string or a list, then only files ending in the substring(s) + listed therein will be included.""" # Load the listfile contents, if we haven't already. This returns a list of File nodes. if not self.loaded: @@ -525,25 +523,33 @@ def FilesUnderSubPath(self, subpath, transitive): self.files = _read_vcpkg_file_list(self.env, self) self.loaded = True - triplet = self.descriptor.get_triplet() + # Compute the appropriate filter check based on 'suffix_filters' + if suffix_filters is None: + matches_filters = lambda f: True + elif type(suffix_filters) is str: + matches_filters = lambda f: f.endswith(suffix_filters) + elif type(suffix_filters) is list: + matches_filters = lambda f: len(filter(lambda s: f.endswith(s), suffix_filters)) > 0 + prefix = self.env.Dir(self.env.subst('$VCPKGROOT/installed/' + self.descriptor.get_triplet() + "/" + subpath)) _vcpkg_print(Debug, 'Looking for files under ' + str(prefix)) matching_files = [] for file in self.files: - if file.is_under(prefix): + if file.is_under(prefix) and matches_filters(file.get_abspath()): _vcpkg_print(Debug, 'Matching file ' + file.get_abspath()) matching_files += [file] if transitive: package_targets_map = get_package_targets_map(self.env) for dep in self.descriptor.package_deps: - matching_files += package_targets_map[dep].FilesUnderSubPath(subpath, transitive) + matching_files += package_targets_map[dep].FilesUnderSubPath(subpath, transitive, suffix_filters) return SCons.Util.NodeList(matching_files) def __str__(self): return "Package: " + super(PackageContents, self).__str__() + def get_package_descriptor(env, spec): package_descriptors_map = get_package_descriptors_map(env) if spec in package_descriptors_map: @@ -566,7 +572,7 @@ def vcpkg_action(target, source, env): def get_vcpkg_deps(node, env, path, arg): deps = [] - if not node.package_deps is None: + if node.package_deps is not None: for pkg in node.package_deps: target = env.VCPkg(pkg) deps += target[0] From 2ffe5d45add48da9459a33a359cbae3a5a90dbb2 Mon Sep 17 00:00:00 2001 From: Ryan Saunders Date: Sun, 18 Sep 2022 22:52:25 -0700 Subject: [PATCH 14/22] Fix a flaw in enumerating package contents, in which packages reachable by multiple paths through the dependency graph get their files reported multiple times. --- SCons/Tool/VCPkgTests.py | 37 +++++++++++++++++++++++++------------ SCons/Tool/vcpkg.py | 11 +++++++++-- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/SCons/Tool/VCPkgTests.py b/SCons/Tool/VCPkgTests.py index 849ecc69cd..4822c4f097 100644 --- a/SCons/Tool/VCPkgTests.py +++ b/SCons/Tool/VCPkgTests.py @@ -430,13 +430,14 @@ def assert_package_files(env, static, actual_files, expected_subpaths): subpath_used[s] = False for f in actual_files: path = f.get_abspath() - used_subpath = None + matched_subpath = None for s in expected_subpaths: if path == env.File(prefix + s).get_abspath(): - assert used_subpath is None, f"Used file subpath '{s}' more than once" - used_subpath = s + assert matched_subpath is None, f"File '{path}' matched more than one subpath ('{s}' and '{used_subpath}')" + assert subpath_used[s] is False, f"Subpath '{s}' matched more than one file" + matched_subpath = s subpath_used[s] = True - assert used_subpath is not None, f"File '{path}' does not match any expected subpath" + assert matched_subpath is not None, f"File '{path}' does not match any expected subpath" for s in expected_subpaths: assert subpath_used[s] is True, f"Suffix '{s}' did not match any file" @@ -616,6 +617,16 @@ def test_enumerating_package_contents(self): 'lib/xyzzy.lib', 'lib/pkgconfig/xyzzy.pc', 'shared/xyzzy/copyright'] + swordFiles = ['bin/sword.dll', + 'bin/sword.pdb', + 'debug/bin/sword.dll', + 'debug/bin/sword.pdb', + 'debug/lib/sword.lib', + 'debug/lib/pkgconfig/sword.pc', + 'include/sword.h', + 'lib/sword.lib', + 'lib/pkgconfig/sword.pc', + 'shared/sword/copyright'] frobotzFiles = ['bin/frobotz.dll', 'bin/frobotz.pdb', 'debug/bin/frobotz.dll', @@ -629,7 +640,8 @@ def test_enumerating_package_contents(self): # By default, we should get libraries under bin/ and lib/ with MakeVCPkgEnv() as env, \ AvailablePackage("xyzzy", "0.1", files = xyzzyFiles), \ - AvailablePackage("frobotz", "1.2.3", files = frobotzFiles, dependencies = ["xyzzy"]): + AvailablePackage("glowing_sword", "0.1", files = swordFiles, dependencies = ["xyzzy"]), \ + AvailablePackage("frobotz", "1.2.3", files = frobotzFiles, dependencies = ["xyzzy", "glowing_sword"]): # For simplicity, override the platform-specific naming for static and shared libraries so that we don't # have to modify the file lists to make the test pass on all platforms env['SHLIBPREFIX'] = '' @@ -638,16 +650,17 @@ def test_enumerating_package_contents(self): env['LIBSUFFIX'] = '.lib' for pkg in env.VCPkg("frobotz"): assert_package_files(env, False, pkg.Headers(), ['include/frobotz.h']) - assert_package_files(env, False, pkg.Headers(transitive = True), ['include/frobotz.h', 'include/xyzzy.h']) + assert_package_files(env, False, pkg.Headers(transitive = True), ['include/frobotz.h', 'include/sword.h', 'include/xyzzy.h']) assert_package_files(env, False, pkg.SharedLibraries(), ['bin/frobotz.dll']) - assert_package_files(env, False, pkg.SharedLibraries(transitive = True), ['bin/frobotz.dll', 'bin/xyzzy.dll']) + assert_package_files(env, False, pkg.SharedLibraries(transitive = True), ['bin/frobotz.dll', 'bin/sword.dll', 'bin/xyzzy.dll']) assert_package_files(env, False, pkg.StaticLibraries(), ['lib/frobotz.lib']) - assert_package_files(env, False, pkg.StaticLibraries(transitive = True), ['lib/frobotz.lib', 'lib/xyzzy.lib']) + assert_package_files(env, False, pkg.StaticLibraries(transitive = True), ['lib/frobotz.lib', 'lib/sword.lib', 'lib/xyzzy.lib']) # with $VCPKGDEBUG = True, we should get libraries under debug/bin/ and debug/lib/ with MakeVCPkgEnv(debug = True) as env, \ AvailablePackage("xyzzy", "0.1", files = xyzzyFiles), \ - AvailablePackage("frobotz", "1.2.3", files = frobotzFiles, dependencies = ["xyzzy"]): + AvailablePackage("glowing_sword", "0.1", files = swordFiles, dependencies = ["xyzzy"]), \ + AvailablePackage("frobotz", "1.2.3", files = frobotzFiles, dependencies = ["xyzzy", "glowing_sword"]): # For simplicity, override the platform-specific naming for static and shared libraries so that we don't # have to modify the file lists to make the test pass on all platforms env['SHLIBPREFIX'] = '' @@ -656,11 +669,11 @@ def test_enumerating_package_contents(self): env['LIBSUFFIX'] = '.lib' for pkg in env.VCPkg("frobotz"): assert_package_files(env, False, pkg.Headers(), ['include/frobotz.h']) - assert_package_files(env, False, pkg.Headers(transitive = True), ['include/frobotz.h', 'include/xyzzy.h']) + assert_package_files(env, False, pkg.Headers(transitive = True), ['include/frobotz.h', 'include/sword.h', 'include/xyzzy.h']) assert_package_files(env, False, pkg.SharedLibraries(), ['debug/bin/frobotz.dll']) - assert_package_files(env, False, pkg.SharedLibraries(transitive = True), ['debug/bin/frobotz.dll', 'debug/bin/xyzzy.dll']) + assert_package_files(env, False, pkg.SharedLibraries(transitive = True), ['debug/bin/frobotz.dll', 'debug/bin/sword.dll', 'debug/bin/xyzzy.dll']) assert_package_files(env, False, pkg.StaticLibraries(), ['debug/lib/frobotz.lib']) - assert_package_files(env, False, pkg.StaticLibraries(transitive = True), ['debug/lib/frobotz.lib', 'debug/lib/xyzzy.lib']) + assert_package_files(env, False, pkg.StaticLibraries(transitive = True), ['debug/lib/frobotz.lib', 'debug/lib/sword.lib', 'debug/lib/xyzzy.lib']) if __name__ == "__main__": diff --git a/SCons/Tool/vcpkg.py b/SCons/Tool/vcpkg.py index 50fdc708c5..959843d7d7 100644 --- a/SCons/Tool/vcpkg.py +++ b/SCons/Tool/vcpkg.py @@ -508,7 +508,7 @@ def SharedLibraries(self, transitive = False): else: return self.FilesUnderSubPath('bin/', transitive, self.env['SHLIBSUFFIX']) - def FilesUnderSubPath(self, subpath, transitive, suffix_filters = None): + def FilesUnderSubPath(self, subpath, transitive, suffix_filters = None, packages_visited = None): """Returns a (possibly empty) list of File nodes belonging to this package that are located under the relative path `subpath` underneath the triplet install directory. If `transitive` is True, then files belonging to upstream dependencies of this package are also included. @@ -539,10 +539,17 @@ def FilesUnderSubPath(self, subpath, transitive, suffix_filters = None): _vcpkg_print(Debug, 'Matching file ' + file.get_abspath()) matching_files += [file] + # If the caller requested us to also recurse into dependencies, do that now. However, don't visit the same + # package more than once (i.e., if it's reachable via multiple paths in the dependency graph) if transitive: + if packages_visited is None: + packages_visited = [] package_targets_map = get_package_targets_map(self.env) for dep in self.descriptor.package_deps: - matching_files += package_targets_map[dep].FilesUnderSubPath(subpath, transitive, suffix_filters) + dep_pkg = package_targets_map[dep] + if dep_pkg not in packages_visited: + matching_files += dep_pkg.FilesUnderSubPath(subpath, transitive, suffix_filters, packages_visited) + packages_visited.append(self) return SCons.Util.NodeList(matching_files) From 076d2d98b4c9f6085faaff0dc9d6e60611d3b8ae Mon Sep 17 00:00:00 2001 From: Ryan Saunders Date: Sun, 18 Sep 2022 22:54:53 -0700 Subject: [PATCH 15/22] Add disabled test for the list-of-packages case --- SCons/Tool/VCPkgTests.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/SCons/Tool/VCPkgTests.py b/SCons/Tool/VCPkgTests.py index 4822c4f097..0351eadaa0 100644 --- a/SCons/Tool/VCPkgTests.py +++ b/SCons/Tool/VCPkgTests.py @@ -45,7 +45,6 @@ # * Test parsing of real vcpkg.exe output # * Test feature super-setting # * Test "static" installs -# * Test transitive dependencies only listing files once class MockPackage: def __init__(self, name, version, dependencies, files): @@ -494,6 +493,15 @@ def test_install_with_no_dependency(self): ExpectInstall(["frobotz"]): env.VCPkg("frobotz") + # This doesn't work right now + def broken_test_install_multiple(self): + """Test that the VCPkg builder installs multiple missing packages specified in a single VCPkg() call""" + with MakeVCPkgEnv() as env, \ + AvailablePackage("abcd", "0.1"), \ + AvailablePackage("efgh", "1.2.3"), \ + ExpectInstall(["abcd", "efgh"]): + env.VCPkg(["abcd", "efgh"]) + def test_duplicate_install(self): """Test that duplicate invocations of the VCPkg builder installs a package only once""" with MakeVCPkgEnv() as env, \ From 4ec11b5d13292d95312817f86694eee70469183a Mon Sep 17 00:00:00 2001 From: Ryan Saunders Date: Mon, 19 Sep 2022 01:07:57 -0700 Subject: [PATCH 16/22] Fix multi-package case. Update documentation. --- SCons/Tool/VCPkgTests.py | 5 ++- SCons/Tool/vcpkg.py | 6 ++- SCons/Tool/vcpkg.xml | 91 ++++++++++++++++++++++++++++++++-------- 3 files changed, 81 insertions(+), 21 deletions(-) diff --git a/SCons/Tool/VCPkgTests.py b/SCons/Tool/VCPkgTests.py index 0351eadaa0..a8719aa9b0 100644 --- a/SCons/Tool/VCPkgTests.py +++ b/SCons/Tool/VCPkgTests.py @@ -493,8 +493,7 @@ def test_install_with_no_dependency(self): ExpectInstall(["frobotz"]): env.VCPkg("frobotz") - # This doesn't work right now - def broken_test_install_multiple(self): + def test_install_multiple(self): """Test that the VCPkg builder installs multiple missing packages specified in a single VCPkg() call""" with MakeVCPkgEnv() as env, \ AvailablePackage("abcd", "0.1"), \ @@ -657,6 +656,7 @@ def test_enumerating_package_contents(self): env['SHLIBSUFFIX'] = '.dll' env['LIBSUFFIX'] = '.lib' for pkg in env.VCPkg("frobotz"): + assert_package_files(env, False, pkg.FilesUnderSubPath(''), frobotzFiles) assert_package_files(env, False, pkg.Headers(), ['include/frobotz.h']) assert_package_files(env, False, pkg.Headers(transitive = True), ['include/frobotz.h', 'include/sword.h', 'include/xyzzy.h']) assert_package_files(env, False, pkg.SharedLibraries(), ['bin/frobotz.dll']) @@ -676,6 +676,7 @@ def test_enumerating_package_contents(self): env['SHLIBSUFFIX'] = '.dll' env['LIBSUFFIX'] = '.lib' for pkg in env.VCPkg("frobotz"): + assert_package_files(env, False, pkg.FilesUnderSubPath(''), frobotzFiles) assert_package_files(env, False, pkg.Headers(), ['include/frobotz.h']) assert_package_files(env, False, pkg.Headers(transitive = True), ['include/frobotz.h', 'include/sword.h', 'include/xyzzy.h']) assert_package_files(env, False, pkg.SharedLibraries(), ['debug/bin/frobotz.dll']) diff --git a/SCons/Tool/vcpkg.py b/SCons/Tool/vcpkg.py index 959843d7d7..620b9b0fb3 100644 --- a/SCons/Tool/vcpkg.py +++ b/SCons/Tool/vcpkg.py @@ -508,7 +508,7 @@ def SharedLibraries(self, transitive = False): else: return self.FilesUnderSubPath('bin/', transitive, self.env['SHLIBSUFFIX']) - def FilesUnderSubPath(self, subpath, transitive, suffix_filters = None, packages_visited = None): + def FilesUnderSubPath(self, subpath, transitive = False, suffix_filters = None, packages_visited = None): """Returns a (possibly empty) list of File nodes belonging to this package that are located under the relative path `subpath` underneath the triplet install directory. If `transitive` is True, then files belonging to upstream dependencies of this package are also included. @@ -627,10 +627,14 @@ def generate(env): else: _max_verbosity = Normal + # single_source = True shouldn't be required, as VCPkgBuilder is capable of handling lists of inputs. + # However, there are appears to be a bug in how Builder._createNodes processes lists of nodes, and + # the result is that VCPkgBuilder only gets the first item in the list. VCPkgBuilder = SCons.Builder.Builder(action = vcpkg_action, source_factory = lambda spec: get_package_descriptor(env, spec), target_factory = lambda desc: PackageContents(env, desc), source_scanner = vcpkg_source_scanner, + single_source = True, suffix = '.list', emitter = vcpkg_emitter) diff --git a/SCons/Tool/vcpkg.xml b/SCons/Tool/vcpkg.xml index 80596ba416..46f78c868a 100644 --- a/SCons/Tool/vcpkg.xml +++ b/SCons/Tool/vcpkg.xml @@ -26,49 +26,104 @@ See its __doc__ string for a discussion of the format. - Sets construction variables for the &vcpkg; package manager + Integrates the vcpkg package manager into SCons. + If the is given on the command line, lots of extra information will be + emitted, indicating what the vcpkg tool is doing and why. - - VCPKGROOT - VCPKGDEBUG - Downloads and builds one or more software packages (plus any dependencies of those packages) via the - &vcpkg; package manager, making the built artifcats available for other builders. + vcpkg package manager, making the built artifacts available + for other builders. -# Install FreeImage, plus any of its dependencies +# Download and build FreeImage, plus all of its dependencies env.VCPkg('freeimage') - &vcpkg; is distributed as a Git repository, containing the &vcpkg; executable (or a script to build it) + vcpkg is distributed as a Git repository, containing the vcpkg executable (or a script to build it) and a "snapshot" of the current versions of all available packages. A typical usage pattern is for your - project to incorporate &vcpkg; as a Git submodule underneath your project (run 'git submodule --help'), + project to incorporate vcpkg as a Git submodule underneath your project (run 'git submodule --help'), though system-wide installation is also supported. - Packages built with &vcpkg; may produce header files, static libraries and shared libraries - (.dll/.so files). The &vcpkg;-internal directores - containing these built artifacts are added to &cv-link-CPPPATH; and &cv-link-LIBPATH; and - &cv-link-PATH;, respectively. + Packages built with vcpkg may produce header files, static libraries and shared libraries + (.dll/.so files), organized in directories underneath + &cv-vcpkg-VCPKGROOT;. The VCPkg builder makes these artifacts available to the SCons build as + straightforwardly as possible: + + + The directory containing header files is automatically added to &cv-link-CPPPATH;. + + + The directory containing static libraries is automatically added to &cv-link-LIBPATH;. + + + + The object returned by invoking the VCPkg builder provides methods for enumerating the + files produced by the package (and optionally, any of its upstream dependencies), allowing + your SConstruct file to do arbitrary, further processing on them. + + + + + + + Since the operating system needs to be able to find any shared libraries that your program depends on, + you will need to ensure that these libraries end up somewhere in the library search path. One way to do + this is to manually add the path to where these shared libraries to the search path, but this has the + downsides of being "manual", and also of potentially breaking if vcpkg ever alters its directory + structure. A better way is to enumerate the shared libraries and &t-install; them into the same + directory as your program builds into: + + + +# Ensure that the 'freeimage' and 'fftw3' packages are built, and then install all shared libraries produced by them +# or any of their dependencies into myVariantDir +for pkg in env.VCPkg(['freeimage', 'fftw3']): + env.Install(myVariantDir, pkg.SharedLibraries(transitive = True)) + + # Of course, packages contain more than shared libraries. While a typical project likely won't need to do this, + # you can enumerate other artifacts from the package by calling other functions: + print("Header files: " + ' '.join(pkg.Headers())) + print("Static libs (incl. dependencies): " + ' '.join(pkg.StaticLibraries(transitive = True))) + print("Everything: " + ' '.join(pkg.FilesUnderSubPath(''))) + print(".txt files under share: " + ' '.join(pkg.FilesUnderSubPath('share/', suffix_filters = '.txt') + + + + An additional benefit of this approach is that it works better with multiple variants: let's say + that you have "debug" and "release" build configurations, building into different variant directories. + By installing the shared libraries into these directories, you can use the corresponding "debug" and + "release" builds of the VCPkg-built libraries without conflict. + + + + Note that the return value from invoking the VCPkg builder is always a list, even if you only specify + a single package to build, as SCons always returns a list from invoking a Builder. Thus, the "for" + loop iterating over the packages is still necessary, even in the single-package case. + + + VCPKGROOT + VCPKGDEBUG + - Specifies the path to the root directory of the &vcpkg; installation. This must be set in the - SConstruct/SConscript file, and must point to an existing &vcpkg; installation. Often, this directory + Specifies the path to the root directory of the vcpkg installation. This must be set in the + SConstruct/SConscript file, and must point to an existing vcpkg installation. Often, this directory will be a Git sub-module of your project, in which case VCPKGROOT will be specified relative to the project root. @@ -83,9 +138,9 @@ env['VCPKGROOT'] = '#/vcpkg' - Specifies whether &vcpkg; should build debug or optimized versions of packages. If True, then "debug" - packages will be built, with full debugging information and most optimizations disabled. If False (or - unset), then packages will be built using optimized settings. + Specifies whether vcpkg should build debug or optimized versions of packages. If True, then "debug" + packages will be built and used, with full debugging information and most optimizations disabled. If + False (or unset), then packages will be built using optimized settings. Note that, while you may choose to set this to match the optimization settings of your project's build, From 8e22ef51c32591f57c85fa4236cb9521408ce382 Mon Sep 17 00:00:00 2001 From: Ryan Saunders Date: Sat, 24 Sep 2022 15:26:47 -0700 Subject: [PATCH 17/22] Add missing XML xlink namespace --- SCons/Tool/vcpkg.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/SCons/Tool/vcpkg.xml b/SCons/Tool/vcpkg.xml index 46f78c868a..f9bd6dc68b 100644 --- a/SCons/Tool/vcpkg.xml +++ b/SCons/Tool/vcpkg.xml @@ -21,6 +21,7 @@ See its __doc__ string for a discussion of the format. From 93630d3a9fc7435154b887d54d3d8a865eb55437 Mon Sep 17 00:00:00 2001 From: Ryan Saunders Date: Sat, 24 Sep 2022 15:44:20 -0700 Subject: [PATCH 18/22] Fix missed variable rename --- SCons/Tool/VCPkgTests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Tool/VCPkgTests.py b/SCons/Tool/VCPkgTests.py index a8719aa9b0..78f7778112 100644 --- a/SCons/Tool/VCPkgTests.py +++ b/SCons/Tool/VCPkgTests.py @@ -432,7 +432,7 @@ def assert_package_files(env, static, actual_files, expected_subpaths): matched_subpath = None for s in expected_subpaths: if path == env.File(prefix + s).get_abspath(): - assert matched_subpath is None, f"File '{path}' matched more than one subpath ('{s}' and '{used_subpath}')" + assert matched_subpath is None, f"File '{path}' matched more than one subpath ('{s}' and '{matched_subpath}')" assert subpath_used[s] is False, f"Subpath '{s}' matched more than one file" matched_subpath = s subpath_used[s] = True From 93233b0b28831df4fc7c37557178ba10e4331b97 Mon Sep 17 00:00:00 2001 From: Ryan Saunders Date: Sat, 24 Sep 2022 15:45:18 -0700 Subject: [PATCH 19/22] Convert VCPkgBuilder to its own class overriding __eq__, in order to pass unit tests --- SCons/Tool/vcpkg.py | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/SCons/Tool/vcpkg.py b/SCons/Tool/vcpkg.py index 620b9b0fb3..c8f71d2ef5 100644 --- a/SCons/Tool/vcpkg.py +++ b/SCons/Tool/vcpkg.py @@ -614,6 +614,25 @@ def vcpkg_emitter(target, source, env): return target, source +class VCPkgBuilder(SCons.Builder.BuilderBase): + def __init__(self, env): + # single_source = True shouldn't be required, as VCPkgBuilder is capable of handling lists of inputs. + # However, there are appears to be a bug in how Builder._createNodes processes lists of nodes, and + # the result is that VCPkgBuilder only gets the first item in the list. + super().__init__(action = SCons.Action.Action(vcpkg_action), + source_factory = lambda spec: get_package_descriptor(env, spec), + target_factory = lambda desc: PackageContents(env, desc), + source_scanner = vcpkg_source_scanner, + single_source = True, + suffix = '.list', + emitter = vcpkg_emitter) + + # Need to override operator ==, since a VCPkgBuilder is bound to a specific instance of Environment, but + # Environment unit tests expect to be able to compare the BUILDERS dictionaries between two Environments. + def __eq__(self, other): + return type(self) == type(other) + + # TODO: static? def generate(env): """Add Builders and construction variables for vcpkg to an Environment.""" @@ -627,18 +646,7 @@ def generate(env): else: _max_verbosity = Normal - # single_source = True shouldn't be required, as VCPkgBuilder is capable of handling lists of inputs. - # However, there are appears to be a bug in how Builder._createNodes processes lists of nodes, and - # the result is that VCPkgBuilder only gets the first item in the list. - VCPkgBuilder = SCons.Builder.Builder(action = vcpkg_action, - source_factory = lambda spec: get_package_descriptor(env, spec), - target_factory = lambda desc: PackageContents(env, desc), - source_scanner = vcpkg_source_scanner, - single_source = True, - suffix = '.list', - emitter = vcpkg_emitter) - - env['BUILDERS']['VCPkg'] = VCPkgBuilder + env['BUILDERS']['VCPkg'] = VCPkgBuilder(env) def exists(env): return 1 From 9147eabc6f8e530d9fda06ba60764526f109e283 Mon Sep 17 00:00:00 2001 From: Ryan Saunders Date: Sat, 24 Sep 2022 16:00:59 -0700 Subject: [PATCH 20/22] Fix naming of cv- link --- SCons/Tool/vcpkg.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Tool/vcpkg.xml b/SCons/Tool/vcpkg.xml index f9bd6dc68b..c9cd9d4d5d 100644 --- a/SCons/Tool/vcpkg.xml +++ b/SCons/Tool/vcpkg.xml @@ -57,7 +57,7 @@ env.VCPkg('freeimage') Packages built with vcpkg may produce header files, static libraries and shared libraries (.dll/.so files), organized in directories underneath - &cv-vcpkg-VCPKGROOT;. The VCPkg builder makes these artifacts available to the SCons build as + &cv-link-VCPKGROOT;. The VCPkg builder makes these artifacts available to the SCons build as straightforwardly as possible: From 7e7727fed769b143a9eb5526b761e576160541eb Mon Sep 17 00:00:00 2001 From: Ryan Saunders Date: Sat, 24 Sep 2022 21:27:44 -0700 Subject: [PATCH 21/22] Fix doc generation --- SCons/Tool/vcpkg.xml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/SCons/Tool/vcpkg.xml b/SCons/Tool/vcpkg.xml index c9cd9d4d5d..3e11c7a301 100644 --- a/SCons/Tool/vcpkg.xml +++ b/SCons/Tool/vcpkg.xml @@ -21,13 +21,12 @@ See its __doc__ string for a discussion of the format. - Integrates the vcpkg package manager into SCons. + Integrates the vcpkg package manager into SCons. If the is given on the command line, lots of extra information will be emitted, indicating what the vcpkg tool is doing and why. @@ -38,7 +37,7 @@ See its __doc__ string for a discussion of the format. Downloads and builds one or more software packages (plus any dependencies of those packages) via the - vcpkg package manager, making the built artifacts available + vcpkg package manager, making the built artifacts available for other builders. From a7f585850a5d32562b0dbb684e2d972cd9be1120 Mon Sep 17 00:00:00 2001 From: Ryan Saunders Date: Mon, 16 Jan 2023 13:44:12 -0800 Subject: [PATCH 22/22] Update xml traversal limit --- SCons/Tool/docbook/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Tool/docbook/__init__.py b/SCons/Tool/docbook/__init__.py index 5cf5e61980..0b2ecdd7a3 100644 --- a/SCons/Tool/docbook/__init__.py +++ b/SCons/Tool/docbook/__init__.py @@ -69,7 +69,7 @@ # lxml etree XSLT global max traversal depth # -lmxl_xslt_global_max_depth = 3100 +lmxl_xslt_global_max_depth = 4000 if has_lxml and lmxl_xslt_global_max_depth: def __lxml_xslt_set_global_max_depth(max_depth):