diff --git a/.github/workflows/nightly-release.yaml b/.github/workflows/nightly-release.yaml index 6014d5433fed0..80eb4d0ec3a7e 100644 --- a/.github/workflows/nightly-release.yaml +++ b/.github/workflows/nightly-release.yaml @@ -65,10 +65,11 @@ jobs: run: | ./scripts/run_bazel.py \ --attempts=5 --jobs-on-last-attempt=4 \ - build -c opt --remote_download_toplevel \ + test -c opt --remote_download_toplevel \ --pre_release=nightly --nightly_date=${{ env.nightly_date }} \ //toolchain/install:prefix_root/bin/carbon \ - //toolchain/install:carbon_toolchain_tar_rule + //toolchain/install:carbon_toolchain_tar_gz_rule \ + //toolchain/install:carbon_toolchain_tar_gz_test - name: Extract the release version run: | @@ -90,4 +91,4 @@ jobs: --notes 'A nightly development build of Carbon.' \ --prerelease \ v${{ env.release_version }} \ - "bazel-bin/toolchain/install/carbon_toolchain-${{ env.release_version }}.tar.bz2" + "bazel-bin/toolchain/install/carbon_toolchain-${{ env.release_version }}.tar.gz" diff --git a/toolchain/install/BUILD b/toolchain/install/BUILD index f04c7a5c95335..996ec8a9385f9 100644 --- a/toolchain/install/BUILD +++ b/toolchain/install/BUILD @@ -6,9 +6,7 @@ load("@bazel_skylib//rules:write_file.bzl", "write_file") load("@llvm-project//llvm:binary_alias.bzl", "binary_alias") load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files", "pkg_mklink", "strip_prefix") -load("@rules_pkg//pkg:tar.bzl", "pkg_tar") -load("@rules_pkg//pkg:zip.bzl", "pkg_zip") -load("pkg_naming.bzl", "pkg_naming_variables") +load("pkg_helpers.bzl", "pkg_naming_variables", "pkg_tar_and_test") load("symlink_filegroup.bzl", "symlink_filegroup") package(default_visibility = ["//visibility:public"]) @@ -32,10 +30,22 @@ write_file( ], ) +# Create symlinks for the core library. Note that this should *not* be depended +# on directly, use `:core_data` instead. symlink_filegroup( - name = "core_data", + name = "symlink_core", srcs = ["//core:prelude"], out_prefix = "prefix_root/lib/carbon/", + visibility = ["//visibility:private"], +) + +# The filegroup to get the core library. +filegroup( + name = "core_data", + srcs = [ + ":install_marker", + ":symlink_core", + ], ) # Copy Clang and LLVM toolchain files into a synthetic LLVM installation under @@ -170,15 +180,17 @@ pkg_files( ] pkg_files( - name = "packaging_core_files", - srcs = [":core_data"], + name = "packaging_data_files", + srcs = [ + ":core_data", + ], strip_prefix = strip_prefix.from_pkg("prefix_root"), ) pkg_filegroup( name = "packaging_files", srcs = [ - ":packaging_core_files", + ":packaging_data_files", ":packaging_exe_files", ] + [ ":packaging_link_lld_alias_" + bin_name @@ -190,28 +202,18 @@ pkg_naming_variables( name = "packaging_variables", ) -# TODO: We should add support for injecting a version string into both the -# output filename and the package directory name. -pkg_tar( - name = "carbon_toolchain_tar_rule", +# We build both a compressed and uncompressed tar file with the same code here. +# This lets us use the tar file in testing as it is fast to create, but ship the +# compressed version as a release. +pkg_tar_and_test( srcs = [":packaging_files"], - extension = "tar.bz2", + name_base = "carbon_toolchain", package_dir = "carbon_toolchain-$(version)", - package_file_name = "carbon_toolchain-$(version).tar.bz2", + package_file_name_base = "carbon_toolchain-$(version)", package_variables = ":packaging_variables", stamp = -1, # Allow `--stamp` builds to produce file timestamps. - tags = ["manual"], # Slow, exclude from wildcard builds. -) - -# TODO: We should add support for injecting a version string into both the -# output filename and the package directory name. -pkg_zip( - name = "carbon_toolchain_zip_rule", - srcs = [":packaging_files"], - out = "carbon_toolchain.zip", - package_dir = "carbon_toolchain-$(version)", - package_file_name = "carbon_toolchain-$(version).zip", - package_variables = ":packaging_variables", - stamp = -1, # Allow `--stamp` builds to produce file timestamps. - tags = ["manual"], # Slow, exclude from wildcard builds. + test_data = [ + ":install_data", + ], + test_install_marker = ":install_marker", ) diff --git a/toolchain/install/pkg_helpers.bzl b/toolchain/install/pkg_helpers.bzl new file mode 100644 index 0000000000000..7e2743d8b7de1 --- /dev/null +++ b/toolchain/install/pkg_helpers.bzl @@ -0,0 +1,69 @@ +# Part of the Carbon Language project, under the Apache License v2.0 with LLVM +# Exceptions. See /LICENSE for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +"""Rule to create variables for package naming.""" + +load("@rules_pkg//pkg:providers.bzl", "PackageVariablesInfo") +load("@rules_pkg//pkg:tar.bzl", "pkg_tar") +load("@rules_python//python:defs.bzl", "py_test") +load("//bazel/version:compute_version.bzl", "VERSION_ATTRS", "compute_version") + +def _pkg_naming_variables_impl(ctx): + # TODO: Add support for digging the target CPU out of the toolchain here, + # remapping it to a more canonical name, and add that to the variables. The + # Bazel target CPU is already directly available, but it isn't likely + # canonical. + # TODO: Include the target OS as well as the target CPU. This likely needs + # similar re-mapping as the CPU does. + return PackageVariablesInfo(values = { + "version": compute_version(ctx), + }) + +pkg_naming_variables = rule( + implementation = _pkg_naming_variables_impl, + attrs = VERSION_ATTRS, +) + +def pkg_tar_and_test(name_base, package_file_name_base, test_data, test_install_marker, **kwargs): + """Create a `pkg_tar` and a test for both `.tar` and `.tar.gz` extensions. + + Args: + name_base: + The base name of the rules and tests. Will have `tar` or `tar_gz` added + and then `_rule` for the `pkg_tar` and `_test` for the test. + package_file_name_base: + The base of the `package_file_name` attribute to `pkg_tar`. The file + extensions will be appended after a `.`. + test_data: + The test data to verify the tar file against. + test_install_marker: + The install marker within the test data to locate the installation. + **kwargs: + Passed to `pkg_tar` for all the rest of its attributes. + """ + for file_ext in ["tar", "tar.gz"]: + target_ext = file_ext.replace(".", "_") + tar_target = name_base + "_" + target_ext + "_rule" + pkg_tar( + name = tar_target, + extension = file_ext, + package_file_name = package_file_name_base + "." + file_ext, + # The compressed tar is slow, exclude building and testing that. + tags = ["manual"] if file_ext == "tar.gz" else [], + **kwargs + ) + + py_test( + name = name_base + "_" + target_ext + "_test", + size = "small", + srcs = ["toolchain_tar_test.py"], + data = [":" + tar_target, test_install_marker] + test_data, + args = [ + "$(location :{})".format(tar_target), + "$(location {})".format(test_install_marker), + ], + main = "toolchain_tar_test.py", + # The compressed tar is slow, exclude building and testing that. + tags = ["manual"] if file_ext == "tar.gz" else [], + ) diff --git a/toolchain/install/pkg_naming.bzl b/toolchain/install/pkg_naming.bzl deleted file mode 100644 index d58977598efb9..0000000000000 --- a/toolchain/install/pkg_naming.bzl +++ /dev/null @@ -1,24 +0,0 @@ -# Part of the Carbon Language project, under the Apache License v2.0 with LLVM -# Exceptions. See /LICENSE for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -"""Rule to create variables for package naming.""" - -load("@rules_pkg//pkg:providers.bzl", "PackageVariablesInfo") -load("//bazel/version:compute_version.bzl", "VERSION_ATTRS", "compute_version") - -def _pkg_naming_variables_impl(ctx): - # TODO: Add support for digging the target CPU out of the toolchain here, - # remapping it to a more canonical name, and add that to the variables. The - # Bazel target CPU is already directly available, but it isn't likely - # canonical. - # TODO: Include the target OS as well as the target CPU. This likely needs - # similar re-mapping as the CPU does. - return PackageVariablesInfo(values = { - "version": compute_version(ctx), - }) - -pkg_naming_variables = rule( - implementation = _pkg_naming_variables_impl, - attrs = VERSION_ATTRS, -) diff --git a/toolchain/install/toolchain_tar_test.py b/toolchain/install/toolchain_tar_test.py new file mode 100644 index 0000000000000..9647ff72cabcf --- /dev/null +++ b/toolchain/install/toolchain_tar_test.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 + +"""Check that a release tar contains the same files as a prefix root.""" + +__copyright__ = """ +Part of the Carbon Language project, under the Apache License v2.0 with LLVM +Exceptions. See /LICENSE for license information. +SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +""" + +import argparse +import sys +from pathlib import Path +import tarfile + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "tar_file", + type=Path, + help="The tar file to test.", + ) + parser.add_argument( + "install_marker", + type=Path, + help="The path of the install marker in a prefix root to test against.", + ) + args = parser.parse_args() + + # Locate the prefix root from the install marker. + if not args.install_marker.exists(): + sys.exit("ERROR: No install marker: " + args.install_marker) + prefix_root_path = args.install_marker.parent.parent.parent + + # First check that every file and directory in the tar file exists in our + # prefix root, and build a set of those paths. + installed_paths = set() + with tarfile.open(args.tar_file) as tar: + for tarinfo in tar: + relative_path = Path(*Path(tarinfo.name).parts[1:]) + installed_paths.add(relative_path) + if not prefix_root_path.joinpath(relative_path).exists(): + sys.exit( + "ERROR: File `{0}` is not in prefix root: `{1}`".format( + tarinfo.name, prefix_root_path + ) + ) + + # If we found an empty tar file, it's always an error. + if len(installed_paths) == 0: + sys.exit("ERROR: Tar file `{0}` was empty.".format(args.tar_file)) + + # Now check that every file and directory in the prefix root is in that set. + for prefix_path in prefix_root_path.glob("**/*"): + relative_path = prefix_path.relative_to(prefix_root_path) + if relative_path not in installed_paths: + sys.exit( + "ERROR: File `{0}` is not in tar file.".format(relative_path) + ) + + +if __name__ == "__main__": + main()