diff --git a/WORKSPACE b/WORKSPACE index 2b01001..8da5d9a 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -11,12 +11,6 @@ git_repository( remote = "git@github.com:bazelbuild/rules_docker.git", ) -git_repository( - name = "io_bazel_rules_k8s", - commit = "d6e1b65317246fe044482f9e042556c77e6893b8", - remote = "git@github.com:bazelbuild/rules_k8s.git", -) - load("@io_bazel_rules_docker//container:container.bzl", "repositories") repositories() @@ -28,10 +22,6 @@ load( _py_image_repos() -load("@io_bazel_rules_k8s//k8s:k8s.bzl", "k8s_defaults", "k8s_repositories") - -k8s_repositories() - load("//terraform:dependencies.bzl", "terraform_repositories") terraform_repositories() diff --git a/examples/README.md b/examples/README.md index 997e67b..391fffd 100644 --- a/examples/README.md +++ b/examples/README.md @@ -93,30 +93,46 @@ build:publish --define IMAGE_CHROOT=index.docker.io/netchris ### [`release/BUILD`](release/BUILD) ```python -load("//terraform:def.bzl", "terraform_module_publisher") +load("//terraform/experimental/ghrelease:def.bzl", "ghrelease", "ghrelease_assets", "ghrelease_test_suite") + +VERSION = "0.2" # To make our terraform module available to others we configure a -# 'terraform_module_publisher' which will -# - Run configured tests -# - Publish relevant docker images -# - Output each module into its own subdirectory in this repo (note -# that we can optionally output to a different repo) -terraform_module_publisher( - name = "publish", - bazelrc_config = "publish", - prepublish_tests = [ - # bazel excludes all tests tagged as 'manual' from wildcard - # patterns, so we explicitly include our e2e test. - "//...", # <- means 'all tests' (which aren't tagged 'manual') - "//examples/test:e2e_integration_test", +# 'ghrelease' which will: +# - Run "preflight-checks" (dependant test suites, etc) +# - Push associated docker images +# - Generate release notes, including a changelog from the previous +# version +# - Increment the current tag/version's patch version (or prerelease +# version if the `--prerelease` flag is used) +# - Create a new GitHub Release with release notes and any extra docs +# - Attach any assets to the new Release +ghrelease( + name = "release", + version = VERSION, + deps = [ + ":prerelease-tests", + ":tf-modules", + ], +) + +ghrelease_assets( + name = "tf-modules", + bazel_flags = ["--config=publish"], + data = [ + "//examples/src:hello-world_ecs", + "//examples/src:hello-world_k8s", ], - published_modules = { - "mymodule": "//examples/src:hello-world_k8s", - "mymodule-ecs": "//examples/src:hello-world_ecs", - }, - # remote = "git@github.com:my-org-terraform-modules/terraform-myproject-modules.git", - # remote_path = "modules", ) + +ghrelease_test_suite( + name = "prerelease-tests", + tests = [ + "//examples/...", + "//examples/test:k8s-e2e_integration_test", + ], +) + ``` Check out the [release directory](release/mymodule) to see the published module! diff --git a/examples/release/BUILD b/examples/release/BUILD index a3fbdfc..2278d1c 100644 --- a/examples/release/BUILD +++ b/examples/release/BUILD @@ -1,15 +1,31 @@ -load("//terraform:def.bzl", "terraform_module_publisher") +load("//terraform/experimental/ghrelease:def.bzl", "ghrelease", "ghrelease_assets", "ghrelease_test_suite") -terraform_module_publisher( - name = "publish", - bazelrc_config = "publish", - prepublish_tests = [ +VERSION = "0.2" + +ghrelease( + name = "release", + args = ["--draft"], + version = VERSION, + deps = [ + ":prerelease-tests", + ":tf-modules", + ], +) + +ghrelease_assets( + name = "tf-modules", + bazel_flags = ["--config=publish"], + data = [ + "//examples/src:hello-world_ecs", + "//examples/src:hello-world_k8s", + ], +) + +ghrelease_test_suite( + name = "prerelease-tests", + tests = [ "//...", "//examples/test:k8s-e2e_integration_test", - #"//examples/test:ecs-e2e_integration_test", + # "//examples/test:ecs-e2e_integration_test" ], - published_modules = { - "mymodule": "//examples/src:hello-world_k8s", - "mymodule-ecs": "//examples/src:hello-world_ecs", - }, ) diff --git a/examples/src/BUILD b/examples/src/BUILD index fe26b23..ccab3a9 100644 --- a/examples/src/BUILD +++ b/examples/src/BUILD @@ -1,4 +1,4 @@ -load("//terraform:def.bzl", "terraform_module", "terraform_workspace") +load("//terraform:def.bzl", "terraform_module") load("//terraform:container.bzl", "image_embedder", "terraform_k8s_manifest") load("@io_bazel_rules_docker//python:image.bzl", "py_image") @@ -32,7 +32,6 @@ terraform_module( "k8s.tf", "main.tf", ], - description = "A hello-world service that is designed to run on Kubernetes", embed = [":k8s-deployment"], visibility = ["//visibility:public"], ) @@ -44,7 +43,6 @@ terraform_module( "ecs-plumbing.tf", "main.tf", ], - description = "A hello-world service that is designed to run on ECS", embed = [":ecs-task-containers"], visibility = ["//visibility:public"], ) diff --git a/examples/src/ecs.tf b/examples/src/ecs.tf index 4bc1676..9d64252 100644 --- a/examples/src/ecs.tf +++ b/examples/src/ecs.tf @@ -1,3 +1,6 @@ +/** + * A hello-world service that is designed to run on ECS! + */ variable replicas { description = "Desired # of ECS replicas" default = 1 diff --git a/examples/src/k8s.tf b/examples/src/k8s.tf index 5ca1aad..8dbc0e3 100644 --- a/examples/src/k8s.tf +++ b/examples/src/k8s.tf @@ -1,4 +1,6 @@ - +/** + * A hello-world service that is designed to run on Kubernetes + */ data kubectl_namespace current {} resource kubernetes_config_map hello_world_server { diff --git a/terraform/container.bzl b/terraform/container.bzl index 0ab6f7c..e6ef464 100644 --- a/terraform/container.bzl +++ b/terraform/container.bzl @@ -1,15 +1,12 @@ load("//terraform:providers.bzl", _ModuleInfo = "ModuleInfo") -load("//terraform/internal:k8s.bzl", _terraform_k8s_manifest = "terraform_k8s_manifest") -load("//terraform/internal:image_embedder.bzl", _image_publisher = "image_publisher") +load("//terraform/internal:k8s_manifest.bzl", "terraform_k8s_manifest") +load("//terraform/internal:image_embedder.bzl", "image_publisher") load( "//terraform/internal:image_embedder_lib.bzl", _embed_images = "embed_images", _image_embedder_attrs = "image_embedder_attrs", ) -terraform_k8s_manifest = _terraform_k8s_manifest -image_publisher = _image_publisher - def _image_embedder_impl(ctx): providers = [] diff --git a/terraform/def.bzl b/terraform/def.bzl index 128d69f..559e635 100644 --- a/terraform/def.bzl +++ b/terraform/def.bzl @@ -1,94 +1,4 @@ -load( - "//terraform/internal:terraform.bzl", - "terraform_plugin", - _terraform_module = "terraform_module", - _terraform_workspace = "terraform_workspace", -) -load("//terraform/internal:test.bzl", "terraform_integration_test") -load( - "//terraform/internal:distribution.bzl", - "terraform_distribution_publisher", - "terraform_module_publisher", - _terraform_distribution_dir = "terraform_distribution_dir", -) -load("//terraform:providers.bzl", "tf_workspace_files_prefix") - -def terraform_distribution_dir(name, deps, **kwargs): - srcs_name = "%s.srcs-list" % name - module_name = "%s.module" % name - - # change "relative" deps to absolute deps - deps_abs = [ - "//" + native.package_name() + dep if dep.startswith(":") else dep - for dep in deps - ] - native.genquery( - name = srcs_name, - opts = ["--noimplicit_deps"], - expression = """kind("source file", deps(set(%s)))""" % " ".join(deps_abs), - scope = deps_abs, - ) - - terraform_module( - name = module_name, - deps = deps_abs, - ) - - _terraform_distribution_dir( - name = name, - srcs_list = ":" + srcs_name, - module = ":" + module_name, - **kwargs - ) - -def _flip_modules_attr(modules): - """ - Translate modules attr from a 'name=>label' dict to 'label=>name' - """ - flipped = {} - for name, label in modules.items(): - if not (label.startswith("@") or label.startswith("//") or label.startswith(":")): - fail("Modules are now specified as 'name=>label'", attr="modules") - # append package path & workspace name as necessary - abs_label = "//" + native.package_name() + label if label.startswith(":") else label - abs_label = native.repository_name() + abs_label if abs_label.startswith("//") else abs_label - if abs_label in flipped: - fail("Modules may only be specified once (%s)" % label, attr = "modules") - flipped[abs_label] = name - return flipped - -def terraform_module(name, modules = {}, **kwargs): - _terraform_module( - name = name, - modules = _flip_modules_attr(modules), - **kwargs - ) - -def terraform_workspace(name, modules = {}, **kwargs): - _terraform_workspace( - name = name, - modules = _flip_modules_attr(modules), - **kwargs - ) - - # create a convenient destroy target which - # CDs to the package dir and runs terraform destroy - native.genrule( - name = "%s.destroy" % name, - outs = ["%s.destroy.sh" % name], - cmd = """echo ' - #!/bin/sh - set -eu - tf_workspace_dir="$$BUILD_WORKSPACE_DIRECTORY/{package}/{tf_workspace_files_prefix}" - if [ -e "$$tf_workspace_dir" ]; then - cd "$$tf_workspace_dir" - exec terraform destroy "$$@" .terraform/tfroot - else - >&2 echo "Could not find terraform workspace dir, so there is nothing to destroy ($$tf_workspace_dir)" - fi - ' > $@""".format( - package = native.package_name(), - tf_workspace_files_prefix = tf_workspace_files_prefix(name), - ), - executable = True, - ) +load("//terraform/internal:terraform.bzl", "terraform_plugin") +load("//terraform/internal:workspace.bzl", "terraform_workspace") +load("//terraform/internal:module.bzl", "terraform_module") +load("//terraform/internal:integration_test.bzl", "terraform_integration_test") diff --git a/terraform/dependencies.bzl b/terraform/dependencies.bzl index 99909a9..8a42c4e 100644 --- a/terraform/dependencies.bzl +++ b/terraform/dependencies.bzl @@ -27,7 +27,7 @@ _EXTERNAL_BINARIES = { "hub": dict( url = "https://github.com/github/hub/releases/download/v{version}/hub-{platform}-amd64-{version}.tgz", path = "hub-{platform}-amd64-{version}/bin/hub", - version = "2.5.1", + version = "2.6.0", ), } @@ -52,8 +52,21 @@ py_library( visibility = ["//visibility:public"], )""", sha256 = "592766c6303207a20efc445587778322d7f73b161bd994f227adaa341ba212ab", - url = ("https://pypi.python.org/packages/4a/85/" + - "db5a2df477072b2902b0eb892feb37d88ac635d36245a72a6a69b23b383a" + - "/PyYAML-3.12.tar.gz"), + url = ("https://pypi.python.org/packages/4a/85/db5a2df477072b2902b0eb892feb37d88ac635d36245a72a6a69b23b383a/PyYAML-3.12.tar.gz"), strip_prefix = "PyYAML-3.12/lib/yaml", ) + if "py_semver" not in native.existing_rules(): + native.new_http_archive( + name = "py_semver", + build_file_content = """ +py_library( + name = "py_semver", + srcs = glob(["*.py"]), + visibility = ["//visibility:public"], + imports = ["semver"], +) +""", + sha256 = "5b09010a66d9a3837211bb7ae5a20d10ba88f8cb49e92cb139a69ef90d5060d8", + url = "https://files.pythonhosted.org/packages/47/13/8ae74584d6dd33a1d640ea27cd656a9f718132e75d759c09377d10d64595/semver-2.8.1.tar.gz", + strip_prefix = "semver-2.8.1", + ) diff --git a/terraform/experimental/ghrelease/BUILD b/terraform/experimental/ghrelease/BUILD new file mode 100644 index 0000000..e69de29 diff --git a/terraform/experimental/ghrelease/def.bzl b/terraform/experimental/ghrelease/def.bzl new file mode 100644 index 0000000..1cfa1b7 --- /dev/null +++ b/terraform/experimental/ghrelease/def.bzl @@ -0,0 +1,3 @@ +load("//terraform/experimental/ghrelease/internal:publisher.bzl", "ghrelease") +load("//terraform/experimental/ghrelease/internal:assets.bzl", "ghrelease_assets") +load("//terraform/experimental/ghrelease/internal:test_suite.bzl", "ghrelease_test_suite") diff --git a/terraform/experimental/ghrelease/internal/BUILD b/terraform/experimental/ghrelease/internal/BUILD new file mode 100644 index 0000000..2d95c5d --- /dev/null +++ b/terraform/experimental/ghrelease/internal/BUILD @@ -0,0 +1,24 @@ +[py_binary( + name = f[:-len(".py")], + srcs = [f], + visibility = ["//visibility:public"], + deps = [":lib"], +) for f in glob(["*_runner.py"])] + +py_library( + name = "lib", + srcs = ["lib.py"], + imports = [ + "rules_terraform/terraform/experimental/ghrelease/internal", + ], + deps = [ + "@py_semver", + ], +) + +py_test( + name = "test", + size = "small", + srcs = ["test.py"], + deps = [":lib"], +) diff --git a/terraform/experimental/ghrelease/internal/assets.bzl b/terraform/experimental/ghrelease/internal/assets.bzl new file mode 100644 index 0000000..4751a99 --- /dev/null +++ b/terraform/experimental/ghrelease/internal/assets.bzl @@ -0,0 +1,105 @@ +load("//terraform/internal:image_embedder_lib.bzl", "create_image_publisher", "image_publisher_aspect", "image_publisher_attrs") + +GhReleaseAssetsInfo = provider( + fields = { + "bazel_flags": "List", + "env": "dict", + "docs": "Depset", + }, +) + +def _impl(ctx): + """ + """ + files = [] + transitive_runfiles = [] + transitive_assets = [] + transitive_docs = [] + + for t in ctx.attr.data: + # todo: handle file targets appropriately + # get assets + transitive_assets.append(t[DefaultInfo].files) + + # grab docs from the 'docs' property if present + og_info = t[OutputGroupInfo] + if hasattr(og_info, "docs"): + if type(og_info.docs) == "list": + transitive_docs.append(depset(direct = og_info.docs)) + else: + transitive_docs.append(og_info.docs) + + # flatten & 'uniquify' our list of asset files + assets = depset(transitive = transitive_assets).to_list() + assets = {k: None for k in assets}.keys() + + # make sure there are no duplicate filenames + filenames = {} + for f in assets: + if f.basename in filenames: + filenames[f.basename].append(f) + else: + filenames[f.basename] = [f] + duplicates = {k: v for k, v in filenames.items() if len(v) > 1} + if len(duplicates) > 0: + fail("Found duplicate file names: %s" % duplicates, attr = "data") + + # create an image publisher + image_publisher = ctx.actions.declare_file(ctx.attr.name + ".image-publisher") + files.append(image_publisher) + publisher_runfiles = create_image_publisher(ctx, image_publisher, ctx.attr.data) + + config_file = ctx.actions.declare_file(ctx.attr.name + ".config.json") + files.append(config_file) + config = struct( + env = ctx.attr.env, + bazel_flags = ctx.attr.bazel_flags, + assets = [f.short_path for f in sorted(assets)], + label = str(ctx.label), + image_publisher = image_publisher.short_path, + ) + ctx.actions.write(config_file, config.to_json()) + ctx.actions.write(ctx.outputs.executable, """#!/usr/bin/env bash + set -euo pipefail + exec "{runner}" "--config={config}" "$@" <&0 + """.format( + runner = ctx.executable._assets_runner.short_path, + config = config_file.short_path, + ), is_executable = True) + transitive_runfiles.append(ctx.attr._assets_runner.data_runfiles) + transitive_runfiles.append(ctx.attr._assets_runner.default_runfiles) + + runfiles = ctx.runfiles(files = files + assets, transitive_files = publisher_runfiles) + for rf in transitive_runfiles: + runfiles = runfiles.merge(rf) + + return [ + DefaultInfo( + files = depset(direct = files), + runfiles = runfiles, + ), + GhReleaseAssetsInfo( + bazel_flags = ctx.attr.bazel_flags, + env = ctx.attr.env, + docs = depset(transitive = transitive_docs) if transitive_docs else None, + ), + ] + +ghrelease_assets = rule( + _impl, + attrs = image_publisher_attrs + { + "bazel_flags": attr.string_list(default = []), + "env": attr.string_dict(default = {}), + "data": attr.label_list( + default = [], + aspects = [image_publisher_aspect], + #allow_files = True, + ), + "_assets_runner": attr.label( + default = Label("//terraform/experimental/ghrelease/internal:assets_runner"), + executable = True, + cfg = "host", + ), + }, + executable = True, +) diff --git a/terraform/experimental/ghrelease/internal/assets_runner.py b/terraform/experimental/ghrelease/internal/assets_runner.py new file mode 100644 index 0000000..3bc4836 --- /dev/null +++ b/terraform/experimental/ghrelease/internal/assets_runner.py @@ -0,0 +1,87 @@ +from __future__ import print_function + +import argparse +import errno +import json +import os +import shutil +import subprocess +import sys +from collections import namedtuple +from os import path + +from lib import BazelFlagsEnvVar + +parser = argparse.ArgumentParser( + description="Builds artifacts & outputs them to the specified directory") + +parser.add_argument( + '--config', action='store', required=True, + type=lambda pathstr: json.load(open(pathstr, "r"), object_hook=lambda d: namedtuple('X', d.keys())(*d.values())), + help=argparse.SUPPRESS) + +parser.add_argument( + '--overwrite', dest='overwrite', action='store_true', + default=False, + help="Overwrite existing files in output_dir") + +parser.add_argument( + '--publish', dest='publish', action='store_true', + default=False, + help="Publish dependant assets such as docker images whose digests may be embedded in transitive assets.") + +parser.add_argument( + 'output_dir', action='store', + help='') + + +def main(args): + """ + """ + # canonicalize 'output_dir' relative to BUILD_WORKING_DIRECTORY + output_dir = args.output_dir + if not path.isabs(output_dir): + output_dir = path.join(os.environ['BUILD_WORKING_DIRECTORY'], output_dir) + output_dir = path.normpath(output_dir) + try: + # create the output dir + os.makedirs(output_dir, mode=0o755) + except OSError as e: + # ignore if existing dir, but raise otherwise + if e.errno != errno.EEXIST: + raise e + + # reinvoke bazel with the correct flags if necessary + correct_flags = json.dumps(args.config.bazel_flags) + if correct_flags != os.getenv(BazelFlagsEnvVar): + new_env = {k: v for k, v in os.environ.items()} + new_env.update(args.config.env) + new_env[BazelFlagsEnvVar] = correct_flags + cmd = ["bazel", "run"] + cmd.extend(args.config.bazel_flags) + cmd.append(args.config.label) + cmd.append("--") + cmd.extend(sys.argv[1:]) + print("Reinvoking bazel with flags: %s" % " ".join(args.config.bazel_flags)) + os.chdir(os.environ['BUILD_WORKING_DIRECTORY']) + os.execvpe(cmd[0], cmd, new_env) + + # invoke docker image publisher + if args.publish: + rc = subprocess.call([args.config.image_publisher]) + if rc != 0: + print("Error, failed to publish images! D:") + exit(rc) + + # copy each of the provided assets to the output dir + for f in args.config.assets: + tgt_path = path.join(output_dir, path.basename(f)) + if path.exists(tgt_path) and not args.overwrite: + print("Error, file already exists in output_dir: '%s'" % path.basename(f)) + exit(1) + src_path = path.realpath(f) + shutil.copyfile(src_path, tgt_path) + + +if __name__ == '__main__': + main(parser.parse_args()) diff --git a/terraform/experimental/ghrelease/internal/lib.py b/terraform/experimental/ghrelease/internal/lib.py new file mode 100644 index 0000000..94ec38c --- /dev/null +++ b/terraform/experimental/ghrelease/internal/lib.py @@ -0,0 +1,357 @@ +from __future__ import print_function + +import atexit +import logging +import os +import re +import shlex +import shutil +import subprocess +from os import path +from subprocess import CalledProcessError +from tempfile import mkdtemp + +import semver +import sys +from semver import VersionInfo +from typing import Union + +# Use this env var to determine if this script was invoked w/ the appropriate bazel flags +BazelFlagsEnvVar = "RULES_TERRAFORM_GHRELEASE_BAZEL_FLAGS" + + +def next_semver(major, minor, prerelease=None, versions=None): + # type: (int, int, str, list) -> str + major = int(major) + minor = int(minor) + + def stuff_we_care_about(v): + """ + we only care about stuff that: + - has our same major+minor version + - is a release OR has our same prerelease token + """ + if v.major != major: + return False + if v.minor != minor: + return False + if not prerelease: + return True + if v.prerelease is None: + return True + prefix = prerelease + "." + if v.prerelease.startswith(prefix): + # also needs to end in an int for us to care about it + return bool(re.match("^\d+$", v.prerelease[len(prefix):])) + return False + + semvers = [ + VersionInfo.parse(v[1:] if v.startswith("v") else v) + for v in versions or [] + ] + semvers = filter(stuff_we_care_about, semvers) + if semvers: + latest = sorted(semvers)[-1] + version = str(latest) + # do we bump patch? + if not latest.prerelease: + version = semver.bump_patch(version) + # is this a final version? + if prerelease: + version = semver.bump_prerelease(version, prerelease) + else: + version = semver.finalize_version(version) + else: + # nothing exists on the current minor version so create a new one + version = "%s.%s.0" % (major, minor) + if prerelease: + version += "-%s.1" % prerelease + return "v" + version + + +class SubprocessHelper(object): + + def __init__(self, args=None, chomp_output=False, **popen_kwargs): + """ + Set defaults for calling a subprocess. + :param args: Will be prepended to all invocations. + :param chomp_output: Remove trailing newline from output. + :param popen_kwargs: Default kwargs for subprocess.Popen + :return: + """ + self._chomp_output = chomp_output + self._rc = None + if type(args) in (str, unicode): + args = shlex.split(args) + self._args = args or [] + if 'stdout' not in popen_kwargs: + popen_kwargs['stdout'] = subprocess.PIPE + if 'stderr' not in popen_kwargs: + popen_kwargs['stderr'] = subprocess.PIPE + self._popen_kwargs = popen_kwargs + + def __call__(self, args, chomp_output=None, success_exit_codes=None, **kwargs): + # type: (Union[list,str], bool, list[int], dict) -> str + """ + Execute the provided command and return its output. + :rtype: str + :param args: Will be appended to the default args & executed + :param chomp_output: Remove trailing newline from output. + :param success_exit_codes: Do not raise an exception when process returns these codes. + :param kwargs: passed through to subprocess.Popen + :return: + """ + success_exit_codes = success_exit_codes or {0} + if type(args) in (str, unicode): + args = shlex.split(args) + args = self._args + args + # add defaults + for k, v in self._popen_kwargs.items(): + if k not in kwargs: + kwargs[k] = v + p = subprocess.Popen(args, **kwargs) + out, err = p.communicate() # type: (str, str) + rc = p.wait() + self._rc = rc + if rc not in success_exit_codes: + raise CalledProcessError(rc, " ".join(args), output=err or out) + chomp = self._chomp_output if chomp_output is None else chomp_output + if chomp and out: + return out.rstrip("\r\n") + else: + return out + + @property + def returncode(self): + # type: () -> int + """ + :return: Exit code of most recently executed command. + """ + return self._rc + + +class ReleaseInfo(VersionInfo): + __slots__ = ('url', 'commit', 'tag') + + def __init__(self, tag, url, commit): + super(ReleaseInfo, self).__init__(**semver.parse( + tag[1:] if tag.startswith('v') else tag)) + self.tag = tag + self.url = url + self.commit = commit + + +class GhHelper: + + def __init__(self, repo_dir, branch, docs_branch, version_major, + version_minor, hub_binary=None): + + self._docs_branch = docs_branch + self._repo_dir = repo_dir + self._branch = branch + self._version_major = version_major + self._version_minor = version_minor + + git = SubprocessHelper('git', chomp_output=True, cwd=repo_dir) + hub = SubprocessHelper(path.abspath(hub_binary) if hub_binary else "hub", + chomp_output=True, cwd=repo_dir) + self._hub = hub + self._git = git + + remote = git("remote") + self._remote_url = git('remote get-url ' + remote) + self._commit = git('rev-parse --verify HEAD') + self._repo_url = hub('browse -u') + # keep only parts of the URL we care about + self._repo_url = "/".join(self._repo_url.split("/")[0:5]) + + tags = {} + self._heads = set() + for line in git('ls-remote --tags --heads ' + + self._remote_url).splitlines(): + if "\trefs/heads/" in line: + commit, head = line.strip().split("\trefs/heads/") + self._heads.add(head) + if "\trefs/tags/" in line: + commit, tag = line.strip().split("\trefs/tags/") + tags[tag] = commit + self._releases = [] + for line in hub('release --format="%T %U"').splitlines(): + tag, url = line.split(" ") + commit = tags[tag] + try: + self._releases.append(ReleaseInfo(tag, url, commit)) + except ValueError: + logging.warning("Could not parse '%s' as semver", tag) + + def get_next_semver(self, prerelease): + return next_semver(self._version_major, self._version_minor, + prerelease, [v.tag for v in self._releases]) + + def check_srcs_match_head(self, srcs, publish): + """ + (warning or err depends if we're publishing) + check & report on: (aka "local" git checks, bc we resolve locally) + - all source files are checked in (accumulate srcfiles while iterating tests/artifacts?) + :param srcs: + :return: + """ + print("check_srcs_match_head ...Unimplemented :(") + + def check_local_tracks_authoritative_branch(self, publish): + """ + remote tracking branch is the authoritative remote+branch (warning or err depends if we're publishing) + :param publish: + :return: + """ + head_ref = self._git('symbolic-ref -q HEAD') + tracked_branch = self._git([ + 'for-each-ref', + '--format=%(upstream:lstrip=3)', + head_ref]) + print("check_local_tracks_authoritative_branch", end=" ...") + if tracked_branch != self._branch: + print("FAILED") + msg = "Local branch does not track authoritative branch '%s'" % self._branch + if publish: + print("FATAL: %s" % msg, file=sys.stderr) + exit(1) + else: + print("WARNING: %s (this will prevent publishing)" % msg, file=sys.stderr) + else: + print("OK") + + def check_head_exists_in_remote(self): + """ + check HEAD commit exists in remote tracking branch + - else push to remote + :return: + """ + print("check_head_exists_in_remote", end=" ...") + sys.stdout.flush() + sys.stderr.flush() + self._git('push') + print("OK") + + def publish_docs(self, docs_dir): + links = [] + for root, dirs, files in os.walk(docs_dir): + for f in files: + abspath = path.normpath(path.join(root, f)) + relpath = abspath[len(docs_dir) + 1:] + links.append("{repo_url}/blob/{commit}/" + relpath) + + # short-circuit if there's nothing to do + if len(links) == 0 and self._docs_branch not in self._heads: + return [] + + # create temp dir + tmpdir = mkdtemp() + atexit.register(shutil.rmtree, tmpdir, ignore_errors=True) + + # convenience wrapper + # - runs git in tmpdir + # - exits with message if there's a problem + # - returns process returncode otherwise + + _git = SubprocessHelper('git', chomp_output=True, cwd=tmpdir) + + if self._docs_branch in self._heads: + # shallow clone docs branch if it exists + _git(["clone", "--depth", "1", + "-b", self._docs_branch, + "--", self._remote_url, tmpdir]) + else: + # else init empty repo + _git("init") + _git("checkout -b " + self._docs_branch) + _git("remote add origin " + self._remote_url) + + # check difference between docs dir and docs branch + worktree = "--work-tree=%s" % docs_dir + + def has_changes(): + _git([worktree, "diff", "--exit-code", + "remotes/origin/" + self._docs_branch], + success_exit_codes=[0, 1]) + return _git.returncode == 1 + + if self._docs_branch not in self._heads or has_changes(): + # add,commit,push docs if there is a difference + _git([worktree, "add", "-A"]) + _git("commit -m 'Updating docs.'") + _git("push -u origin " + self._docs_branch) + + commit = _git("rev-parse --verify HEAD") + + return [l.format( + commit=commit, + repo_url=self._repo_url, + ) for l in links] + + def generate_releasenotes(self, docs_links=None, asset_srcs=None): + # type: (list, set) -> str + """ + :param docs_links: List of links to docs associated with this release + :param asset_srcs: (Unimplemented) Set of files. The changelog will + be filtered to include only commits which involve these files. + :return: + """ + changelog_tpl = "- [`%h`]({repo_url}/commit/%H) %s" + docs_tpl = """### Docs +{links}""" + + from_tag = None + from_commit = None + # find latest release (for our MAJOR.MINOR version) + # - get all releases, latest to earliest + # - return first release that has <= MAJOR and <= MINOR versions + for r in sorted(self._releases, reverse=True): + if r.major <= self._version_major and r.minor <= self._version_minor: + from_tag = r.tag + from_commit = r.commit + + output_parts = [] + if docs_links: + docs_links_md = [ + "- [{filename}]({url})".format( + filename=l.split('/')[-1], + url=l) + for l in docs_links + ] + output_parts.append(docs_tpl.format( + links="\n".join(docs_links_md))) + + if from_commit: + changelog = "### Changes Since `%s`:\n" % from_tag + changelog += self._git([ + "log", + "--format=%s" % changelog_tpl.format(repo_url=self._repo_url), + "%s..%s" % (from_commit, self._commit) + ]) + output_parts.append(changelog) + + return "\n\n".join(output_parts) + + def publish_release(self, assets_dir, release_notes, tag, draft): + args = ["release", "create"] + if draft: + args += ["--draft"] + if "-" in tag: + args += ["--prerelease"] + for root, dirs, files in os.walk(assets_dir): + for f in files: + args += ["--attach=%s" % path.join(root, f)] + args += [tag] + args += ["--commitish=%s" % self._commit] + args += ["--message=%s\n\n%s" % (tag, release_notes)] + if sys.stdout.isatty(): + args.append("--browse") + + self._hub(args, stdout=sys.stdout, stderr=sys.stderr) + # get the new tag (if this wasn't a draft) + if not draft: + try: + self._git('fetch --tags') + except CalledProcessError as e: + logging.warning("Could not pull tags after publishing: %s", str(e)) diff --git a/terraform/experimental/ghrelease/internal/publisher.bzl b/terraform/experimental/ghrelease/internal/publisher.bzl new file mode 100644 index 0000000..61a231e --- /dev/null +++ b/terraform/experimental/ghrelease/internal/publisher.bzl @@ -0,0 +1,119 @@ +load(":test_suite.bzl", "GhReleaseTestSuiteInfo") +load(":assets.bzl", "GhReleaseAssetsInfo") + +def _parse_version(version): + v = version + if v.startswith("v"): + v = v[1:] + parts = v.split(".") + if (len(parts) == 2 and + parts[0].isdigit() and + parts[1].isdigit()): + return struct(major = parts[0], minor = parts[1]) + fail("Expected '[v]MAJOR.MINOR' but got '%s'" % version, attr = "version") + +def _impl(ctx): + """ + """ + files = [] + transitive_runfiles = [] + + transitive_docs = [] + asset_configs = [] + test_configs = [] + + for dep in ctx.attr.deps: + if GhReleaseAssetsInfo in dep: + nfo = dep[GhReleaseAssetsInfo] + transitive_docs.append(nfo.docs) + asset_configs.append(struct( + label = str(dep.label), + env = nfo.env, + bazel_flags = nfo.bazel_flags, + )) + if GhReleaseTestSuiteInfo in dep: + i = dep[GhReleaseTestSuiteInfo] + test_configs.append(struct( + label = str(dep.label), + )) + + docs = depset(direct = ctx.files.docs, transitive = transitive_docs).to_list() + docs = {k: None for k in docs}.keys() + + # make sure there are no duplicate filenames + filenames = {} + for f in docs: + if f.basename in filenames: + filenames[f.basename].append(f) + else: + filenames[f.basename] = [f] + duplicates = {k: v for k, v in filenames.items() if len(v) > 1} + if len(duplicates) > 0: + fail("Found duplicate file names: %s" % duplicates, attr = "docs") + + args_file = ctx.actions.declare_file(ctx.attr.name + ".args") + files.append(args_file) + ctx.actions.write(args_file, "\n".join(ctx.attr.args)) + + config_file = ctx.actions.declare_file(ctx.attr.name + ".config.json") + files.append(config_file) + config = struct( + asset_configs = asset_configs, + test_configs = test_configs, + docs = [f.short_path for f in docs], + docs_branch = ctx.attr.docs_branch, + branch = ctx.attr.branch, + version = _parse_version(ctx.attr.version), + hub = ctx.executable._tool_hub.short_path, + ) + ctx.actions.write(config_file, config.to_json()) + ctx.actions.write( + ctx.outputs.executable, + """#!/usr/bin/env bash + set -euo pipefail + exec "%s" "--config=$0.config.json" "@$0.args" "$@" <&0 + """ % ctx.executable._publisher_runner.short_path, + ) + transitive_runfiles.append(ctx.attr._publisher_runner.data_runfiles) + transitive_runfiles.append(ctx.attr._publisher_runner.default_runfiles) + transitive_runfiles.append(ctx.attr._tool_hub.data_runfiles) + transitive_runfiles.append(ctx.attr._tool_hub.default_runfiles) + + runfiles = ctx.runfiles(files = files + docs) + for rf in transitive_runfiles: + runfiles = runfiles.merge(rf) + + return [ + DefaultInfo( + files = depset(direct = files), + runfiles = runfiles, + ), + ] + +ghrelease = rule( + _impl, + attrs = { + "deps": attr.label_list( + default = [], + providers = [ + [GhReleaseAssetsInfo], + [GhReleaseTestSuiteInfo], + ], + ), + "version": attr.string(mandatory = True), + "branch": attr.string(default = "master"), + "docs_branch": attr.string(default = "docs"), + "docs": attr.label_list(default = [], allow_files = True), + "_publisher_runner": attr.label( + default = Label("//terraform/experimental/ghrelease/internal:publisher_runner"), + executable = True, + cfg = "host", + ), + "_tool_hub": attr.label( + default = "@tool_hub", + executable = True, + cfg = "host", + ), + }, + executable = True, +) diff --git a/terraform/experimental/ghrelease/internal/publisher_runner.py b/terraform/experimental/ghrelease/internal/publisher_runner.py new file mode 100644 index 0000000..c298526 --- /dev/null +++ b/terraform/experimental/ghrelease/internal/publisher_runner.py @@ -0,0 +1,147 @@ +from __future__ import print_function + +import argparse +import atexit +import json +import os +import shutil +import subprocess +import sys +import tempfile +from collections import namedtuple +from os import path + +from lib import BazelFlagsEnvVar, GhHelper, SubprocessHelper + + +def str2bool(v): + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Boolean value expected.') + + +parser = argparse.ArgumentParser( + fromfile_prefix_chars='@', + description="Runs pre-flight checks before publishing a new GitHub Release.") + +parser.add_argument( + '--config', action='store', required=True, + type=lambda pathstr: json.load(open(pathstr, "r"), object_hook=lambda d: namedtuple('X', d.keys())(*d.values())), + help=argparse.SUPPRESS) + +parser.add_argument( + '--draft', type=str2bool, nargs='?', const=True, default=False, + help="") + +parser.add_argument( + '--prerelease', type=str2bool, nargs='?', const=True, default=False, + help="") + +parser.add_argument( + '--prerelease_identifier', action='store', default="pre", + help="Eg. alpha,beta,rc,pre") + +parser.add_argument( + '--publish', dest='publish', action='store_true', + default=False, + help="Publish this release to GitHub after running pre-flight checks.") + +_bazel = SubprocessHelper('bazel', cwd=os.environ['BUILD_WORKSPACE_DIRECTORY']) + + +def run_test_suites(test_configs): + # TODO(ceason): run_test_suites() should return a list of all source+build files relevant to the executed tests + srcs = set() + for t in test_configs: + descriptor, script = tempfile.mkstemp() + atexit.register(os.remove, script) + print("Running test suite %s" % t.label) + _bazel(['run', '--script_path', script, t.label]) + os.close(descriptor) + rc = subprocess.call([script]) + if rc != 0: + exit(rc) + return srcs + + +def build_assets(asset_configs, assets_dir, tag, publish): + # TODO(ceason): build_assets() should return two lists of files (source,build) relevant to the built assets + srcs = set() + build_srcs = set() + + copy_assets_args = [assets_dir] + if publish: + copy_assets_args.append("--publish") + + for a in asset_configs: + descriptor, copy_assets_script = tempfile.mkstemp() + atexit.register(os.remove, copy_assets_script) + + new_env = {k: v for k, v in os.environ.items()} + new_env.update(a.env) + new_env[BazelFlagsEnvVar] = json.dumps(a.bazel_flags) + args = ["run"] + args += a.bazel_flags + args += ["--script_path", copy_assets_script] + args += [a.label] + print("Building assets %s" % a.label, end=" ...") + _bazel(args, env=new_env) + print("OK") + os.close(descriptor) + rc = subprocess.call([copy_assets_script] + copy_assets_args, env=new_env) + if rc != 0: + exit(rc) + return srcs, build_srcs + + +def main(args): + """ + """ + workspace_dir = os.environ['BUILD_WORKSPACE_DIRECTORY'] + + # create temp dirs + docs_dir = tempfile.mkdtemp() + assets_dir = tempfile.mkdtemp() + atexit.register(shutil.rmtree, docs_dir, ignore_errors=True) + atexit.register(shutil.rmtree, assets_dir, ignore_errors=True) + + # copy docs + for f in args.config.docs: + tgt_path = path.join(docs_dir, path.basename(f)) + if path.exists(tgt_path): + print("Error, docs file already exists: '%s'" % path.basename(f)) + exit(1) + shutil.copyfile(f, tgt_path) + + # run tests + test_srcs = run_test_suites(args.config.test_configs) + + # git-related preflight checks + git = GhHelper(workspace_dir, args.config.branch, args.config.docs_branch, + version_major=args.config.version.major, + version_minor=args.config.version.minor, + hub_binary=args.config.hub) + git.check_local_tracks_authoritative_branch(args.publish) + + # build the assets + tag = git.get_next_semver(args.prerelease_identifier if args.prerelease else None) + asset_srcs, build_srcs = build_assets(args.config.asset_configs, assets_dir, tag, args.publish) + git.check_srcs_match_head(asset_srcs | test_srcs | build_srcs, args.publish) + + # publish assets & tag as a new GH release + if args.publish: + print("Publishing release to %s" % git._repo_url, file=sys.stderr) + git.check_head_exists_in_remote() + docs_links = git.publish_docs(docs_dir) # ie push them to docs_branch + release_notes = git.generate_releasenotes(docs_links, asset_srcs) + git.publish_release(assets_dir, release_notes, tag, args.draft) + else: + print("Finished running preflight checks. Run with '--publish' flag " + "to publish this as a release.", file=sys.stderr) + + +if __name__ == '__main__': + main(parser.parse_args()) diff --git a/terraform/experimental/ghrelease/internal/test.py b/terraform/experimental/ghrelease/internal/test.py new file mode 100644 index 0000000..c03e8fe --- /dev/null +++ b/terraform/experimental/ghrelease/internal/test.py @@ -0,0 +1,57 @@ +from __future__ import print_function + +import unittest +from random import shuffle + +from lib import next_semver + + +class TestNextSemver(unittest.TestCase): + def test(self): + vs = [ + # no prereleases + "v1.6.0", + "v1.6.1", + + # only prereleases + "v1.7.1-beta.1", + "v1.7.1-alpha.0", + "v1.7.1-rc.3", + + # pre and releases + "v1.8.0", + "v1.8.1", + "v1.8.1-beta.1", + "v1.8.1-alpha.0", + "v1.8.1-rc.3", + + # asdf + "v0.2.0-rc.0", + "v0.2.0-rc.1", + "v0.2.0-rc.2", + ] + shuffle(vs) + + # asdf + self.assertEqual(next_semver(0, 2, "rc", vs), "v0.2.0-rc.3") + + # next release + self.assertEqual(next_semver(1, 7, versions=vs), "v1.7.1") + self.assertEqual(next_semver(1, 6, versions=vs), "v1.6.2") + self.assertEqual(next_semver(1, 8, versions=vs), "v1.8.2") + + # new prerelease + self.assertEqual(next_semver(1, 6, "alpha", vs), "v1.6.2-alpha.1") + + # next prerelease + self.assertEqual(next_semver(1, 7, "beta", vs), "v1.7.1-beta.2") + + # new release + self.assertEqual(next_semver(1, 9, versions=vs), "v1.9.0") + + # no existing versions + self.assertEqual(next_semver(1, 6, "alpha"), "v1.6.0-alpha.1") + + +if __name__ == '__main__': + unittest.main() diff --git a/terraform/experimental/ghrelease/internal/test_suite.bzl b/terraform/experimental/ghrelease/internal/test_suite.bzl new file mode 100644 index 0000000..0720309 --- /dev/null +++ b/terraform/experimental/ghrelease/internal/test_suite.bzl @@ -0,0 +1,59 @@ +GhReleaseTestSuiteInfo = provider( + fields = { + }, +) + +def _impl(ctx): + """ + """ + files = [] + transitive_runfiles = [] + + transitive_runfiles.append(ctx.attr._test_suite_runner.data_runfiles) + transitive_runfiles.append(ctx.attr._test_suite_runner.default_runfiles) + + config_file = ctx.actions.declare_file(ctx.attr.name + ".config.json") + files.append(config_file) + config = struct( + env = ctx.attr.env, + bazel_flags = ctx.attr.bazel_flags, + tests = ctx.attr.tests, + ) + ctx.actions.write(config_file, config.to_json()) + + executable = ctx.actions.declare_file(ctx.attr.name + ".runner.bash") + ctx.actions.write(executable, """#!/usr/bin/env bash + set -euo pipefail + exec "{runner}" "--config={config}" "$@" <&0 + """.format( + runner = ctx.executable._test_suite_runner.short_path, + config = config_file.short_path, + ), is_executable = True) + + runfiles = ctx.runfiles(files = files) + for rf in transitive_runfiles: + runfiles = runfiles.merge(rf) + + return [ + DefaultInfo( + files = depset(direct = files), + executable = executable, + runfiles = runfiles, + ), + GhReleaseTestSuiteInfo(), + ] + +ghrelease_test_suite = rule( + _impl, + attrs = { + "bazel_flags": attr.string_list(default = []), + "env": attr.string_dict(default = {}), + "tests": attr.string_list(default = []), + "_test_suite_runner": attr.label( + default = Label("//terraform/experimental/ghrelease/internal:test_suite_runner"), + executable = True, + cfg = "host", + ), + }, + executable = True, +) diff --git a/terraform/experimental/ghrelease/internal/test_suite_runner.py b/terraform/experimental/ghrelease/internal/test_suite_runner.py new file mode 100644 index 0000000..3c62a0a --- /dev/null +++ b/terraform/experimental/ghrelease/internal/test_suite_runner.py @@ -0,0 +1,42 @@ +from __future__ import print_function + +import argparse +import json +import os +import subprocess + +parser = argparse.ArgumentParser( + description="") + +parser.add_argument( + '--config', action='store', required=True, + type=lambda path: json.load(open(path, "r")), + help=argparse.SUPPRESS) + + +def main(args): + """ + """ + tests = args.config['tests'] + bazel_flags = args.config['bazel_flags'] + env = args.config['env'] + + if len(tests) == 0: + raise ValueError("Config does not contain any tests") + args = ["bazel", "test"] + args.extend(bazel_flags) + args.append("--") + args.extend(tests) + workspace_dir = os.environ['BUILD_WORKSPACE_DIRECTORY'] + environment = {k: v for k, v in os.environ.items()} + environment.update(env) + rc = subprocess.call(args, cwd=workspace_dir, env=environment) + if rc == 4: + # return code 4 means no tests found, so it's "successful" + exit(0) + else: + exit(rc) + + +if __name__ == '__main__': + main(parser.parse_args()) diff --git a/terraform/experimental/publisher.sh.tpl b/terraform/experimental/publisher.sh.tpl deleted file mode 100755 index fd3fc05..0000000 --- a/terraform/experimental/publisher.sh.tpl +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/env bash -[ "$DEBUG" = "1" ] && set -x -set -euo pipefail -err_report() { echo "errexit on line $(caller)" >&2; } -trap err_report ERR - -HUB_ARGS=(%{hub_args}) -PREPUBLISH_TESTS=(%{prepublish_tests}) -PREPUBLISH_BUILDS=(%{prepublish_builds}) -VERSION="%{version}" -BRANCH="%{branch}" -PUBLISH_ENV=(%{env}) -hub="$PWD/%{tool_hub}" -PRERELEASE_IDENTIFIER="rc" -BAZELRC_CONFIG="%{bazelrc_config}" - -bazelcfg="" -if [ -n "$BAZELRC_CONFIG" ]; then - bazelcfg="--config=$BAZELRC_CONFIG" -fi - -_bazel(){ - # return code 4 means no tests found, so it's "successful" - local rc=0; - if bazel "$@"; then rc=0; else rc=$?; fi - case "$rc" in - 0|4) ;; - *) exit $rc ;; - esac -} -_git(){ - local cmdout=$(mktemp) - if git "$@" > "$cmdout" 2>&1; then - rm -rf "$cmdout" - else - rc=$? - >&2 cat "$cmdout" - rm -rf "$cmdout" - exit $rc - fi -} - -# Parses provided semver eg "v1.0.2-rc.1+1539276024" to: -# - SEMVER_MAJOR 1 -# - SEMVER_MINOR 0 -# - SEMVER_PATCH 2 -# - SEMVER_PRE rc -# - SEMVER_PRE_VERSION 1 -# - SEMVER_BUILDINFO 1539276024 -parse_semver(){ - local version=${1-$(cat)} - SEMVER_BUILDINFO=$(awk -F'+' '{print $2}' <<< "$version") - version=$(awk -F'+' '{print $1}' <<< "$version") - SEMVER_PRE=$(awk -F'-' '{print $2}' <<< "$version") - SEMVER_PRE_VERSION=$(awk -F'.' '{print $2}' <<< "$SEMVER_PRE") - SEMVER_PRE=$(awk -F'.' '{print $1}' <<< "$SEMVER_PRE") - version=$(awk -F'-' '{print $1}' <<< "$version") - SEMVER_MAJOR=$(awk -F'.' '{print $1}' <<< "$version") - SEMVER_MAJOR=${SEMVER_MAJOR#v*} - SEMVER_MINOR=$(awk -F'.' '{print $2}' <<< "$version") - SEMVER_PATCH=$(awk -F'.' '{print $3}' <<< "$version") -} - -# Print out a new semver tag to use -# Tag format is "v$VERSION.$((highest_released_patch++))-rc.$((latest_rc++))" -# -# $1 - Current version -# STDIN - List of existing versions (eg 'git tag' or 'hub release') -next_prerelease(){ - local existing="$(sort --version-sort --reverse)" - parse_semver "$1" - local current_major=$SEMVER_MAJOR - local current_minor=$SEMVER_MINOR - # figure out next patch version for our major+minor ("latest published + 1") - local next_patch_version=0 - while read v; do - parse_semver "$v" - if [ -z "$SEMVER_PRE" ]; then - next_patch_version=$((${SEMVER_PATCH:-"-1"} + 1)) - break - fi - done < <(grep "^[v]$current_major\.$current_minor\." <<< "$existing") - - # figure out the next prerelease version for our major+minor+next_patch - parse_semver "$(grep "^[v]$current_major\.$current_minor\.$next_patch_version-$PRERELEASE_IDENTIFIER\." <<< "$existing" | head -1)" - local next_pre_version=$((${SEMVER_PRE_VERSION:-"-1"} + 1)) - - echo -n "v$current_major.$current_minor.$next_patch_version-$PRERELEASE_IDENTIFIER.$next_pre_version" -} - -pushd "$BUILD_WORKSPACE_DIRECTORY" > /dev/null - -if [ ${#PREPUBLISH_BUILDS[@]} -gt 0 ]; then - >&2 echo "Building: $(printf "\n %s" "${PREPUBLISH_BUILDS[@]}")" - _bazel build -- "${PREPUBLISH_BUILDS[@]}" -fi - -if [ ${#PREPUBLISH_TESTS[@]} -gt 0 ]; then - >&2 echo "Testing: $(printf "\n %s" "${PREPUBLISH_TESTS[@]}")" - _bazel test -- "${PREPUBLISH_TESTS[@]}" -fi - -# todo: make sure all build-relevant files are committed -# - enumerate build-relevant files & check that current content == HEAD content -# - 'git add' as necessary & prompt user to commit index - -# make sure current commit exists in upstream branch -_git push -commit=$(git rev-parse --verify HEAD) - -# open in browser if run from terminal -if [ -t 1 ]; then - HUB_ARGS+=("--browse") -fi - -_git fetch --tags -tag=$(git tag|next_prerelease "$VERSION") - -release_message=$(mktemp) -trap "rm -rf $release_message" EXIT - -# find the published tag previous to our new tag -current_published_tag="" -#current_published_tag=$( -# parse_semver $VERSION -# (git tag|grep -E "^[v]?[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$" -# echo "$tag" -# )\ -# |sort --version-sort --reverse\ -# |grep -FA1 "$tag"\ -# |grep -vF "$tag" -#) - -repo_url=$("$hub" browse -u) -echo "$tag" > "$release_message" -if [ -n "$current_published_tag" ]; then - echo " -### Changes Since \`$current_published_tag\`: -" >> "$release_message" - git log --format="- [\`%h\`]($repo_url/commit/%H) %s" "$current_published_tag".."$commit" >> "$release_message" -fi - -"$hub" release create "${HUB_ARGS[@]}" --commitish="$commit" --file="$release_message" "$tag" -_git fetch --tags diff --git a/terraform/experimental/publishing.bzl b/terraform/experimental/publishing.bzl deleted file mode 100644 index 73f1ad5..0000000 --- a/terraform/experimental/publishing.bzl +++ /dev/null @@ -1,116 +0,0 @@ -def _resolve_label(label): - # append package path & workspace name as necessary - if label.startswith(":"): - label = native.package_name() + label - if label.startswith("//"): - label = native.repository_name() + label - return label - -def _quoted_bash_string(str): - return '"%s"' % str.replace('"', '\\"') - -def _impl(ctx): - """ - """ - - runfiles = [] - transitive_runfiles = [] - hub_args = [] - - transitive_runfiles.append(ctx.attr._tool_hub.default_runfiles.files) - if ctx.attr.draft: - hub_args.append("--draft") - if ctx.attr.prerelease: - hub_args.append("--prerelease") - - for file in ctx.files.assets: - runfiles.append(file) - hub_args.append("--attach=$PWD/" + file.short_path) - - ctx.actions.expand_template( - template = ctx.file._launcher_template, - output = ctx.outputs.executable, - substitutions = { - "%{hub_args}": " ".join([_quoted_bash_string(arg) for arg in hub_args]), - "%{tool_hub}": ctx.executable._tool_hub.short_path, - "%{bazelrc_config}": ctx.attr.bazelrc_config or "", - "%{prepublish_builds}": " ".join([ - _quoted_bash_string(l) - for l in sorted(ctx.attr.prepublish_builds) - ]), - "%{prepublish_tests}": " ".join([ - _quoted_bash_string(l) - for l in sorted(ctx.attr.tests) - ]), - "%{version}": ctx.attr.version, - "%{branch}": ctx.attr.version, - "%{env}": " ".join([ - "'%s=%s'" % (k, _quoted_bash_string(v)) - for k, v in ctx.attr.env - ]), - }, - ) - - return [DefaultInfo( - runfiles = ctx.runfiles( - files = runfiles, - transitive_files = depset(transitive = transitive_runfiles), - ), - )] - -_github_release_publisher = rule( - implementation = _impl, - attrs = { - # "assets_map": attr.string_dict(mandatory = True), - "assets": attr.label_list(allow_files = True, mandatory = True), - "version": attr.string(default = "0.0", doc = "Major & minor semver version; does NOT include patch version (which is automatically incremented/generated)"), - "bazelrc_config": attr.string(default = ""), - "branch": attr.string(default = "master", doc = "Only allow releases to be published from this branch."), - "prepublish_builds": attr.string_list( - doc = "Ensure these target patterns build prior to publishing (eg make sure '//...' builds)", - default = ["//..."], - ), - "tests": attr.string_list( - doc = "Ensure these tests pass prior to publishing (eg '//...', plus explicitly enumerating select tests tagged as 'manual')", - default = ["//..."], - ), - "draft": attr.bool(default = False, doc = "Create a draft release"), - "prerelease": attr.bool(default = False, doc = "Create a pre-release"), - "env": attr.string_dict( - doc = "Environment variables set when publishing (useful in conjunction with '--workspace_status_command' script)", - default = {}, - ), - "_launcher_template": attr.label( - executable = True, - cfg = "host", - allow_single_file = True, - default = ":publisher.sh.tpl", - ), - "_tool_hub": attr.label( - default = "@tool_hub", - executable = True, - cfg = "host", - ), - }, - executable = True, -) - -def github_release(name, assets = [], **kwargs): - """ - """ - - # translate assets map to a mapping of name-to-labelname and list of asset labels - assets = [_resolve_label(l) for l in assets] - - _github_release_publisher( - name = name, - assets = assets, - **kwargs - ) - - native.genquery( - name = name + ".srcs-list", - opts = ["--noimplicit_deps"], - expression = """kind("source file", deps(set(%s)))""" % " ".join(assets), - scope = assets, - ) diff --git a/terraform/experimental/publishing.md b/terraform/experimental/publishing.md index 5638a0e..eb2d6f7 100644 --- a/terraform/experimental/publishing.md +++ b/terraform/experimental/publishing.md @@ -2,8 +2,8 @@ ## GH Release Sequence #### Local Pre-flight checks +- test all `ghrelease_test_suite`s - build all configured builds+tests -- test all `release_tests` - check that all transitive (BUILD+source)files for artifacts & tests match with HEAD - else prompt user to commit specified files - ?check that no '--override_repository' (use '--announce_rc' flag to see effective flags per-config?) @@ -42,6 +42,7 @@ Required attrs: Optional attrs: - `branch` (default master) - `default_flags` List of default flags (eg --prerelease,--draft,etc) +- `docs` List of files to include as docs - `docs_branch` (default docs) Any provided docs will be added to the HEAD of this branch - `github_domain` (default github.com) @@ -51,12 +52,7 @@ Attrs: - `tests` list of test patterns to run - `tags` maybe find tests dynamically based on specified tags(/-negation)? -#### `release_asset` -Attrs: -- `bazel_flags` -- `srcs` - -#### `release_docs` +#### `release_assets` Attrs: - `bazel_flags` - `srcs` diff --git a/terraform/internal/BUILD b/terraform/internal/BUILD index fe84322..be58d6c 100644 --- a/terraform/internal/BUILD +++ b/terraform/internal/BUILD @@ -4,12 +4,6 @@ exports_files(glob([ "*.bzl", ])) -py_binary( - name = "render_tf", - srcs = ["render_tf.py"], - visibility = ["//visibility:public"], -) - py_binary( name = "render_workspace", srcs = ["render_workspace.py"], @@ -38,13 +32,6 @@ py_binary( ], ) -py_binary( - name = "k8s", - srcs = ["k8s.py"], - visibility = ["//visibility:public"], - deps = ["@yaml"], -) - py_binary( name = "k8s_manifest", srcs = ["k8s_manifest.py"], diff --git a/terraform/internal/distribution.bzl b/terraform/internal/distribution.bzl deleted file mode 100644 index 029701e..0000000 --- a/terraform/internal/distribution.bzl +++ /dev/null @@ -1,189 +0,0 @@ -load("//terraform:providers.bzl", "DistributionDirInfo", "ModuleInfo", "PluginInfo") -load(":terraform.bzl", "terraform_module") -load("//terraform/internal:terraform_lib.bzl", "create_terraform_renderer", "runfiles_path", "tf_renderer_attrs") -load("//terraform/internal:image_embedder_lib.bzl", "create_image_publisher", "image_publisher_aspect", "image_publisher_attrs") - -def _distribution_dir_impl(ctx): - """ - """ - runfiles = [] - transitive_runfiles = [] - renderer_args = [] - - module = ctx.attr.module[ModuleInfo] - transitive_runfiles.append(ctx.attr.module.data_runfiles.files) - - # add tools' runfiles - transitive_runfiles.append(ctx.attr._terraform_docs.data_runfiles.files) - - runfiles.append(ctx.file.srcs_list) - - # bundle the renderer with args for the content of this tf module - render_tf = ctx.actions.declare_file("%s.render-tf" % ctx.attr.name) - transitive_runfiles.append(create_terraform_renderer(ctx, render_tf, module)) - - # publish container images - image_publisher = ctx.actions.declare_file(ctx.attr.name + ".image-publisher") - transitive_runfiles.append(create_image_publisher( - ctx, - image_publisher, - [ctx.attr.module], - )) - - # expand the runner template - ctx.actions.expand_template( - template = ctx.file._template, - substitutions = { - "%{workspace_name}": ctx.workspace_name, - "%{srcs_list_path}": ctx.file.srcs_list.short_path, - "%{readme_description}": ctx.attr.description or module.description, - "%{render_tf}": render_tf.short_path, - "%{publish_images}": image_publisher.short_path, - }, - output = ctx.outputs.executable, - ) - - return [DefaultInfo( - runfiles = ctx.runfiles( - files = runfiles, - transitive_files = depset(transitive = transitive_runfiles), - collect_data = True, - collect_default = True, - ), - ), DistributionDirInfo()] - -terraform_distribution_dir = rule( - implementation = _distribution_dir_impl, - attrs = dict( - image_publisher_attrs.items() + tf_renderer_attrs.items(), - srcs_list = attr.label(mandatory = True, single_file = True), - description = attr.string(default = ""), - module = attr.label( - mandatory = True, - providers = [ModuleInfo], - aspects = [image_publisher_aspect], - ), - _template = attr.label( - default = Label("//terraform/internal:distribution_dir.sh.tpl"), - single_file = True, - allow_files = True, - ), - _terraform_docs = attr.label( - default = Label("@tool_terraform_docs"), - executable = True, - cfg = "host", - ), - _terraform = attr.label( - default = Label("@tool_terraform"), - executable = True, - cfg = "host", - ), - ), - executable = True, -) - -def _distribution_publisher_impl(ctx): - """ - """ - runfiles = [] - transitive_runfiles = [] - env_vars = [] - distrib_dir_targets = [] - - for dep in ctx.attr.deps: - transitive_runfiles.append(dep.data_runfiles.files) - transitive_runfiles.append(dep.default_runfiles.files) - distrib_dir_targets.append("%s=%s" % (dep.label.name, dep.label)) - - for name, value in ctx.attr.env.items(): - if "'" in name or "\\" in name or "=" in name: - fail("Env var names may not contain the following characters: \\,',= (got '%s') " % name, attr = "env") - if "'" in value or "\\" in value: - fail("Env var values may not contain the following characters: \\,' (got '%s') " % name, attr = "env") - env_vars.append("'%s=%s'" % (name, value)) - - # expand the runner template - ctx.actions.expand_template( - template = ctx.file._template, - substitutions = { - "%{env_vars}": " ".join(env_vars), - "%{prepublish_tests}": " ".join(["'%s'" % t for t in ctx.attr.prepublish_tests or []]), - "%{prepublish_builds}": " ".join(["'%s'" % t for t in ctx.attr.prepublish_builds or []]), - "%{distrib_dir_targets}": " ".join(distrib_dir_targets), - "%{package}": ctx.label.package, - "%{remote}": ctx.attr.remote or "", - "%{remote_path}": ctx.attr.remote_path or "", - "%{bazelrc_config}": ctx.attr.bazelrc_config or "", - "%{remote_branch}": ctx.attr.remote_branch or "master", - }, - output = ctx.outputs.executable, - ) - - return [DefaultInfo( - runfiles = ctx.runfiles( - files = runfiles, - transitive_files = depset(transitive = transitive_runfiles), - collect_data = True, - collect_default = True, - ), - )] - -terraform_distribution_publisher = rule( - implementation = _distribution_publisher_impl, - attrs = { - "remote": attr.string(doc = "Git remote URI. Publish to this repo instead of a local directory."), - "remote_path": attr.string(), - "bazelrc_config": attr.string(default = ""), - "remote_branch": attr.string(default = "master"), - "deps": attr.label_list( - mandatory = True, - providers = [DistributionDirInfo], - ), - "_template": attr.label( - default = Label("//terraform/internal:publisher.sh.tpl"), - single_file = True, - allow_files = True, - ), - "prepublish_builds": attr.string_list( - doc = "Ensure these target patterns build prior to publishing (eg make sure '//...' builds)", - default = ["//..."], - ), - "prepublish_tests": attr.string_list( - doc = "Ensure these tests pass prior to publishing (eg '//...', plus explicitly enumerating select tests tagged as 'manual')", - default = ["//..."], - ), - "env": attr.string_dict( - doc = "Environment variables set when publishing (useful in conjunction with '--workspace_status_command' script)", - default = {}, - ), - }, - executable = True, -) - -def terraform_module_publisher(name, published_modules = {}, **kwargs): - """ - """ - - # for each output path, create a 'distribution_dir' & srcs list - dist_dirs = [] - for path, label in published_modules.items(): - label_abs = "//" + native.package_name() + label if label.startswith(":") else label - srcs_name = "%s.srcs-list" % path - native.genquery( - name = srcs_name, - opts = ["--noimplicit_deps"], - expression = """kind("source file", deps(%s))""" % label_abs, - scope = [label_abs], - ) - dist_dirs.append(":%s" % path) - terraform_distribution_dir( - name = path, - module = label, - srcs_list = srcs_name, - ) - - terraform_distribution_publisher( - name = name, - deps = dist_dirs, - **kwargs - ) diff --git a/terraform/internal/distribution_dir.sh.tpl b/terraform/internal/distribution_dir.sh.tpl deleted file mode 100755 index daa2026..0000000 --- a/terraform/internal/distribution_dir.sh.tpl +++ /dev/null @@ -1,161 +0,0 @@ -#!/usr/bin/env bash -[ "$DEBUG" = "1" ] && set -x -set -euo pipefail -err_report() { echo "errexit on line $(caller)" >&2; } -trap err_report ERR - -# 'rules_k8s' needs to have PYTHON_RUNFILES set -export PYTHON_RUNFILES=${PYTHON_RUNFILES:=$0.runfiles} -RUNFILES=${BASH_SOURCE[0]}.runfiles -WORKSPACE_NAME="%{workspace_name}" -SRCS_LIST_PATH=$RUNFILES/$WORKSPACE_NAME/%{srcs_list_path} -README_DESCRIPTION="%{readme_description}" -terraform_docs="$RUNFILES/tool_terraform_docs/binary" -render_tf="%{render_tf}" -publish_images="%{publish_images}" - -for arg in "$@"; do case $arg in - --tgt-dir=*) - ARG_TGT_DIR="${arg#*=}" - ;; - *) - >&2 echo "Unknown option '$arg'" - exit 1 - ;; -esac done - -: ${ARG_TGT_DIR?"Missing Required argument --tgt-dir"} - - - -put-file(){ - local filename=$1; shift - local content="$(cat)" - local temp_file=$(mktemp) - echo "$content" > "$temp_file" - # Error if the destination file already exists & it's NOT the same content - if [ -e "$filename" ]; then - local filediff=$(diff "$filename" "$temp_file") - if [ -n "$filediff" ]; then - 2>&1 echo "ERROR, duplicate files; $filename already exists & is different: $filediff" - exit 1 - fi - fi - mkdir -p $(dirname "$filename") - mv -T "$temp_file" "$filename" - chmod +w "$filename" -} - - -generate-changelog(){ - local output_dir=$1; shift - # change dir to the repo root - pushd $(cd "$BUILD_WORKSPACE_DIRECTORY" && git rev-parse --show-toplevel) > /dev/null - - # get the remote's URL (in order of precedence) - local remote_url - if remote_url=$(git remote get-url upstream 2>/dev/null); then - : - elif remote_url=$(git remote get-url origin 2>/dev/null); then - : - else - remote_url=$(git remote get-url $(git remote)) - fi - - # "normalize" the remote URL for easier parsing of its components - case "$remote_url" in - # https://github.com/ceason/rules_terraform.git - https://*) remote_url=${remote_url#https://*} ;; - http://*) remote_url=${remote_url#http://*} ;; - - # ssh://git@bitbucket.org/ceason/rules_terraform.git - ssh://git@*) remote_url=${remote_url#ssh://git@*} ;; - - # git@github.com:ceason/rules_terraform.git - git@*) remote_url=${remote_url#git@*} ;; - *) - >&2 echo "Unrecognized format for git remote url '$remote_url'" - exit 1 - ;; - esac - - # parse the components of the remote - remote_url=${remote_url%.git} - remote_host=$( awk -F'[:/]' '{print $1}' <<< "$remote_url") - remote_account=$(awk -F'[:/]' '{print $2}' <<< "$remote_url") - remote_repo=$( awk -F'[:/]' '{print $3}' <<< "$remote_url") - - local commit_link_prefix="https://$remote_host/$remote_account/$remote_repo/commit" - - # find all commits relevant to the source files - local filtered_commits=$(mktemp) - local filtered_commits_unique=$(mktemp) - local output_file=$output_dir/CHANGELOG.md - while read filename; do - git log --pretty='%H' --follow -- "$filename" >> "$filtered_commits" - done < <(grep '^//' "$SRCS_LIST_PATH"|sed 's,^//,,g; s,:,/,g') - sort -u --output="$filtered_commits_unique" "$filtered_commits" - - # create an ordered changelog, including only the filtered commits - while read commit; do - if grep -q $commit $filtered_commits_unique; then - git show -s --date=short --format="- _%cd_ [\`%h\`]($commit_link_prefix/%H) %s" $commit >> "$output_file" - fi - done < <(git log --pretty='%H') - rm -rf $filtered_commits - rm -rf $filtered_commits_unique - popd > /dev/null -} - -generate-readme(){ - local output_dir=$1; shift - # change dir to the repo root - pushd "$output_dir" > /dev/null - - # get terraform inputs/outputs (if there are any) - local terraform_info=$("$terraform_docs" md .) - local terraform_section="" - if [ -n "$terraform_info" ]; then - terraform_section="# Terraform - -$terraform_info -" - fi - - # write the readme - cat < For full changelog see [CHANGELOG.md](CHANGELOG.md) -EOF - popd > /dev/null -} - - -main(){ - # build up the new release dir in a separate location - local STAGING_DIR=$(mktemp -d) - - $render_tf --output_dir "$STAGING_DIR" --plugin_dir "$STAGING_DIR/terraform.d/plugins" - generate-changelog "$STAGING_DIR" - generate-readme "$STAGING_DIR" - - "$publish_images" - - # replace the target dir with the successfully populated staging dir - mkdir -p "$ARG_TGT_DIR" - chmod -R +w "$ARG_TGT_DIR" - rm -rf "$ARG_TGT_DIR" - mkdir -p $(dirname "$ARG_TGT_DIR") - mv "$STAGING_DIR" "$ARG_TGT_DIR" - chmod -R +rw "$ARG_TGT_DIR" - - # Add this stuff to git - (cd "$ARG_TGT_DIR" && git add .) -} - - -main "$@" \ No newline at end of file diff --git a/terraform/internal/image_embedder_lib.bzl b/terraform/internal/image_embedder_lib.bzl index 86f2a1e..d9d3b3d 100644 --- a/terraform/internal/image_embedder_lib.bzl +++ b/terraform/internal/image_embedder_lib.bzl @@ -111,13 +111,15 @@ def create_image_publisher(ctx, output, aspect_targets): image_specs = [] for t in aspect_targets: - if PublishableTargetsInfo in t: - targets = t[PublishableTargetsInfo].targets - if targets != None: - for target in targets.to_list(): - info = target[ImagePublishInfo] - transitive_runfiles.append(info.runfiles) - image_specs.extend(info.image_specs) + # TODO(ceason): identify file targets in a more robust way + if PublishableTargetsInfo not in t and str(t).startswith(" str - return """resource kubectl_generic_object %s_%s { - yaml = "${file("${path.module}/%s")}" -} -""" % ( - self._obj['metadata']['name'].lower(), - self._obj['kind'].lower(), - self.filename) - - def content(self): - # type: () -> str - # strip namespace from all provided objects - self._obj['metadata'].pop('namespace', None) - return yaml.dump(self._obj, default_flow_style=False) - - def __lt__(self, other): - return self.filename < other.filename - - -def write_tf(args): - k8s_objects = [] - for path in args.file: - with open(path, 'r') as f: - for obj in yaml.load_all(f.read()): - k8s_objects.append(KubectlGenericObject(obj)) - # create terraform resources for all of the objects - tf = "\n".join([o.terraform_resource() for o in sorted(k8s_objects)]) - with open(args.output, "w") as f: - f.write(tf) - - -def write_k8s(args): - with open(args.input_file, "r") as in_file: - for item in yaml.load_all(in_file): - obj = KubectlGenericObject(item) - with open(obj.filename, "w") as out_file: - out_file.write(obj.content()) - - -def main(): - args = parser.parse_args() - if args.command == "write-tf": - write_tf(args) - elif args.command == "write-k8s": - write_k8s(args) - else: - raise ValueError("Invalid command") - - -if __name__ == '__main__': - main() diff --git a/terraform/internal/module.bzl b/terraform/internal/module.bzl index a2d912b..1e3aa49 100644 --- a/terraform/internal/module.bzl +++ b/terraform/internal/module.bzl @@ -81,7 +81,11 @@ rm -rf "$module_dir" return struct( terraform_module_info = module_info, - providers = [module_info, DefaultInfo(files = depset(direct = [ctx.outputs.out]))], + providers = [ + module_info, + DefaultInfo(files = depset(direct = [ctx.outputs.out])), + OutputGroupInfo(docs = [ctx.outputs.docs_md]), + ], ) def _module_attrs(aspects = []): diff --git a/terraform/internal/publisher.sh.tpl b/terraform/internal/publisher.sh.tpl deleted file mode 100755 index ad4aebf..0000000 --- a/terraform/internal/publisher.sh.tpl +++ /dev/null @@ -1,166 +0,0 @@ -#!/usr/bin/env bash -[ "$DEBUG" = "1" ] && set -x -set -euo pipefail -err_report() { echo "errexit on line $(caller)" >&2; } -trap err_report ERR - -# register cleanup traps here, then execute them on EXIT! -ITS_A_TRAP=() -cleanup(){ - set +e # we want to keep executing cleanup hooks even if one fails - local JOBS="$(jobs -rp)" - if [ -n "${JOBS}" ]; then - kill $JOBS - wait $JOBS 2>/dev/null - fi - # walk the hooks in reverse order (run most recently registered first) - for (( idx=${#ITS_A_TRAP[@]}-1 ; idx>=0 ; idx-- )) ; do - local cmd="${ITS_A_TRAP[idx]}" - (eval "$cmd") - done -} -trap cleanup EXIT - -export RUNFILES=${RUNFILES:=$0.runfiles} -export PYTHON_RUNFILES=${PYTHON_RUNFILES:=$0.runfiles} -RELEASE_VARS=(%{env_vars}) -PREPUBLISH_TESTS=(%{prepublish_tests}) -PREPUBLISH_BUILDS=(%{prepublish_builds}) -DISTRIB_DIR_TARGETS=(%{distrib_dir_targets}) -REMOTE="%{remote}" -REMOTE_PATH="%{remote_path}" -REMOTE_BRANCH="%{remote_branch}" -BAZELRC_CONFIG="%{bazelrc_config}" - -bazelcfg="" -if [ -n "$BAZELRC_CONFIG" ]; then - bazelcfg="--config=$BAZELRC_CONFIG" -fi - -cd "$BUILD_WORKSPACE_DIRECTORY" - - -# This script will... -# - make sure everything builds -# - run the configured tests -# - then update all releasefiles directories! -# todo: check that all targets' source files are checked in before committing/pushing the release directory - -_bazel(){ - # return code 4 means no tests found, so it's "successful" - local rc=0; - if bazel "$@"; then rc=0; else rc=$?; fi - case "$rc" in - 0|4) ;; - *) exit $rc ;; - esac -} - -_git(){ - local cmdout=$(mktemp) - if git "$@" > "$cmdout" 2>&1; then - rm -rf "$cmdout" - else - rc=$? - >&2 cat "$cmdout" - rm -rf "$cmdout" - exit $rc - fi -} - -if [ ${#PREPUBLISH_BUILDS[@]} -gt 0 ]; then - >&2 echo "Building: $(printf "\n %s" "${PREPUBLISH_BUILDS[@]}")" - _bazel build -- "${PREPUBLISH_BUILDS[@]}" -fi - -if [ ${#PREPUBLISH_TESTS[@]} -gt 0 ]; then - >&2 echo "Testing: $(printf "\n %s" "${PREPUBLISH_TESTS[@]}")" - _bazel test -- "${PREPUBLISH_TESTS[@]}" -fi - -# create all of the releasefiles update scripts (without running them), -# then run them in parallel -distdir_scripts=() -trap 'for f in ${distdir_scripts[@]}; do rm -rf $f; done' EXIT -for item in "${DISTRIB_DIR_TARGETS[@]}"; do - name=$(cut -d'=' -f1 <<< "$item") - label=$(cut -d'=' -f2 <<< "$item") - script=$(mktemp) - distdir_scripts+=("${name}=${script}") - ITS_A_TRAP+=("rm -rf '$script'") - cmdout=$(mktemp) - if env "${RELEASE_VARS[@]}" bazel run "$bazelcfg" --script_path=$script "$label" > "$cmdout" 2>&1; then - rm -rf "$cmdout" - else - rc=$? - >&2 cat "$cmdout" - rm -rf "$cmdout" - exit $rc - fi -done - -# figure out what the "output root" is (ie is it our own repo, or a remote one...) -OUTPUT_ROOT=$BUILD_WORKSPACE_DIRECTORY/%{package} -UPDATE_DIR_MESSAGE="Updating release directory" -if [ -n "${REMOTE:=""}" ]; then - # if it's a remote repo, clone down to a cache dir - sanitized_remote=$(tr '@/.:' '_' <<< "$REMOTE") - repo_dir="$HOME/.cache/rules_terraform_publisher/$sanitized_remote/$REMOTE_BRANCH" - if [ ! -e "$repo_dir" ]; then - git clone $REMOTE -b "$REMOTE_BRANCH" "$repo_dir" - else - pushd "$repo_dir" > /dev/null - _git reset --hard - _git clean -fxd - _git pull - popd > /dev/null - fi - OUTPUT_ROOT=$repo_dir/$REMOTE_PATH - UPDATE_DIR_MESSAGE="$UPDATE_DIR_MESSAGE ($REMOTE)" -fi - ->&2 echo "$UPDATE_DIR_MESSAGE" -returncodes=$(mktemp -d) -ITS_A_TRAP+=("rm -rf $returncodes") -for item in "${distdir_scripts[@]}"; do - name=$(cut -d'=' -f1 <<< "$item") - script=$(cut -d'=' -f2 <<< "$item") - ( set +e - "$script" --tgt-dir="$OUTPUT_ROOT/$name" - echo -n "$?" > "$returncodes/$BASHPID" - ) & -done -wait -for file in "$returncodes"/*; do - rc=$(cat "$file") - if [ "$rc" -ne 0 ]; then - exit $rc - fi -done - -# Prompt user to publish changes if there are any -cd "$OUTPUT_ROOT" -cd "$(git rev-parse --show-toplevel)" -if git diff --quiet -w --cached; then - echo "There are no changes to the release files" -else - git status - while read -r -p "Would you like to publish these changes? -[y]es, [n]o, show [d]iff: "; do - case ${REPLY,,} in - y|yes) - git commit -m "Updating release dir" - git push - break - ;; - n|no) echo "Exiting without updating releasefiles"; break;; - d|diff) - git diff -w --cached - git status - ;; - *) echo "Invalid option '$REPLY'";; - esac - done -fi - - diff --git a/terraform/internal/render_tf.py b/terraform/internal/render_tf.py deleted file mode 100644 index a69c093..0000000 --- a/terraform/internal/render_tf.py +++ /dev/null @@ -1,164 +0,0 @@ -import argparse -import os -import shutil -import subprocess -import tempfile - -import errno -import yaml - -parser = argparse.ArgumentParser( - fromfile_prefix_chars='@', - description='Render a Terraform workspace & associated plugins dir') - -parser.add_argument( - '--k8s_object', action='append', metavar=('output_prefix', 'resolver'), nargs=2, default=[], - help='File that when executed, outputs a yaml stream of k8s objects') - -parser.add_argument( - '--file_generator', action='append', metavar=('output_prefix', 'executable'), nargs=2, default=[], - help='File that when executed, outputs files') - -parser.add_argument( - '--file', action='append', metavar=('tgt_path', 'src'), nargs=2, default=[], - help="'src' file will be copied to 'tgt_path', relative to 'output_dir'") - -parser.add_argument( - '--plugin_file', action='append', metavar=('tgt_path', 'src'), nargs=2, default=[], - help="'src' file will be copied to 'tgt_path', relative to 'plugin_dir'") - -parser.add_argument( - '--output_dir', action='store', - help='Target directory for the output. This will be used as the terraform root.') - -parser.add_argument( - '--plugin_dir', action='store', - help='Location to place terraform plugin files (eg .terraform/plugins, terraform.d/plugins, etc ). If unspecified, no plugin files will be output.') - -parser.add_argument( - '--symlink_plugins', dest='symlink_plugins', action='store_true', - default=False, - help="Symlink plugin files into the output directory rather than copying them (note: not currently implemented)") - - -def put_file(output_path, src=None, content=None, overwrite=False): - """ - - :param overwrite: - :param output_path: - :param src: Source file, will be copied to output_path (mutually exclusive with 'content') - :param content: File content which will be written to output_path (mutually exclusive with 'src') - :return: - """ - if src and content: - raise ValueError("Only one of 'src' or 'content' may be specified.") - if not (src or content): - raise ValueError("Must specify 'src' or 'content'.") - # make sure file doesn't already exist - if os.path.isfile(output_path): - if overwrite: - os.remove(output_path) - else: - raise AssertionError("Target file already exists: '%s'" % output_path) - # create the parent dir - try: - os.makedirs(os.path.dirname(output_path), mode=0755) - except OSError as e: - # ignore if existing dir, but raise otherwise - if e.errno != errno.EEXIST: - raise - # copy or put le file! - if src: - shutil.copyfile(src, output_path) - shutil.copymode(src, output_path) - else: - with open(output_path, "w") as f: - f.write(content) - - -def main(args): - for tgt, src in args.file: - tgt_abs = args.output_dir + "/" + tgt - put_file(tgt_abs, src) - - # only write plugin files if the plugin_dir was specified - if args.plugin_dir: - for tgt, src in args.plugin_file: - tgt_abs = args.plugin_dir + "/" + tgt - put_file(tgt_abs, src, overwrite=True) - - # for each file_generator.. - # - create a tmpdir - # - run generator in the tmpdir - # - 'put_file' each generated file back to the output dir - # - remove the tmpdir - for item in args.file_generator: - prefix, executable = item - tmpdir = tempfile.mkdtemp() - executable = executable.replace('${RUNFILES}', os.environ["RUNFILES"]) - try: - subprocess.check_call([executable], cwd=tmpdir) - except subprocess.CalledProcessError as e: - print(executable) - shutil.rmtree(tmpdir) - exit(e.returncode) - # walk the dir and 'put_file' each file - for dirpath, dirnames, files in os.walk(tmpdir): - if dirpath == tmpdir: - relative_dirpath = "" - elif dirpath.startswith(tmpdir + os.sep): - relative_dirpath = dirpath[len(tmpdir):] - else: - raise ValueError("Unreachable") - for filename in files: - try: - put_file(args.output_dir + os.sep + prefix + os.sep + relative_dirpath + os.sep + filename, - src=tmpdir + os.sep + filename) - except: - shutil.rmtree(tmpdir) - raise - shutil.rmtree(tmpdir) - - # for each k8s_object., - # - split resolver output to individual object files - # - write files underneath appropriate directory - # - accumulate resources into terraform file - k8s_tf_files = {} - for item in args.k8s_object: - prefix, resolver = item - try: - stdout = subprocess.check_output([resolver]) - except subprocess.CalledProcessError as e: - exit(e.returncode) - for k8s_object in yaml.load_all(stdout): - # strip 'namespace' if it's present - k8s_object['metadata'].pop('namespace', None) - content = yaml.dump(k8s_object, default_flow_style=False) - filename = "{name}-{kind}.yaml".format( - name=k8s_object['metadata']['name'], - kind=k8s_object['kind'].lower(), - ) - tgtfile = "{tgt_dir}/{prefix}/{filename}".format( - prefix=prefix, - filename=filename, - tgt_dir=args.output_dir, - ) - put_file(tgtfile, content=content) - - # accumulate per-prefix TF resources file - tf_file = prefix + "/k8s_objects.tf" - k8s_tf_files[tf_file] = k8s_tf_files.get(tf_file, "") + """ -resource kubectl_generic_object {name}_{kind} {{ - yaml = "${{file("${{path.module}}/{filename}")}}" -}} -""".format( - name=k8s_object['metadata']['name'].lower(), - kind=k8s_object['kind'].lower(), - filename=filename) - # write out each module's tf file - for path, content in k8s_tf_files.items(): - put_file(args.output_dir + "/" + path, content=content) - - -if __name__ == '__main__': - main(parser.parse_args()) diff --git a/terraform/internal/terraform.bzl b/terraform/internal/terraform.bzl index 07a47f8..79369a1 100644 --- a/terraform/internal/terraform.bzl +++ b/terraform/internal/terraform.bzl @@ -3,9 +3,9 @@ load("//terraform/internal:terraform_lib.bzl", "create_terraform_renderer", "run load( "//terraform/internal:image_embedder_lib.bzl", _create_image_publisher = "create_image_publisher", + _embed_images = "embed_images", _image_publisher_aspect = "image_publisher_aspect", _image_publisher_attrs = "image_publisher_attrs", - _embed_images = "embed_images", ) def _plugin_impl(ctx): @@ -32,171 +32,3 @@ terraform_plugin = rule( "windows_amd64": attr.label(allow_single_file = True), }, ) - -def _module_impl(ctx): - """ - """ - - runfiles = [] - transitive_runfiles = [] - transitive_plugins = [] - file_map = {} - file_generators = [] - - # aggregate files - for f in ctx.files.srcs: - label = f.owner or ctx.label - prefix = label.package + "/" - path = f.short_path[len(prefix):] - file_map[path] = f - runfiles.append(f) - - # todo: validate that no files (src or embedded) collide with 'modules' attribute (eg module is - # only thing allowed to populate its subpath) - # - test that `filepath` does not begin with `module+"/"` for all configured modules - embeds = [] - embeds.extend(ctx.attr.embed or []) - if ctx.attr.deps: - print("Attribute 'deps' is deprecated. Use 'embed' instead (%s)" % ctx.label) - embeds.extend(ctx.attr.deps or []) - for dep in embeds: - transitive_runfiles.append(dep.default_runfiles.files) - mi = dep[ModuleInfo] - if hasattr(mi, "file_generators"): - file_generators.extend(mi.file_generators) - if hasattr(mi, "plugins"): - transitive_plugins.append(mi.plugins) - if hasattr(mi, "files"): - for filename, file in mi.files.items(): - if filename in file_map and file_map[filename] != file: - fail("Cannot embed file '%s' from module '%s' because it already comes from '%s'" % ( - filename, - dep.label, - file_map[filename].short_path, - ), attr = "embed") - file_map[filename] = file - - for m, module_name in ctx.attr.modules.items(): - transitive_runfiles.append(m.default_runfiles.files) - mi = m[ModuleInfo] - if hasattr(mi, "plugins"): - transitive_plugins.append(mi.plugins) - if hasattr(mi, "file_generators"): - # make the imported module's generated files relative to its own subdirectory - for o in mi.file_generators: - file_generators.append(struct( - executable = o.executable, - output_prefix = "%s/%s" % (module_name, o.output_prefix), - )) - if hasattr(mi, "files"): - # make the imported module's files relative to its own subdirectory - for filename, file in mi.files.items(): - newname = "%s/%s" % (module_name, filename) - if newname in file_map and file_map[newname] != file: - fail("Cannot embed file '%s' from module '%s' because it already comes from '%s'" % ( - filename, - dep.label, - file_map[filename].short_path, - ), attr = "embed") - file_map[newname] = file - - # dedupe any file generators we got - file_generators = {k: None for k in file_generators}.keys() - - module_info = ModuleInfo( - files = file_map, - file_generators = file_generators, - plugins = depset(direct = ctx.attr.plugins or [], transitive = transitive_plugins), - description = ctx.attr.description, - ) - - providers = [module_info] - - # if this is a workspace, create a launcher - if ctx.attr._is_workspace: - image_publisher = ctx.actions.declare_file(ctx.attr.name + ".image-publisher") - runfiles.append(image_publisher) - transitive_runfiles.append(_create_image_publisher( - ctx, - image_publisher, - ctx.attr.deps + ctx.attr.embed + ctx.attr.modules.keys(), - )) - - # bundle the renderer with args for the content of this tf module - render_tf = ctx.actions.declare_file("%s.render-tf" % ctx.attr.name) - transitive_runfiles.append(create_terraform_renderer(ctx, render_tf, module_info)) - ctx.actions.expand_template( - template = ctx.file._workspace_launcher_template, - output = ctx.outputs.executable, - substitutions = { - "%{package}": ctx.label.package, - "%{tf_workspace_files_prefix}": tf_workspace_files_prefix(ctx.attr.name), - "%{render_tf}": render_tf.short_path, - "%{artifact_publishers}": image_publisher.short_path, - }, - ) - providers.append(WorkspaceInfo(render_tf = render_tf)) - - return providers + [DefaultInfo(runfiles = ctx.runfiles( - files = runfiles, - transitive_files = depset(transitive = transitive_runfiles), - ))] - -def _common_attrs(aspects = []): - return tf_renderer_attrs + { - "srcs": attr.label_list( - allow_files = True, - aspects = aspects, - ), - "deps": attr.label_list( - doc = "Deprecated. Use 'embed' instead ('embed' is functionally identical to 'deps', but seems more semantically correct).", - allow_rules = [ - "_k8s_object", - "terraform_module", - ], - aspects = aspects, - ), - "embed": attr.label_list( - # we should use "providers" instead, but "k8s_object" does not - # currently (2018-9-8) support them - doc = "Merge the content of other s (or other 'ModuleInfo' providing deps) into this one.", - providers = [ModuleInfo], - aspects = aspects, - ), - "modules": attr.label_keyed_string_dict( - # hack: disabling provider check until doc generator supports 'providers' attribute - # see https://github.com/bazelbuild/skydoc/blob/master/skydoc/stubs/attr.py#L180 - providers = [ModuleInfo], - aspects = aspects, - ), - "description": attr.string( - doc = "Optional description of module.", - default = "", - ), - "plugins": attr.label_list( - doc = "Custom Terraform plugins that this module requires.", - providers = [PluginInfo], - ), - } - -terraform_module = rule( - implementation = _module_impl, - attrs = dict( - _common_attrs().items(), - # hack: this flag lets us share the same implementation function as 'terraform_module' - _is_workspace = attr.bool(default = False), - ), -) - -terraform_workspace = rule( - implementation = _module_impl, - executable = True, - attrs = _common_attrs([_image_publisher_aspect]) + _image_publisher_attrs + { - # hack: this flag lets us share the same implementation function as 'terraform_module' - "_is_workspace": attr.bool(default = True), - "_workspace_launcher_template" : attr.label( - allow_single_file = True, - default = "//terraform/internal:workspace_launcher.sh.tpl", - ), - }, -) diff --git a/terraform/internal/test.bzl b/terraform/internal/test.bzl deleted file mode 100644 index d108ee3..0000000 --- a/terraform/internal/test.bzl +++ /dev/null @@ -1,76 +0,0 @@ -load("//terraform:providers.bzl", "WorkspaceInfo", "tf_workspace_files_prefix") -load("//terraform/internal:image_embedder_lib.bzl", "create_image_publisher", "image_publisher_aspect", "image_publisher_attrs") - -def _integration_test_impl(ctx): - """ - """ - - runfiles = [] - transitive_runfiles = [] - - transitive_runfiles.append(ctx.attr._runner_template.data_runfiles.files) - transitive_runfiles.append(ctx.attr._stern.data_runfiles.files) - transitive_runfiles.append(ctx.attr.srctest.data_runfiles.files) - transitive_runfiles.append(ctx.attr.terraform_workspace.data_runfiles.files) - render_tf = ctx.attr.terraform_workspace[WorkspaceInfo].render_tf - - image_publisher = ctx.actions.declare_file(ctx.attr.name + ".image-publisher") - transitive_runfiles.append(create_image_publisher( - ctx, - image_publisher, - [ctx.attr.srctest, ctx.attr.terraform_workspace], - )) - - ctx.actions.expand_template( - template = ctx.file._runner_template, - substitutions = { - "%{render_tf}": render_tf.short_path, - "%{srctest}": ctx.executable.srctest.short_path, - "%{stern}": ctx.executable._stern.short_path, - "%{tf_workspace_files_prefix}": tf_workspace_files_prefix(ctx.attr.terraform_workspace), - "%{pretest_publishers}": image_publisher.short_path, - }, - output = ctx.outputs.executable, - is_executable = True, - ) - - return [DefaultInfo( - runfiles = ctx.runfiles( - files = runfiles, - transitive_files = depset(transitive = transitive_runfiles), - # collect_data = True, - # collect_default = True, - ), - )] - -# Wraps the source test with infrastructure spinup and teardown -terraform_integration_test = rule( - test = True, - implementation = _integration_test_impl, - attrs = image_publisher_attrs + { - "terraform_workspace": attr.label( - doc = "TF Workspace to spin up before testing & tear down after testing.", - mandatory = True, - executable = True, - cfg = "host", - providers = [WorkspaceInfo], - aspects = [image_publisher_aspect], - ), - "srctest": attr.label( - doc = "Label of source test to wrap", - mandatory = True, - executable = True, - cfg = "target", # 'host' does not work for jvm source tests, because it launches with @embedded_jdk//:jar instead of @local_jdk//:jar - aspects = [image_publisher_aspect], - ), - "_runner_template": attr.label( - default = "//terraform/internal:test_runner.sh.tpl", - allow_single_file = True, - ), - "_stern": attr.label( - executable = True, - cfg = "host", - default = "@tool_stern", - ), - }, -) diff --git a/terraform/internal/test_runner.sh.tpl b/terraform/internal/test_runner.sh.tpl deleted file mode 100755 index b587048..0000000 --- a/terraform/internal/test_runner.sh.tpl +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash -[ "$DEBUG" = "1" ] && set -x -set -euo pipefail -err_report() { echo "errexit on line $(caller)" >&2; } -trap err_report ERR - -export RUNFILES=${RUNFILES_DIR} - -# register cleanup traps here, then execute them on EXIT! -ITS_A_TRAP=() -cleanup(){ - set +e # we want to keep executing cleanup hooks even if one fails - local JOBS="$(jobs -rp)" - if [ -n "${JOBS}" ]; then - kill $JOBS - wait $JOBS 2>/dev/null - fi - # walk the hooks in reverse order (run most recently registered first) - for (( idx=${#ITS_A_TRAP[@]}-1 ; idx>=0 ; idx-- )) ; do - local cmd="${ITS_A_TRAP[idx]}" - (eval "$cmd") - done -} -trap cleanup EXIT - -render_tf="%{render_tf}" -stern="$PWD/%{stern}" -SRCTEST="%{srctest}" -tf_workspace_files_prefix="%{tf_workspace_files_prefix}" -PRETEST_PUBLISHERS=(%{pretest_publishers}) - -# run pretest publishers (eg docker image publisher) -for publisher in "${PRETEST_PUBLISHERS[@]}"; do - "$publisher" -done - -mkdir -p "$tf_workspace_files_prefix" - -: ${TMPDIR:=/tmp} -: ${TF_PLUGIN_CACHE_DIR:=$TMPDIR/rules_terraform/plugin-cache} -export TF_PLUGIN_CACHE_DIR -mkdir -p "$TF_PLUGIN_CACHE_DIR" - -# guess the kubeconfig location if it isn't already set -: ${KUBECONFIG:="/Users/$USER/.kube/config:/home/$USER/.kube/config"} -export KUBECONFIG - -# render the tf to a tempdir -tfroot=$TEST_TMPDIR/tf/tfroot -tfplan=$TEST_TMPDIR/tf/tfplan -tfstate=$TEST_TMPDIR/tf/tfstate.json -rm -rf "$TEST_TMPDIR/tf" -mkdir -p "$TEST_TMPDIR/tf" -chmod 700 $(dirname "$tfstate") -"$render_tf" --output_dir "$tfroot" --plugin_dir "$tf_workspace_files_prefix/.terraform/plugins" --symlink_plugins - -# init and validate terraform -pushd "$tf_workspace_files_prefix" > /dev/null -timeout 20 terraform init -input=false "$tfroot" -timeout 20 terraform validate "$tfroot" -# if the kubectl provider is used then create a namespace for the test -if [ "$(find .terraform/plugins/ -type f \( -name 'terraform-provider-kubernetes_*' -o -name 'terraform-provider-kubectl_*' \)|wc -l)" -gt 0 ]; then - kubectl config view --merge --raw --flatten > "$TEST_TMPDIR/kubeconfig.yaml" - ITS_A_TRAP+=("rm -rf '$TEST_TMPDIR/kubeconfig.yaml'") - kube_context=$(kubectl config current-context) - export KUBECONFIG="$TEST_TMPDIR/kubeconfig.yaml" - test_namespace=$(mktemp --dry-run test-XXXXXXXXX|tr '[:upper:]' '[:lower:]') - kubectl create namespace "$test_namespace" - ITS_A_TRAP+=("kubectl delete namespace $test_namespace --wait=false") - kubectl config set-context $kube_context --namespace=$test_namespace - # tail stuff with stern in the background - "$stern" '.*' --tail 1 --color always & -fi -timeout 20 terraform plan -out="$tfplan" -input=false "$tfroot" -popd > /dev/null - -# apply the terraform -ITS_A_TRAP+=("cd '$PWD/$tf_workspace_files_prefix' && terraform destroy -state='$tfstate' -auto-approve -refresh=false") -pushd "$tf_workspace_files_prefix" > /dev/null -terraform apply -state-out="$tfstate" -auto-approve "$tfplan" -popd > /dev/null - -# run the test & await its completion -"$SRCTEST" "$@" diff --git a/terraform/internal/workspace.bzl b/terraform/internal/workspace.bzl index 44a5d5f..45cc95d 100644 --- a/terraform/internal/workspace.bzl +++ b/terraform/internal/workspace.bzl @@ -92,3 +92,23 @@ def terraform_workspace(name, modules = {}, **kwargs): modules = flip_modules_attr(modules), **kwargs ) + + # create a convenient destroy target which + # CDs to the package dir and runs terraform destroy + native.genrule( + name = "%s.destroy" % name, + outs = ["%s.destroy.sh" % name], + cmd = """echo '#!/usr/bin/env bash + set -euo pipefail + terraform="$$BUILD_WORKSPACE_DIRECTORY/{package}/{tf_workspace_files_prefix}/.terraform/terraform.sh" + if [ -e "$$terraform" ]; then + exec "$$terraform" destroy "$$@" + else + >&2 echo "Could not find terraform wrapper, so there is nothing to destroy! ($$terraform)" + fi + ' > $@""".format( + package = native.package_name(), + tf_workspace_files_prefix = tf_workspace_files_prefix(), + ), + executable = True, + ) diff --git a/terraform/internal/workspace_launcher.sh.tpl b/terraform/internal/workspace_launcher.sh.tpl deleted file mode 100644 index 5d133dc..0000000 --- a/terraform/internal/workspace_launcher.sh.tpl +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env bash -[ "$DEBUG" = "1" ] && set -x -set -euo pipefail -err_report() { echo "errexit on line $(caller)" >&2; } -trap err_report ERR - -# 'rules_k8s' needs to have PYTHON_RUNFILES set -export PYTHON_RUNFILES=${PYTHON_RUNFILES:=$(cd $0.runfiles && pwd)} -: ${TMPDIR:=/tmp} -export TF_PLUGIN_CACHE_DIR="${TF_PLUGIN_CACHE_DIR:=$TMPDIR/rules_terraform/plugin-cache}" -mkdir -p "$TF_PLUGIN_CACHE_DIR" - -tf_workspace_dir="$BUILD_WORKSPACE_DIRECTORY/%{package}/%{tf_workspace_files_prefix}" -tfroot="$tf_workspace_dir/.terraform/tfroot" -plugin_dir="$tf_workspace_dir/.terraform/plugins" -render_tf="%{render_tf}" -ARTIFACT_PUBLISHERS=(%{artifact_publishers}) - -export RUNFILES=${RUNFILES-$(cd "$0.runfiles" && pwd)} - -_terraform_quiet(){ - local output=$(mktemp) - chmod 600 $output - if terraform "$@" > "$output" 2>&1; then - rm -rf $output - return 0 - else - >&2 cat $output - rm -rf $output - exit 1 - fi -} - -# figure out which command we are running (default to 'apply') -if [ $# -gt 0 ]; then - command=$1; shift -else - command="apply" -fi - -# publish artifacts if we're running apply (eg docker image publisher) -if [ "$command" == "apply" ]; then - for publisher in "${ARTIFACT_PUBLISHERS[@]}"; do - "$publisher" - done -fi - -case "$command" in -# these commands -# - don't rerender the tfroot/plugins dirs -destroy) - cd "$tf_workspace_dir" - exec terraform "$command" "$@" "$tfroot" - ;; - -# these commands -# - operate on an already existing state -# - don't take 'tfroot' as an arg -# - thus don't require rendering tfroot/plugin dirs -workspace|import|output|taint|untaint|state|debug) - cd "$tf_workspace_dir" - exec terraform "$command" "$@" - ;; - -# these commands -# - render the tfroot/plugins dirs -providers) - rm -rf "$tfroot" - "$render_tf" --output_dir "$tfroot" --plugin_dir "$plugin_dir" --symlink - cd "$tf_workspace_dir" - exec terraform "$command" "$@" "$tfroot" - ;; - -# all other commands -# - render the tfroot/plugins dirs -# - initialize tf -*) - rm -rf "$tfroot" - "$render_tf" --output_dir "$tfroot" --plugin_dir "$plugin_dir" --symlink - cd "$tf_workspace_dir" - _terraform_quiet init -input=false "$tfroot" - exec terraform "$command" "$@" "$tfroot" - ;; - -esac \ No newline at end of file