diff --git a/Makefile b/Makefile index d1eeb85..012ef66 100644 --- a/Makefile +++ b/Makefile @@ -25,12 +25,14 @@ YQ := $(PWD)/venv/bin/yq -y PIPREQS := $(PWD)/venv/bin/pipreqs BLACK := $(PWD)/venv/bin/black +num_cpus = $(shell lscpu | awk '/^CPU.s/{ print $$2 }') + DEBUG ?= 0 ifeq ($(DEBUG), 1) - PYTESTFLAGS =-rA + PYTESTFLAGS =-rA --log-cli-level=DEBUG VERBOSITY=5 else - PYTESTFLAGS ="" + PYTESTFLAGS =--log-cli-level=CRITICAL VERBOSITY=0 endif @@ -44,14 +46,15 @@ lint: poetry run black -q --check --exclude venv/ --color --diff . test: - VERBOSE=$(VERBOSITY) py.test $(PYTESTFLAGS) -rA -vvvv tests/ \ - --log-format="%(asctime)s %(levelname)s %(message)s" \ - --log-date-format="%Y-%m-%d %H:%M:%S" \ - --show-capture=all + VERBOSE=$(VERBOSITY) $(PYTEST) -n $(num_cpus) $(PYTESTFLAGS) -vvvv tests/ + # VERBOSE=$(VERBOSITY) $(PYTEST) -n $(num_cpus) $(PYTESTFLAGS) -rA -vvvv tests/ \ + # --log-format="%(asctime)s %(levelname)s %(message)s" \ + # --log-date-format="%Y-%m-%d %H:%M:%S" \ + # --show-capture=all venv: pip3 install -U pip --break-system-packages - pip3 install -U virtualenv --break-system-packages + # --show-capture=all python3 -mvenv venv/ requirements: diff --git a/poetry.lock b/poetry.lock index 498cabd..b57411d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -343,6 +343,20 @@ files = [ {file = "distlib-0.3.8.tar.gz", hash = "sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64"}, ] +[[package]] +name = "execnet" +version = "2.1.1" +description = "execnet: rapid multi-Python deployment" +optional = false +python-versions = ">=3.8" +files = [ + {file = "execnet-2.1.1-py3-none-any.whl", hash = "sha256:26dee51f1b80cebd6d0ca8e74dd8745419761d3bef34163928cbebbdc4749fdc"}, + {file = "execnet-2.1.1.tar.gz", hash = "sha256:5189b52c6121c24feae288166ab41b32549c7e2348652736540b9e6e7d4e72e3"}, +] + +[package.extras] +testing = ["hatch", "pre-commit", "pytest", "tox"] + [[package]] name = "filelock" version = "3.16.1" @@ -864,6 +878,37 @@ pytest = ">=6.2.5" [package.extras] dev = ["pre-commit", "pytest-asyncio", "tox"] +[[package]] +name = "pytest-xdist" +version = "3.6.1" +description = "pytest xdist plugin for distributed testing, most importantly across multiple CPUs" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest_xdist-3.6.1-py3-none-any.whl", hash = "sha256:9ed4adfb68a016610848639bb7e02c9352d5d9f03d04809919e2dafc3be4cca7"}, + {file = "pytest_xdist-3.6.1.tar.gz", hash = "sha256:ead156a4db231eec769737f57668ef58a2084a34b2e55c4a8fa20d861107300d"}, +] + +[package.dependencies] +execnet = ">=2.1" +pytest = ">=7.0.0" + +[package.extras] +psutil = ["psutil (>=3.0)"] +setproctitle = ["setproctitle"] +testing = ["filelock"] + +[[package]] +name = "python-decouple" +version = "3.8" +description = "Strict separation of settings from code." +optional = false +python-versions = "*" +files = [ + {file = "python-decouple-3.8.tar.gz", hash = "sha256:ba6e2657d4f376ecc46f77a3a615e058d93ba5e465c01bbe57289bfb7cce680f"}, + {file = "python_decouple-3.8-py3-none-any.whl", hash = "sha256:d0d45340815b25f4de59c974b855bb38d03151d81b037d9e3f463b0c9f8cbd66"}, +] + [[package]] name = "pyyaml" version = "6.0.2" @@ -1201,4 +1246,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = ">=3.11.1,<4.0" -content-hash = "f57f0349b7c5e572072b301d3f515f297a59363b072b679fca441a8137812aad" +content-hash = "cbe898185a47d8da4623f3b5be3801c5b2cc7fd812ee8409f1903a2d0a0e054d" diff --git a/pyproject.toml b/pyproject.toml index 0b6b9e7..81d5667 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ python = ">=3.11.1,<4.0" colorlog = ">=5.0.1,<7.0.0" jinja2 = "^3.0.1" pretty_traceback = "*" +python-decouple = "^3.8" [tool.poetry.group.test.dependencies] black = "*" @@ -47,6 +48,10 @@ requests-mock = "*" toml = "*" tox = ">=4" + +[tool.poetry.group.dev.dependencies] +pytest-xdist = "^3.6.1" + [tool.poetry.scripts] tfwrapper = 'terrapyne:main' @@ -54,6 +59,12 @@ tfwrapper = 'terrapyne:main' requires = ["poetry-core>=1.1.0"] build-backend = "poetry.core.masonry.api" +[tool.pytest.ini_options] +log_cli = true +log_cli_level = "INFO" +log_cli_format = "%(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s)" +log_cli_date_format = "%Y-%m-%d %H:%M:%S" + [tool.black] # Make sure to match flake8's max-line-length. line-length = 130 # black's default diff --git a/src/terrapyne/terrapyne.py b/src/terrapyne/terrapyne.py index a02fda2..325e8e9 100644 --- a/src/terrapyne/terrapyne.py +++ b/src/terrapyne/terrapyne.py @@ -14,30 +14,34 @@ import json +class TerraformException(Exception): + pass + + class Terraform: - def __init__(self, required_version=None): + def __init__(self, required_version=None, environment_variables={}): self.executable = which("terraform") or next(Path("~/.local/bin").expanduser().glob("terraform")) self.environment_variables = { "TF_IN_AUTOMATION": "1", - "TF_LOG": "trace", - "TF_LOG_PATH": "./terraform.log", "TF_INPUT": "0", + "NO_COLOR": "1", "TF_CLI_ARGS": "-no-color", "TF_CLI_ARGS_init": "-input=false -no-color", "TF_CLI_ARGS_validate": "-no-color", "TF_CLI_ARGS_plan": "-input=false -no-color", "TF_CLI_ARGS_apply": "-input=false -no-color -auto-approve", "TF_CLI_ARGS_destroy": "-input=false -no-color -auto-approve", - "NO_COLOR": "1", - } + } | self.generate_environment_variables(environment_variables) + # "TF_LOG": "trace", + # "TF_LOG_PATH": "./terraform.log", # "TF_PLUGIN_CACHE_DIR": "./tf-cache", self.tfplan_name = "current.tfplan" # Name by project log.debug(f"terraform executable: {self.executable}") if required_version is not None: if self.version != required_version: - raise Exception(f"required version of terraform check failed: {self.version} != {required_version}") + raise TerraformException(f"required version of terraform check failed: {self.version} != {required_version}") @cached_property def version(self): @@ -54,37 +58,33 @@ def version(self): def init(self, args=None) -> Tuple[str, str, int]: return self.exec( cmd=["init", *(args or [])], - expect_stdout=["Terraform has been successfully initialized!"], ) def validate(self, args=None) -> Tuple[str, str, int]: return self.exec( cmd=["validate", *(args or [])], ignore_exit_code=True, - expect_stdout=["Success! The configuration is valid."], ) - def plan(self, args=None) -> Tuple[str, str, int]: + def plan(self, args=None, environment_variables={}) -> Tuple[str, str, int]: return self.exec( cmd=["plan", *(args or [])], - # expect_stdout=[ - # "No changes. Your infrastructure matches the configuration." - # ], + environment_variables=self.generate_environment_variables(environment_variables), ) - def apply(self, args=None) -> Tuple[str, str, int]: - if not os.path.exists(self.tfplan_name): + def apply(self, args=None, environment_variables={}) -> Tuple[str, str, int]: + if not Path(self.tfplan_name).exists(): self.init(args=["-backend=false"]) - self.plan(args=[f"-out={self.tfplan_name}"]) + self.plan( + args=[f"-out={self.tfplan_name}"], + environment_variables=self.generate_environment_variables(environment_variables), + ) return self.exec( cmd=["apply", *(args or [])], - # expect_stdout=[ - # "Apply complete!", - # "No changes. Your infrastructure matches the configuration.", - # ], + environment_variables=self.generate_environment_variables(environment_variables), ) - def output(self, args=None) -> Tuple[str, str, int]: + def output(self, args=None) -> Tuple[Any, str, int]: o, e, c = self.exec( cmd=["output", *(args or ["-json"])], ) @@ -93,19 +93,8 @@ def output(self, args=None) -> Tuple[str, str, int]: def state(self, args=None) -> Tuple[str, str, int]: return self.exec( cmd=["state", *(args or [""])], - # expect_stdout=[ - # "No changes. Your infrastructure matches the configuration." - # ], ) - def get_resources(self) -> dict[Any, Any]: - o, _, _ = self.dump() - return o["resources"] - - def get_outputs(self) -> dict[Any, Any]: - o, _, _ = self.dump() - return o["outputs"] - def dump(self, args=None) -> Tuple[dict[Any, Any], str, int]: o, e, c = self.exec( cmd=["state", "pull", *(args or [""])], @@ -113,19 +102,30 @@ def dump(self, args=None) -> Tuple[dict[Any, Any], str, int]: return json.loads(o), e, c def destroy(self, args=None) -> Tuple[str, str, int]: - return self.exec( - cmd=["destroy", *(args or [])] - # expect_stdout=[ - # "Apply complete!", - # "No changes. Your infrastructure matches the configuration.", - # ], - ) + return self.exec(cmd=["destroy", *(args or [])]) def fmt(self) -> Tuple[str, str, int]: return self.exec( cmd=["fmt", "-recursive"], ) + def get_resources(self) -> dict[Any, Any]: + o, _, _ = self.dump() + return o["resources"] + + def get_outputs(self) -> dict[Any, Any]: + o, _, _ = self.dump() + return o["outputs"] + + def generate_environment_variables(self, in_vars) -> dict[str, str]: + updated = {} + for key in in_vars: + newkey = key + if key.startswith("TF_VAR"): + newkey = re.sub("^TF_VAR_", "", key) + updated[f"TF_VAR_{newkey}"] = in_vars[key] + return updated + def make_layout(self) -> None: for tf_file in [ "main.tf", @@ -164,41 +164,40 @@ def exec( input="", expect_exit_code=0, ignore_exit_code=False, - expect_stdout=None, - expect_stderr=None, + environment_variables={}, ) -> Tuple[str, str, int]: - log.debug(f"terraform.exec({cmd}) with {self.executable}") cmd.insert(0, self.executable) - log.debug(f"terraform.exec({cmd})") + log.debug(f"terraform.exec({cmd}) with {self.executable}") - os.environ.update(self.environment_variables) + process_env_vars = {} + for key in os.environ: + if key.startswith("TF_VAR_"): + process_env_vars[key] = os.environ[key] p = Popen( cmd or ["terraform", "version"], stdout=PIPE, stdin=PIPE, stderr=PIPE, + env=(self.environment_variables | process_env_vars | environment_variables), ) stdout, stderr = p.communicate(input=input.encode()) stdout = stdout.decode().strip() stderr = stderr.decode().strip() exit_code = p.returncode - log.debug(f"stdout: {stdout}") - log.debug(f"stderr: {stderr}") - log.debug(f"exit_code: {exit_code}") + logmsg = " ".join( + [ + "terraform.exec:", + f"exit_code:[{exit_code}]", + f"stdout:[{stdout}]", + f"stderr:[{stderr}]", + f"cwd:[{os.getcwd()}]", + ] + ) + log.debug(logmsg) if ignore_exit_code is not True and exit_code != expect_exit_code: - raise Exception(f"[Error]: exit_code {exit_code} running '{cmd}': {stderr}") - - if expect_stdout is not None: - for string in expect_stdout: - if string not in stdout: - raise Exception(f"stdout did not contain expected string: {string}") - - if expect_stderr is not None: - for string in expect_stderr: - if string not in stderr: - raise Exception(f"stderr did not contain expected string: {string}") + raise TerraformException(f"[Error]: exit_code {exit_code} running '{cmd}': {stderr}") return stdout, stderr, exit_code diff --git a/tests/test_version.py b/tests/test_version.py index 6d1b008..addecf0 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -4,105 +4,89 @@ """ """ +from pathlib import Path import shutil import sys import os import tempfile import platform import json +from decouple import config +from unittest import mock -# import pytest - -sys.path.append(os.path.dirname(os.path.realpath(__file__)) + "/../src") +sys.path.append(Path(__file__ + "/../src").resolve()) import terrapyne import terrapyne.logging import logging as log terraform = terrapyne.Terraform() -VERBOSITY = int(os.environ.get("VERBOSE", 0)) -VERBOSITY = 5 if int(os.environ.get("DEBUG", 0)) >= 1 else VERBOSITY +VERBOSITY = int(config("VERBOSITY", 0)) +VERBOSE = int(config("VERBOSE", 0)) +DEBUG = int(config("DEBUG", 0)) +VERBOSITY = 5 if DEBUG != 0 else VERBOSE or VERBOSITY class TestImport: def test_terrapyne_import(self): with terrapyne.logging.cli_log_config(verbose=VERBOSITY, logger=log.root): assert terraform - log.debug(f"log terrform version: {terraform.version}") assert terraform.version - log.debug(f"log terrform executable: {terraform.executable}") assert terraform.executable def test_terrapyne_required_version(self, tf_required_version): with terrapyne.logging.cli_log_config(verbose=VERBOSITY, logger=log.root): terraform = terrapyne.Terraform(required_version=tf_required_version) assert terraform.version - log.debug(f"log terrform version: {terraform.version}") - assert len(terraform.platform.split('_')) == 2 - # "{}_{}".format( - # platform.system().lower(), platform.release().split("-")[-1] - # ), f"terraform platform string ({terraform.platform}) appears invalid" - log.debug(f"log terrform platform: {terraform.platform}") + assert len(terraform.platform.split("_")) == 2 def test_terrapyne_blank_layout(self): with terrapyne.logging.cli_log_config(verbose=VERBOSITY, logger=log.root): with tempfile.TemporaryDirectory() as tmpdir: os.chdir(tmpdir) terraform.make_layout() - os.mkdir(terraform.environment_variables.get("TF_PLUGIN_CACHE_DIR", "tf-cache")) + # Path(terraform.environment_variables.get("TF_PLUGIN_CACHE_DIR", "tf-cache")).mkdir() shutil.copy("terraform.tf", "/tmp/terraform-1.tf") _fmt_out = terraform.fmt() - log.debug(f"fmt: {_fmt_out}") shutil.copy("terraform.tf", "/tmp/terraform-2.tf") _init_out = terraform.init() - log.debug(f"init: {_init_out}") _validate_out = terraform.validate() - log.debug(f"validate: {_validate_out}") _plan_out = terraform.plan() - log.debug(f"plan: {_plan_out}") _apply_out = terraform.apply() - log.debug(f"apply: {_apply_out}") _destroy_out = terraform.destroy() - log.debug(f"destroy: {_destroy_out}") assert "0 added" in _apply_out[0] assert "0 changed" in _apply_out[0] assert "0 destroyed" in _apply_out[0] - assert True def test_terrapyne_minimal_layout(self): with terrapyne.logging.cli_log_config(verbose=VERBOSITY, logger=log.root): with tempfile.TemporaryDirectory() as tmpdir: os.chdir(tmpdir) terraform.make_layout() - os.mkdir(terraform.environment_variables.get("TF_PLUGIN_CACHE_DIR", "tf-cache")) + # Path(terraform.environment_variables.get("TF_PLUGIN_CACHE_DIR", "tf-cache")).mkdir() _init_out = terraform.init() - log.debug(f"init: {_init_out}") with open("outputs.tf", "a") as f: f.write( """ - resource "null_resource" "example" { - provisioner "local-exec" { - command = "echo This command will execute whenever the configuration changes" - } - # Using triggers to force execution on every apply - triggers = { - always_run = timestamp() - } + resource "local_file" "foo" { + content = "foo!" + filename = "${path.module}/foo.bar" } output "example" { - value = null_resource.example + sensitive = true + value = local_file.foo } output "foo" { @@ -112,36 +96,75 @@ def test_terrapyne_minimal_layout(self): ) _apply_out = terraform.apply() - log.debug(f"apply: {_apply_out}") - + assert Path("foo.bar").exists() assert "1 added" in _apply_out[0] assert "0 changed" in _apply_out[0] assert "0 destroyed" in _apply_out[0] _output_out = terraform.output() - log.debug(f"output: {_output_out}") assert _output_out[0]["foo"]["value"] == "bar" _outputs_out = terraform.get_outputs() - log.debug(f"outputs: {_outputs_out}") - assert _outputs_out["foo"]["value"] == "bar" + assert _outputs_out["example"]["value"]["content"] == "foo!" _resources_out = terraform.get_resources() - log.debug(f"resources: {_resources_out}") - assert _resources_out[0]["instances"] + assert _resources_out[0]["instances"][0]["attributes"]["content"] == "foo!" + assert _resources_out[0]["instances"][0]["attributes"]["filename"] == "./foo.bar" - _destroy_out = terraform.destroy() - log.debug(f"destroy: {_destroy_out}") + Path("./foo.bar").unlink() + _apply_out = terraform.apply() + assert Path("foo.bar").exists() + assert "1 added" in _apply_out[0] + _destroy_out = terraform.destroy() assert "1 destroyed" in _destroy_out[0] - assert True + + def test_terrapyne_env_vars(self, capsys): + with terrapyne.logging.cli_log_config(verbose=VERBOSITY, logger=log.root): + envvars = {"TF_LOG": "trace", "TF_LOG_PATH": "tf-log.log", "foo": "nbar"} + terraform = terrapyne.Terraform(environment_variables=envvars) + with tempfile.TemporaryDirectory() as tmpdir: + os.chdir(tmpdir) + terraform.make_layout() + + with open("outputs.tf", "a") as f: + f.write( + """ + variable "foo" { + type = string + } + output "foo" { + value = var.foo + } + """ + ) + + # default env vars + _apply_out = terraform.apply() + _output_out = terraform.output() + assert _output_out[0]["foo"]["value"] == "nbar" + + # per apply env vars + _apply_out = terraform.apply(environment_variables={"foo": "env_bar"}) + _output_out = terraform.output() + assert _output_out[0]["foo"]["value"] == "env_bar" + + # external env vars + with mock.patch.dict('os.environ', {'TF_VAR_foo': 'external_bar'}, clear=True): + _apply_out = terraform.apply() + _output_out = terraform.output() + assert _output_out[0]["foo"]["value"] == "external_bar" + + # different env vars per apply + _apply_out = terraform.apply() + _output_out = terraform.output() + assert _output_out[0]["foo"]["value"] == "nbar" if __name__ == "__main__": with terrapyne.logging.cli_log_config(verbose=VERBOSITY, logger=log.root): - print("hey") - log.critical("Everything passed") + log.info("Starting tests manually") TestImport().test_terrapyne_import() - TestImport().test_terrapyne_required_version() + TestImport().test_terrapyne_required_version("1.9.7") TestImport().test_terrapyne_blank_layout() TestImport().test_terrapyne_minimal_layout()