From e5f07b51555fbd40eb10fb80d18e97817b59d1fa Mon Sep 17 00:00:00 2001 From: anzoman Date: Thu, 5 Nov 2020 14:00:53 +0100 Subject: [PATCH] Reshape opera package CLI command This commit is connected to the previous three ones, where the new opera package CLI command has been introduced. The current development process shows clearly that we don't need so complicated behaviour for the package command (e.g. tosca library is just an idea and is not yet implemented in opera) and that reaching its goal (i.e. compressed CSAR) can be simplified. And that is exactly what we did here. "The best things come in small packages." and the same can happen with TOSCA CSARs, containing the blueprint of the application that we wanna orchestrate. The opera package CLI command can be used to create a valid TOSCA CSAR by packaging together the TOSCA service template and all accompanying files. In general, opera package receives a directory (where user's TOSCA templates are located) and produces a compressed CSAR file. The command can create the CSAR if there is at least one TOSCA YAML file in the input folder. If the CSAR structure is already present (if TOSCA-Metadata/TOSCA.meta exists and all other TOSCA CSAR constraints are satisfied) the CSAR is created without an additional temporary directory. And if not, the files are copied to the tempdir, where the CSAR structure is created and at the end the tempdir is compressed. The input folder is the mandatory positional argument, but there are also other command flags that can be used: 1. --service-template/-t is the name of the main TOSCA template from the input folder that will be used as CSAR entrypoint. When not set, the root YAML file from the folder is used. 2. --output/-o is the CSAR file path output that will be created. If not set, the output CSAR file will be named using UUID. 3. --format/-f is the option that specifies the compressed file format. Currently, there are two choices here: 'zip' or 'tar' with 'zip' being the default compression method. 4. --verbose/-v turns on debug mode (currently not implemented). The last thing to mention within this commit is that we added some integration test cases to test the functionality of the new command. --- src/opera/commands/package.py | 90 ++++++++------- src/opera/parser/tosca/csar.py | 109 ++++++++++++++++-- .../cli_commands/playbooks/create.yaml | 3 + .../cli_commands/playbooks/delete.yaml | 3 + tests/integration/cli_commands/runme.sh | 8 +- tests/integration/misc_tosca_types/runme.sh | 16 ++- 6 files changed, 166 insertions(+), 63 deletions(-) diff --git a/src/opera/commands/package.py b/src/opera/commands/package.py index c67ef74b..9abf6794 100644 --- a/src/opera/commands/package.py +++ b/src/opera/commands/package.py @@ -1,12 +1,10 @@ import argparse -from pathlib import Path, PurePath -import shutil -import tempfile -from zipfile import ZipFile + +from pathlib import Path from opera.error import DataError, ParseError -from opera.parser import tosca -from opera.storage import Storage +from opera.parser.tosca.csar import CloudServiceArchive +from opera.utils import generate_random_pathname def add_parser(subparsers): @@ -15,46 +13,46 @@ def add_parser(subparsers): help="Package service template and all accompanying files into a CSAR" ) parser.add_argument( - "--output", "-O", type=argparse.FileType("w"), - help="Output file (the CSAR)", + "--service-template", "-t", + help="Name or path to the TOSCA service template " + "file from the root of the input folder", ) parser.add_argument( - "--library-source", "-L", - help="Path to the library files") - parser.add_argument("service_template", - type=argparse.FileType("r"), - help="Path to the root of the service template") - parser.set_defaults(func=package) - - -def copy_libraries(dest_path, missing_imports, library_source): - for imp in missing_imports: - lib_candidate = library_source / imp.parent - csar_prefix = imp.parent - print(f"candidate: {lib_candidate} -> {dest_path / csar_prefix}") - shutil.copytree(lib_candidate, dest_path / csar_prefix) - + "--output", "-o", + help="Output CSAR file path", + ) + parser.add_argument( + "--format", "-f", choices=("zip", "tar"), + default="zip", help="CSAR compressed file format", + ) + parser.add_argument("service_template_folder", + help="Path to the root of the service template or " + "folder you want to create the TOSCA CSAR from") + parser.set_defaults(func=_parser_callback) -def copy_service(dest_path, service_path): - print(f"Service: {service_path} -> {dest_path}") - shutil.copytree(service_path, dest_path, symlinks=True) +def _parser_callback(args): + if not Path(args.service_template_folder).is_dir(): + raise argparse.ArgumentTypeError("Directory {} is not a valid path!" + .format(args.service_template_folder)) -def compress_service(source_path, output): - with ZipFile(output.name, "w") as csar: - for path in source_path.glob("**/*"): - pathname = str(path) - rel_pathname = str(path.relative_to(source_path)) - csar.write(pathname, arcname=rel_pathname) + # if the output is set use it, if not create a random file name with UUID + if args.output: + # remove file extension if needed (".zip" or ".tar") + # because shutil.make_archive already adds the extension + if args.output.endswith("." + args.format): + csar_output = str(Path(args.output).with_suffix('')) + else: + csar_output = str(Path(args.output)) + else: + # generate a create a unique random file name + csar_output = generate_random_pathname("opera-package-") -def package(args): - service_template_path = PurePath(args.service_template.name) - service_path = service_template_path.parent - library_path = PurePath(args.library_source) try: - missing_imports = tosca.load_for_imports_list(Path.cwd(), - service_template_path) + output_package = package(args.service_template_folder, csar_output, + args.service_template, args.format) + print("CSAR was created and packed to '{}'.".format(output_package)) except ParseError as e: print("{}: {}".format(e.loc, e)) return 1 @@ -62,10 +60,14 @@ def package(args): print(str(e)) return 1 - with tempfile.TemporaryDirectory(prefix="opera-") as tempdir: - dest_path = Path(tempdir) / "service" - copy_service(dest_path, service_path) - copy_libraries(dest_path, missing_imports, library_path) - compress_service(dest_path, args.output) - return 0 + + +def package(input_dir: str, csar_output: str, service_template: str, + csar_format: str) -> str: + """ + :raises ParseError: + :raises DataError: + """ + csar = CloudServiceArchive(input_dir) + return csar.package_csar(csar_output, service_template, csar_format) diff --git a/src/opera/parser/tosca/csar.py b/src/opera/parser/tosca/csar.py index 9b3210ee..7fee4e62 100644 --- a/src/opera/parser/tosca/csar.py +++ b/src/opera/parser/tosca/csar.py @@ -2,7 +2,7 @@ import yaml from pathlib import Path -from tempfile import TemporaryDirectory +from tempfile import TemporaryDirectory, mktemp from zipfile import ZipFile from opera.error import ParseError @@ -15,6 +15,92 @@ def __init__(self, csar_name): self._root_yaml_template = None self._metadata = None + def package_csar(self, output, service_template=None, csar_format="zip"): + try: + meta_file_path = Path( + self._csar_name) / "TOSCA-Metadata" / "TOSCA.meta" + + if not service_template: + root_yaml_files = [] + root_yaml_files.extend(Path(self._csar_name).glob('*.yaml')) + root_yaml_files.extend(Path(self._csar_name).glob('*.yml')) + + if not meta_file_path.exists() and len(root_yaml_files) != 1: + raise ParseError( + "You didn't specify the CSAR TOSCA entrypoint with " + "'-t/--service-template' option. Therefore there " + "should be one YAML file in the root of the CSAR to " + "select it as the entrypoint. More than one YAML has " + "been found: {}. Please select one of the files as " + "the CSAR entrypoint using '-t/--service-template' " + "flag or remove all the excessive YAML files.".format( + list(map(str, root_yaml_files))), self) + service_template = root_yaml_files[0].name + else: + if not Path(self._csar_name).joinpath( + service_template).exists(): + raise ParseError('The supplied TOSCA service template ' + 'file "{}" does not exist in folder ' + '"{}".'.format(service_template, + self._csar_name), self) + + if meta_file_path.exists(): + # check existing TOSCA.meta file + with meta_file_path.open() as meta_file: + self._tosca_meta = yaml.safe_load(meta_file) + + self._validate_csar_version() + self._validate_tosca_meta_file_version() + + template_entry = self._get_entry_definitions() + if service_template and template_entry != service_template: + raise ParseError('The file entry "{}" defined within ' + '"Entry-Definitions" in ' + '"TOSCA-Metadata/TOSCA.meta" does not ' + 'match with the file name "{}" supplied ' + 'in service_template CLI argument.'. + format(template_entry, service_template), + self) + + # check if 'Entry-Definitions' points to an existing + # template file in the CSAR + if not Path(self._csar_name).joinpath(template_entry).exists(): + raise ParseError('The file "{}" defined within ' + '"Entry-Definitions" in ' + '"TOSCA-Metadata/TOSCA.meta" does ' + 'not exist.'.format(template_entry), self) + return shutil.make_archive(output, csar_format, + self._csar_name) + else: + # use tempdir because we don't want to modify user's folder + # with TemporaryDirectory(prefix="opera-package-") as tempdir + # cannot be used because shutil.copytree would fail due to the + # existing temporary folder (this happens only when running + # with python version lower than 3.8) + tempdir = mktemp(prefix="opera-package-") + # create tempdir and copy in all the needed CSAR files + shutil.copytree(self._csar_name, tempdir) + + # create TOSCA-Metadata/TOSCA.meta file using the specified + # TOSCA service template or directory root YAML file + content = """ + TOSCA-Meta-File-Version: 1.1 + CSAR-Version: 1.1 + Created-By: xOpera TOSCA orchestrator + Entry-Definitions: {} + """.format(service_template) + + meta_file_folder = Path(tempdir) / "TOSCA-Metadata" + meta_file = (meta_file_folder / "TOSCA.meta") + + meta_file_folder.mkdir() + meta_file.touch() + meta_file.write_text(content) + + return shutil.make_archive(output, csar_format, tempdir) + except Exception as e: + raise ParseError("Error when creating CSAR: {}".format(e), self) + def unpackage_csar(self, output_dir, csar_format="zip"): # unpack the CSAR to the specified location shutil.unpack_archive(self._csar_name, output_dir, csar_format) @@ -33,16 +119,15 @@ def validate_csar(self): self._validate_csar_version() self._validate_tosca_meta_file_version() template_entry = self._get_entry_definitions() - self._get_created_by() - self._get_other_definitions() # check if 'Entry-Definitions' points to an existing # template file in the CSAR if not Path(tempdir).joinpath(template_entry).exists(): - raise ParseError( - 'The file "{}" defined within "Entry-Definitions" ' - 'in "TOSCA-Metadata/TOSCA.meta" does not exist.' - .format(template_entry), self) + raise ParseError('The file "{}" defined within ' + '"Entry-Definitions" in ' + '"TOSCA-Metadata/TOSCA.meta" does ' + 'not exist.'.format(template_entry), + self) return template_entry else: @@ -52,7 +137,7 @@ def validate_csar(self): root_yaml_files.extend(Path(tempdir).glob('*.yml')) if len(root_yaml_files) != 1: - raise ParseError("There should be one root level yaml " + raise ParseError("There should be one root level YAML " "file in the root of the CSAR: {}." .format(root_yaml_files), self) @@ -62,7 +147,6 @@ def validate_csar(self): self._get_template_version() self._get_template_name() - self._get_author() return root_yaml_files[0].name except Exception as e: @@ -72,8 +156,8 @@ def _validate_csar_version(self): csar_version = self._tosca_meta.get('CSAR-Version') if csar_version and csar_version != 1.1: raise ParseError('CSAR-Version entry in the CSAR {} is ' - 'required to denote version 1.1".'.format( - self._csar_name), self) + 'required to denote version 1.1".'. + format(self._csar_name), self) return csar_version def _validate_tosca_meta_file_version(self): @@ -94,9 +178,11 @@ def _get_entry_definitions(self): self._csar_name), self) return self._tosca_meta.get('Entry-Definitions') + # prepared for future, currently not used since this is an optional keyname def _get_created_by(self): return self._tosca_meta.get('Created-By') + # prepared for future, currently not used since this is an optional keyname def _get_other_definitions(self): return self._tosca_meta.get('Other-Definitions') @@ -114,5 +200,6 @@ def _get_template_name(self): 'template_name in metadata".'.format(self._csar_name), self) return self._metadata.get('template_name') + # prepared for future, currently not used since this is an optional keyname def _get_author(self): return self._metadata.get('template_author') diff --git a/tests/integration/cli_commands/playbooks/create.yaml b/tests/integration/cli_commands/playbooks/create.yaml index 40fb603d..7ee988fd 100644 --- a/tests/integration/cli_commands/playbooks/create.yaml +++ b/tests/integration/cli_commands/playbooks/create.yaml @@ -9,3 +9,6 @@ - name: Say hello from node's property debug: msg: "{{ my_property_input }}" + + - name: Start a long task (needed for the interruption testing) + shell: sleep 10 \ No newline at end of file diff --git a/tests/integration/cli_commands/playbooks/delete.yaml b/tests/integration/cli_commands/playbooks/delete.yaml index 473d5af4..813b1a6e 100644 --- a/tests/integration/cli_commands/playbooks/delete.yaml +++ b/tests/integration/cli_commands/playbooks/delete.yaml @@ -5,3 +5,6 @@ - name: Say helo and bye bye debug: msg: "Hello again and goodbye my friend!" + + - name: Start a long task (needed for the interruption testing) + shell: sleep 10 \ No newline at end of file diff --git a/tests/integration/cli_commands/runme.sh b/tests/integration/cli_commands/runme.sh index 23381847..4a68f085 100755 --- a/tests/integration/cli_commands/runme.sh +++ b/tests/integration/cli_commands/runme.sh @@ -75,10 +75,10 @@ $opera_executable init --clean service.yaml info_out="$($opera_executable info --format json)" test "$(echo "$info_out" | jq -r .status)" = "initialized" -# deploy service template, but interrupt after 1 second +# deploy service template, but interrupt after 2 seconds $opera_executable deploy & DEPLOY_TIMEOUT_PID=$! -sleep 1s && kill -SIGKILL $DEPLOY_TIMEOUT_PID +sleep 2s && kill -SIGKILL $DEPLOY_TIMEOUT_PID # test opera info status after deploy info_out="$($opera_executable info --format json)" @@ -186,10 +186,10 @@ $opera_executable outputs -p ./csar-test-dir info_out="$($opera_executable info -p ./csar-test-dir -f json)" test "$(echo "$info_out" | jq -r .status)" = "deployed" -# undeploy the CSAR, but interrupt after 1 second +# undeploy the CSAR, but interrupt after 2 seconds $opera_executable undeploy -p ./csar-test-dir & UNDEPLOY_TIMEOUT_PID=$! -sleep 1s && kill -SIGKILL $UNDEPLOY_TIMEOUT_PID +sleep 2s && kill -SIGKILL $UNDEPLOY_TIMEOUT_PID # test opera info status after undeploy info_out="$($opera_executable info -p ./csar-test-dir -f json)" diff --git a/tests/integration/misc_tosca_types/runme.sh b/tests/integration/misc_tosca_types/runme.sh index cfadc049..3c405101 100755 --- a/tests/integration/misc_tosca_types/runme.sh +++ b/tests/integration/misc_tosca_types/runme.sh @@ -5,7 +5,6 @@ set -euo pipefail opera_executable="$1" # perform integration test with TOSCA service template -# test opera info contents before everything $opera_executable info --format json $opera_executable validate -i inputs.yaml service-template.yaml $opera_executable info --format yaml @@ -20,6 +19,7 @@ $opera_executable info # integration test with compressed CSAR # warning: opera init is deprecated and could be removed in the future +# prepare the TOSCA CSAR zip file manually rm -rf .opera mkdir -p csar-test zip -r test.csar service-template.yaml modules TOSCA-Metadata @@ -36,15 +36,23 @@ $opera_executable outputs $opera_executable info -f json $opera_executable undeploy $opera_executable info -# deploy compressed CSAR again (without opera init) -$opera_executable deploy -i inputs.yaml -c -f test.csar +# remove manually created CSAR and create a new zipped CSAR with opera package +cd .. +rm -rf csar-test +$opera_executable package -t service-template.yaml -o csar-test/test.zip . +cp inputs.yaml csar-test +cd csar-test +# validate and deploy compressed CSAR again (without opera init) +$opera_executable validate -i inputs.yaml test.zip +$opera_executable info -f yaml +$opera_executable deploy -i inputs.yaml -c -f test.zip $opera_executable info -f json $opera_executable outputs $opera_executable info -f json $opera_executable undeploy $opera_executable info # use opera unpackage and unpack the CSAR and deploy the extracted TOSCA files -$opera_executable unpackage -d unpacked-csar test.csar +$opera_executable unpackage -d unpacked-csar test.zip $opera_executable info cp inputs.yaml unpacked-csar cd unpacked-csar