Reshape opera package CLI command

This commit is connected to the previous three ones, where the new
opera package CLI command has been introduced. The current development
process shows clearly that we don't need so complicated behaviour for
the package command (e.g. tosca library is just an idea and is not yet
implemented in opera) and that reaching its goal (i.e. compressed CSAR)
can be simplified. And that is exactly what we did here.

"The best things come in small packages." and the same can happen with
TOSCA CSARs, containing the blueprint of the application that we wanna
orchestrate. The opera package CLI command can be used to create a
valid TOSCA CSAR by packaging together the TOSCA service template and
all accompanying files. In general, opera package receives a directory
(where user's TOSCA templates are located) and produces a compressed
CSAR file. The command can create the CSAR if there is at least one
TOSCA YAML file in the input folder. If the CSAR structure is already
present (if TOSCA-Metadata/TOSCA.meta exists and all other TOSCA CSAR
constraints are satisfied) the CSAR is created without an additional
temporary directory. And if not, the files are copied to the tempdir,
where the CSAR structure is created and at the end the tempdir is
compressed. The input folder is the mandatory positional argument, but
there are also other command flags that can be used:

  1. --service-template/-t is the name of the main TOSCA template
     from the input folder that will be used as CSAR entrypoint.
     When not set, the root YAML file from the folder is used.

  2. --output/-o is the CSAR file path output that will be created.
     If not set, the output CSAR file will be named using UUID.

  3. --format/-f is the option that specifies the compressed file
     format. Currently, there are two choices here: 'zip' or 'tar'
     with 'zip' being the default compression method.

  4. --verbose/-v turns on debug mode (currently not implemented).

The last thing to mention within this commit is that we added some
integration test cases to test the functionality of the new command.

src/opera/commands/package.py

@@ -1,12 +1,10 @@
 import argparse
-from pathlib import Path, PurePath
-import shutil
-import tempfile
-from zipfile import ZipFile
+
+from pathlib import Path
 
 from opera.error import DataError, ParseError
-from opera.parser import tosca
-from opera.storage import Storage
+from opera.parser.tosca.csar import CloudServiceArchive
+from opera.utils import generate_random_pathname
 
 
 def add_parser(subparsers):
@@ -15,57 +13,61 @@ def add_parser(subparsers):
         help="Package service template and all accompanying files into a CSAR"
     )
     parser.add_argument(
-        "--output", "-O", type=argparse.FileType("w"),
-        help="Output file (the CSAR)",
+        "--service-template", "-t",
+        help="Name or path to the TOSCA service template "
+             "file from the root of the input folder",
     )
     parser.add_argument(
-        "--library-source", "-L",
-        help="Path to the library files")
-    parser.add_argument("service_template",
-                        type=argparse.FileType("r"),
-                        help="Path to the root of the service template")
-    parser.set_defaults(func=package)
-
-
-def copy_libraries(dest_path, missing_imports, library_source):
-    for imp in missing_imports:
-        lib_candidate = library_source / imp.parent
-        csar_prefix = imp.parent
-        print(f"candidate: {lib_candidate} -> {dest_path / csar_prefix}")
-        shutil.copytree(lib_candidate, dest_path / csar_prefix)
-
+        "--output", "-o",
+        help="Output CSAR file path",
+    )
+    parser.add_argument(
+        "--format", "-f", choices=("zip", "tar"),
+        default="zip", help="CSAR compressed file format",
+    )
+    parser.add_argument("service_template_folder",
+                        help="Path to the root of the service template or "
+                             "folder you want to create the TOSCA CSAR from")
+    parser.set_defaults(func=_parser_callback)
 
-def copy_service(dest_path, service_path):
-    print(f"Service: {service_path} -> {dest_path}")
-    shutil.copytree(service_path, dest_path, symlinks=True)
 
+def _parser_callback(args):
+    if not Path(args.service_template_folder).is_dir():
+        raise argparse.ArgumentTypeError("Directory {} is not a valid path!"
+                                         .format(args.service_template_folder))
 
-def compress_service(source_path, output):
-    with ZipFile(output.name, "w") as csar:
-        for path in source_path.glob("**/*"):
-            pathname = str(path)
-            rel_pathname = str(path.relative_to(source_path))
-            csar.write(pathname, arcname=rel_pathname)
+    # if the output is set use it, if not create a random file name with UUID
+    if args.output:
+        # remove file extension if needed (".zip" or ".tar")
+        # because shutil.make_archive already adds the extension
+        if args.output.endswith("." + args.format):
+            csar_output = str(Path(args.output).with_suffix(''))
+        else:
+            csar_output = str(Path(args.output))
 
+    else:
+        # generate a create a unique random file name
+        csar_output = generate_random_pathname("opera-package-")
 
-def package(args):
-    service_template_path = PurePath(args.service_template.name)
-    service_path = service_template_path.parent
-    library_path = PurePath(args.library_source)
     try:
-        missing_imports = tosca.load_for_imports_list(Path.cwd(),
-                                                      service_template_path)
+        output_package = package(args.service_template_folder, csar_output,
+                                 args.service_template, args.format)
+        print("CSAR was created and packed to '{}'.".format(output_package))
     except ParseError as e:
         print("{}: {}".format(e.loc, e))
         return 1
     except DataError as e:
         print(str(e))
         return 1
 
-    with tempfile.TemporaryDirectory(prefix="opera-") as tempdir:
-        dest_path = Path(tempdir) / "service"
-        copy_service(dest_path, service_path)
-        copy_libraries(dest_path, missing_imports, library_path)
-        compress_service(dest_path, args.output)
-
     return 0
+
+
+def package(input_dir: str, csar_output: str, service_template: str,
+            csar_format: str) -> str:
+    """
+    :raises ParseError:
+    :raises DataError:
+    """
+    csar = CloudServiceArchive(input_dir)
+    return csar.package_csar(csar_output, service_template, csar_format)

src/opera/parser/tosca/csar.py

@@ -2,7 +2,7 @@
 import yaml
 
 from pathlib import Path
-from tempfile import TemporaryDirectory
+from tempfile import TemporaryDirectory, mktemp
 from zipfile import ZipFile
 
 from opera.error import ParseError
@@ -15,6 +15,92 @@ def __init__(self, csar_name):
         self._root_yaml_template = None
         self._metadata = None
 
+    def package_csar(self, output, service_template=None, csar_format="zip"):
+        try:
+            meta_file_path = Path(
+                self._csar_name) / "TOSCA-Metadata" / "TOSCA.meta"
+
+            if not service_template:
+                root_yaml_files = []
+                root_yaml_files.extend(Path(self._csar_name).glob('*.yaml'))
+                root_yaml_files.extend(Path(self._csar_name).glob('*.yml'))
+
+                if not meta_file_path.exists() and len(root_yaml_files) != 1:
+                    raise ParseError(
+                        "You didn't specify the CSAR TOSCA entrypoint with "
+                        "'-t/--service-template' option. Therefore there "
+                        "should be one YAML file in the root of the CSAR to "
+                        "select it as the entrypoint. More than one YAML has "
+                        "been found: {}. Please select one of the files as "
+                        "the CSAR entrypoint using '-t/--service-template' "
+                        "flag or remove all the excessive YAML files.".format(
+                            list(map(str, root_yaml_files))), self)
+                service_template = root_yaml_files[0].name
+            else:
+                if not Path(self._csar_name).joinpath(
+                        service_template).exists():
+                    raise ParseError('The supplied TOSCA service template '
+                                     'file "{}" does not exist in folder '
+                                     '"{}".'.format(service_template,
+                                                    self._csar_name), self)
+
+            if meta_file_path.exists():
+                # check existing TOSCA.meta file
+                with meta_file_path.open() as meta_file:
+                    self._tosca_meta = yaml.safe_load(meta_file)
+
+                self._validate_csar_version()
+                self._validate_tosca_meta_file_version()
+
+                template_entry = self._get_entry_definitions()
+                if service_template and template_entry != service_template:
+                    raise ParseError('The file entry "{}" defined within '
+                                     '"Entry-Definitions" in '
+                                     '"TOSCA-Metadata/TOSCA.meta" does not '
+                                     'match with the file name "{}" supplied '
+                                     'in service_template CLI argument.'.
+                                     format(template_entry, service_template),
+                                     self)
+
+                # check if 'Entry-Definitions' points to an existing
+                # template file in the CSAR
+                if not Path(self._csar_name).joinpath(template_entry).exists():
+                    raise ParseError('The file "{}" defined within '
+                                     '"Entry-Definitions" in '
+                                     '"TOSCA-Metadata/TOSCA.meta" does '
+                                     'not exist.'.format(template_entry), self)
+                return shutil.make_archive(output, csar_format,
+                                           self._csar_name)
+            else:
+                # use tempdir because we don't want to modify user's folder
+                # with TemporaryDirectory(prefix="opera-package-") as tempdir
+                # cannot be used because shutil.copytree would fail due to the
+                # existing temporary folder (this happens only when running
+                # with python version lower than 3.8)
+                tempdir = mktemp(prefix="opera-package-")
+                # create tempdir and copy in all the needed CSAR files
+                shutil.copytree(self._csar_name, tempdir)
+
+                # create TOSCA-Metadata/TOSCA.meta file using the specified
+                # TOSCA service template or directory root YAML file
+                content = """
+                          TOSCA-Meta-File-Version: 1.1
+                          CSAR-Version: 1.1
+                          Created-By: xOpera TOSCA orchestrator
+                          Entry-Definitions: {}
+                          """.format(service_template)
+
+                meta_file_folder = Path(tempdir) / "TOSCA-Metadata"
+                meta_file = (meta_file_folder / "TOSCA.meta")
+
+                meta_file_folder.mkdir()
+                meta_file.touch()
+                meta_file.write_text(content)
+
+                return shutil.make_archive(output, csar_format, tempdir)
+        except Exception as e:
+            raise ParseError("Error when creating CSAR: {}".format(e), self)
+
     def unpackage_csar(self, output_dir, csar_format="zip"):
         # unpack the CSAR to the specified location
         shutil.unpack_archive(self._csar_name, output_dir, csar_format)
@@ -33,16 +119,15 @@ def validate_csar(self):
                     self._validate_csar_version()
                     self._validate_tosca_meta_file_version()
                     template_entry = self._get_entry_definitions()
-                    self._get_created_by()
-                    self._get_other_definitions()
 
                     # check if 'Entry-Definitions' points to an existing
                     # template file in the CSAR
                     if not Path(tempdir).joinpath(template_entry).exists():
-                        raise ParseError(
-                            'The file "{}" defined within "Entry-Definitions" '
-                            'in "TOSCA-Metadata/TOSCA.meta" does not exist.'
-                            .format(template_entry), self)
+                        raise ParseError('The file "{}" defined within '
+                                         '"Entry-Definitions" in '
+                                         '"TOSCA-Metadata/TOSCA.meta" does '
+                                         'not exist.'.format(template_entry),
+                                         self)
 
                     return template_entry
                 else:
@@ -52,7 +137,7 @@ def validate_csar(self):
                     root_yaml_files.extend(Path(tempdir).glob('*.yml'))
 
                     if len(root_yaml_files) != 1:
-                        raise ParseError("There should be one root level yaml "
+                        raise ParseError("There should be one root level YAML "
                                          "file in the root of the CSAR: {}."
                                          .format(root_yaml_files), self)
 
@@ -62,7 +147,6 @@ def validate_csar(self):
 
                     self._get_template_version()
                     self._get_template_name()
-                    self._get_author()
 
                     return root_yaml_files[0].name
             except Exception as e:
@@ -72,8 +156,8 @@ def _validate_csar_version(self):
         csar_version = self._tosca_meta.get('CSAR-Version')
         if csar_version and csar_version != 1.1:
             raise ParseError('CSAR-Version entry in the CSAR {} is '
-                             'required to denote version 1.1".'.format(
-                                self._csar_name), self)
+                             'required to denote version 1.1".'.
+                             format(self._csar_name), self)
         return csar_version
 
     def _validate_tosca_meta_file_version(self):
@@ -94,9 +178,11 @@ def _get_entry_definitions(self):
                     self._csar_name), self)
         return self._tosca_meta.get('Entry-Definitions')
 
+    # prepared for future, currently not used since this is an optional keyname
     def _get_created_by(self):
         return self._tosca_meta.get('Created-By')
 
+    # prepared for future, currently not used since this is an optional keyname
     def _get_other_definitions(self):
         return self._tosca_meta.get('Other-Definitions')
 
@@ -114,5 +200,6 @@ def _get_template_name(self):
                 'template_name in metadata".'.format(self._csar_name), self)
         return self._metadata.get('template_name')
 
+    # prepared for future, currently not used since this is an optional keyname
     def _get_author(self):
         return self._metadata.get('template_author')

tests/integration/cli_commands/playbooks/create.yaml

@@ -9,3 +9,6 @@
     - name: Say hello from node's property
       debug:
         msg: "{{ my_property_input }}"
+
+    - name: Start a long task (needed for the interruption testing)
+      shell: sleep 10

tests/integration/cli_commands/playbooks/delete.yaml

@@ -5,3 +5,6 @@
     - name: Say helo and bye bye
       debug:
         msg: "Hello again and goodbye my friend!"
+
+    - name: Start a long task (needed for the interruption testing)
+      shell: sleep 10

tests/integration/cli_commands/runme.sh

@@ -75,10 +75,10 @@ $opera_executable init --clean service.yaml
 info_out="$($opera_executable info --format json)"
 test "$(echo "$info_out" | jq -r .status)" = "initialized"
 
-# deploy service template, but interrupt after 1 second
+# deploy service template, but interrupt after 2 seconds
 $opera_executable deploy &
 DEPLOY_TIMEOUT_PID=$!
-sleep 1s && kill -SIGKILL $DEPLOY_TIMEOUT_PID
+sleep 2s && kill -SIGKILL $DEPLOY_TIMEOUT_PID
 
 # test opera info status after deploy
 info_out="$($opera_executable info --format json)"
@@ -186,10 +186,10 @@ $opera_executable outputs -p ./csar-test-dir
 info_out="$($opera_executable info -p ./csar-test-dir -f json)"
 test "$(echo "$info_out" | jq -r .status)" = "deployed"
 
-# undeploy the CSAR, but interrupt after 1 second
+# undeploy the CSAR, but interrupt after 2 seconds
 $opera_executable undeploy -p ./csar-test-dir &
 UNDEPLOY_TIMEOUT_PID=$!
-sleep 1s && kill -SIGKILL $UNDEPLOY_TIMEOUT_PID
+sleep 2s && kill -SIGKILL $UNDEPLOY_TIMEOUT_PID
 
 # test opera info status after undeploy
 info_out="$($opera_executable info -p ./csar-test-dir -f json)"

tests/integration/misc_tosca_types/runme.sh

@@ -5,7 +5,6 @@ set -euo pipefail
 opera_executable="$1"
 
 # perform integration test with TOSCA service template
-# test opera info contents before everything
 $opera_executable info --format json
 $opera_executable validate -i inputs.yaml service-template.yaml
 $opera_executable info --format yaml
@@ -20,6 +19,7 @@ $opera_executable info
 
 # integration test with compressed CSAR
 # warning: opera init is deprecated and could be removed in the future
+# prepare the TOSCA CSAR zip file manually
 rm -rf .opera
 mkdir -p csar-test
 zip -r test.csar service-template.yaml modules TOSCA-Metadata
@@ -36,15 +36,23 @@ $opera_executable outputs
 $opera_executable info -f json
 $opera_executable undeploy
 $opera_executable info
-# deploy compressed CSAR again (without opera init)
-$opera_executable deploy -i inputs.yaml -c -f test.csar
+# remove manually created CSAR and create a new zipped CSAR with opera package
+cd ..
+rm -rf csar-test
+$opera_executable package -t service-template.yaml -o csar-test/test.zip .
+cp inputs.yaml csar-test
+cd csar-test
+# validate and deploy compressed CSAR again (without opera init)
+$opera_executable validate -i inputs.yaml test.zip
+$opera_executable info -f yaml
+$opera_executable deploy -i inputs.yaml -c -f test.zip
 $opera_executable info -f json
 $opera_executable outputs
 $opera_executable info -f json
 $opera_executable undeploy
 $opera_executable info
 # use opera unpackage and unpack the CSAR and deploy the extracted TOSCA files
-$opera_executable unpackage -d unpacked-csar test.csar
+$opera_executable unpackage -d unpacked-csar test.zip
 $opera_executable info
 cp inputs.yaml unpacked-csar
 cd unpacked-csar