Implemented PDS asset handling #3524

- introduced new usecase for PDS job execution
  (makes it easier for developers to find the relevant parts)

sechub-commons-archive/src/main/java/com/mercedesbenz/sechub/commons/archive/ArchiveSupport.java

@@ -4,6 +4,7 @@
 import static java.util.Objects.requireNonNull;
 
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -329,6 +330,28 @@ public void compressFolder(ArchiveType type, File folder, File targetArchiveFile
 
     }
 
+    /**
+     * Just extracts the given archive file to target folder, without any
+     * adjustments.
+     *
+     * @param archiveType  archive type to use
+     * @param archiveFile  file which shall be extracted
+     * @param targetFolder target folder where extraction shall be done
+     * @throws IOException
+     */
+    public ArchiveExtractionResult extractFileAsIsToFolder(ArchiveType archiveType, File archiveFile, File targetFolder,
+            ArchiveExtractionConstraints archiveExtractionConstraints) throws IOException {
+        try (FileInputStream sourceInputStream = new FileInputStream(archiveFile)) {
+            String sourceLocation = archiveFile.getAbsolutePath();
+            var result = switch (archiveType) {
+            case TAR -> extractTar(sourceInputStream, sourceLocation, targetFolder, null, archiveExtractionConstraints);
+            case ZIP -> extractZip(sourceInputStream, sourceLocation, targetFolder, null, archiveExtractionConstraints);
+            default -> throw new IllegalArgumentException("Archive type: " + archiveType + " is not supported!");
+            };
+            return result;
+        }
+    }
+
     private void compressRecursively(String basePath, ArchiveOutputStream outputStream, File file, ArchiveType type, String pathAddition,
             CreationPathContext creationPathContext) throws IOException {
 
@@ -407,13 +430,13 @@ private void compressRecursively(String basePath, ArchiveOutputStream outputStre
     }
 
     private ArchiveExtractionResult extractTar(InputStream sourceInputStream, String sourceLocation, File outputDir,
-            SecHubFileStructureDataProvider fileStructureProvider, ArchiveExtractionConstraints archiveExtractionConstraints) throws IOException {
+            SecHubFileStructureDataProvider fileStructurDataProvider, ArchiveExtractionConstraints archiveExtractionConstraints) throws IOException {
         try (ArchiveInputStream archiveInputStream = new ArchiveStreamFactory().createArchiveInputStream(ArchiveType.TAR.getType(), sourceInputStream)) {
             if (!(archiveInputStream instanceof TarArchiveInputStream)) {
                 throw new IOException("Cannot extract: " + sourceLocation + " because it is not a tar tar");
             }
             try (SafeArchiveInputStream safeArchiveInputStream = new SafeArchiveInputStream(archiveInputStream, archiveExtractionConstraints)) {
-                return extract(safeArchiveInputStream, sourceLocation, outputDir, fileStructureProvider);
+                return extract(safeArchiveInputStream, sourceLocation, outputDir, fileStructurDataProvider);
             }
         } catch (ArchiveException e) {
             throw new IOException("Was not able to extract tar:" + sourceLocation + " at " + outputDir, e);
@@ -422,11 +445,11 @@ private ArchiveExtractionResult extractTar(InputStream sourceInputStream, String
     }
 
     private ArchiveExtractionResult extractZip(InputStream sourceInputStream, String sourceLocation, File outputDir,
-            SecHubFileStructureDataProvider configuration, ArchiveExtractionConstraints archiveExtractionConstraints) throws IOException {
+            SecHubFileStructureDataProvider fileStructurDataProvider, ArchiveExtractionConstraints archiveExtractionConstraints) throws IOException {
         try (ArchiveInputStream archiveInputStream = new ArchiveStreamFactory().createArchiveInputStream(ArchiveType.ZIP.getType(), sourceInputStream);
                 SafeArchiveInputStream safeArchiveInputStream = new SafeArchiveInputStream(archiveInputStream, archiveExtractionConstraints)) {
 
-            return extract(safeArchiveInputStream, sourceLocation, outputDir, configuration);
+            return extract(safeArchiveInputStream, sourceLocation, outputDir, fileStructurDataProvider);
 
         } catch (ArchiveException e) {
             throw new IOException("Was not able to extract tar:" + sourceLocation + " at " + outputDir, e);
@@ -460,7 +483,7 @@ public boolean isZipFileStream(InputStream inputStream) {
     }
 
     private ArchiveExtractionResult extract(SafeArchiveInputStream safeArchiveInputStream, String sourceLocation, File outputDir,
-            SecHubFileStructureDataProvider fileStructureProvider) throws ArchiveException, IOException {
+            SecHubFileStructureDataProvider fileStructurDataProvider) throws ArchiveException, IOException {
 
         ArchiveExtractionResult result = new ArchiveExtractionResult();
         result.targetLocation = outputDir.getAbsolutePath();
@@ -473,7 +496,7 @@ private ArchiveExtractionResult extract(SafeArchiveInputStream safeArchiveInputS
                 throw new IllegalStateException("Entry path is null - cannot be handled!");
             }
 
-            ArchiveTransformationData data = createTransformationData(fileStructureProvider, name);
+            ArchiveTransformationData data = createTransformationData(fileStructurDataProvider, name);
             if (data == null) {
                 continue;
             }

sechub-commons-pds/src/main/java/com/mercedesbenz/sechub/commons/pds/PDSDefaultRuntimeKeyConstants.java

@@ -30,6 +30,8 @@ public class PDSDefaultRuntimeKeyConstants {
 
     public static final String RT_KEY_PDS_JOB_BINARIES_TAR_FILE = "pds.job.binaries.tar.file";
 
+    public static final String RT_KEY_PDS_JOB_EXTRACTED_ASSETS_FOLDER = "pds.job.extracted.assets.folder";
+
     public static final String RT_KEY_PDS_JOB_EXTRACTED_SOURCES_FOLDER = "pds.job.extracted.sources.folder";
 
     public static final String RT_KEY_PDS_JOB_HAS_EXTRACTED_BINARIES = "pds.job.has.extracted.binaries";

sechub-commons-pds/src/main/java/com/mercedesbenz/sechub/commons/pds/PDSLauncherScriptEnvironmentConstants.java

@@ -27,6 +27,7 @@ public class PDSLauncherScriptEnvironmentConstants {
     @Deprecated
     public static final String PDS_JOB_SOURCECODE_UNZIPPED_FOLDER = "PDS_JOB_SOURCECODE_UNZIPPED_FOLDER";
 
+    public static final String PDS_JOB_EXTRACTED_ASSETS_FOLDER = "PDS_JOB_EXTRACTED_ASSETS_FOLDER";
     public static final String PDS_JOB_EXTRACTED_SOURCES_FOLDER = "PDS_JOB_EXTRACTED_SOURCES_FOLDER";
     public static final String PDS_JOB_HAS_EXTRACTED_SOURCES = "PDS_JOB_HAS_EXTRACTED_SOURCES";
     public static final String PDS_JOB_SOURCECODE_ZIP_FILE = "PDS_JOB_SOURCECODE_ZIP_FILE";

sechub-commons-pds/src/main/java/com/mercedesbenz/sechub/commons/pds/data/PDSTemplateMetaData.java

@@ -7,28 +7,28 @@
 
 public class PDSTemplateMetaData {
 
-    private String template;
-    private TemplateType type;
+    private String templateId;
+    private TemplateType templateType;
     private PDSAssetData assetData;
 
     public PDSAssetData getAssetData() {
         return assetData;
     }
 
-    public String getTemplate() {
-        return template;
+    public String getTemplateId() {
+        return templateId;
     }
 
-    public TemplateType getType() {
-        return type;
+    public TemplateType getTemplateType() {
+        return templateType;
     }
 
-    public void setType(TemplateType type) {
-        this.type = type;
+    public void setTemplateType(TemplateType type) {
+        this.templateType = type;
     }
 
-    public void setTemplate(String templateId) {
-        this.template = templateId;
+    public void setTemplateId(String templateId) {
+        this.templateId = templateId;
     }
 
     public void setAssetData(PDSAssetData assetData) {
@@ -37,7 +37,7 @@ public void setAssetData(PDSAssetData assetData) {
 
     @Override
     public int hashCode() {
-        return Objects.hash(assetData, template, type);
+        return Objects.hash(assetData, templateId, templateType);
     }
 
     @Override
@@ -49,34 +49,34 @@ public boolean equals(Object obj) {
         if (getClass() != obj.getClass())
             return false;
         PDSTemplateMetaData other = (PDSTemplateMetaData) obj;
-        return Objects.equals(assetData, other.assetData) && Objects.equals(template, other.template) && type == other.type;
+        return Objects.equals(assetData, other.assetData) && Objects.equals(templateId, other.templateId) && templateType == other.templateType;
     }
 
     @Override
     public String toString() {
-        return "PDSTemplateMetaData [" + (template != null ? "template=" + template + ", " : "") + (type != null ? "type=" + type + ", " : "")
-                + (assetData != null ? "assetData=" + assetData : "") + "]";
+        return "PDSTemplateMetaData [" + (templateId != null ? "template=" + templateId + ", " : "")
+                + (templateType != null ? "type=" + templateType + ", " : "") + (assetData != null ? "assetData=" + assetData : "") + "]";
     }
 
     public static class PDSAssetData {
-        private String asset;
-        private String file;
+        private String assetId;
+        private String fileName;
         private String checksum;
 
-        public String getAsset() {
-            return asset;
+        public String getAssetId() {
+            return assetId;
         }
 
-        public void setAsset(String asset) {
-            this.asset = asset;
+        public void setAssetId(String assetId) {
+            this.assetId = assetId;
         }
 
-        public String getFile() {
-            return file;
+        public String getFileName() {
+            return fileName;
         }
 
-        public void setFile(String file) {
-            this.file = file;
+        public void setFileName(String fileName) {
+            this.fileName = fileName;
         }
 
         public String getChecksum() {
@@ -89,7 +89,7 @@ public void setChecksum(String checksum) {
 
         @Override
         public int hashCode() {
-            return Objects.hash(asset, checksum, file);
+            return Objects.hash(assetId, checksum, fileName);
         }
 
         @Override
@@ -101,12 +101,12 @@ public boolean equals(Object obj) {
             if (getClass() != obj.getClass())
                 return false;
             PDSAssetData other = (PDSAssetData) obj;
-            return Objects.equals(asset, other.asset) && Objects.equals(checksum, other.checksum) && Objects.equals(file, other.file);
+            return Objects.equals(assetId, other.assetId) && Objects.equals(checksum, other.checksum) && Objects.equals(fileName, other.fileName);
         }
 
         @Override
         public String toString() {
-            return "PDSAssetData [" + (asset != null ? "asset=" + asset + ", " : "") + (file != null ? "file=" + file + ", " : "")
+            return "PDSAssetData [" + (assetId != null ? "assetId=" + assetId + ", " : "") + (fileName != null ? "fileName=" + fileName + ", " : "")
                     + (checksum != null ? "checksum=" + checksum : "") + "]";
         }
 

sechub-commons-pds/src/test/java/com/mercedesbenz/sechub/commons/pds/data/PDSTemplateMetaDataTest.java

@@ -1,10 +1,15 @@
 // SPDX-License-Identifier: MIT
 package com.mercedesbenz.sechub.commons.pds.data;
 
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.util.List;
+
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 
 import com.mercedesbenz.sechub.commons.model.JSONConverter;
+import com.mercedesbenz.sechub.commons.pds.data.PDSTemplateMetaData.PDSAssetData;
 import com.mercedesbenz.sechub.test.TestFileReader;
 
 class PDSTemplateMetaDataTest {
@@ -16,8 +21,17 @@ void examples_in_doc_are_valid(String fileName) {
         String json = TestFileReader.readTextFromFile("./../sechub-doc/src/docs/asciidoc/documents/shared/snippet/" + fileName);
 
         /* execute */
-        JSONConverter.get().fromJSONtoListOf(PDSTemplateMetaData.class, json);
+        List<PDSTemplateMetaData> result = JSONConverter.get().fromJSONtoListOf(PDSTemplateMetaData.class, json);
 
+        /* test */
+        assertEquals(1, result.size());
+        PDSTemplateMetaData entry = result.iterator().next();
+        assertNotNull(entry.getTemplateId());
+        PDSAssetData assetData = entry.getAssetData();
+        assertNotNull(assetData);
+        assertNotNull(assetData.getAssetId());
+        assertNotNull(assetData.getChecksum());
+        assertNotNull(assetData.getFileName());
     }
 
 }

sechub-doc/src/docs/asciidoc/documents/pds/pds_config.adoc

@@ -48,7 +48,7 @@ Either use system properties
 pds.techuser.userid
 pds.techuser.apitoken
 ----
-or env entries
+or environment variables
 ----
 PDS_TECHUSER_USERID
 PDS_TECHUSER_APITOKEN
@@ -187,6 +187,12 @@ how the data is gathered.
 |PDS_JOB_SOURCECODE_ZIP_FILE        | The absolute path to the uploaded "sourcecode.zip" file
 |PDS_JOB_EXTRACTED_SOURCES_FOLDER   | When auto extracting is enabled (default) the uploaded source code is extracted to this folder
 |PDS_JOB_EXTRACTED_BINARIES_FOLDER  | When auto extracting is enabled (default) the uploaded binaries are extracted to this folder
+|PDS_JOB_EXTRACTED_ASSETS_FOLDER    | The absolute path to the extracted assets. +
+ + 
+                                      Files for template types are located in dedicated sub folders which are named by the id of the template type. + +
+ +
+                                      For example: WEBSCAN_LOGIN template type has id: `webscan-login`. To access the file `custom-login.groovy` a script
+                                      or a wrapper application can simply fetch the file via `$PDS_JOB_EXTRACTED_ASSETS_FOLDER/webscan-login/custom-login.groovy`. 
 |PDS_SCAN_TARGET_URL                | Target URL for current scan (e.g webscan). Will not be set in all scan types. E.g. for a code scan this environemnt variable will not be available
 |PDS_SCAN_CONFIGURATION             | Contains the SecHub configuration as JSON _(but reduced to current scan type, so e.g. a web scan will have no code scan configuration data available)_ +
                                       +
@@ -216,10 +222,15 @@ include::../gen/gen_pds_executor_config_parameters.adoc[]
 ==== File locations
 
 ===== Upload
-`$PDS_JOB_WORKSPACE_LOCATION/upload/`
+Content from uploaded user archives is extracted to:
+
+`PDS_JOB_EXTRACTED_SOURCES_FOLDER`,
+
+`PDS_JOB_EXTRACTED_BINARIES_FOLDER`
+
+Content from uploaded asset files is extracted to:
+ `PDS_JOB_EXTRACTED_ASSETS_FOLDER`,
 
-Automatically unzipped content is available inside +
-`$PDS_JOB_WORKSPACE_LOCATION/upload/unzipped`
 
 ===== Output
 

sechub-doc/src/docs/asciidoc/documents/shared/snippet/pds-param-template-metadata-example1.json

@@ -1,9 +1,9 @@
 [ {
-  "template" : "single-singon",
-  "type" : "webscan-login",
+  "templateId" : "single-singon",
+  "templateType" : "webscan-login",
   "assetData" : {
-    "asset" : "custom-webscan-setup",
-    "file" : "WEBSCAN_PRODUCT_ID.zip",
+    "assetId" : "custom-webscan-setup",
+    "fileName" : "WEBSCAN_PRODUCT_ID.zip",
     "checksum" : "434c6c6ec1b0ed9844149069d7d45ac18e72505b"
   }
 } ]

sechub-doc/src/docs/asciidoc/documents/shared/snippet/pds-param-template-metadata-syntax.json

@@ -1,11 +1,11 @@
   [ //<1>
     {
-        "template" : "templateId", //<2>
-        "type": "WEBSCAN_LOGIN", //<3>
+        "templateId" : "templateId", //<2>
+        "tempplateType": "WEBSCAN_LOGIN", //<3>
 
         "assetData" : { //<4>
-                "asset" : "assetId", //<5>
-                "file" : "fileName", //<6>
+                "assetId" : "assetId", //<5>
+                "fileName" : "fileName", //<6>
                 "checksum" : "fileChecksum" //<7>
         }
 

sechub-integrationtest/pds/product-scripts/integrationtest-webscan.sh

@@ -20,6 +20,11 @@ dumpPDSVariables
 # We added pds.config.templates.metadata.list as optional parameter here for testing
 # So we can dump the variable here - used in scenario12 integration test
 dumpVariable "PDS_CONFIG_TEMPLATE_METADATA_LIST"
+
+ASSET_FILE1="$PDS_JOB_EXTRACTED_ASSETS_FOLDER/webscan-login/testfile1.txt"
+TEST_CONTENT_FROM_ASSETFILE=$(cat $ASSET_FILE1) 
+# Afterwards TEST_CONTENT_FROM_ASSETFILE=i am "testfile1.txt" for scenario12 integration tests
+dumpVariable "TEST_CONTENT_FROM_ASSETFILE"
 
 if [[ "$PDS_TEST_KEY_VARIANTNAME" = "a" ]]; then
 

sechub-integrationtest/src/test/java/com/mercedesbenz/sechub/integrationtest/scenario12/PDSWebScanJobScenario12IntTest.java

@@ -62,17 +62,20 @@ public class PDSWebScanJobScenario12IntTest {
      *
      * The tests checks following:
      *
-     * - pds_web_scan_has_expected_info_finding_with_given_target_url_and_product2_level_information_and_sechub_web_config_parts
-     * - pds web scan has expected info finding, with
-     *    - given target url
+     * - PDS web scan has expected info finding, with
+     *    - given target URL
      *    - product level information
-     *    - sechub web configuration parts
+     *    - SecHub web configuration parts
      *
      *  - PDS parameter for template meta data configuration is correct and transmitted to PDS
      *    The parameter "pds.config.template.metadata.list" is normally not available inside
      *    the scripts, but for testing we added the parameter inside server configuration so it
      *    will be added to script level and can be checked by TestAPI
      *
+     *  - PDS will download and extract the uploaded asset file automatically and the
+     *    extracted content is available inside the test bash script (executed by PDS)
+     *
+     *
      * @formatter:on
      */
     @Test
@@ -163,13 +166,13 @@ public void pds_web_scan_can_be_executed_and_works() throws Exception {
         assertTrue(includes.contains("/customer/<*>"));
         assertTrue(excludes.contains("<*>/admin/<*>"));
 
-        // config must contain the expected headers
+        // web configuration must contain the expected headers
         assertExpectedHeaders(webConfiguration);
 
-        // config must contain the expected client certificate
+        // web configuration must contain the expected client certificate
         assertExpectedClientCertificate(webConfiguration);
 
-        // config must contain the expected openApi definition
+        // web configuration must contain the expected openApi definition
         assertExpectedOpenApiDefinition(webConfiguration);
 
         /* additional testing : messages*/
@@ -185,9 +188,10 @@ public void pds_web_scan_can_be_executed_and_works() throws Exception {
         Map<String, String> variables = fetchPDSVariableTestOutputMap(pdsJobUUID);
 
         String expectedMetaDataListJson = """
-                [{"template":"template-scenario12-1","type":"WEBSCAN_LOGIN","assetData":{"asset":"asset-s12-pds-inttest-webscan","file":"PDS_INTTEST_PRODUCT_WEBSCAN.zip","checksum":"ff06430bfc2d8c698ab8effa41b914525b8cca1c1eecefa76d248b25cc598fba"}}]
+                [{"templateId":"template-scenario12-1","templateType":"WEBSCAN_LOGIN","assetData":{"assetId":"asset-s12-pds-inttest-webscan","fileName":"PDS_INTTEST_PRODUCT_WEBSCAN.zip","checksum":"ff06430bfc2d8c698ab8effa41b914525b8cca1c1eecefa76d248b25cc598fba"}}]
                 """.trim();
         assertThat(variables.get("PDS_CONFIG_TEMPLATE_METADATA_LIST")).isEqualTo(expectedMetaDataListJson);
+        assertThat(variables.get("TEST_CONTENT_FROM_ASSETFILE")).isEqualTo("i am \"testfile1.txt\" for scenario12 integration tests");
         /* @formatter:on */
     }