Native unzip (#856)

* Native unzip Signed-off-by: Prabhu Subramanian <[email protected]> --------- Signed-off-by: Prabhu Subramanian <[email protected]>
CycloneDX · Jan 30, 2024 · 7e1fc4c · 7e1fc4c
1 parent 4a427f7
commit 7e1fc4c
Show file tree

Hide file tree

Showing 6 changed files with 143 additions and 72 deletions.
diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml
@@ -67,10 +67,7 @@ jobs:
           deno lint
           mkdir build
           ${{ matrix.build }}
-      - uses: actions/upload-artifact@v4
-        with:
-          name: ${{ matrix.artifact }}
-          path: ${{ matrix.artifact }}
+        continue-on-error: true
       # - name: Release
       #   uses: softprops/action-gh-release@v1
       #   if: startsWith(github.ref, 'refs/tags/')

diff --git a/docker.js b/docker.js
@@ -684,7 +684,7 @@ export const extractTar = async (fullImageName, dir) => {
         preserveOwner: false,
         noMtime: true,
         noChmod: true,
-        strict: true,
+        strict: false,
         C: dir,
         portable: true,
         onwarn: () => {},
@@ -696,6 +696,9 @@ export const extractTar = async (fullImageName, dir) => {
             path.includes("etc/") ||
             path.includes("logs/") ||
             path.includes("dev/") ||
+            path.includes("usr/share/zoneinfo/") ||
+            path.includes("usr/share/doc/") ||
+            path.includes("usr/share/i18n/") ||
             [
               "BlockDevice",
               "CharacterDevice",
@@ -727,6 +730,12 @@ export const extractTar = async (fullImageName, dir) => {
       console.log("------------");
       console.log(err);
       console.log("------------");
+    } else if (err.code === "TAR_BAD_ARCHIVE") {
+      if (DEBUG_MODE) {
+        console.log(`Archive ${fullImageName} is empty. Skipping.`);
+      }
+    } else {
+      console.log(err);
     }
     return false;
   }
@@ -832,13 +841,30 @@ export const extractFromManifest = async (
     }
     const lastLayer = layers[layers.length - 1];
     for (const layer of layers) {
+      try {
+        if (!lstatSync(join(tempDir, layer)).isFile()) {
+          console.log(
+            `Skipping layer ${layer} since it is not a readable file.`
+          );
+          continue;
+        }
+      } catch (e) {
+        console.log(`Skipping layer ${layer} since it is not a readable file.`);
+        continue;
+      }
       if (DEBUG_MODE) {
         console.log(`Extracting layer ${layer} to ${allLayersExplodedDir}`);
       }
       try {
         await extractTar(join(tempDir, layer), allLayersExplodedDir);
       } catch (err) {
-        console.log(err);
+        if (err.code === "TAR_BAD_ARCHIVE") {
+          if (DEBUG_MODE) {
+            console.log(`Layer ${layer} is empty.`);
+          }
+        } else {
+          console.log(err);
+        }
       }
     }
     if (manifest.Config) {
@@ -1058,15 +1084,23 @@ export const getPkgPathList = (exportData, lastWorkingDir) => {
     }
   }
   if (lastWorkingDir && lastWorkingDir !== "") {
-    knownSysPaths.push(lastWorkingDir);
+    if (
+      !lastWorkingDir.includes("/opt/") &&
+      !lastWorkingDir.includes("/home/")
+    ) {
+      knownSysPaths.push(lastWorkingDir);
+    }
     // Some more common app dirs
-    if (!lastWorkingDir.startsWith("/app")) {
+    if (!lastWorkingDir.includes("/app/")) {
       knownSysPaths.push(join(allLayersExplodedDir, "/app"));
     }
-    if (!lastWorkingDir.startsWith("/data")) {
+    if (!lastWorkingDir.includes("/layers/")) {
+      knownSysPaths.push(join(allLayersExplodedDir, "/layers"));
+    }
+    if (!lastWorkingDir.includes("/data/")) {
       knownSysPaths.push(join(allLayersExplodedDir, "/data"));
     }
-    if (!lastWorkingDir.startsWith("/srv")) {
+    if (!lastWorkingDir.includes("/srv/")) {
       knownSysPaths.push(join(allLayersExplodedDir, "/srv"));
     }
   }

diff --git a/index.js b/index.js
@@ -948,39 +948,42 @@ export const createJarBom = async (path, options) => {
       false,
       true
     );
+  }
+  if (path.endsWith(".jar")) {
+    jarFiles = [resolve(path)];
   } else {
     jarFiles = getAllFiles(
       path,
       (options.multiProject ? "**/" : "") + "*.[jw]ar",
       options
     );
-    // Jenkins plugins
-    const hpiFiles = getAllFiles(
-      path,
-      (options.multiProject ? "**/" : "") + "*.hpi",
-      options
-    );
-    if (hpiFiles.length) {
-      jarFiles = jarFiles.concat(hpiFiles);
+  }
+  // Jenkins plugins
+  const hpiFiles = getAllFiles(
+    path,
+    (options.multiProject ? "**/" : "") + "*.hpi",
+    options
+  );
+  if (hpiFiles.length) {
+    jarFiles = jarFiles.concat(hpiFiles);
+  }
+  const tempDir = mkdtempSync(join(tmpdir(), "jar-deps-"));
+  for (const jar of jarFiles) {
+    if (DEBUG_MODE) {
+      console.log(`Parsing ${jar}`);
     }
-    const tempDir = mkdtempSync(join(tmpdir(), "jar-deps-"));
-    for (const jar of jarFiles) {
-      if (DEBUG_MODE) {
-        console.log(`Parsing ${jar}`);
-      }
-      const dlist = await extractJarArchive(jar, tempDir);
-      if (dlist && dlist.length) {
-        pkgList = pkgList.concat(dlist);
-      }
-      if (pkgList.length) {
-        pkgList = await getMvnMetadata(pkgList);
-      }
+    const dlist = await extractJarArchive(jar, tempDir);
+    if (dlist && dlist.length) {
+      pkgList = pkgList.concat(dlist);
     }
-    // Clean up
-    if (tempDir && tempDir.startsWith(tmpdir()) && rmSync) {
-      rmSync(tempDir, { recursive: true, force: true });
+    if (pkgList.length) {
+      pkgList = await getMvnMetadata(pkgList);
     }
   }
+  // Clean up
+  if (tempDir && tempDir.startsWith(tmpdir()) && rmSync) {
+    rmSync(tempDir, { recursive: true, force: true });
+  }
   pkgList = pkgList.concat(convertJarNSToPackages(nsMapping));
   return buildBomNSData(options, pkgList, "maven", {
     src: path,
@@ -1002,7 +1005,7 @@ export const createJavaBom = async (path, options) => {
   // This is subsequently referred to in the dependencies list
   let parentComponent = {};
   // war/ear mode
-  if (path.endsWith(".war")) {
+  if (path.endsWith(".war") || path.endsWith(".jar")) {
     // Check if the file exists
     if (existsSync(path)) {
       if (DEBUG_MODE) {
@@ -4800,7 +4803,12 @@ export const createMultiXBom = async (pathList, options) => {
       }
     }
   } // for
-  if (options.lastWorkingDir && options.lastWorkingDir !== "") {
+  if (
+    options.lastWorkingDir &&
+    options.lastWorkingDir !== "" &&
+    !options.lastWorkingDir.includes("/opt/") &&
+    !options.lastWorkingDir.includes("/home/")
+  ) {
     bomData = await createJarBom(options.lastWorkingDir, options);
     if (
       bomData &&

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@cyclonedx/cdxgen",
-  "version": "10.0.0",
+  "version": "10.0.1",
   "description": "Creates CycloneDX Software Bill of Materials (SBOM) from source or container image",
   "homepage": "http://github.com/cyclonedx/cdxgen",
   "author": "Prabhu Subramanian <[email protected]>",

diff --git a/utils.js b/utils.js
@@ -7026,7 +7026,7 @@ export const extractJarArchive = async function (
     existsSync(manifestname)
   ) {
     tempDir = dirname(jarFile);
-  } else if (!existsSync(join(tempDir, fname))) {
+  } else if (!existsSync(join(tempDir, fname)) && lstatSync(jarFile).isFile()) {
     // Only copy if the file doesn't exist
     copyFileSync(jarFile, join(tempDir, fname), constants.COPYFILE_FICLONE);
   }
@@ -7040,36 +7040,57 @@ export const extractJarArchive = async function (
       "bin"
     )}`;
   }
-  if (jarFile.endsWith(".war") || jarFile.endsWith(".hpi")) {
-    const jarResult = spawnSync("jar", ["-xf", join(tempDir, fname)], {
-      encoding: "utf-8",
-      cwd: tempDir,
-      shell: isWin,
-      env
-    });
-    if (jarResult.status !== 0) {
-      console.error(jarResult.stdout, jarResult.stderr);
-      console.log(
-        "Check if JRE is installed and the jar command is available in the PATH."
-      );
+  if (
+    jarFile.endsWith(".war") ||
+    jarFile.endsWith(".hpi") ||
+    jarFile.endsWith(".jar")
+  ) {
+    try {
+      const zip = new StreamZip.async({ file: join(tempDir, fname) });
+      await zip.extract(null, tempDir);
+      await zip.close();
+    } catch (e) {
+      console.log(`Unable to extract ${join(tempDir, fname)}. Skipping.`);
       return pkgList;
     }
     jarFiles = getAllFiles(join(tempDir, "WEB-INF", "lib"), "**/*.jar");
     if (jarFile.endsWith(".hpi")) {
       jarFiles.push(jarFile);
     }
+    // Some jar files could also have more jar files inside BOOT-INF directory
+    const jarFiles2 = getAllFiles(join(tempDir, "BOOT-INF", "lib"), "**/*.jar");
+    if (jarFiles && jarFiles2.length) {
+      jarFiles = jarFiles.concat(jarFiles2);
+    }
+    // Fallback. If our jar file didn't include any jar
+    if (jarFile.endsWith(".jar") && !jarFiles.length) {
+      jarFiles = [join(tempDir, fname)];
+    }
   } else {
     jarFiles = [join(tempDir, fname)];
   }
+  if (DEBUG_MODE) {
+    console.log(`List of jars: ${jarFiles}`);
+  }
   if (jarFiles && jarFiles.length) {
     for (const jf of jarFiles) {
+      // If the jar file doesn't exist at the point of use, skip it
+      if (!existsSync(jf)) {
+        if (DEBUG_MODE) {
+          console.log(jf, "is not a readable file");
+        }
+        continue;
+      }
       pomname = jf.replace(".jar", ".pom");
       const jarname = basename(jf);
       // Ignore test jars
       if (
         jarname.endsWith("-tests.jar") ||
         jarname.endsWith("-test-sources.jar")
       ) {
+        if (DEBUG_MODE) {
+          console.log(`Skipping tests jar ${jarname}`);
+        }
         continue;
       }
       const manifestDir = join(tempDir, "META-INF");
@@ -7081,16 +7102,20 @@ export const extractJarArchive = async function (
       if (existsSync(pomname)) {
         jarResult = { status: 0 };
       } else {
-        jarResult = spawnSync("jar", ["-xf", jf, "META-INF"], {
-          encoding: "utf-8",
-          cwd: tempDir,
-          shell: isWin,
-          env
-        });
+        // Unzip natively
+        try {
+          const zip = new StreamZip.async({ file: jf });
+          await zip.extract(null, tempDir);
+          await zip.close();
+          jarResult = { status: 0 };
+        } catch (e) {
+          if (DEBUG_MODE) {
+            console.log(`Unable to extract ${jf}. Skipping.`);
+          }
+          jarResult = { status: 1 };
+        }
       }
-      if (jarResult.status !== 0) {
-        console.error(jarResult.stdout, jarResult.stderr);
-      } else {
+      if (jarResult.status === 0) {
         // When maven descriptor is available take group, name and version from pom.properties
         // META-INF/maven/${groupId}/${artifactId}/pom.properties
         // see https://maven.apache.org/shared/maven-archiver/index.html
@@ -7114,7 +7139,7 @@ export const extractJarArchive = async function (
             const res = await cdxgenAgent.get(searchurl, {
               responseType: "json",
               timeout: {
-                lookup: 200,
+                lookup: 1000,
                 connect: 5000,
                 secureConnect: 5000,
                 socket: 1000,
@@ -7132,7 +7157,11 @@ export const extractJarArchive = async function (
             }
           } catch (err) {
             if (err && err.message && !err.message.includes("404")) {
-              if (DEBUG_MODE) {
+              if (err.message.includes("Timeout")) {
+                console.log(
+                  "Maven search appears to be unavailable. Search will be skipped for all remaining packages."
+                );
+              } else if (DEBUG_MODE) {
                 console.log(err);
               }
               search_maven_org_errors++;
@@ -7266,20 +7295,23 @@ export const extractJarArchive = async function (
             console.log(`Ignored jar ${jarname}`, name, version);
           }
         }
-        try {
-          if (rmSync && existsSync(join(tempDir, "META-INF"))) {
-            // Clean up META-INF
-            rmSync(join(tempDir, "META-INF"), {
-              recursive: true,
-              force: true
-            });
-          }
-        } catch (err) {
-          // ignore cleanup errors
+      }
+      try {
+        if (rmSync && existsSync(join(tempDir, "META-INF"))) {
+          // Clean up META-INF
+          rmSync(join(tempDir, "META-INF"), {
+            recursive: true,
+            force: true
+          });
         }
+      } catch (err) {
+        // ignore cleanup errors
       }
     } // for
   } // if
+  if (jarFiles.length !== pkgList.length) {
+    console.log(`Obtained only ${pkgList.length} from ${jarFiles.length} jars`);
+  }
   return pkgList;
 };