From 7e1fc4c2d16ca93e46bd35d987b8a1f51f04afd6 Mon Sep 17 00:00:00 2001 From: prabhu Date: Tue, 30 Jan 2024 16:22:31 +0000 Subject: [PATCH] Native unzip (#856) * Native unzip Signed-off-by: Prabhu Subramanian --------- Signed-off-by: Prabhu Subramanian --- .github/workflows/nodejs.yml | 5 +- docker.js | 46 +++++++++++++--- index.js | 58 +++++++++++--------- package-lock.json | 4 +- package.json | 2 +- utils.js | 100 +++++++++++++++++++++++------------ 6 files changed, 143 insertions(+), 72 deletions(-) diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index 418455068..fdd010dfa 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -67,10 +67,7 @@ jobs: deno lint mkdir build ${{ matrix.build }} - - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.artifact }} - path: ${{ matrix.artifact }} + continue-on-error: true # - name: Release # uses: softprops/action-gh-release@v1 # if: startsWith(github.ref, 'refs/tags/') diff --git a/docker.js b/docker.js index 5ab813865..1220bbab1 100644 --- a/docker.js +++ b/docker.js @@ -684,7 +684,7 @@ export const extractTar = async (fullImageName, dir) => { preserveOwner: false, noMtime: true, noChmod: true, - strict: true, + strict: false, C: dir, portable: true, onwarn: () => {}, @@ -696,6 +696,9 @@ export const extractTar = async (fullImageName, dir) => { path.includes("etc/") || path.includes("logs/") || path.includes("dev/") || + path.includes("usr/share/zoneinfo/") || + path.includes("usr/share/doc/") || + path.includes("usr/share/i18n/") || [ "BlockDevice", "CharacterDevice", @@ -727,6 +730,12 @@ export const extractTar = async (fullImageName, dir) => { console.log("------------"); console.log(err); console.log("------------"); + } else if (err.code === "TAR_BAD_ARCHIVE") { + if (DEBUG_MODE) { + console.log(`Archive ${fullImageName} is empty. Skipping.`); + } + } else { + console.log(err); } return false; } @@ -832,13 +841,30 @@ export const extractFromManifest = async ( } const lastLayer = layers[layers.length - 1]; for (const layer of layers) { + try { + if (!lstatSync(join(tempDir, layer)).isFile()) { + console.log( + `Skipping layer ${layer} since it is not a readable file.` + ); + continue; + } + } catch (e) { + console.log(`Skipping layer ${layer} since it is not a readable file.`); + continue; + } if (DEBUG_MODE) { console.log(`Extracting layer ${layer} to ${allLayersExplodedDir}`); } try { await extractTar(join(tempDir, layer), allLayersExplodedDir); } catch (err) { - console.log(err); + if (err.code === "TAR_BAD_ARCHIVE") { + if (DEBUG_MODE) { + console.log(`Layer ${layer} is empty.`); + } + } else { + console.log(err); + } } } if (manifest.Config) { @@ -1058,15 +1084,23 @@ export const getPkgPathList = (exportData, lastWorkingDir) => { } } if (lastWorkingDir && lastWorkingDir !== "") { - knownSysPaths.push(lastWorkingDir); + if ( + !lastWorkingDir.includes("/opt/") && + !lastWorkingDir.includes("/home/") + ) { + knownSysPaths.push(lastWorkingDir); + } // Some more common app dirs - if (!lastWorkingDir.startsWith("/app")) { + if (!lastWorkingDir.includes("/app/")) { knownSysPaths.push(join(allLayersExplodedDir, "/app")); } - if (!lastWorkingDir.startsWith("/data")) { + if (!lastWorkingDir.includes("/layers/")) { + knownSysPaths.push(join(allLayersExplodedDir, "/layers")); + } + if (!lastWorkingDir.includes("/data/")) { knownSysPaths.push(join(allLayersExplodedDir, "/data")); } - if (!lastWorkingDir.startsWith("/srv")) { + if (!lastWorkingDir.includes("/srv/")) { knownSysPaths.push(join(allLayersExplodedDir, "/srv")); } } diff --git a/index.js b/index.js index b2f29b2ed..f98ac5e41 100644 --- a/index.js +++ b/index.js @@ -948,39 +948,42 @@ export const createJarBom = async (path, options) => { false, true ); + } + if (path.endsWith(".jar")) { + jarFiles = [resolve(path)]; } else { jarFiles = getAllFiles( path, (options.multiProject ? "**/" : "") + "*.[jw]ar", options ); - // Jenkins plugins - const hpiFiles = getAllFiles( - path, - (options.multiProject ? "**/" : "") + "*.hpi", - options - ); - if (hpiFiles.length) { - jarFiles = jarFiles.concat(hpiFiles); + } + // Jenkins plugins + const hpiFiles = getAllFiles( + path, + (options.multiProject ? "**/" : "") + "*.hpi", + options + ); + if (hpiFiles.length) { + jarFiles = jarFiles.concat(hpiFiles); + } + const tempDir = mkdtempSync(join(tmpdir(), "jar-deps-")); + for (const jar of jarFiles) { + if (DEBUG_MODE) { + console.log(`Parsing ${jar}`); } - const tempDir = mkdtempSync(join(tmpdir(), "jar-deps-")); - for (const jar of jarFiles) { - if (DEBUG_MODE) { - console.log(`Parsing ${jar}`); - } - const dlist = await extractJarArchive(jar, tempDir); - if (dlist && dlist.length) { - pkgList = pkgList.concat(dlist); - } - if (pkgList.length) { - pkgList = await getMvnMetadata(pkgList); - } + const dlist = await extractJarArchive(jar, tempDir); + if (dlist && dlist.length) { + pkgList = pkgList.concat(dlist); } - // Clean up - if (tempDir && tempDir.startsWith(tmpdir()) && rmSync) { - rmSync(tempDir, { recursive: true, force: true }); + if (pkgList.length) { + pkgList = await getMvnMetadata(pkgList); } } + // Clean up + if (tempDir && tempDir.startsWith(tmpdir()) && rmSync) { + rmSync(tempDir, { recursive: true, force: true }); + } pkgList = pkgList.concat(convertJarNSToPackages(nsMapping)); return buildBomNSData(options, pkgList, "maven", { src: path, @@ -1002,7 +1005,7 @@ export const createJavaBom = async (path, options) => { // This is subsequently referred to in the dependencies list let parentComponent = {}; // war/ear mode - if (path.endsWith(".war")) { + if (path.endsWith(".war") || path.endsWith(".jar")) { // Check if the file exists if (existsSync(path)) { if (DEBUG_MODE) { @@ -4800,7 +4803,12 @@ export const createMultiXBom = async (pathList, options) => { } } } // for - if (options.lastWorkingDir && options.lastWorkingDir !== "") { + if ( + options.lastWorkingDir && + options.lastWorkingDir !== "" && + !options.lastWorkingDir.includes("/opt/") && + !options.lastWorkingDir.includes("/home/") + ) { bomData = await createJarBom(options.lastWorkingDir, options); if ( bomData && diff --git a/package-lock.json b/package-lock.json index 9bb59a0b7..9e62f9ca2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@cyclonedx/cdxgen", - "version": "10.0.0", + "version": "10.0.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@cyclonedx/cdxgen", - "version": "10.0.0", + "version": "10.0.1", "license": "Apache-2.0", "dependencies": { "@babel/parser": "^7.23.9", diff --git a/package.json b/package.json index a734ee5a3..2c919149c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@cyclonedx/cdxgen", - "version": "10.0.0", + "version": "10.0.1", "description": "Creates CycloneDX Software Bill of Materials (SBOM) from source or container image", "homepage": "http://github.com/cyclonedx/cdxgen", "author": "Prabhu Subramanian ", diff --git a/utils.js b/utils.js index d9dd64d55..d81d4fd97 100644 --- a/utils.js +++ b/utils.js @@ -7026,7 +7026,7 @@ export const extractJarArchive = async function ( existsSync(manifestname) ) { tempDir = dirname(jarFile); - } else if (!existsSync(join(tempDir, fname))) { + } else if (!existsSync(join(tempDir, fname)) && lstatSync(jarFile).isFile()) { // Only copy if the file doesn't exist copyFileSync(jarFile, join(tempDir, fname), constants.COPYFILE_FICLONE); } @@ -7040,29 +7040,47 @@ export const extractJarArchive = async function ( "bin" )}`; } - if (jarFile.endsWith(".war") || jarFile.endsWith(".hpi")) { - const jarResult = spawnSync("jar", ["-xf", join(tempDir, fname)], { - encoding: "utf-8", - cwd: tempDir, - shell: isWin, - env - }); - if (jarResult.status !== 0) { - console.error(jarResult.stdout, jarResult.stderr); - console.log( - "Check if JRE is installed and the jar command is available in the PATH." - ); + if ( + jarFile.endsWith(".war") || + jarFile.endsWith(".hpi") || + jarFile.endsWith(".jar") + ) { + try { + const zip = new StreamZip.async({ file: join(tempDir, fname) }); + await zip.extract(null, tempDir); + await zip.close(); + } catch (e) { + console.log(`Unable to extract ${join(tempDir, fname)}. Skipping.`); return pkgList; } jarFiles = getAllFiles(join(tempDir, "WEB-INF", "lib"), "**/*.jar"); if (jarFile.endsWith(".hpi")) { jarFiles.push(jarFile); } + // Some jar files could also have more jar files inside BOOT-INF directory + const jarFiles2 = getAllFiles(join(tempDir, "BOOT-INF", "lib"), "**/*.jar"); + if (jarFiles && jarFiles2.length) { + jarFiles = jarFiles.concat(jarFiles2); + } + // Fallback. If our jar file didn't include any jar + if (jarFile.endsWith(".jar") && !jarFiles.length) { + jarFiles = [join(tempDir, fname)]; + } } else { jarFiles = [join(tempDir, fname)]; } + if (DEBUG_MODE) { + console.log(`List of jars: ${jarFiles}`); + } if (jarFiles && jarFiles.length) { for (const jf of jarFiles) { + // If the jar file doesn't exist at the point of use, skip it + if (!existsSync(jf)) { + if (DEBUG_MODE) { + console.log(jf, "is not a readable file"); + } + continue; + } pomname = jf.replace(".jar", ".pom"); const jarname = basename(jf); // Ignore test jars @@ -7070,6 +7088,9 @@ export const extractJarArchive = async function ( jarname.endsWith("-tests.jar") || jarname.endsWith("-test-sources.jar") ) { + if (DEBUG_MODE) { + console.log(`Skipping tests jar ${jarname}`); + } continue; } const manifestDir = join(tempDir, "META-INF"); @@ -7081,16 +7102,20 @@ export const extractJarArchive = async function ( if (existsSync(pomname)) { jarResult = { status: 0 }; } else { - jarResult = spawnSync("jar", ["-xf", jf, "META-INF"], { - encoding: "utf-8", - cwd: tempDir, - shell: isWin, - env - }); + // Unzip natively + try { + const zip = new StreamZip.async({ file: jf }); + await zip.extract(null, tempDir); + await zip.close(); + jarResult = { status: 0 }; + } catch (e) { + if (DEBUG_MODE) { + console.log(`Unable to extract ${jf}. Skipping.`); + } + jarResult = { status: 1 }; + } } - if (jarResult.status !== 0) { - console.error(jarResult.stdout, jarResult.stderr); - } else { + if (jarResult.status === 0) { // When maven descriptor is available take group, name and version from pom.properties // META-INF/maven/${groupId}/${artifactId}/pom.properties // see https://maven.apache.org/shared/maven-archiver/index.html @@ -7114,7 +7139,7 @@ export const extractJarArchive = async function ( const res = await cdxgenAgent.get(searchurl, { responseType: "json", timeout: { - lookup: 200, + lookup: 1000, connect: 5000, secureConnect: 5000, socket: 1000, @@ -7132,7 +7157,11 @@ export const extractJarArchive = async function ( } } catch (err) { if (err && err.message && !err.message.includes("404")) { - if (DEBUG_MODE) { + if (err.message.includes("Timeout")) { + console.log( + "Maven search appears to be unavailable. Search will be skipped for all remaining packages." + ); + } else if (DEBUG_MODE) { console.log(err); } search_maven_org_errors++; @@ -7266,20 +7295,23 @@ export const extractJarArchive = async function ( console.log(`Ignored jar ${jarname}`, name, version); } } - try { - if (rmSync && existsSync(join(tempDir, "META-INF"))) { - // Clean up META-INF - rmSync(join(tempDir, "META-INF"), { - recursive: true, - force: true - }); - } - } catch (err) { - // ignore cleanup errors + } + try { + if (rmSync && existsSync(join(tempDir, "META-INF"))) { + // Clean up META-INF + rmSync(join(tempDir, "META-INF"), { + recursive: true, + force: true + }); } + } catch (err) { + // ignore cleanup errors } } // for } // if + if (jarFiles.length !== pkgList.length) { + console.log(`Obtained only ${pkgList.length} from ${jarFiles.length} jars`); + } return pkgList; };