From d24715cafbf5578d61796cb4368b7187251ef9a1 Mon Sep 17 00:00:00 2001 From: prabhu Date: Wed, 19 Jun 2024 10:44:08 +0100 Subject: [PATCH] parsedeps improvements (#146) Signed-off-by: Prabhu Subramanian --- build.sbt | 2 +- codemeta.json | 2 +- src/main/scala/io/appthreat/atom/Atom.scala | 2 +- .../parsedeps/PythonDependencyParser.scala | 9 ++- .../atom/PythonDependencyScannerTests.scala | 66 ++++++++++++++++++- wrapper/nodejs/package-lock.json | 14 ++-- wrapper/nodejs/package.json | 6 +- 7 files changed, 85 insertions(+), 16 deletions(-) diff --git a/build.sbt b/build.sbt index 629a13c..e0174bf 100644 --- a/build.sbt +++ b/build.sbt @@ -1,6 +1,6 @@ name := "atom" ThisBuild / organization := "io.appthreat" -ThisBuild / version := "2.0.12" +ThisBuild / version := "2.0.13" ThisBuild / scalaVersion := "3.4.1" val chenVersion = "2.0.11" diff --git a/codemeta.json b/codemeta.json index c6a1087..13db1e2 100644 --- a/codemeta.json +++ b/codemeta.json @@ -7,7 +7,7 @@ "downloadUrl": "https://github.com/AppThreat/atom", "issueTracker": "https://github.com/AppThreat/atom/issues", "name": "atom", - "version": "2.0.12", + "version": "2.0.13", "description": "Atom is a novel intermediate representation for next-generation code analysis.", "applicationCategory": "code-analysis", "keywords": [ diff --git a/src/main/scala/io/appthreat/atom/Atom.scala b/src/main/scala/io/appthreat/atom/Atom.scala index 4696651..69c6441 100644 --- a/src/main/scala/io/appthreat/atom/Atom.scala +++ b/src/main/scala/io/appthreat/atom/Atom.scala @@ -184,7 +184,7 @@ object Atom: ) cmd("parsedeps") .text("Extract dependencies from the build file and imports") - .action((_, *) => AtomParseDepsConfig().withRemoveAtom(true)) + .action((_, *) => AtomParseDepsConfig().withRemoveAtom(false)) cmd("data-flow") .text("Extract backward data-flow slices") .action((_, _) => AtomDataFlowConfig().withDataDependencies(true)) diff --git a/src/main/scala/io/appthreat/atom/parsedeps/PythonDependencyParser.scala b/src/main/scala/io/appthreat/atom/parsedeps/PythonDependencyParser.scala index 8722417..2a79159 100644 --- a/src/main/scala/io/appthreat/atom/parsedeps/PythonDependencyParser.scala +++ b/src/main/scala/io/appthreat/atom/parsedeps/PythonDependencyParser.scala @@ -25,13 +25,17 @@ object PythonDependencyParser extends XDependencyParser: private def parseSetupPy(cpg: Cpg): Set[ModuleWithVersion] = val dataFlowEnabled = cpg.metaData.overlays.contains(OssDataFlow.overlayName) - val requirementsPattern = """([\[\]/.\w_-]+)\s?((=>|<=|==|>=|=<|<|>|!=).*)""".r + val requirementsPattern = """([\[\]/.\w_-]+)\s?((=>|<=|==|>=|=<|<|>|!=|~=).*)""".r def dataSourcesToRequires = (cpg.literal ++ cpg.identifier) .where(_.file.name(".*setup.py")) .where(_.argumentName("install_requires")) .collectAll[CfgNode] + def installRequires = cpg.call.where(_.file.name(".*setup.py")).where(_.argumentName( + "install_requires" + )).argument.collectAll[Literal] + def setupCall = cpg.call("setup").where(_.file.name(".*setup.py")) def findOriginalDeclaration(xs: Traversal[CfgNode]): Iterable[Literal] = @@ -52,10 +56,11 @@ object PythonDependencyParser extends XDependencyParser: .to(Iterable) val initialTraversal = if dataFlowEnabled then setupCall.reachableBy(dataSourcesToRequires) - else dataSourcesToRequires + else (dataSourcesToRequires ++ installRequires) findOriginalDeclaration(initialTraversal) .map(x => X2Cpg.stripQuotes(x.code)) .map { + case requirementsPattern(name, versionSpecifiers, _) if versionSpecifiers.contains("==") => val versions = versionSpecifiers.split(',').toSeq diff --git a/src/test/scala/io/appthreat/atom/PythonDependencyScannerTests.scala b/src/test/scala/io/appthreat/atom/PythonDependencyScannerTests.scala index ffd8e41..8bcc84b 100644 --- a/src/test/scala/io/appthreat/atom/PythonDependencyScannerTests.scala +++ b/src/test/scala/io/appthreat/atom/PythonDependencyScannerTests.scala @@ -67,7 +67,8 @@ class PythonDependencyScannerTests extends PySrc2CpgFixture(withOssDataflow = fa | "google-api-core[grpc] >= 1.34.0, <3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,!=2.10.*", | "proto-plus >= 1.22.0, <2.0.0dev", | "proto-plus >= 1.22.2, <2.0.0dev; python_version>='3.11'", - | "protobuf>=3.19.5,<5.0.0dev,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5" + | "protobuf>=3.19.5,<5.0.0dev,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5", + | "dbt-core~=1.7,<1.8" |] |test_requirements = [ | "pytest-httpbin==2.0.0", @@ -205,6 +206,7 @@ class PythonDependencyScannerTests extends PySrc2CpgFixture(withOssDataflow = fa ModuleWithVersion("PickyThing", "2.4c1", "<1.6,>1.9,!=1.9.6,<2.0a0", ""), ModuleWithVersion("certifi", "", ">=2017.4.17", ""), ModuleWithVersion("charset_normalizer", "", ">=2,<4", ""), + ModuleWithVersion("dbt-core", "", "~=1.7,<1.8", ""), ModuleWithVersion("google-api-core[grpc]", "", ">= 1.34.0, <3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,!=2.10.*", ""), ModuleWithVersion("idna", "", ">=2.5,<4", ""), ModuleWithVersion("os", "", "", "os.path"), @@ -221,4 +223,66 @@ class PythonDependencyScannerTests extends PySrc2CpgFixture(withOssDataflow = fa } } + "dependencies from the `impacket` library" should { + lazy val cpg = code( + """ + |#!/usr/bin/env python + |# $Id$ + | + |import glob + |import os + |import platform + | + |from setuptools import setup + | + |PACKAGE_NAME = "impacket2" + | + |if platform.system() != 'Darwin': + | data_files = [(os.path.join('share', 'doc', PACKAGE_NAME), ['README.md', 'LICENSE']+glob.glob('doc/*'))] + |else: + | data_files = [] + | + |def read(fname): + | return open(os.path.join(os.path.dirname(__file__), fname)).read() + | + |setup(name = PACKAGE_NAME, + | version = "0.9.21-dev", + | package_dir={'': 'src'}, + | platforms = ["Unix"], + | packages=['impacket2', 'impacket2.dcerpc', 'impacket2.examples', 'impacket2.dcerpc.v5', 'impacket2.dcerpc.v5.dcom', + | 'impacket2.krb5', 'impacket2.ldap', 'impacket2.examples.ntlmrelayx', + | 'impacket2.examples.ntlmrelayx.clients', 'impacket2.examples.ntlmrelayx.servers', + | 'impacket2.examples.ntlmrelayx.servers.socksplugins', 'impacket2.examples.ntlmrelayx.utils', + | 'impacket2.examples.ntlmrelayx.attacks'], + | data_files = data_files, + | install_requires=['pyasn1>=0.2.3', 'pycryptodomex', 'pyOpenSSL>=0.13.1', 'six', 'ldap3==2.5.1', 'ldapdomaindump>=0.9.0', 'flask>=1.0'], + | extras_require={ + | 'pyreadline:sys_platform=="win32"': [], + | 'python_version<"2.7"': [ 'argparse' ], + | }, + | classifiers = [ + | "Programming Language :: Python :: 3.6", + | "Programming Language :: Python :: 2.7", + | "Programming Language :: Python :: 2.6", + | ] + |) + | + |""".stripMargin, + "setup.py" + ) + + "have the modules scanned successfully" in { + val scanResult = PythonDependencyParser.parse(cpg) + scanResult.modules shouldBe List( + ModuleWithVersion("flask", "", ">=1.0", ""), + ModuleWithVersion("ldap3", "2.5.1", "", ""), + ModuleWithVersion("ldapdomaindump", "", ">=0.9.0", ""), + ModuleWithVersion("pyOpenSSL", "", ">=0.13.1", ""), + ModuleWithVersion("pyasn1", "", ">=0.2.3", ""), + ModuleWithVersion("pycryptodomex", "", "", ""), + ModuleWithVersion("six", "", "", "") + ) + } + } + } diff --git a/wrapper/nodejs/package-lock.json b/wrapper/nodejs/package-lock.json index e28e23e..eca499e 100644 --- a/wrapper/nodejs/package-lock.json +++ b/wrapper/nodejs/package-lock.json @@ -1,15 +1,15 @@ { "name": "@appthreat/atom", - "version": "2.0.12", + "version": "2.0.13", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@appthreat/atom", - "version": "2.0.12", + "version": "2.0.13", "license": "Apache-2.0", "dependencies": { - "@babel/parser": "^7.24.4", + "@babel/parser": "^7.24.7", "typescript": "^5.4.5", "yargs": "^17.7.2" }, @@ -19,7 +19,7 @@ "phpastgen": "phpastgen.js" }, "devDependencies": { - "eslint": "^8.57.0" + "eslint": "8.57.0" }, "engines": { "node": ">=16.0.0" @@ -35,9 +35,9 @@ } }, "node_modules/@babel/parser": { - "version": "7.24.4", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.24.4.tgz", - "integrity": "sha512-zTvEBcghmeBma9QIGunWevvBAp4/Qu9Bdq+2k0Ot4fVMD6v3dsC9WOcRSKk7tRRyBM/53yKMJko9xOatGQAwSg==", + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.24.7.tgz", + "integrity": "sha512-9uUYRm6OqQrCqQdG1iCBwBPZgN8ciDBro2nIOFaiRz1/BCxaI7CNvQbDHvsArAC7Tw9Hda/B3U+6ui9u4HWXPw==", "bin": { "parser": "bin/babel-parser.js" }, diff --git a/wrapper/nodejs/package.json b/wrapper/nodejs/package.json index df1a649..e40829f 100644 --- a/wrapper/nodejs/package.json +++ b/wrapper/nodejs/package.json @@ -1,6 +1,6 @@ { "name": "@appthreat/atom", - "version": "2.0.12", + "version": "2.0.13", "description": "Create atom (⚛) representation for your application, packages and libraries", "exports": "./index.js", "type": "module", @@ -9,12 +9,12 @@ "lint": "eslint *.mjs *.js" }, "dependencies": { - "@babel/parser": "^7.24.4", + "@babel/parser": "^7.24.7", "typescript": "^5.4.5", "yargs": "^17.7.2" }, "devDependencies": { - "eslint": "^8.57.0" + "eslint": "8.57.0" }, "bin": { "atom": "./index.js",