joern 2 (#36)

* joern 2 Signed-off-by: Prabhu Subramanian <[email protected]> * joern 2 Signed-off-by: Prabhu Subramanian <[email protected]> * joern 2 Signed-off-by: Prabhu Subramanian <[email protected]> * joern 2 Signed-off-by: Prabhu Subramanian <[email protected]> --------- Signed-off-by: Prabhu Subramanian <[email protected]>
AppThreat · Jun 5, 2023 · cbba92c · cbba92c
1 parent ed2ed61
commit cbba92c
Show file tree

Hide file tree

Showing 12 changed files with 23 additions and 234 deletions.
diff --git a/.github/workflows/exetests.yml b/.github/workflows/exetests.yml
@@ -170,18 +170,6 @@ jobs:
         with:
           repository: 'ShiftLeftSecurity/shiftleft-go-example'
           path: 'repotests/shiftleft-go-example'
-      - uses: actions/checkout@v3
-        with:
-          repository: 'prabhu/shiftleft-scala-example'
-          path: 'repotests/shiftleft-scala-example'
-      - uses: actions/checkout@v3
-        with:
-          repository: 'HooliCorp/vulnerable_net_core'
-          path: 'repotests/vulnerable_net_core'
-      - uses: actions/checkout@v3
-        with:
-          repository: 'HooliCorp/Goatly.NET'
-          path: 'repotests/Goatly.NET'
       - uses: actions/checkout@v3
         with:
           repository: 'HooliCorp/DjanGoat'

diff --git a/.github/workflows/joern2.yml b/.github/workflows/joern2.yml
@@ -34,5 +34,5 @@ jobs:
             /tmp/querydb.json
             querydb/target/querydb.zip
             target/joern-cli.zip
-            target/joern-cli.zip.
+            target/joern-cli.zip.sha512
             joern-install.sh
diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml
@@ -72,13 +72,14 @@ jobs:
         with:
           images: |
             ghcr.io/appthreat/cpggen
+            ghcr.io/appthreat/cpggen-oss
 
       - name: Build and push Docker images
         uses: docker/build-push-action@v4
         with:
           context: .
           file: Dockerfile
-          platforms: linux/amd64
+          platforms: linux/amd64,linux/arm64
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
@@ -120,41 +121,7 @@ jobs:
           labels: ${{ steps.meta2.outputs.labels }}
           cache-from: type=gha,scope=cpggen-alma8
           cache-to: type=gha,mode=max,scope=cpggen-alma8
-  oss-container:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-      packages: write
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - name: Log in to the Container registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Extract metadata (tags, labels) for Docker
-        id: meta3
-        uses: docker/metadata-action@v4
-        with:
-          images: |
-            ghcr.io/appthreat/cpggen-oss
-      - name: Build and push Docker OSS images
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          file: ci/Dockerfile-oss
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ${{ steps.meta3.outputs.tags }}
-          labels: ${{ steps.meta3.outputs.labels }}
-          cache-from: type=gha,scope=cpggen-oss
-          cache-to: type=gha,mode=max,scope=cpggen-oss
   slim-container:
     runs-on: ubuntu-latest
     permissions:

diff --git a/.github/workflows/repotests.yml b/.github/workflows/repotests.yml
@@ -42,22 +42,6 @@ jobs:
         with:
           repository: 'ShiftLeftSecurity/shiftleft-ts-example'
           path: 'repotests/shiftleft-ts-example'
-      - uses: actions/checkout@v3
-        with:
-          repository: 'ShiftLeftSecurity/shiftleft-go-example'
-          path: 'repotests/shiftleft-go-example'
-      - uses: actions/checkout@v3
-        with:
-          repository: 'prabhu/shiftleft-scala-example'
-          path: 'repotests/shiftleft-scala-example'
-      - uses: actions/checkout@v3
-        with:
-          repository: 'HooliCorp/vulnerable_net_core'
-          path: 'repotests/vulnerable_net_core'
-      - uses: actions/checkout@v3
-        with:
-          repository: 'HooliCorp/Goatly.NET'
-          path: 'repotests/Goatly.NET'
       - uses: actions/checkout@v3
         with:
           repository: 'HooliCorp/DjanGoat'
@@ -80,15 +64,15 @@ jobs:
           docker build -t ghcr.io/appthreat/atomgen -f ci/Dockerfile-atom .
           docker build -t ghcr.io/appthreat/cpggen .
           docker run --rm -v /tmp:/tmp -v $(pwd):/app:rw -t ghcr.io/appthreat/cpggen cpggen -i /app/repotests/shiftleft-java-example -o /tmp/all_cpgs/shiftleft-java-example
-          docker run --rm -v /tmp:/tmp -v $(pwd):/app:rw -t ghcr.io/appthreat/cpggen joern -J-Xmx7G --script /app/contrib/joern_scripts/cpg-methods.sc --params payload=/tmp/all_cpgs/shiftleft-java-example/shiftleft-java-example-java.cpg.bin,resultFile=/tmp/all_cpgs/shiftleft-java-example/java-cpg-methods.json
+          docker run --rm -v /tmp:/tmp -v $(pwd):/app:rw -t ghcr.io/appthreat/cpggen joern -J-Xmx7G --script /app/contrib/joern_scripts/cpg-methods.sc --param payload=/tmp/all_cpgs/shiftleft-java-example/shiftleft-java-example-java.cpg.bin --param resultFile=/tmp/all_cpgs/shiftleft-java-example/java-cpg-methods.json
           if [ -e "/tmp/all_cpgs/shiftleft-java-example/java-cpg-methods.json" ]; then
             echo "Java cpg test was successful"
           else
             echo "Java cpg test was not successful"
             exit 1
           fi
           docker run --rm -v /tmp:/tmp -v $(pwd):/app:rw -t ghcr.io/appthreat/atomgen -i /app/repotests/shiftleft-java-example -o /tmp/all_cpgs/shiftleft-java-example
-          docker run --rm -v /tmp:/tmp -v $(pwd):/app:rw -t ghcr.io/appthreat/cpggen joern -J-Xmx7G --script /app/contrib/joern_scripts/cpg-methods.sc --params payload=/tmp/all_cpgs/shiftleft-java-example/shiftleft-java-example-java.⚛,resultFile=/tmp/all_cpgs/shiftleft-java-example/java-cpg-methods.json
+          docker run --rm -v /tmp:/tmp -v $(pwd):/app:rw -t ghcr.io/appthreat/cpggen joern -J-Xmx7G --script /app/contrib/joern_scripts/cpg-methods.sc --param payload=/tmp/all_cpgs/shiftleft-java-example/shiftleft-java-example-java.⚛ --param resultFile=/tmp/all_cpgs/shiftleft-java-example/java-cpg-methods.json
           if [ -e "/tmp/all_cpgs/shiftleft-java-example/java-cpg-methods.json" ]; then
             echo "Java atom test was successful"
           else
@@ -145,7 +129,7 @@ jobs:
             echo "Binary cpg test was not successful"
             exit 1
           fi
-          docker run --rm -v /tmp:/tmp -v $(pwd):/app:rw -t ghcr.io/appthreat/cpggen joern -J-Xmx7G --script /app/contrib/joern_scripts/cpg-methods.sc --params payload=/tmp/all_cpgs/juicy-malware/juicy_malware_linux_amd_64-binary.cpg.bin,resultFile=/tmp/all_cpgs/juicy-malware/binary-cpg-methods.json
+          docker run --rm -v /tmp:/tmp -v $(pwd):/app:rw -t ghcr.io/appthreat/cpggen joern -J-Xmx7G --script /app/contrib/joern_scripts/cpg-methods.sc --param payload=/tmp/all_cpgs/juicy-malware/juicy_malware_linux_amd_64-binary.cpg.bin --param resultFile=/tmp/all_cpgs/juicy-malware/binary-cpg-methods.json
           if [ -e "/tmp/all_cpgs/juicy-malware/binary-cpg-methods.json" ]; then
             echo "Binary cpg test was successful"
           else

diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml
@@ -34,18 +34,6 @@ jobs:
       with:
         repository: 'ShiftLeftSecurity/shiftleft-ts-example'
         path: 'repotests/shiftleft-ts-example'
-    - uses: actions/checkout@v3
-      with:
-        repository: 'ShiftLeftSecurity/shiftleft-go-example'
-        path: 'repotests/shiftleft-go-example'
-    - uses: actions/checkout@v3
-      with:
-        repository: 'HooliCorp/vulnerable_net_core'
-        path: 'repotests/vulnerable_net_core'
-    - uses: actions/checkout@v3
-      with:
-        repository: 'HooliCorp/Goatly.NET'
-        path: 'repotests/Goatly.NET'
     - uses: actions/checkout@v3
       with:
         repository: 'HooliCorp/DjanGoat'

diff --git a/Dockerfile b/Dockerfile
@@ -4,7 +4,7 @@ LABEL maintainer="appthreat" \
       org.opencontainers.image.authors="Team AppThreat <[email protected]>" \
       org.opencontainers.image.source="https://github.com/appthreat/cpggen" \
       org.opencontainers.image.url="https://github.com/appthreat/cpggen" \
-      org.opencontainers.image.version="1.5.2" \
+      org.opencontainers.image.version="1.6.0" \
       org.opencontainers.image.vendor="AppThreat" \
       org.opencontainers.image.licenses="Apache-2.0" \
       org.opencontainers.image.title="cpggen" \
@@ -13,28 +13,22 @@ LABEL maintainer="appthreat" \
 
 ARG TARGETPLATFORM
 
-ENV JOERN_HOME=/usr/local/bin \
+ENV JOERN_HOME=/opt/joern-cli \
     LC_ALL=en_US.UTF-8 \
     LANG=en_US.UTF-8 \
     LANGUAGE=en_US.UTF-8 \
-    GOROOT=/usr/local/go \
-    GO_VERSION=1.19.9 \
     SBT_VERSION=1.9.0 \
     GRADLE_VERSION=8.1.1 \
     GRADLE_HOME=/opt/gradle-8.1.1 \
     GRADLE_OPTS="-Dorg.gradle.daemon=false" \
     JAVA_HOME="/etc/alternatives/jre_17" \
     JAVA_17_HOME="/etc/alternatives/jre_17" \
-    JAVA_8_HOME="/usr/lib/jvm/jre-1.8.0" \
-    CGO_ENABLED=1 \
-    GO111MODULE="" \
-    GOOS="linux" \
     PYTHONUNBUFFERED=1 \
     PYTHONIOENCODING="utf-8" \
     DOTNET_CLI_TELEMETRY_OPTOUT=1 \
     JOERN_DATAFLOW_TRACKED_WIDTH=128 \
     ANDROID_HOME=/opt/android-sdk-linux \
-    PATH=${PATH}:/opt/joern/joern-cli:/opt/joern/joern-cli/bin:/usr/local/go/bin:/usr/local/bin:/root/.local/bin:/opt/sbt/bin:/usr/local/go/pkg/tool/linux_amd64:${JAVA_HOME}/bin:${ANDROID_HOME}/cmdline-tools/latest/bin:${ANDROID_HOME}/tools:${ANDROID_HOME}/tools/bin:${ANDROID_HOME}/platform-tools:
+    PATH=${PATH}:/opt/joern-cli:/opt/joern-cli/bin:/usr/local/bin:/root/.local/bin:/opt/sbt/bin:${JAVA_HOME}/bin:${ANDROID_HOME}/cmdline-tools/latest/bin:${ANDROID_HOME}/tools:${ANDROID_HOME}/tools/bin:${ANDROID_HOME}/platform-tools:
 
 COPY . /usr/local/src/
 
@@ -54,21 +48,17 @@ RUN set -e; \
     echo -e "[nodejs]\nname=nodejs\nstream=20\nprofiles=\nstate=enabled\n" > /etc/dnf/modules.d/nodejs.module \
     && microdnf module enable maven php -y \
     && microdnf install -y gcc gcc-c++ libstdc++-devel git-core php php-cli python3.11 python3.11-devel python3.11-pip pcre2 which tar zip unzip sudo \
-        java-17-openjdk-headless java-1.8.0-openjdk-headless maven ncurses jq krb5-libs libicu openssl-libs compat-openssl11 zlib \
-        dotnet-sdk-7.0 dotnet-targeting-pack-7.0 dotnet-templates-7.0 dotnet-hostfxr-7.0 nodejs graphviz graphviz-gd graphviz-python3 glibc-common glibc-all-langpacks xorg-x11-fonts-75dpi \
+        java-17-openjdk-headless maven ncurses jq krb5-libs libicu openssl-libs compat-openssl11 zlib \
+        nodejs graphviz graphviz-gd graphviz-python3 glibc-common glibc-all-langpacks xorg-x11-fonts-75dpi \
     && alternatives --install /usr/bin/python3 python /usr/bin/python3.11 1 \
     && python3 --version \
     && python3 -m pip install --upgrade pip \
     && curl -LO https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6.1-2/wkhtmltox-0.12.6.1-2.almalinux9.${ARCH_NAME}.rpm \
     && rpm -ivh wkhtmltox-0.12.6.1-2.almalinux9.${ARCH_NAME}.rpm \
     && rm wkhtmltox-0.12.6.1-2.almalinux9.${ARCH_NAME}.rpm \
-    && curl -LO "https://dl.google.com/go/go${GO_VERSION}.linux-${OS_ARCH_SUFFIX}.tar.gz" \
-    && tar -C /usr/local -xzf go${GO_VERSION}.linux-${OS_ARCH_SUFFIX}.tar.gz \
-    && rm go${GO_VERSION}.linux-${OS_ARCH_SUFFIX}.tar.gz \
-    && go install github.com/magefile/mage@latest \
-    && curl -LO https://github.com/appthreat/joern/releases/latest/download/joern-install.sh \
-    && chmod +x ./joern-install.sh \
-    && ./joern-install.sh --without-plugins \
+    && curl -LO https://github.com/appthreat/cpggen/releases/latest/download/joern-cli.zip \
+    && unzip -q joern-cli.zip -d /opt/ \
+    && rm joern-cli.zip \
     && curl -LO "https://services.gradle.org/distributions/gradle-${GRADLE_VERSION}-bin.zip" \
     && unzip -q gradle-${GRADLE_VERSION}-bin.zip -d /opt/ \
     && chmod +x /opt/gradle-${GRADLE_VERSION}/bin/gradle \
@@ -94,7 +84,6 @@ RUN set -e; \
     && python3 -m pip install --no-cache-dir poetry \
     && poetry config virtualenvs.create false \
     && cd /usr/local/src/ && poetry install --no-cache --without dev \
-    && rm /joern-cli.zip /joern-install.sh \
     && rm -rf /var/cache/yum \
     && microdnf clean all
 

diff --git a/ci/Dockerfile-alma8 b/ci/Dockerfile-alma8
@@ -4,7 +4,7 @@ LABEL maintainer="appthreat" \
       org.opencontainers.image.authors="Team AppThreat <[email protected]>" \
       org.opencontainers.image.source="https://github.com/appthreat/cpggen" \
       org.opencontainers.image.url="https://github.com/appthreat/cpggen" \
-      org.opencontainers.image.version="1.5.2" \
+      org.opencontainers.image.version="1.6.0" \
       org.opencontainers.image.vendor="AppThreat" \
       org.opencontainers.image.licenses="Apache-2.0" \
       org.opencontainers.image.title="cpggen" \
@@ -13,45 +13,36 @@ LABEL maintainer="appthreat" \
 
 ARG TARGETPLATFORM
 
-ENV JOERN_HOME=/usr/local/bin \
+ENV JOERN_HOME=/opt/joern-cli \
     LC_ALL=en_US.UTF-8 \
     LANG=en_US.UTF-8 \
     LANGUAGE=en_US.UTF-8 \
-    GOROOT=/usr/local/go \
-    GO_VERSION=1.19.9 \
     SBT_VERSION=1.9.0 \
     GRADLE_VERSION=8.1.1 \
     GRADLE_HOME=/opt/gradle-8.1.1 \
     GRADLE_OPTS="-Dorg.gradle.daemon=false" \
     JAVA_HOME="/etc/alternatives/jre_17" \
     JAVA_17_HOME="/etc/alternatives/jre_17" \
     JAVA_8_HOME="/usr/lib/jvm/jre-1.8.0" \
-    CGO_ENABLED=1 \
-    GO111MODULE="" \
-    GOOS="linux" \
     PYTHONUNBUFFERED=1 \
     PYTHONIOENCODING="utf-8" \
     DOTNET_CLI_TELEMETRY_OPTOUT=1 \
     JOERN_DATAFLOW_TRACKED_WIDTH=128 \
-    PATH=${PATH}:/opt/joern/joern-cli:/opt/joern/joern-cli/bin:/usr/local/go/bin:/usr/local/bin:/root/.local/bin:/opt/sbt/bin:/usr/local/go/pkg/tool/linux_amd64:${JAVA_HOME}/bin:
+    PATH=${PATH}:/opt/joern-cli:/opt/joern-cli/bin:/usr/local/bin:/root/.local/bin:/opt/sbt/bin:${JAVA_HOME}/bin:
 
 COPY . /usr/local/src/
 
 RUN echo -e "[nodejs]\nname=nodejs\nstream=18\nprofiles=\nstate=enabled\n" > /etc/dnf/modules.d/nodejs.module \
     && microdnf module enable maven php -y \
     && microdnf install -y gcc gcc-c++ libstdc++-devel git-core php php-cli python38 python38-devel pcre2 which tar zip unzip sudo \
         java-17-openjdk-headless java-1.8.0-openjdk-headless maven ncurses jq krb5-libs libicu openssl-libs compat-openssl10 zlib \
-        dotnet-sdk-7.0 dotnet-targeting-pack-7.0 dotnet-templates-7.0 dotnet-hostfxr-7.0 nodejs graphviz glibc-common glibc-all-langpacks xorg-x11-fonts-75dpi \
+        nodejs graphviz glibc-common glibc-all-langpacks xorg-x11-fonts-75dpi \
     && curl -LO https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6.1-2/wkhtmltox-0.12.6.1-2.almalinux8.x86_64.rpm \
     && if [ "$TARGETPLATFORM" = "linux/amd64" ]; then rpm -ivh wkhtmltox-0.12.6.1-2.almalinux8.x86_64.rpm; fi \
     && rm wkhtmltox-0.12.6.1-2.almalinux8.x86_64.rpm \
-    && curl -LO "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz" \
-    && tar -C /usr/local -xzf go${GO_VERSION}.linux-amd64.tar.gz \
-    && rm go${GO_VERSION}.linux-amd64.tar.gz \
-    && go install github.com/magefile/mage@latest \
-    && curl -LO https://github.com/appthreat/joern/releases/latest/download/joern-install.sh \
-    && chmod +x ./joern-install.sh \
-    && ./joern-install.sh --without-plugins \
+    && curl -LO https://github.com/appthreat/cpggen/releases/latest/download/joern-cli.zip \
+    && unzip -q joern-cli.zip -d /opt/ \
+    && rm joern-cli.zip \
     && curl -LO "https://services.gradle.org/distributions/gradle-${GRADLE_VERSION}-bin.zip" \
     && unzip -q gradle-${GRADLE_VERSION}-bin.zip -d /opt/ \
     && chmod +x /opt/gradle-${GRADLE_VERSION}/bin/gradle \
@@ -68,7 +59,6 @@ RUN echo -e "[nodejs]\nname=nodejs\nstream=18\nprofiles=\nstate=enabled\n" > /et
     && python3 -m pip install --no-cache-dir poetry==1.3.2 \
     && poetry config virtualenvs.create false \
     && cd /usr/local/src/ && poetry install --no-cache --without dev \
-    && rm /joern-cli.zip /joern-install.sh \
     && rm -rf /var/cache/yum \
     && microdnf clean all
 

diff --git a/ci/Dockerfile-atom b/ci/Dockerfile-atom
@@ -4,7 +4,7 @@ LABEL maintainer="appthreat" \
       org.opencontainers.image.authors="Team AppThreat <[email protected]>" \
       org.opencontainers.image.source="https://github.com/appthreat/cpggen" \
       org.opencontainers.image.url="https://github.com/appthreat/cpggen" \
-      org.opencontainers.image.version="1.5.2" \
+      org.opencontainers.image.version="1.6.0" \
       org.opencontainers.image.vendor="AppThreat" \
       org.opencontainers.image.licenses="Apache-2.0" \
       org.opencontainers.image.title="cpggen" \