From 20caae15aa92bf47890a2a64ab30dafdafb5c39e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 11:19:08 +0200 Subject: [PATCH 01/56] Configured different OS for PR tests We had some issues with file paths on varying OS in the past. Let's make this a little bit safer here. @mpollmeier Do we also have to adapt the release workflow test section somehow? --- .github/workflows/pr.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 0200254d6..c7da37c84 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -2,7 +2,10 @@ name: PR on: pull_request jobs: pr: - runs-on: ubuntu-20.04 + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] steps: - uses: actions/checkout@v2 with: From 8761e735bfc9b36a024948a7ebb219f35471a7dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 11:27:23 +0200 Subject: [PATCH 02/56] Update pr.yml --- .github/workflows/pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index c7da37c84..bdb08a00b 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -11,7 +11,7 @@ jobs: with: fetch-depth: 1 - name: Set up JDK 11 - uses: actions/setup-java@v2 + uses: olafurpg/setup-scala@v11 with: distribution: 'adopt' java-version: 8 From ffde46d55ab8fd75591b5fa4671ef895ecc3249f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 11:30:40 +0200 Subject: [PATCH 03/56] Update pr.yml --- .github/workflows/pr.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index bdb08a00b..2644d53ed 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -10,11 +10,10 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 1 - - name: Set up JDK 11 + - name: Set up JDK 8 and SBT uses: olafurpg/setup-scala@v11 with: - distribution: 'adopt' - java-version: 8 + java-version: adopt@1.8 - uses: actions/cache@v2 with: path: | From fb1675121beb585f6f32975918c963d17a5a0a41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 11:33:49 +0200 Subject: [PATCH 04/56] Update pr.yml --- .github/workflows/pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 2644d53ed..663d2eaca 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -21,4 +21,4 @@ jobs: ~/.coursier key: ${{ runner.os }}-sbt-${{ hashfiles('**/build.sbt') }} - name: Compile and run tests - run: sbt clean test + run: sbt clean stage test From 22616fdac04b2b450657002611c28daee29b70f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 11:42:47 +0200 Subject: [PATCH 05/56] Update pr.yml --- .github/workflows/pr.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 663d2eaca..761d12715 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -5,7 +5,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-20.04, windows-2019, macos-10.15] steps: - uses: actions/checkout@v2 with: @@ -21,4 +21,4 @@ jobs: ~/.coursier key: ${{ runner.os }}-sbt-${{ hashfiles('**/build.sbt') }} - name: Compile and run tests - run: sbt clean stage test + run: sbt clean test From d5beb1905492d3da500ff3eea5f2b5e4591d0674 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 13:13:30 +0200 Subject: [PATCH 06/56] Trying to debug this ... --- .../io/shiftleft/js2cpg/cpg/passes/CfgIntegrationTest.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/CfgIntegrationTest.scala b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/CfgIntegrationTest.scala index 200d3cc4d..49a7e08c9 100644 --- a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/CfgIntegrationTest.scala +++ b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/CfgIntegrationTest.scala @@ -1296,7 +1296,9 @@ class CfgIntegrationTest extends AnyWordSpec with Matchers { File.usingTemporaryDirectory("js2cpgCfgIntegrationTest") { workspace => cpg = callFrontend(workspace, code) match { - case Failure(exception) => fail(exception) + case Failure(exception) => + println(exception) + fail(exception) case Success(cpg) => new CfgCreationPass(cpg).createAndApply() cpg From e37e908181349684f03fd19ee39788e156d0ec04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 13:18:24 +0200 Subject: [PATCH 07/56] Update pr.yml --- .github/workflows/pr.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 761d12715..08df90e24 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -21,4 +21,6 @@ jobs: ~/.coursier key: ${{ runner.os }}-sbt-${{ hashfiles('**/build.sbt') }} - name: Compile and run tests - run: sbt clean test + run: sbt -v clean +test + shell: bash + From de4106dcdf9e268361d317416db7a59264bc2a05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 13:33:01 +0200 Subject: [PATCH 08/56] To be truly OS independent --- js2cpg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js2cpg.sh b/js2cpg.sh index e616b2f77..929cff296 100755 --- a/js2cpg.sh +++ b/js2cpg.sh @@ -1,6 +1,6 @@ #!/usr/bin/env sh -SCRIPT_ABS_PATH=$(readlink -f "$0") +SCRIPT_ABS_PATH=$( cd "$(dirname "$0")" ; pwd -P ) SCRIPT_ABS_DIR=$(dirname $SCRIPT_ABS_PATH) $SCRIPT_ABS_DIR/target/universal/stage/bin/js2cpg -Dlog4j.configurationFile=$SCRIPT_ABS_DIR/src/main/resources/log4j2.xml $@ From b9cb57a00597dd41f16f5dd7dfa4472c73f2f7b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 13:46:21 +0200 Subject: [PATCH 09/56] Update js2cpg.sh --- js2cpg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js2cpg.sh b/js2cpg.sh index 929cff296..651e5ade7 100755 --- a/js2cpg.sh +++ b/js2cpg.sh @@ -1,6 +1,6 @@ #!/usr/bin/env sh -SCRIPT_ABS_PATH=$( cd "$(dirname "$0")" ; pwd -P ) +SCRIPT_ABS_PATH=$(realpath $0) SCRIPT_ABS_DIR=$(dirname $SCRIPT_ABS_PATH) $SCRIPT_ABS_DIR/target/universal/stage/bin/js2cpg -Dlog4j.configurationFile=$SCRIPT_ABS_DIR/src/main/resources/log4j2.xml $@ From fbfe4cd994ce7f196b18ee6d61572759bc92063b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 13:46:34 +0200 Subject: [PATCH 10/56] Update js2cpg.sh --- js2cpg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js2cpg.sh b/js2cpg.sh index 651e5ade7..add9edebc 100755 --- a/js2cpg.sh +++ b/js2cpg.sh @@ -1,6 +1,6 @@ #!/usr/bin/env sh -SCRIPT_ABS_PATH=$(realpath $0) +SCRIPT_ABS_PATH=$(realpath "$0") SCRIPT_ABS_DIR=$(dirname $SCRIPT_ABS_PATH) $SCRIPT_ABS_DIR/target/universal/stage/bin/js2cpg -Dlog4j.configurationFile=$SCRIPT_ABS_DIR/src/main/resources/log4j2.xml $@ From 6aaab7a33accc0a8b0274b8457def35b97d0752e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 13:53:24 +0200 Subject: [PATCH 11/56] Update js2cpg.sh --- js2cpg.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/js2cpg.sh b/js2cpg.sh index add9edebc..ac99ef18c 100755 --- a/js2cpg.sh +++ b/js2cpg.sh @@ -1,6 +1,8 @@ #!/usr/bin/env sh -SCRIPT_ABS_PATH=$(realpath "$0") -SCRIPT_ABS_DIR=$(dirname $SCRIPT_ABS_PATH) +dir=$(cd -P -- "$(dirname -- "$0")" && pwd -P) +scriptpath="$dir/js2cpg/target/universal/stage/bin/js2cpg" -$SCRIPT_ABS_DIR/target/universal/stage/bin/js2cpg -Dlog4j.configurationFile=$SCRIPT_ABS_DIR/src/main/resources/log4j2.xml $@ +$scriptpath \ + -Dlog4j.configurationFile=$dir/js2cpg/src/main/resources/log4j2.xml \ + $@ From cb845a128fcbf97fec066664d3e87974a6bb8585 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 13:58:40 +0200 Subject: [PATCH 12/56] Update js2cpg.sh --- js2cpg.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/js2cpg.sh b/js2cpg.sh index ac99ef18c..cee1e7f0b 100755 --- a/js2cpg.sh +++ b/js2cpg.sh @@ -1,8 +1,8 @@ #!/usr/bin/env sh dir=$(cd -P -- "$(dirname -- "$0")" && pwd -P) -scriptpath="$dir/js2cpg/target/universal/stage/bin/js2cpg" +scriptpath="$dir/target/universal/stage/bin/js2cpg" $scriptpath \ - -Dlog4j.configurationFile=$dir/js2cpg/src/main/resources/log4j2.xml \ + -Dlog4j.configurationFile=$dir/src/main/resources/log4j2.xml \ $@ From cce260aabb75f3c6526e5e8a408e389b39a7d996 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 14:03:22 +0200 Subject: [PATCH 13/56] Update pr.yml --- .github/workflows/pr.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 08df90e24..aea82577a 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -21,6 +21,4 @@ jobs: ~/.coursier key: ${{ runner.os }}-sbt-${{ hashfiles('**/build.sbt') }} - name: Compile and run tests - run: sbt -v clean +test - shell: bash - + run: sbt clean test From 01f7fe5db7bff2f98fe47a021f60699a64b2f30e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 14:15:22 +0200 Subject: [PATCH 14/56] Update pr.yml --- .github/workflows/pr.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index aea82577a..74b63fe15 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -22,3 +22,5 @@ jobs: key: ${{ runner.os }}-sbt-${{ hashfiles('**/build.sbt') }} - name: Compile and run tests run: sbt clean test + shell: bash + From ae14e61412643fde11e8e2db1d17a42b22e2b4f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 14:19:54 +0200 Subject: [PATCH 15/56] Update js2cpg.sh --- js2cpg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js2cpg.sh b/js2cpg.sh index cee1e7f0b..7ec566c47 100755 --- a/js2cpg.sh +++ b/js2cpg.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env sh +#!/bin/bash dir=$(cd -P -- "$(dirname -- "$0")" && pwd -P) scriptpath="$dir/target/universal/stage/bin/js2cpg" From d37f29b541d951b652d676f84198b33e28e5876c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 14:23:04 +0200 Subject: [PATCH 16/56] Update js2cpg.sh --- js2cpg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js2cpg.sh b/js2cpg.sh index 7ec566c47..cee1e7f0b 100755 --- a/js2cpg.sh +++ b/js2cpg.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env sh dir=$(cd -P -- "$(dirname -- "$0")" && pwd -P) scriptpath="$dir/target/universal/stage/bin/js2cpg" From a818c751fbe557abe7965d856e9ec531ec1ee245 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 14:25:57 +0200 Subject: [PATCH 17/56] Update IntegrationTestFixture.scala --- .../io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala index cee18ca52..795752c01 100644 --- a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala +++ b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala @@ -57,7 +57,7 @@ abstract class IntegrationTestFixture { ExternalCommand .run( - s"./js2cpg.sh ${workspace.pathAsString} --output ${cpgPath.pathAsString} --no-ts --no-babel") + s"js2cpg.sh ${workspace.pathAsString} --output ${cpgPath.pathAsString} --no-ts --no-babel") .map { _ => CpgLoader .loadFromOverflowDb( From 5d2404ff428d68d928ceb6f803a1470b96547b6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 14:32:56 +0200 Subject: [PATCH 18/56] Update IntegrationTestFixture.scala --- .../cpg/passes/IntegrationTestFixture.scala | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala index 795752c01..15398a19c 100644 --- a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala +++ b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala @@ -18,24 +18,13 @@ object IntegrationTestFixture { } private object ExternalCommand { - private val windowsSystemPrefix = "Windows" - private val osNameProperty = "os.name" - def run(command: String): Try[String] = { val result = mutable.ArrayBuffer.empty[String] val lineHandler: String => Unit = line => { result.addOne(line) } - val systemString = System.getProperty(osNameProperty) - val shellPrefix = - if (systemString != null && systemString.startsWith(windowsSystemPrefix)) { - "cmd" :: "/c" :: Nil - } else { - "sh" :: "-c" :: Nil - } - - Process(shellPrefix :+ command, new io.File(js2cpgPath.pathAsString)) + Process(Seq(command), new io.File(js2cpgPath.pathAsString)) .!(ProcessLogger(lineHandler, lineHandler)) match { case 0 => Success(result.mkString(System.lineSeparator())) @@ -57,7 +46,7 @@ abstract class IntegrationTestFixture { ExternalCommand .run( - s"js2cpg.sh ${workspace.pathAsString} --output ${cpgPath.pathAsString} --no-ts --no-babel") + s"bash js2cpg.sh ${workspace.pathAsString} --output ${cpgPath.pathAsString} --no-ts --no-babel") .map { _ => CpgLoader .loadFromOverflowDb( From a42dc89558919c4a8282cb331370d9e01cae7016 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 14:37:25 +0200 Subject: [PATCH 19/56] Update IntegrationTestFixture.scala --- .../cpg/passes/IntegrationTestFixture.scala | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala index 15398a19c..cee18ca52 100644 --- a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala +++ b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala @@ -18,13 +18,24 @@ object IntegrationTestFixture { } private object ExternalCommand { + private val windowsSystemPrefix = "Windows" + private val osNameProperty = "os.name" + def run(command: String): Try[String] = { val result = mutable.ArrayBuffer.empty[String] val lineHandler: String => Unit = line => { result.addOne(line) } - Process(Seq(command), new io.File(js2cpgPath.pathAsString)) + val systemString = System.getProperty(osNameProperty) + val shellPrefix = + if (systemString != null && systemString.startsWith(windowsSystemPrefix)) { + "cmd" :: "/c" :: Nil + } else { + "sh" :: "-c" :: Nil + } + + Process(shellPrefix :+ command, new io.File(js2cpgPath.pathAsString)) .!(ProcessLogger(lineHandler, lineHandler)) match { case 0 => Success(result.mkString(System.lineSeparator())) @@ -46,7 +57,7 @@ abstract class IntegrationTestFixture { ExternalCommand .run( - s"bash js2cpg.sh ${workspace.pathAsString} --output ${cpgPath.pathAsString} --no-ts --no-babel") + s"./js2cpg.sh ${workspace.pathAsString} --output ${cpgPath.pathAsString} --no-ts --no-babel") .map { _ => CpgLoader .loadFromOverflowDb( From 6e21bd7b9c043964f6c2e20aafcd8d71581cea68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 14:42:16 +0200 Subject: [PATCH 20/56] Update IntegrationTestFixture.scala --- .../js2cpg/cpg/passes/IntegrationTestFixture.scala | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala index cee18ca52..16565dd7c 100644 --- a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala +++ b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala @@ -18,23 +18,13 @@ object IntegrationTestFixture { } private object ExternalCommand { - private val windowsSystemPrefix = "Windows" - private val osNameProperty = "os.name" - def run(command: String): Try[String] = { val result = mutable.ArrayBuffer.empty[String] val lineHandler: String => Unit = line => { result.addOne(line) } - val systemString = System.getProperty(osNameProperty) - val shellPrefix = - if (systemString != null && systemString.startsWith(windowsSystemPrefix)) { - "cmd" :: "/c" :: Nil - } else { - "sh" :: "-c" :: Nil - } - + val shellPrefix = "sh" :: "-c" :: Nil Process(shellPrefix :+ command, new io.File(js2cpgPath.pathAsString)) .!(ProcessLogger(lineHandler, lineHandler)) match { case 0 => From 415850fa6fb97b8cb5d22c88302a0031e41c038a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 14:58:20 +0200 Subject: [PATCH 21/56] Update IntegrationTestFixture.scala --- .../js2cpg/cpg/passes/IntegrationTestFixture.scala | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala index 16565dd7c..0198d45e2 100644 --- a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala +++ b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala @@ -18,13 +18,23 @@ object IntegrationTestFixture { } private object ExternalCommand { + private val windowsSystemPrefix = "Windows" + private val osNameProperty = "os.name" + def run(command: String): Try[String] = { val result = mutable.ArrayBuffer.empty[String] val lineHandler: String => Unit = line => { result.addOne(line) } - val shellPrefix = "sh" :: "-c" :: Nil + val systemString = System.getProperty(osNameProperty) + val shellPrefix = + if (systemString != null && systemString.startsWith(windowsSystemPrefix)) { + "cmd" :: "/c" :: Nil + } else { + "sh" :: "-c" :: Nil + } + Process(shellPrefix :+ command, new io.File(js2cpgPath.pathAsString)) .!(ProcessLogger(lineHandler, lineHandler)) match { case 0 => From 03f0bf9d30efc9c30ae6daaa27b49f4485e7eee5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 15:00:15 +0200 Subject: [PATCH 22/56] Update pr.yml --- .github/workflows/pr.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 74b63fe15..04c242617 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -21,6 +21,4 @@ jobs: ~/.coursier key: ${{ runner.os }}-sbt-${{ hashfiles('**/build.sbt') }} - name: Compile and run tests - run: sbt clean test - shell: bash - + run: sbt "; clean; +test" From c1f201c8def7615c5cdd8851b4f79ed7f53098b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 15:04:43 +0200 Subject: [PATCH 23/56] Update pr.yml --- .github/workflows/pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 04c242617..751a88545 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -21,4 +21,4 @@ jobs: ~/.coursier key: ${{ runner.os }}-sbt-${{ hashfiles('**/build.sbt') }} - name: Compile and run tests - run: sbt "; clean; +test" + run: sbt -v "clean; +test;" From ec4d53f9e018b2f18009f24f284cb10761521a0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 15:06:42 +0200 Subject: [PATCH 24/56] Update pr.yml --- .github/workflows/pr.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 751a88545..d6ddf2b7b 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -22,3 +22,5 @@ jobs: key: ${{ runner.os }}-sbt-${{ hashfiles('**/build.sbt') }} - name: Compile and run tests run: sbt -v "clean; +test;" + shell: bash + From 03327739fe94d5bec7c35a2584b0f0b00162881a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 15:11:18 +0200 Subject: [PATCH 25/56] Update IntegrationTestFixture.scala --- .../io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala index 0198d45e2..58d75a833 100644 --- a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala +++ b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala @@ -57,7 +57,7 @@ abstract class IntegrationTestFixture { ExternalCommand .run( - s"./js2cpg.sh ${workspace.pathAsString} --output ${cpgPath.pathAsString} --no-ts --no-babel") + s"js2cpg.sh ${workspace.pathAsString} --output ${cpgPath.pathAsString} --no-ts --no-babel") .map { _ => CpgLoader .loadFromOverflowDb( From 2aa1c5ca111e04f2e1f797e1a656e8cee44009b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 15:12:22 +0200 Subject: [PATCH 26/56] Update js2cpg.sh --- js2cpg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js2cpg.sh b/js2cpg.sh index cee1e7f0b..7ec566c47 100755 --- a/js2cpg.sh +++ b/js2cpg.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env sh +#!/bin/bash dir=$(cd -P -- "$(dirname -- "$0")" && pwd -P) scriptpath="$dir/target/universal/stage/bin/js2cpg" From be92b7f461b4a955fd279f092868442d24b9e934 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 15:19:46 +0200 Subject: [PATCH 27/56] Update IntegrationTestFixture.scala --- .../js2cpg/cpg/passes/IntegrationTestFixture.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala index 58d75a833..6c07772b1 100644 --- a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala +++ b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala @@ -28,14 +28,14 @@ object IntegrationTestFixture { } val systemString = System.getProperty(osNameProperty) - val shellPrefix = + val (shellPrefix, cmd) = if (systemString != null && systemString.startsWith(windowsSystemPrefix)) { - "cmd" :: "/c" :: Nil + ("sh" :: Nil, command.replace("./", "")) } else { - "sh" :: "-c" :: Nil + ("sh" :: "-c" :: Nil, command) } - Process(shellPrefix :+ command, new io.File(js2cpgPath.pathAsString)) + Process(shellPrefix :+ cmd, new io.File(js2cpgPath.pathAsString)) .!(ProcessLogger(lineHandler, lineHandler)) match { case 0 => Success(result.mkString(System.lineSeparator())) @@ -57,7 +57,7 @@ abstract class IntegrationTestFixture { ExternalCommand .run( - s"js2cpg.sh ${workspace.pathAsString} --output ${cpgPath.pathAsString} --no-ts --no-babel") + s"./js2cpg.sh ${workspace.pathAsString} --output ${cpgPath.pathAsString} --no-ts --no-babel") .map { _ => CpgLoader .loadFromOverflowDb( From cf5b513dded7f1c01f7babf9971a6c349827873a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 15:25:17 +0200 Subject: [PATCH 28/56] Update IntegrationTestFixture.scala --- .../io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala index 6c07772b1..7b0e6b9a6 100644 --- a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala +++ b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala @@ -30,7 +30,7 @@ object IntegrationTestFixture { val systemString = System.getProperty(osNameProperty) val (shellPrefix, cmd) = if (systemString != null && systemString.startsWith(windowsSystemPrefix)) { - ("sh" :: Nil, command.replace("./", "")) + ("bash" :: Nil, command.replace("./", "")) } else { ("sh" :: "-c" :: Nil, command) } From a0c6631f211c90d36c0972edbdb159750e0de2d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 15:29:16 +0200 Subject: [PATCH 29/56] Update IntegrationTestFixture.scala --- .../io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala index 7b0e6b9a6..7dc33001c 100644 --- a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala +++ b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala @@ -30,7 +30,7 @@ object IntegrationTestFixture { val systemString = System.getProperty(osNameProperty) val (shellPrefix, cmd) = if (systemString != null && systemString.startsWith(windowsSystemPrefix)) { - ("bash" :: Nil, command.replace("./", "")) + ("sh" :: "-c" :: Nil, command.replace("./", ".\")) } else { ("sh" :: "-c" :: Nil, command) } From b4ec973b26b103f707dc65274da7f1d5a45705d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 15:32:55 +0200 Subject: [PATCH 30/56] Update IntegrationTestFixture.scala --- .../io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala index 7dc33001c..633e8674b 100644 --- a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala +++ b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala @@ -30,7 +30,7 @@ object IntegrationTestFixture { val systemString = System.getProperty(osNameProperty) val (shellPrefix, cmd) = if (systemString != null && systemString.startsWith(windowsSystemPrefix)) { - ("sh" :: "-c" :: Nil, command.replace("./", ".\")) + ("sh" :: "-c" :: Nil, command.replace("./", ".\\")) } else { ("sh" :: "-c" :: Nil, command) } From 904231950ee9e69531e0b39f23300faf26131c09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 15:39:01 +0200 Subject: [PATCH 31/56] Update IntegrationTestFixture.scala --- .../js2cpg/cpg/passes/IntegrationTestFixture.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala index 633e8674b..2d4aa3dbe 100644 --- a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala +++ b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala @@ -28,14 +28,14 @@ object IntegrationTestFixture { } val systemString = System.getProperty(osNameProperty) - val (shellPrefix, cmd) = + val cmd = if (systemString != null && systemString.startsWith(windowsSystemPrefix)) { - ("sh" :: "-c" :: Nil, command.replace("./", ".\\")) + "sh" :: "-c" :: ".\\" + command :: Nil } else { - ("sh" :: "-c" :: Nil, command) + "sh" :: "-c" :: "./" + command :: Nil } - Process(shellPrefix :+ cmd, new io.File(js2cpgPath.pathAsString)) + Process(cmd, new io.File(js2cpgPath.pathAsString)) .!(ProcessLogger(lineHandler, lineHandler)) match { case 0 => Success(result.mkString(System.lineSeparator())) @@ -57,7 +57,7 @@ abstract class IntegrationTestFixture { ExternalCommand .run( - s"./js2cpg.sh ${workspace.pathAsString} --output ${cpgPath.pathAsString} --no-ts --no-babel") + s"js2cpg.sh ${workspace.pathAsString} --output ${cpgPath.pathAsString} --no-ts --no-babel") .map { _ => CpgLoader .loadFromOverflowDb( From 2c7571ddcc5b825ca5ffc311f834541bf66f7b09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 15:46:09 +0200 Subject: [PATCH 32/56] Update IntegrationTestFixture.scala --- .../io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala index 2d4aa3dbe..3af3684f7 100644 --- a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala +++ b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala @@ -30,7 +30,7 @@ object IntegrationTestFixture { val systemString = System.getProperty(osNameProperty) val cmd = if (systemString != null && systemString.startsWith(windowsSystemPrefix)) { - "sh" :: "-c" :: ".\\" + command :: Nil + "sh" :: command :: Nil } else { "sh" :: "-c" :: "./" + command :: Nil } From f4843c1694431d9e6f49ccc2bbbf519231102c04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 15:53:25 +0200 Subject: [PATCH 33/56] Update IntegrationTestFixture.scala --- .../js2cpg/cpg/passes/IntegrationTestFixture.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala index 3af3684f7..c511c06da 100644 --- a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala +++ b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala @@ -30,9 +30,9 @@ object IntegrationTestFixture { val systemString = System.getProperty(osNameProperty) val cmd = if (systemString != null && systemString.startsWith(windowsSystemPrefix)) { - "sh" :: command :: Nil + "sh" :: "-c" :: "\"" + command + "\"" :: Nil } else { - "sh" :: "-c" :: "./" + command :: Nil + "sh" :: "-c" :: command :: Nil } Process(cmd, new io.File(js2cpgPath.pathAsString)) @@ -57,7 +57,7 @@ abstract class IntegrationTestFixture { ExternalCommand .run( - s"js2cpg.sh ${workspace.pathAsString} --output ${cpgPath.pathAsString} --no-ts --no-babel") + s"./js2cpg.sh ${workspace.pathAsString} --output ${cpgPath.pathAsString} --no-ts --no-babel") .map { _ => CpgLoader .loadFromOverflowDb( From 1cb703dd97c40429c101841affbe1d9e313c45f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 17:22:37 +0200 Subject: [PATCH 34/56] Update IntegrationTestFixture.scala --- .../io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala index c511c06da..54d6573cb 100644 --- a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala +++ b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala @@ -30,7 +30,7 @@ object IntegrationTestFixture { val systemString = System.getProperty(osNameProperty) val cmd = if (systemString != null && systemString.startsWith(windowsSystemPrefix)) { - "sh" :: "-c" :: "\"" + command + "\"" :: Nil + "cmd" :: "/c" :: "\"" + command + "\"" :: Nil } else { "sh" :: "-c" :: command :: Nil } From 8e59ecdf49b6dcd2f20631216a05b65205b7f723 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 17:33:09 +0200 Subject: [PATCH 35/56] Update IntegrationTestFixture.scala --- .../shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala index 54d6573cb..5848ffa38 100644 --- a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala +++ b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala @@ -32,7 +32,7 @@ object IntegrationTestFixture { if (systemString != null && systemString.startsWith(windowsSystemPrefix)) { "cmd" :: "/c" :: "\"" + command + "\"" :: Nil } else { - "sh" :: "-c" :: command :: Nil + "sh" :: "-c" :: "./" + command :: Nil } Process(cmd, new io.File(js2cpgPath.pathAsString)) @@ -57,7 +57,7 @@ abstract class IntegrationTestFixture { ExternalCommand .run( - s"./js2cpg.sh ${workspace.pathAsString} --output ${cpgPath.pathAsString} --no-ts --no-babel") + s"js2cpg.sh ${workspace.pathAsString} --output ${cpgPath.pathAsString} --no-ts --no-babel") .map { _ => CpgLoader .loadFromOverflowDb( From ee44358fa34563de5cbb63fdee3d19610a75d104 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 17:50:36 +0200 Subject: [PATCH 36/56] Update IntegrationTestFixture.scala --- .../js2cpg/cpg/passes/IntegrationTestFixture.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala index 5848ffa38..c511c06da 100644 --- a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala +++ b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala @@ -30,9 +30,9 @@ object IntegrationTestFixture { val systemString = System.getProperty(osNameProperty) val cmd = if (systemString != null && systemString.startsWith(windowsSystemPrefix)) { - "cmd" :: "/c" :: "\"" + command + "\"" :: Nil + "sh" :: "-c" :: "\"" + command + "\"" :: Nil } else { - "sh" :: "-c" :: "./" + command :: Nil + "sh" :: "-c" :: command :: Nil } Process(cmd, new io.File(js2cpgPath.pathAsString)) @@ -57,7 +57,7 @@ abstract class IntegrationTestFixture { ExternalCommand .run( - s"js2cpg.sh ${workspace.pathAsString} --output ${cpgPath.pathAsString} --no-ts --no-babel") + s"./js2cpg.sh ${workspace.pathAsString} --output ${cpgPath.pathAsString} --no-ts --no-babel") .map { _ => CpgLoader .loadFromOverflowDb( From 115efdeae5bb57abc1683412d07bc21c32afba98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 17:55:27 +0200 Subject: [PATCH 37/56] Update IntegrationTestFixture.scala --- .../io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala index c511c06da..b7a5fbb63 100644 --- a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala +++ b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala @@ -30,7 +30,7 @@ object IntegrationTestFixture { val systemString = System.getProperty(osNameProperty) val cmd = if (systemString != null && systemString.startsWith(windowsSystemPrefix)) { - "sh" :: "-c" :: "\"" + command + "\"" :: Nil + "sh" :: "-c" :: "\"" + command.replace("\\", "\\\\") + "\"" :: Nil } else { "sh" :: "-c" :: command :: Nil } From d3742b9141dc8b02ca6e89ab16255e43f10be728 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 18:08:24 +0200 Subject: [PATCH 38/56] Update IntegrationTestFixture.scala --- .../io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala index b7a5fbb63..2f64918be 100644 --- a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala +++ b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala @@ -30,6 +30,7 @@ object IntegrationTestFixture { val systemString = System.getProperty(osNameProperty) val cmd = if (systemString != null && systemString.startsWith(windowsSystemPrefix)) { + println("Command was: " + command.replace("\\", "\\\\")) "sh" :: "-c" :: "\"" + command.replace("\\", "\\\\") + "\"" :: Nil } else { "sh" :: "-c" :: command :: Nil From a8a12733e425254c5cb65e5cadc98d30f2fee985 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 18:18:01 +0200 Subject: [PATCH 39/56] Update IntegrationTestFixture.scala --- .../shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala index 2f64918be..e364d6c7b 100644 --- a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala +++ b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala @@ -30,7 +30,8 @@ object IntegrationTestFixture { val systemString = System.getProperty(osNameProperty) val cmd = if (systemString != null && systemString.startsWith(windowsSystemPrefix)) { - println("Command was: " + command.replace("\\", "\\\\")) + println("Command raw: " + command) + println("Command replaced: " + command.replace("\\", "\\\\")) "sh" :: "-c" :: "\"" + command.replace("\\", "\\\\") + "\"" :: Nil } else { "sh" :: "-c" :: command :: Nil From f23ad89418eea037745bdfc58b93c8e93173e5c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 18:23:45 +0200 Subject: [PATCH 40/56] Update IntegrationTestFixture.scala --- .../shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala index e364d6c7b..c511c06da 100644 --- a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala +++ b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala @@ -30,9 +30,7 @@ object IntegrationTestFixture { val systemString = System.getProperty(osNameProperty) val cmd = if (systemString != null && systemString.startsWith(windowsSystemPrefix)) { - println("Command raw: " + command) - println("Command replaced: " + command.replace("\\", "\\\\")) - "sh" :: "-c" :: "\"" + command.replace("\\", "\\\\") + "\"" :: Nil + "sh" :: "-c" :: "\"" + command + "\"" :: Nil } else { "sh" :: "-c" :: command :: Nil } From ba0edad3e41179b7daf8012ed9c04e367b3aa9c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Sep 2021 18:24:22 +0200 Subject: [PATCH 41/56] Update pr.yml --- .github/workflows/pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index d6ddf2b7b..03f6f317e 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -21,6 +21,6 @@ jobs: ~/.coursier key: ${{ runner.os }}-sbt-${{ hashfiles('**/build.sbt') }} - name: Compile and run tests - run: sbt -v "clean; +test;" + run: sbt -v -Dfile.encoding=UTF-8 "clean; +test;" shell: bash From 2871ce528d5cecc3c360e9534dd6669b3f6e25f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leutha=CC=88user?= Date: Thu, 16 Sep 2021 07:06:46 +0200 Subject: [PATCH 42/56] Another try ... --- build.sbt | 8 --- .../cpg/passes/IntegrationTestFixture.scala | 69 ------------------- js2cpg.sh | 10 ++- .../cpg/passes/CfgCreationPassTest.scala | 32 ++++----- 4 files changed, 17 insertions(+), 102 deletions(-) delete mode 100644 cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala rename cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/CfgIntegrationTest.scala => src/test/scala/io/shiftleft/js2cpg/cpg/passes/CfgCreationPassTest.scala (99%) diff --git a/build.sbt b/build.sbt index b7f141947..a6ec9e5e3 100644 --- a/build.sbt +++ b/build.sbt @@ -4,8 +4,6 @@ val gitCommitString = SettingKey[String]("gitSha") enablePlugins(JavaAppPackaging, GitVersioning, BuildInfoPlugin) -Test / test := ((Test / test) dependsOn (cfgIntegrationTests / Test / test)).value - lazy val Fast = config("fast").extend(Test) configs(Fast) inConfig(Fast)(Defaults.testTasks) @@ -122,11 +120,5 @@ lazy val js2cpg = (project in file(".")).settings( buildInfoPackage := "io.shiftleft.js2cpg.core" ) -lazy val cfgIntegrationTests = project.settings( - commonSettings, - publish / skip := true, - Test / test := ((Test / test) dependsOn (js2cpg / stage)).value -) - Universal / packageName := name.value Universal / topLevelDirectory := None diff --git a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala b/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala deleted file mode 100644 index c511c06da..000000000 --- a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/IntegrationTestFixture.scala +++ /dev/null @@ -1,69 +0,0 @@ -package io.shiftleft.js2cpg.cpg.passes - -import better.files.File -import overflowdb.Config -import io.shiftleft.codepropertygraph.Cpg -import io.shiftleft.codepropertygraph.cpgloading.{CpgLoader, CpgLoaderConfig} - -import java.io -import scala.collection.mutable -import scala.sys.process.{Process, ProcessLogger} -import scala.util.{Failure, Success, Try} - -object IntegrationTestFixture { - - private val js2cpgPath = File(".") match { - case f if f.canonicalPath.endsWith("cfgIntegrationTests") => f.parent - case f => f - } - - private object ExternalCommand { - private val windowsSystemPrefix = "Windows" - private val osNameProperty = "os.name" - - def run(command: String): Try[String] = { - val result = mutable.ArrayBuffer.empty[String] - val lineHandler: String => Unit = line => { - result.addOne(line) - } - - val systemString = System.getProperty(osNameProperty) - val cmd = - if (systemString != null && systemString.startsWith(windowsSystemPrefix)) { - "sh" :: "-c" :: "\"" + command + "\"" :: Nil - } else { - "sh" :: "-c" :: command :: Nil - } - - Process(cmd, new io.File(js2cpgPath.pathAsString)) - .!(ProcessLogger(lineHandler, lineHandler)) match { - case 0 => - Success(result.mkString(System.lineSeparator())) - case _ => - Failure(new RuntimeException(result.mkString(System.lineSeparator()))) - } - } - } -} - -abstract class IntegrationTestFixture { - - import IntegrationTestFixture.ExternalCommand - - protected def callFrontend(workspace: File, code: String): Try[Cpg] = { - val file = workspace / "test.js" - val cpgPath = workspace / "cpg.bin.zip" - file.write(code) - - ExternalCommand - .run( - s"./js2cpg.sh ${workspace.pathAsString} --output ${cpgPath.pathAsString} --no-ts --no-babel") - .map { _ => - CpgLoader - .loadFromOverflowDb( - CpgLoaderConfig.withDefaults.withOverflowConfig( - Config.withDefaults.withStorageLocation(cpgPath.pathAsString))) - } - } - -} diff --git a/js2cpg.sh b/js2cpg.sh index 7ec566c47..e616b2f77 100755 --- a/js2cpg.sh +++ b/js2cpg.sh @@ -1,8 +1,6 @@ -#!/bin/bash +#!/usr/bin/env sh -dir=$(cd -P -- "$(dirname -- "$0")" && pwd -P) -scriptpath="$dir/target/universal/stage/bin/js2cpg" +SCRIPT_ABS_PATH=$(readlink -f "$0") +SCRIPT_ABS_DIR=$(dirname $SCRIPT_ABS_PATH) -$scriptpath \ - -Dlog4j.configurationFile=$dir/src/main/resources/log4j2.xml \ - $@ +$SCRIPT_ABS_DIR/target/universal/stage/bin/js2cpg -Dlog4j.configurationFile=$SCRIPT_ABS_DIR/src/main/resources/log4j2.xml $@ diff --git a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/CfgIntegrationTest.scala b/src/test/scala/io/shiftleft/js2cpg/cpg/passes/CfgCreationPassTest.scala similarity index 99% rename from cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/CfgIntegrationTest.scala rename to src/test/scala/io/shiftleft/js2cpg/cpg/passes/CfgCreationPassTest.scala index 49a7e08c9..163d74e41 100644 --- a/cfgIntegrationTests/src/test/scala/io/shiftleft/js2cpg/cpg/passes/CfgIntegrationTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/cpg/passes/CfgCreationPassTest.scala @@ -3,24 +3,20 @@ package io.shiftleft.js2cpg.cpg.passes import better.files.File import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated._ +import io.shiftleft.js2cpg.core.Report +import io.shiftleft.passes.IntervalKeyPool import io.shiftleft.semanticcpg.language._ import io.shiftleft.semanticcpg.passes.CfgCreationPass -import io.shiftleft.semanticcpg.passes.cfgcreation.Cfg.{ - AlwaysEdge, - CaseEdge, - CfgEdgeType, - FalseEdge, - TrueEdge -} +import io.shiftleft.semanticcpg.passes.cfgcreation.Cfg._ import overflowdb.traversal._ import overflowdb._ + import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec import scala.jdk.CollectionConverters._ -import scala.util.{Failure, Success} -class CfgIntegrationTest extends AnyWordSpec with Matchers { +class CfgCreationPassTest extends AnyWordSpec with Matchers { "CFG generation for simple fragments" should { "have correct structure for block expression" in { @@ -1290,19 +1286,17 @@ class CfgIntegrationTest extends AnyWordSpec with Matchers { } } - private class CfgFixture(code: String) extends IntegrationTestFixture { + private class CfgFixture(code: String) { - private var cpg: Cpg = Cpg.emptyCpg + private val cpg: Cpg = Cpg.emptyCpg File.usingTemporaryDirectory("js2cpgCfgIntegrationTest") { workspace => - cpg = callFrontend(workspace, code) match { - case Failure(exception) => - println(exception) - fail(exception) - case Success(cpg) => - new CfgCreationPass(cpg).createAndApply() - cpg - } + val file = workspace / "test.js" + file.write(code) + val keyPool = new IntervalKeyPool(1001, 2000) + val filenames = List((file.path, file.parent.path)) + new AstCreationPass(workspace, filenames, cpg, keyPool, new Report()).createAndApply() + new CfgCreationPass(cpg).createAndApply() } private def matchCode(node: Node, code: String): Boolean = { From 489d2d6e33deef497e33d3c15fca1b1a45e93171 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leutha=CC=88user?= Date: Thu, 16 Sep 2021 07:29:39 +0200 Subject: [PATCH 43/56] Small refactoring. --- .../shiftleft/js2cpg/io/PrivateModulesTest.scala | 8 ++++---- .../preprocessing/TranspilationRunnerTest.scala | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/test/scala/io/shiftleft/js2cpg/io/PrivateModulesTest.scala b/src/test/scala/io/shiftleft/js2cpg/io/PrivateModulesTest.scala index 980066994..a4c09b012 100644 --- a/src/test/scala/io/shiftleft/js2cpg/io/PrivateModulesTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/io/PrivateModulesTest.scala @@ -22,7 +22,7 @@ class PrivateModulesTest extends AnyWordSpec with Matchers { "Handling for private modules" should { "copy and generate js files correctly for a simple project" in { - val projectPath = getClass.getResource("/privatemodules").getPath + val projectPath = getClass.getResource("/privatemodules").getFile File.usingTemporaryDirectory() { tmpDir: File => val tmpProjectPath = File(projectPath).copyToDirectory(tmpDir) @@ -40,7 +40,7 @@ class PrivateModulesTest extends AnyWordSpec with Matchers { } "copy and generate js files correctly for a simple project with additional private modules" in { - val projectPath = getClass.getResource("/privatemodules").getPath + val projectPath = getClass.getResource("/privatemodules").getFile File.usingTemporaryDirectory() { tmpDir: File => val tmpProjectPath = File(projectPath).copyToDirectory(tmpDir) @@ -67,7 +67,7 @@ class PrivateModulesTest extends AnyWordSpec with Matchers { } "copy and generate js files correctly for a simple project with filter" in { - val projectPath = getClass.getResource("/privatemodules").getPath + val projectPath = getClass.getResource("/privatemodules").getFile File.usingTemporaryDirectory() { tmpDir: File => val tmpProjectPath = File(projectPath).copyToDirectory(tmpDir) @@ -91,7 +91,7 @@ class PrivateModulesTest extends AnyWordSpec with Matchers { } "copy and generate js files correctly for a simple project with no private module being references" in { - val projectPath = getClass.getResource("/ignoreprivatemodules").getPath + val projectPath = getClass.getResource("/ignoreprivatemodules").getFile File.usingTemporaryDirectory() { tmpDir: File => val tmpProjectPath = File(projectPath).copyToDirectory(tmpDir) diff --git a/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala b/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala index 35a6c3a90..0c535c242 100644 --- a/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala @@ -34,7 +34,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { "TranspilationRunner" should { "generate js files correctly for a simple Babel project" in { - val projectPath = getClass.getResource("/babel").getPath + val projectPath = getClass.getResource("/babel").getFile File.usingTemporaryDirectory() { tmpDir: File => File.usingTemporaryDirectory() { transpileOutDir: File => val tmpProjectPath = File(projectPath).copyToDirectory(tmpDir) @@ -55,7 +55,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { } "generate and use sourcemap files correctly" in { - val projectPath = getClass.getResource("/typescript").getPath + val projectPath = getClass.getResource("/typescript").getFile File.usingTemporaryDirectory() { tmpDir: File => val tmpProjectPath = File(projectPath).copyToDirectory(tmpDir) @@ -73,7 +73,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { } "generate js files correctly for a simple Typescript project" in { - val projectPath = getClass.getResource("/typescript").getPath + val projectPath = getClass.getResource("/typescript").getFile File.usingTemporaryDirectory() { tmpDir: File => File.usingTemporaryDirectory() { transpileOutDir: File => val tmpProjectPath = File(projectPath).copyToDirectory(tmpDir) @@ -115,7 +115,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { } "generate js files correctly for a simple Typescript project including test files" in { - val projectPath = getClass.getResource("/typescript").getPath + val projectPath = getClass.getResource("/typescript").getFile File.usingTemporaryDirectory() { tmpDir: File => val tmpProjectPath = File(projectPath).copyToDirectory(tmpDir) @@ -136,7 +136,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { } "generate js files correctly for a simple Vue.js project" in { - val projectPath = getClass.getResource("/vue").getPath + val projectPath = getClass.getResource("/vue").getFile File.usingTemporaryDirectory() { tmpDir: File => val tmpProjectPath = File(projectPath).copyToDirectory(tmpDir) @@ -153,7 +153,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { } "generate js file correctly for a EJS template file" in { - val projectPath = getClass.getResource("/ejs").getPath + val projectPath = getClass.getResource("/ejs").getFile File.usingTemporaryDirectory() { tmpDir: File => val tmpProjectPath = File(projectPath).copyToDirectory(tmpDir) @@ -172,7 +172,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { } "generate js file correctly for a pug template file" in { - val projectPath = getClass.getResource("/pug").getPath + val projectPath = getClass.getResource("/pug").getFile File.usingTemporaryDirectory() { tmpDir: File => val tmpProjectPath = File(projectPath).copyToDirectory(tmpDir) From 92e9ff5835a12928d27db096a3ccc742fcaaffb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leutha=CC=88user?= Date: Thu, 16 Sep 2021 07:42:58 +0200 Subject: [PATCH 44/56] Another try. --- .../shiftleft/js2cpg/io/PrivateModulesTest.scala | 8 ++++---- .../preprocessing/TranspilationRunnerTest.scala | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/test/scala/io/shiftleft/js2cpg/io/PrivateModulesTest.scala b/src/test/scala/io/shiftleft/js2cpg/io/PrivateModulesTest.scala index a4c09b012..b88bf0117 100644 --- a/src/test/scala/io/shiftleft/js2cpg/io/PrivateModulesTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/io/PrivateModulesTest.scala @@ -22,7 +22,7 @@ class PrivateModulesTest extends AnyWordSpec with Matchers { "Handling for private modules" should { "copy and generate js files correctly for a simple project" in { - val projectPath = getClass.getResource("/privatemodules").getFile + val projectPath = getClass.getResource("/privatemodules").toURI File.usingTemporaryDirectory() { tmpDir: File => val tmpProjectPath = File(projectPath).copyToDirectory(tmpDir) @@ -40,7 +40,7 @@ class PrivateModulesTest extends AnyWordSpec with Matchers { } "copy and generate js files correctly for a simple project with additional private modules" in { - val projectPath = getClass.getResource("/privatemodules").getFile + val projectPath = getClass.getResource("/privatemodules").toURI File.usingTemporaryDirectory() { tmpDir: File => val tmpProjectPath = File(projectPath).copyToDirectory(tmpDir) @@ -67,7 +67,7 @@ class PrivateModulesTest extends AnyWordSpec with Matchers { } "copy and generate js files correctly for a simple project with filter" in { - val projectPath = getClass.getResource("/privatemodules").getFile + val projectPath = getClass.getResource("/privatemodules").toURI File.usingTemporaryDirectory() { tmpDir: File => val tmpProjectPath = File(projectPath).copyToDirectory(tmpDir) @@ -91,7 +91,7 @@ class PrivateModulesTest extends AnyWordSpec with Matchers { } "copy and generate js files correctly for a simple project with no private module being references" in { - val projectPath = getClass.getResource("/ignoreprivatemodules").getFile + val projectPath = getClass.getResource("/ignoreprivatemodules").toURI File.usingTemporaryDirectory() { tmpDir: File => val tmpProjectPath = File(projectPath).copyToDirectory(tmpDir) diff --git a/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala b/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala index 0c535c242..a49fdfe27 100644 --- a/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala @@ -34,7 +34,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { "TranspilationRunner" should { "generate js files correctly for a simple Babel project" in { - val projectPath = getClass.getResource("/babel").getFile + val projectPath = getClass.getResource("/babel").toURI File.usingTemporaryDirectory() { tmpDir: File => File.usingTemporaryDirectory() { transpileOutDir: File => val tmpProjectPath = File(projectPath).copyToDirectory(tmpDir) @@ -55,7 +55,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { } "generate and use sourcemap files correctly" in { - val projectPath = getClass.getResource("/typescript").getFile + val projectPath = getClass.getResource("/typescript").toURI File.usingTemporaryDirectory() { tmpDir: File => val tmpProjectPath = File(projectPath).copyToDirectory(tmpDir) @@ -73,7 +73,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { } "generate js files correctly for a simple Typescript project" in { - val projectPath = getClass.getResource("/typescript").getFile + val projectPath = getClass.getResource("/typescript").toURI File.usingTemporaryDirectory() { tmpDir: File => File.usingTemporaryDirectory() { transpileOutDir: File => val tmpProjectPath = File(projectPath).copyToDirectory(tmpDir) @@ -115,7 +115,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { } "generate js files correctly for a simple Typescript project including test files" in { - val projectPath = getClass.getResource("/typescript").getFile + val projectPath = getClass.getResource("/typescript").toURI File.usingTemporaryDirectory() { tmpDir: File => val tmpProjectPath = File(projectPath).copyToDirectory(tmpDir) @@ -136,7 +136,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { } "generate js files correctly for a simple Vue.js project" in { - val projectPath = getClass.getResource("/vue").getFile + val projectPath = getClass.getResource("/vue").toURI File.usingTemporaryDirectory() { tmpDir: File => val tmpProjectPath = File(projectPath).copyToDirectory(tmpDir) @@ -153,7 +153,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { } "generate js file correctly for a EJS template file" in { - val projectPath = getClass.getResource("/ejs").getFile + val projectPath = getClass.getResource("/ejs").toURI File.usingTemporaryDirectory() { tmpDir: File => val tmpProjectPath = File(projectPath).copyToDirectory(tmpDir) @@ -172,7 +172,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { } "generate js file correctly for a pug template file" in { - val projectPath = getClass.getResource("/pug").getFile + val projectPath = getClass.getResource("/pug").toURI File.usingTemporaryDirectory() { tmpDir: File => val tmpProjectPath = File(projectPath).copyToDirectory(tmpDir) From 1dd969b2390c7d11415a4748eeb0a42f0ed19aac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= Date: Thu, 16 Sep 2021 08:26:08 +0200 Subject: [PATCH 45/56] Another try. --- build.sbt | 2 +- .../js2cpg/io/EmScriptenCleaner.scala | 2 +- .../js2cpg/helpers/TestHelpers.scala | 30 ------------------- .../js2cpg/io/EmScriptenCleanerTest.scala | 10 +++---- .../js2cpg/io/PrivateModulesTest.scala | 21 ++++++++----- .../TranspilationRunnerTest.scala | 9 +++--- 6 files changed, 25 insertions(+), 49 deletions(-) delete mode 100644 src/test/scala/io/shiftleft/js2cpg/helpers/TestHelpers.scala diff --git a/build.sbt b/build.sbt index a6ec9e5e3..3fde1b5d5 100644 --- a/build.sbt +++ b/build.sbt @@ -1,4 +1,4 @@ -val cpgVersion = "1.3.304" +val cpgVersion = "1.3.314" val gitCommitString = SettingKey[String]("gitSha") diff --git a/src/main/scala/io/shiftleft/js2cpg/io/EmScriptenCleaner.scala b/src/main/scala/io/shiftleft/js2cpg/io/EmScriptenCleaner.scala index 735054e75..dc7ce0651 100644 --- a/src/main/scala/io/shiftleft/js2cpg/io/EmScriptenCleaner.scala +++ b/src/main/scala/io/shiftleft/js2cpg/io/EmScriptenCleaner.scala @@ -14,7 +14,7 @@ object EmScriptenCleaner { val endIndex = lines.indexWhere(EMSCRIPTEN_END_FUNCS.matches) if (startIndex != -1 && endIndex != -1 && endIndex > startIndex) { (lines.slice(0, startIndex) ++ - Seq.fill(endIndex - startIndex - 1)("\n") ++ + Seq.fill(endIndex - startIndex - 1)(System.lineSeparator()) ++ lines.slice(endIndex + 1, lines.length)).iterator } else { lines.iterator diff --git a/src/test/scala/io/shiftleft/js2cpg/helpers/TestHelpers.scala b/src/test/scala/io/shiftleft/js2cpg/helpers/TestHelpers.scala deleted file mode 100644 index ff38938fb..000000000 --- a/src/test/scala/io/shiftleft/js2cpg/helpers/TestHelpers.scala +++ /dev/null @@ -1,30 +0,0 @@ -package io.shiftleft.js2cpg.helpers - -import com.oracle.js.parser.Source -import com.oracle.js.parser.ir.FunctionNode -import io.shiftleft.js2cpg.parser.{JavaScriptParser, JsSource} -import io.shiftleft.js2cpg.util.SourceWrapper._ - -import scala.util.Using - -// Put operations that are shared across different test cases here -object TestHelpers { - - private def parseInternal(src: JsSource): (JsSource, FunctionNode) = - JavaScriptParser.parseFromSource(src).swap - - def parse(code: String): (JsSource, FunctionNode) = { - val src = Source.sourceFor("test", code).toJsSource() - parseInternal(src) - } - - def parseFromUrl(fileUrl: String): (JsSource, FunctionNode) = { - Using.resource(scala.io.Source.fromFile(fileUrl)) { content => - val code = content.getLines().mkString("\n") - val src = Source.sourceFor(fileUrl, code).toJsSource() - parseInternal(src) - } - - } - -} diff --git a/src/test/scala/io/shiftleft/js2cpg/io/EmScriptenCleanerTest.scala b/src/test/scala/io/shiftleft/js2cpg/io/EmScriptenCleanerTest.scala index 2954fbded..b54193187 100644 --- a/src/test/scala/io/shiftleft/js2cpg/io/EmScriptenCleanerTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/io/EmScriptenCleanerTest.scala @@ -11,7 +11,7 @@ class EmScriptenCleanerTest extends AnyWordSpec with Matchers { """ |console.log("Hello"); |console.log("World!");""".stripMargin - val result = EmScriptenCleaner.clean(code.linesIterator).mkString("\n") + val result = EmScriptenCleaner.clean(code.linesIterator).mkString(System.lineSeparator()) result shouldBe code } @@ -21,7 +21,7 @@ class EmScriptenCleanerTest extends AnyWordSpec with Matchers { |// EMSCRIPTEN_START_FUNCS |console.log("Hello"); |console.log("World!");""".stripMargin - val result = EmScriptenCleaner.clean(code.linesIterator).mkString("\n") + val result = EmScriptenCleaner.clean(code.linesIterator).mkString(System.lineSeparator()) result shouldBe code } @@ -31,7 +31,7 @@ class EmScriptenCleanerTest extends AnyWordSpec with Matchers { |console.log("Hello"); |console.log("World!"); |// EMSCRIPTEN_END_FUNCS""".stripMargin - val result = EmScriptenCleaner.clean(code.linesIterator).mkString("\n") + val result = EmScriptenCleaner.clean(code.linesIterator).mkString(System.lineSeparator()) result shouldBe code } @@ -42,7 +42,7 @@ class EmScriptenCleanerTest extends AnyWordSpec with Matchers { |console.log("Hello"); |console.log("World!"); |// EMSCRIPTEN_START_FUNCS""".stripMargin - val result = EmScriptenCleaner.clean(code.linesIterator).mkString("\n") + val result = EmScriptenCleaner.clean(code.linesIterator).mkString(System.lineSeparator()) result shouldBe code } @@ -66,7 +66,7 @@ class EmScriptenCleanerTest extends AnyWordSpec with Matchers { | |otherCode(); |foobar();""".stripMargin - val result = EmScriptenCleaner.clean(code.linesIterator).mkString("\n") + val result = EmScriptenCleaner.clean(code.linesIterator).mkString(System.lineSeparator()) result shouldBe expected } } diff --git a/src/test/scala/io/shiftleft/js2cpg/io/PrivateModulesTest.scala b/src/test/scala/io/shiftleft/js2cpg/io/PrivateModulesTest.scala index b88bf0117..72a93c56c 100644 --- a/src/test/scala/io/shiftleft/js2cpg/io/PrivateModulesTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/io/PrivateModulesTest.scala @@ -3,13 +3,15 @@ package io.shiftleft.js2cpg.io import better.files.File import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.cpgloading.{CpgLoader, CpgLoaderConfig} -import io.shiftleft.codepropertygraph.generated.{PropertyNames, NodeTypes} +import io.shiftleft.codepropertygraph.generated.{NodeTypes, PropertyNames} import io.shiftleft.js2cpg.core.Js2CpgMain import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec import overflowdb._ import overflowdb.traversal.TraversalSource +import java.util.regex.Pattern + class PrivateModulesTest extends AnyWordSpec with Matchers { private def fileNames(cpg: Cpg): List[String] = { @@ -35,7 +37,8 @@ class PrivateModulesTest extends AnyWordSpec with Matchers { CpgLoaderConfig.withDefaults.withOverflowConfig( Config.withDefaults.withStorageLocation(cpgPath))) - fileNames(cpg) should contain allElementsOf Set("@privateA/a.js", "@privateB/b.js") + fileNames(cpg) should contain allElementsOf Set(s"@privateA${java.io.File.separator}a.js", + s"@privateB${java.io.File.separator}b.js") } } @@ -59,10 +62,12 @@ class PrivateModulesTest extends AnyWordSpec with Matchers { CpgLoaderConfig.withDefaults.withOverflowConfig( Config.withDefaults.withStorageLocation(cpgPath))) - fileNames(cpg) should contain allElementsOf Set("@privateA/a.js", - "@privateB/b.js", - "@privateC/c.js", - "privateD/d.js") + fileNames(cpg) should contain allElementsOf Set( + s"@privateA${java.io.File.separator}a.js", + s"@privateB${java.io.File.separator}b.js", + s"@privateC${java.io.File.separator}c.js", + s"privateD${java.io.File.separator}d.js" + ) } } @@ -78,7 +83,7 @@ class PrivateModulesTest extends AnyWordSpec with Matchers { cpgPath, "--no-babel", "--exclude-regex", - ".*@privateA/a.js")) + s".*@privateA${Pattern.quote(java.io.File.separator)}a.js")) val cpg = CpgLoader @@ -86,7 +91,7 @@ class PrivateModulesTest extends AnyWordSpec with Matchers { CpgLoaderConfig.withDefaults.withOverflowConfig( Config.withDefaults.withStorageLocation(cpgPath))) - fileNames(cpg) should contain only "@privateB/b.js" + fileNames(cpg) should contain only s"@privateB${java.io.File.separator}b.js" } } diff --git a/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala b/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala index a49fdfe27..79e419300 100644 --- a/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala @@ -106,7 +106,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { jsFilesAfterTranspilation.map( f => File(f._1).contentAsString - .split("\n") + .split(System.lineSeparator()) .head // we ignore the sourcemap reference comment here .mkString .stripLineEnd)) shouldBe "console.log(\"Hello World!\");" @@ -130,8 +130,8 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { Config.withDefaults.withStorageLocation(cpgPath))) fileNames(cpg) should contain allElementsOf Set("a.ts", "b.ts", - "tests/a.test.ts", - "tests/b.spec.ts") + s"tests${java.io.File.separator}a.test.ts", + s"tests${java.io.File.separator}b.spec.ts") } } @@ -148,7 +148,8 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { .loadFromOverflowDb( CpgLoaderConfig.withDefaults.withOverflowConfig( Config.withDefaults.withStorageLocation(cpgPath))) - fileNames(cpg) should contain allElementsOf Set("src/main.js", "src/App.vue") + fileNames(cpg) should contain allElementsOf Set(s"src${java.io.File.separator}main.js", + s"src${java.io.File.separator}App.vue") } } From 1406e4ad1592518007666b88f0dca41ac4c9209c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= Date: Thu, 16 Sep 2021 08:29:42 +0200 Subject: [PATCH 46/56] Another try. --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 3fde1b5d5..a6ec9e5e3 100644 --- a/build.sbt +++ b/build.sbt @@ -1,4 +1,4 @@ -val cpgVersion = "1.3.314" +val cpgVersion = "1.3.304" val gitCommitString = SettingKey[String]("gitSha") From 21e184cd016ad9a7bb8dce7006d74286f814bff8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= Date: Thu, 16 Sep 2021 08:48:46 +0200 Subject: [PATCH 47/56] Another try. --- .../io/shiftleft/js2cpg/io/ExcludeTest.scala | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/test/scala/io/shiftleft/js2cpg/io/ExcludeTest.scala b/src/test/scala/io/shiftleft/js2cpg/io/ExcludeTest.scala index 21bed7176..220293d21 100644 --- a/src/test/scala/io/shiftleft/js2cpg/io/ExcludeTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/io/ExcludeTest.scala @@ -3,14 +3,16 @@ package io.shiftleft.js2cpg.io import better.files.File import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.cpgloading.{CpgLoader, CpgLoaderConfig} -import io.shiftleft.codepropertygraph.generated.{PropertyNames, NodeTypes} -import io.shiftleft.js2cpg.core.{Js2CpgMain, Js2cpgArgumentsParser} +import io.shiftleft.codepropertygraph.generated.{NodeTypes, PropertyNames} +import io.shiftleft.js2cpg.core.{Js2cpgArgumentsParser, Js2CpgMain} import org.scalatest.matchers.should.Matchers import org.scalatest.prop.TableDrivenPropertyChecks import org.scalatest.wordspec.AnyWordSpec import overflowdb._ import overflowdb.traversal.TraversalSource +import java.util.regex.Pattern + object ExcludeTest { private implicit class ToArg(val arg: String) extends AnyVal { def toArg: String = s"--$arg" @@ -22,7 +24,7 @@ class ExcludeTest extends AnyWordSpec with Matchers with TableDrivenPropertyChec import ExcludeTest._ import Js2cpgArgumentsParser._ - private val projectUnderTestPath: String = getClass.getResource("/excludes").getPath + private val projectUnderTestPath = File(getClass.getResource("/excludes").toURI).pathAsString private def fileNames(cpg: Cpg): List[String] = TraversalSource(cpg.graph).label(NodeTypes.FILE).property(PropertyNames.NAME).toList @@ -44,7 +46,8 @@ class ExcludeTest extends AnyWordSpec with Matchers with TableDrivenPropertyChec CpgLoaderConfig.withDefaults.withOverflowConfig( Config.withDefaults.withStorageLocation(cpgPath))) - fileNames(cpg) should contain theSameElementsAs expectedFiles + fileNames(cpg) should contain theSameElementsAs expectedFiles.map( + _.replace("/", java.io.File.separator)) } } @@ -90,7 +93,9 @@ class ExcludeTest extends AnyWordSpec with Matchers with TableDrivenPropertyChec Seq(EXCLUDE_REGEX.toArg, ".*(index|b)\\..*"), Set("a.js", "folder/c.js", "foo.bar/d.js")), (s"exclude a complete folder with ${EXCLUDE_REGEX.toArg}", - Seq(EXCLUDE_REGEX.toArg, ".*/folder/.*"), + Seq( + EXCLUDE_REGEX.toArg, + s".*${Pattern.quote(java.io.File.separator)}folder${Pattern.quote(java.io.File.separator)}.*"), Set("index.js", "a.js", "foo.bar/d.js")), // -- // Tests for mixed arguments From 104c5323ca075185e9e92a743e04fc2a5a3636a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= Date: Thu, 16 Sep 2021 09:11:26 +0200 Subject: [PATCH 48/56] Another try. --- src/main/scala/io/shiftleft/js2cpg/io/FileUtils.scala | 6 +++--- src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/main/scala/io/shiftleft/js2cpg/io/FileUtils.scala b/src/main/scala/io/shiftleft/js2cpg/io/FileUtils.scala index da0af5c73..2d227f48d 100644 --- a/src/main/scala/io/shiftleft/js2cpg/io/FileUtils.scala +++ b/src/main/scala/io/shiftleft/js2cpg/io/FileUtils.scala @@ -5,14 +5,14 @@ import better.files.File import java.io.Reader import java.math.BigInteger import java.nio.charset.{CharsetDecoder, CodingErrorAction} -import java.nio.file.{FileVisitResult, Files, Path, SimpleFileVisitor} +import java.nio.file.{Files, FileVisitResult, Path, SimpleFileVisitor} import io.shiftleft.js2cpg.core.Config import io.shiftleft.js2cpg.io.FileDefaults._ import org.slf4j.LoggerFactory import java.nio.file.attribute.BasicFileAttributes import scala.collection.concurrent.TrieMap -import scala.collection.{SortedMap, mutable} +import scala.collection.{mutable, SortedMap} import scala.io.{BufferedSource, Codec, Source} import scala.jdk.CollectionConverters._ @@ -33,7 +33,7 @@ object FileUtils { /** * Cleans the given path as String and removes unwanted elements that - * occure during transpilation on the Windows platform and/or CI environments. + * occur during transpilation on the Windows platform and/or CI environments. */ def cleanPath(sourceFileName: String): String = { val replaceDots = sourceFileName.replace("../", "") diff --git a/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala b/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala index 1d9e5bef9..effdfba62 100644 --- a/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala +++ b/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala @@ -95,9 +95,8 @@ class JsSource(val srcDir: File, val projectDir: Path, val source: Source) { // on MacOS the path to the tmp dir is already there File("/") / replacedName } else { - // on all other OS we have to prepend it - if (srcDir.pathAsString.startsWith(tmpDir)) File(tmpDir) / replacedName - else File("/") / replacedName + // on all other OS we are done + srcDir.parent / sourceFileName } srcFilePath } From 72c5238bbe90c39a56db627f37afdef328447870 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= Date: Thu, 16 Sep 2021 09:17:40 +0200 Subject: [PATCH 49/56] Another try. --- src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala b/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala index effdfba62..58c0ce913 100644 --- a/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala +++ b/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala @@ -95,8 +95,9 @@ class JsSource(val srcDir: File, val projectDir: Path, val source: Source) { // on MacOS the path to the tmp dir is already there File("/") / replacedName } else { - // on all other OS we are done - srcDir.parent / sourceFileName + // on all other OS we have to prepend it + if (srcDir.pathAsString.startsWith(tmpDir)) File(tmpDir) / replacedName + else srcDir.parent / sourceFileName } srcFilePath } From 78f1e8ccaed427b4171f3a1f814d17c203338c57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= Date: Thu, 16 Sep 2021 09:19:57 +0200 Subject: [PATCH 50/56] Another try. --- .../scala/io/shiftleft/js2cpg/parser/JavaScriptParser.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/io/shiftleft/js2cpg/parser/JavaScriptParser.scala b/src/main/scala/io/shiftleft/js2cpg/parser/JavaScriptParser.scala index fa4f87664..2a7e136a6 100644 --- a/src/main/scala/io/shiftleft/js2cpg/parser/JavaScriptParser.scala +++ b/src/main/scala/io/shiftleft/js2cpg/parser/JavaScriptParser.scala @@ -81,7 +81,7 @@ object JavaScriptParser { val lines = jsSource.source.getContent.toString.linesIterator.toSeq val replaceIndex = lines.lastIndexWhere(l => importRegex.matches(l.trim())) + 1 val (head, rest) = lines.splitAt(replaceIndex) - val fixedCode = (head ++ nodeJsFixWrapper(rest)).mkString("\n") + val fixedCode = (head ++ nodeJsFixWrapper(rest)).mkString(System.lineSeparator()) val source = Source.sourceFor(jsSource.filePath, fixedCode) source.toJsSource(jsSource.srcDir, jsSource.projectDir) } From 5bc02e0679d57b555de07b9aee72a066bb2232a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= Date: Thu, 16 Sep 2021 09:22:29 +0200 Subject: [PATCH 51/56] Another try. --- src/main/scala/io/shiftleft/js2cpg/core/Config.scala | 2 +- .../scala/io/shiftleft/js2cpg/preprocessing/EjsTranspiler.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/io/shiftleft/js2cpg/core/Config.scala b/src/main/scala/io/shiftleft/js2cpg/core/Config.scala index 9c3c2f83a..9e1a0afdf 100644 --- a/src/main/scala/io/shiftleft/js2cpg/core/Config.scala +++ b/src/main/scala/io/shiftleft/js2cpg/core/Config.scala @@ -74,7 +74,7 @@ case class Config(srcDir: String = "", val slIngoreFilePath = Paths.get(srcDir, Config.SL_IGNORE_FILE) val result = Using(FileUtils.bufferedSourceFromFile(slIngoreFilePath)) { bufferedSource => val content = FileUtils.contentFromBufferedSource(bufferedSource) - content.split("\n").toSeq.map(createPathForIgnore) + content.split(System.lineSeparator()).toSeq.map(createPathForIgnore) } result match { diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/EjsTranspiler.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/EjsTranspiler.scala index 7341792d5..0b3bc694d 100644 --- a/src/main/scala/io/shiftleft/js2cpg/preprocessing/EjsTranspiler.scala +++ b/src/main/scala/io/shiftleft/js2cpg/preprocessing/EjsTranspiler.scala @@ -81,7 +81,7 @@ class EjsTranspiler(override val config: Config, override val projectPath: Path) result.append(extractedJsCode) } } - (result.mkString("\n"), sourceMap) + (result.mkString(System.lineSeparator()), sourceMap) } override def shouldRun(): Boolean = config.templateTranspiling && ejsFiles.nonEmpty From 31e2b5748f0965910a2dd72aa8ac0789d6ca7df2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= Date: Thu, 16 Sep 2021 09:52:16 +0200 Subject: [PATCH 52/56] Another try. --- src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala b/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala index 58c0ce913..de1a1d66a 100644 --- a/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala +++ b/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala @@ -93,11 +93,10 @@ class JsSource(val srcDir: File, val projectDir: Path, val source: Source) { val replacedName = FileUtils.cleanPath(sourceFileName) val srcFilePath: File = if (replacedName.contains(tmpDir)) { // on MacOS the path to the tmp dir is already there - File("/") / replacedName + File(replacedName) } else { // on all other OS we have to prepend it - if (srcDir.pathAsString.startsWith(tmpDir)) File(tmpDir) / replacedName - else srcDir.parent / sourceFileName + File(tmpDir) / replacedName } srcFilePath } From d9457fbd213ea23176d6f59a728df54a838f5de7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= Date: Thu, 16 Sep 2021 10:09:44 +0200 Subject: [PATCH 53/56] Another try. --- .github/workflows/pr.yml | 3 +-- src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala | 8 ++++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 03f6f317e..b657e0215 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -5,7 +5,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04, windows-2019, macos-10.15] + os: [ubuntu-20.04, windows-2019, macos-10.15, macos-11] steps: - uses: actions/checkout@v2 with: @@ -23,4 +23,3 @@ jobs: - name: Compile and run tests run: sbt -v -Dfile.encoding=UTF-8 "clean; +test;" shell: bash - diff --git a/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala b/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala index de1a1d66a..413f245f4 100644 --- a/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala +++ b/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala @@ -95,8 +95,12 @@ class JsSource(val srcDir: File, val projectDir: Path, val source: Source) { // on MacOS the path to the tmp dir is already there File(replacedName) } else { - // on all other OS we have to prepend it - File(tmpDir) / replacedName + // on all other OS we have to prepend it with special handling for Windows: + if (replacedName.contains("AppData\\Local\\Temp")) { + File("/Users") / replacedName + } else { + File(tmpDir) / replacedName + } } srcFilePath } From 0353b3edead24121163ed162e33261ec524af589 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= Date: Thu, 16 Sep 2021 10:25:25 +0200 Subject: [PATCH 54/56] Another try. --- src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala b/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala index 413f245f4..ca45f2832 100644 --- a/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala +++ b/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala @@ -93,10 +93,10 @@ class JsSource(val srcDir: File, val projectDir: Path, val source: Source) { val replacedName = FileUtils.cleanPath(sourceFileName) val srcFilePath: File = if (replacedName.contains(tmpDir)) { // on MacOS the path to the tmp dir is already there - File(replacedName) + File("/") / replacedName } else { // on all other OS we have to prepend it with special handling for Windows: - if (replacedName.contains("AppData\\Local\\Temp")) { + if (replacedName.contains("AppData/Local/Temp")) { File("/Users") / replacedName } else { File(tmpDir) / replacedName From 32ae5a5a9befb253c1c6dfce90024e1aaec4d914 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= Date: Thu, 16 Sep 2021 10:28:20 +0200 Subject: [PATCH 55/56] For debugging. --- src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala b/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala index ca45f2832..8b90dd356 100644 --- a/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala +++ b/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala @@ -91,6 +91,11 @@ class JsSource(val srcDir: File, val projectDir: Path, val source: Source) { srcDir / replacedName case _ => val replacedName = FileUtils.cleanPath(sourceFileName) + logger.debug(s""" + |>>> constructSourceFilePath: + |TMP: '$tmpDir' + |replacedName: '$replacedName' + |""".stripMargin) val srcFilePath: File = if (replacedName.contains(tmpDir)) { // on MacOS the path to the tmp dir is already there File("/") / replacedName From 9073afde22428ae70a48170a399869a08e1a06e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= Date: Thu, 16 Sep 2021 10:45:53 +0200 Subject: [PATCH 56/56] Another try. --- src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala b/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala index 8b90dd356..24642e3b6 100644 --- a/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala +++ b/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala @@ -91,18 +91,13 @@ class JsSource(val srcDir: File, val projectDir: Path, val source: Source) { srcDir / replacedName case _ => val replacedName = FileUtils.cleanPath(sourceFileName) - logger.debug(s""" - |>>> constructSourceFilePath: - |TMP: '$tmpDir' - |replacedName: '$replacedName' - |""".stripMargin) val srcFilePath: File = if (replacedName.contains(tmpDir)) { // on MacOS the path to the tmp dir is already there File("/") / replacedName } else { // on all other OS we have to prepend it with special handling for Windows: if (replacedName.contains("AppData/Local/Temp")) { - File("/Users") / replacedName + srcDir.root / "Users" / replacedName } else { File(tmpDir) / replacedName }