Run tests in parallel

.github/workflows/test.yml

@@ -1,10 +1,6 @@
 name: JFrog CLI Security Tests
 on:
   push:
-    branches:
-      - '**'
-    tags-ignore:
-      - '**'
   # Triggers the workflow on labeled PRs only.
   pull_request_target:
     types: [ labeled ]
@@ -13,17 +9,71 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.ref }}
   cancel-in-progress: true
 jobs:
-  test:
+  Pretest:
     if: contains(github.event.pull_request.labels.*.name, 'safe to test') || github.event_name == 'push'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Remove 'safe to test' label
+        uses: actions-ecosystem/action-remove-labels@v1
+        if: ${{ github.event_name != 'push' }}
+        with:
+          labels: "safe to test"
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Install Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.21.x
+
+      - name: Go Cache
+        uses: actions/cache@v3
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ runner.os }}-go-
+
+      - name: Check code compilation
+        run: go vet -v ./...
+
+  tests:
+    needs: Pretest
+    name: ${{ matrix.suite.name }} Tests (${{ matrix.os }})
     runs-on: ${{ matrix.os }}-latest
+    env:
+      JFROG_CLI_LOG_LEVEL: "DEBUG"
+      GOPROXY: direct
+      GRADLE_OPTS: -Dorg.gradle.daemon=false
     strategy:
       fail-fast: false
       matrix:
+        suite:
+        - name: 'Unit'
+          testFlags: '--test.unit'
+
+        - name: 'Artifactory Integration'
+          testFlags: '--test.artifactory --test.unit=false'
+
+        - name: 'Xray Integration'
+          testFlags: '--test.xray --test.unit=false'
+
+        - name: 'Audit'
+          testFlags: '--test.audit --test.unit=false'
+
+        - name: 'Scans'
+          testFlags: '--test.scan --test.unit=false'
+          ubuntuTestFlags: '--test.scan --test.dockerScan --test.unit=false'
+
         os: [ ubuntu, windows, macos ]
-    env:
-      GOPROXY: direct
-      GRADLE_OPTS: -Dorg.gradle.daemon=false
     steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
       # Install dependencies
       - name: Install Go
         uses: actions/setup-go@v3
@@ -62,15 +112,7 @@ jobs:
         uses: gradle/gradle-build-action@v2
         with:
           gradle-version: 7.6
-      # Checkout code
-      - name: Checkout code
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
+
       # Test
-      - name: Run security tests (without Docker Scan)
-        run: go test -v github.com/jfrog/jfrog-cli-security --timeout 0 --test.security --jfrog.url=${{ secrets.PLATFORM_URL }} --jfrog.adminToken=${{ secrets.PLATFORM_ADMIN_TOKEN }} --jfrog.user=${{ secrets.PLATFORM_USER }} --test.containerRegistry=${{ secrets.CONTAINER_REGISTRY }} --ci.runId=${{ runner.os }}-xray
-        if: ${{ matrix.os != 'ubuntu' }}
-      - name: Run security tests (with Docker Scan, only on Ubuntu)
-        run: go test -v github.com/jfrog/jfrog-cli-security --timeout 0 --test.security --test.dockerScan --jfrog.url=${{ secrets.PLATFORM_URL }} --jfrog.adminToken=${{ secrets.PLATFORM_ADMIN_TOKEN }} --test.containerRegistry=${{ secrets.CONTAINER_REGISTRY }} --ci.runId=${{ runner.os }}-xray
-        if: ${{ matrix.os == 'ubuntu' }}
+      - name: Run Tests
+        run: go test -v github.com/jfrog/jfrog-cli-security ${{ matrix.os == 'ubuntu' && matrix.suite.ubuntuTestFlags || matrix.suite.testFlags }} --jfrog.url=${{ secrets.PLATFORM_URL }} --jfrog.adminToken=${{ secrets.PLATFORM_ADMIN_TOKEN }} --jfrog.user=${{ secrets.PLATFORM_USER }} --test.containerRegistry=${{ secrets.CONTAINER_REGISTRY }} --ci.runId=${{ runner.os }}-xray -v -race --timeout 30m -cover

CONTRIBUTING.md

@@ -71,10 +71,15 @@ go test -v github.com/jfrog/jfrog-cli-security [test-types] [flags]
 
 ### The available test types are:
 
-| Type                 | Description        |
-| -------------------- | ------------------ |
-| `-test.security`     | [Default: true] Security commands integration tests  |
-| `-test.dockerScan`   | [Optional] Docker scan integration tests  |
+| Type                 | Description                                 |
+| -------------------- | ------------------------------------------- |
+| `-test.unit`         | [Default: true] Unit tests                  |
+| `-test.artifactory`  | [Optional] Artifactory integration tests    |
+| `-test.xray`         | [Optional] Xray commands integration tests  |
+| `-test.audit`        | [Optional] Audit command integration tests  |
+| `-test.scan`         | [Optional] Scan commands integration tests  |
+| `-test.dockerScan`   | [Optional] Docker scan integration tests    |
+| `-test.all`          | [Optional] Activate all available tests     |
 
 ### Docker Scan tests
 

artifactory_test.go

@@ -2,12 +2,13 @@ package main
 
 import (
 	"errors"
-	"github.com/stretchr/testify/require"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/jfrog/jfrog-cli-core/v2/utils/dependencies"
 	"github.com/jfrog/jfrog-client-go/utils/io/fileutils"
 
@@ -33,6 +34,7 @@ import (
 // We perform validation on dependency resolution from an Artifactory server during the construction of the dependency tree during 'audit' flow.
 // This process involves resolving all dependencies required by the project.
 func TestDependencyResolutionFromArtifactory(t *testing.T) {
+	securityTestUtils.InitArtifactoryTest(t)
 	testCases := []struct {
 		testProjectPath []string
 		resolveRepoName string
@@ -208,6 +210,7 @@ func clearOrRedirectLocalCacheIfNeeded(t *testing.T, projectType project.Project
 }
 
 func TestDownloadAnalyzerManagerIfNeeded(t *testing.T) {
+	securityTestUtils.InitArtifactoryTest(t)
 	// Configure a new JFrog CLI home dir.
 	tempDirPath, createTempDirCallback := coreTests.CreateTempDirWithCallbackAndAssert(t)
 	defer createTempDirCallback()

audit_test.go

@@ -35,7 +35,7 @@ func TestXrayAuditNpmSimpleJson(t *testing.T) {
 }
 
 func testXrayAuditNpm(t *testing.T, format string) string {
-	securityTestUtils.InitSecurityTest(t, scangraph.GraphScanMinXrayVersion)
+	securityTestUtils.InitAuditTest(t, scangraph.GraphScanMinXrayVersion)
 	tempDirPath, createTempDirCallback := coreTests.CreateTempDirWithCallbackAndAssert(t)
 	defer createTempDirCallback()
 	npmProjectPath := filepath.Join(filepath.FromSlash(securityTestUtils.GetTestResourcesPath()), "projects", "package-managers", "npm", "npm")
@@ -61,7 +61,7 @@ func TestXrayAuditPnpmSimpleJson(t *testing.T) {
 }
 
 func testXrayAuditPnpm(t *testing.T, format string) string {
-	securityTestUtils.InitSecurityTest(t, scangraph.GraphScanMinXrayVersion)
+	securityTestUtils.InitAuditTest(t, scangraph.GraphScanMinXrayVersion)
 	tempDirPath, createTempDirCallback := coreTests.CreateTempDirWithCallbackAndAssert(t)
 	defer createTempDirCallback()
 	npmProjectPath := filepath.Join(filepath.FromSlash(securityTestUtils.GetTestResourcesPath()), "projects", "package-managers", "npm", "npm-no-lock")
@@ -117,7 +117,7 @@ func TestXrayAuditYarnV1SimpleJson(t *testing.T) {
 }
 
 func testXrayAuditYarn(t *testing.T, projectDirName string, yarnCmd func()) {
-	securityTestUtils.InitSecurityTest(t, scangraph.GraphScanMinXrayVersion)
+	securityTestUtils.InitAuditTest(t, scangraph.GraphScanMinXrayVersion)
 	tempDirPath, createTempDirCallback := coreTests.CreateTempDirWithCallbackAndAssert(t)
 	defer createTempDirCallback()
 	yarnProjectPath := filepath.Join(filepath.FromSlash(securityTestUtils.GetTestResourcesPath()), "projects", "package-managers", "yarn", projectDirName)
@@ -232,7 +232,7 @@ func TestXrayAuditNugetSimpleJson(t *testing.T) {
 }
 
 func testXrayAuditNuget(t *testing.T, projectName, format string, restoreTech string) string {
-	securityTestUtils.InitSecurityTest(t, scangraph.GraphScanMinXrayVersion)
+	securityTestUtils.InitAuditTest(t, scangraph.GraphScanMinXrayVersion)
 	tempDirPath, createTempDirCallback := coreTests.CreateTempDirWithCallbackAndAssert(t)
 	defer createTempDirCallback()
 	projectPath := filepath.Join(filepath.FromSlash(securityTestUtils.GetTestResourcesPath()), "projects", "package-managers", "nuget", projectName)
@@ -261,7 +261,7 @@ func TestXrayAuditGradleSimpleJson(t *testing.T) {
 }
 
 func testXrayAuditGradle(t *testing.T, format string) string {
-	securityTestUtils.InitSecurityTest(t, scangraph.GraphScanMinXrayVersion)
+	securityTestUtils.InitAuditTest(t, scangraph.GraphScanMinXrayVersion)
 	tempDirPath, createTempDirCallback := coreTests.CreateTempDirWithCallbackAndAssert(t)
 	defer createTempDirCallback()
 	gradleProjectPath := filepath.Join(filepath.FromSlash(securityTestUtils.GetTestResourcesPath()), "projects", "package-managers", "gradle", "gradle")
@@ -285,7 +285,7 @@ func TestXrayAuditMavenSimpleJson(t *testing.T) {
 }
 
 func testXrayAuditMaven(t *testing.T, format string) string {
-	securityTestUtils.InitSecurityTest(t, scangraph.GraphScanMinXrayVersion)
+	securityTestUtils.InitAuditTest(t, scangraph.GraphScanMinXrayVersion)
 	tempDirPath, createTempDirCallback := coreTests.CreateTempDirWithCallbackAndAssert(t)
 	defer createTempDirCallback()
 	mvnProjectPath := filepath.Join(filepath.FromSlash(securityTestUtils.GetTestResourcesPath()), "projects", "package-managers", "maven", "maven")
@@ -299,7 +299,7 @@ func testXrayAuditMaven(t *testing.T, format string) string {
 }
 
 func TestXrayAuditNoTech(t *testing.T) {
-	securityTestUtils.InitSecurityTest(t, scangraph.GraphScanMinXrayVersion)
+	securityTestUtils.InitAuditTest(t, scangraph.GraphScanMinXrayVersion)
 	tempDirPath, createTempDirCallback := coreTests.CreateTempDirWithCallbackAndAssert(t)
 	defer createTempDirCallback()
 	prevWd := securityTestUtils.ChangeWD(t, tempDirPath)
@@ -310,7 +310,7 @@ func TestXrayAuditNoTech(t *testing.T) {
 }
 
 func TestXrayAuditMultiProjects(t *testing.T) {
-	securityTestUtils.InitSecurityTest(t, scangraph.GraphScanMinXrayVersion)
+	securityTestUtils.InitAuditTest(t, scangraph.GraphScanMinXrayVersion)
 	tempDirPath, createTempDirCallback := coreTests.CreateTempDirWithCallbackAndAssert(t)
 	defer createTempDirCallback()
 	multiProject := filepath.Join(filepath.FromSlash(securityTestUtils.GetTestResourcesPath()), "projects")
@@ -350,7 +350,7 @@ func TestXrayAuditPipSimpleJsonWithRequirementsFile(t *testing.T) {
 }
 
 func testXrayAuditPip(t *testing.T, format, requirementsFile string) string {
-	securityTestUtils.InitSecurityTest(t, scangraph.GraphScanMinXrayVersion)
+	securityTestUtils.InitAuditTest(t, scangraph.GraphScanMinXrayVersion)
 	tempDirPath, createTempDirCallback := coreTests.CreateTempDirWithCallbackAndAssert(t)
 	defer createTempDirCallback()
 	pipProjectPath := filepath.Join(filepath.FromSlash(securityTestUtils.GetTestResourcesPath()), "projects", "package-managers", "python", "pip", "pip-project")
@@ -379,7 +379,7 @@ func TestXrayAuditPipenvSimpleJson(t *testing.T) {
 }
 
 func testXrayAuditPipenv(t *testing.T, format string) string {
-	securityTestUtils.InitSecurityTest(t, scangraph.GraphScanMinXrayVersion)
+	securityTestUtils.InitAuditTest(t, scangraph.GraphScanMinXrayVersion)
 	tempDirPath, createTempDirCallback := coreTests.CreateTempDirWithCallbackAndAssert(t)
 	defer createTempDirCallback()
 	pipenvProjectPath := filepath.Join(filepath.FromSlash(securityTestUtils.GetTestResourcesPath()), "projects", "package-managers", "python", "pipenv", "pipenv-project")
@@ -403,7 +403,7 @@ func TestXrayAuditPoetrySimpleJson(t *testing.T) {
 }
 
 func testXrayAuditPoetry(t *testing.T, format string) string {
-	securityTestUtils.InitSecurityTest(t, scangraph.GraphScanMinXrayVersion)
+	securityTestUtils.InitAuditTest(t, scangraph.GraphScanMinXrayVersion)
 	tempDirPath, createTempDirCallback := coreTests.CreateTempDirWithCallbackAndAssert(t)
 	defer createTempDirCallback()
 	poetryProjectPath := filepath.Join(filepath.FromSlash(securityTestUtils.GetTestResourcesPath()), "projects", "package-managers", "python", "poetry", "poetry-project")
@@ -445,7 +445,7 @@ func TestXrayAuditJasNoViolationsSimpleJson(t *testing.T) {
 }
 
 func testXrayAuditJas(t *testing.T, format string, project string) string {
-	securityTestUtils.InitSecurityTest(t, scangraph.GraphScanMinXrayVersion)
+	securityTestUtils.InitAuditTest(t, scangraph.GraphScanMinXrayVersion)
 	tempDirPath, createTempDirCallback := coreTests.CreateTempDirWithCallbackAndAssert(t)
 	defer createTempDirCallback()
 	projectDir := filepath.Join(filepath.FromSlash(securityTestUtils.GetTestResourcesPath()), filepath.Join("projects", project))
@@ -462,7 +462,7 @@ func testXrayAuditJas(t *testing.T, format string, project string) string {
 }
 
 func TestXrayAuditDetectTech(t *testing.T) {
-	securityTestUtils.InitSecurityTest(t, scangraph.GraphScanMinXrayVersion)
+	securityTestUtils.InitAuditTest(t, scangraph.GraphScanMinXrayVersion)
 	tempDirPath, createTempDirCallback := coreTests.CreateTempDirWithCallbackAndAssert(t)
 	defer createTempDirCallback()
 	mvnProjectPath := filepath.Join(filepath.FromSlash(securityTestUtils.GetTestResourcesPath()), "projects", "package-managers", "maven", "maven")
@@ -480,7 +480,7 @@ func TestXrayAuditDetectTech(t *testing.T) {
 }
 
 func TestXrayRecursiveScan(t *testing.T) {
-	securityTestUtils.InitSecurityTest(t, scangraph.GraphScanMinXrayVersion)
+	securityTestUtils.InitAuditTest(t, scangraph.GraphScanMinXrayVersion)
 	tempDirPath, createTempDirCallback := coreTests.CreateTempDirWithCallbackAndAssert(t)
 	defer createTempDirCallback()
 	projectDir := filepath.Join(filepath.FromSlash(securityTestUtils.GetTestResourcesPath()), "projects", "package-managers")

jfrogclisecurity_test.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"flag"
 	"fmt"
 
 	"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
@@ -17,13 +16,13 @@ import (
 )
 
 func TestMain(m *testing.M) {
-	setupIntegrationTests()
+	setupTests()
 	result := m.Run()
-	tearDownIntegrationTests()
+	tearDownTests()
 	os.Exit(result)
 }
 
-func setupIntegrationTests() {
+func setupTests() {
 	// Disable usage report.
 	if err := os.Setenv(coreutils.ReportUsage, "false"); err != nil {
 		clientLog.Error(fmt.Sprintf("Couldn't set env: %s. Error: %s", coreutils.ReportUsage, err.Error()))
@@ -35,15 +34,15 @@ func setupIntegrationTests() {
 		os.Exit(1)
 	}
 	// General
-	flag.Parse()
 	log.SetDefaultLogger()
-	// Init
+	configTests.InitTestFlags()
+	// Init Integration tests
 	utils.InitTestCliDetails()
 	utils.AuthenticateArtifactory()
 	utils.CreateRequiredRepositories()
 }
 
-func tearDownIntegrationTests() {
+func tearDownTests() {
 	// Important - Virtual repositories must be deleted first
 	utils.DeleteRepos(configTests.CreatedVirtualRepositories)
 	utils.DeleteRepos(configTests.CreatedNonVirtualRepositories)

scans_test.go

@@ -74,13 +74,13 @@ func TestXrayBinaryScanSimpleJsonWithProgress(t *testing.T) {
 }
 
 func testXrayBinaryScan(t *testing.T, format string) string {
-	securityTestUtils.InitSecurityTest(t, scangraph.GraphScanMinXrayVersion)
+	securityTestUtils.InitScanTest(t, scangraph.GraphScanMinXrayVersion)
 	binariesPath := filepath.Join(filepath.FromSlash(securityTestUtils.GetTestResourcesPath()), "projects", "binaries", "*")
 	return securityTests.PlatformCli.RunCliCmdWithOutput(t, "scan", binariesPath, "--licenses", "--format="+format)
 }
 
 func TestXrayBinaryScanWithBypassArchiveLimits(t *testing.T) {
-	securityTestUtils.InitSecurityTest(t, scan.BypassArchiveLimitsMinXrayVersion)
+	securityTestUtils.InitScanTest(t, scan.BypassArchiveLimitsMinXrayVersion)
 	unsetEnv := clientTestUtils.SetEnvWithCallbackAndAssert(t, "JF_INDEXER_COMPRESS_MAXENTITIES", "10")
 	defer unsetEnv()
 	binariesPath := filepath.Join(filepath.FromSlash(securityTestUtils.GetTestResourcesPath()), "projects", "binaries", "*")
@@ -131,11 +131,8 @@ func TestDockerScan(t *testing.T) {
 }
 
 func initNativeDockerWithXrayTest(t *testing.T) func() {
-	if !*securityTests.TestDockerScan || !*securityTests.TestSecurity {
-		t.Skip("Skipping Docker scan test. To run Xray Docker test add the '-test.dockerScan=true' and '-test.security=true' options.")
-	}
+	securityTestUtils.InitDockerScanTest(t, scan.DockerScanMinXrayVersion)
 	oldHomeDir := os.Getenv(coreutils.HomeDir)
-	securityTestUtils.ValidateXrayVersion(t, scan.DockerScanMinXrayVersion)
 	// Create server config to use with the command.
 	securityTestUtils.CreateJfrogHomeConfig(t, true)
 	// Add docker scan mock command
@@ -236,7 +233,7 @@ func createTestWatch(t *testing.T) (string, func()) {
 // Curation tests
 
 func TestCurationAudit(t *testing.T) {
-	securityTestUtils.InitSecurityTest(t, "")
+	securityTestUtils.InitScanTest(t, "")
 	tempDirPath, createTempDirCallback := coreTests.CreateTempDirWithCallbackAndAssert(t)
 	defer createTempDirCallback()
 	multiProject := filepath.Join(filepath.FromSlash(securityTestUtils.GetTestResourcesPath()), "projects", "package-managers", "npm")