test(cli-e2e): move helper functions, clean up tests and setup

e2e/cli-e2e/.eslintrc.json

@@ -0,0 +1,15 @@
+{
+  "extends": ["../../.eslintrc.json"],
+  "ignorePatterns": ["!**/*", "code-pushup.config.ts"],
+  "overrides": [
+    {
+      "files": ["*.ts", "*.tsx"],
+      "parserOptions": {
+        "project": ["e2e/cli-e2e/tsconfig.*?.json"]
+      },
+      "rules": {
+        "@nx/enforce-module-boundaries": "off"
+      }
+    }
+  ]
+}

e2e/cli-e2e/mocks/code-pushup.config.js → e2e/cli-e2e/mocks/fixtures/code-pushup.config.js

@@ -1,6 +1,5 @@
-// TODO: import plugins using NPM package names using local registry: https://github.com/flowup/quality-metrics-cli/issues/33
-import eslintPlugin from '../../../dist/packages/plugin-eslint';
-import lighthousePlugin from '../../../dist/packages/plugin-lighthouse';
+import eslintPlugin from '../../../../dist/packages/plugin-eslint';
+import lighthousePlugin from '../../../../dist/packages/plugin-lighthouse';
 
 export default {
   upload: {

e2e/cli-e2e/mocks/code-pushup.config.mjs → e2e/cli-e2e/mocks/fixtures/code-pushup.config.mjs

@@ -1,6 +1,5 @@
-// TODO: import plugins using NPM package names using local registry: https://github.com/flowup/quality-metrics-cli/issues/33
-import eslintPlugin from '../../../dist/packages/plugin-eslint';
-import lighthousePlugin from '../../../dist/packages/plugin-lighthouse';
+import eslintPlugin from '../../../../dist/packages/plugin-eslint';
+import lighthousePlugin from '../../../../dist/packages/plugin-lighthouse';
 
 export default {
   upload: {

e2e/cli-e2e/mocks/code-pushup.config.ts → e2e/cli-e2e/mocks/fixtures/code-pushup.config.ts

@@ -1,7 +1,6 @@
-// TODO: import plugins using NPM package names using local registry: https://github.com/flowup/quality-metrics-cli/issues/33
-import eslintPlugin from '../../../dist/packages/plugin-eslint';
-import lighthousePlugin from '../../../dist/packages/plugin-lighthouse';
-import { CoreConfig } from '../../../packages/models/src';
+import eslintPlugin from '../../../../dist/packages/plugin-eslint';
+import lighthousePlugin from '../../../../dist/packages/plugin-lighthouse';
+import { CoreConfig } from '../../../../packages/models/src';
 
 export default {
   upload: {

e2e/cli-e2e/project.json

@@ -4,6 +4,13 @@
   "sourceRoot": "examples/cli-e2e/src",
   "projectType": "application",
   "targets": {
+    "lint": {
+      "executor": "@nx/linter:eslint",
+      "outputs": ["{options.outputFile}"],
+      "options": {
+        "lintFilePatterns": ["e2e/cli-e2e/**/*.ts"]
+      }
+    },
     "e2e": {
       "executor": "@nx/vite:test",
       "outputs": ["{options.reportsDirectory}"],

e2e/cli-e2e/tests/__snapshots__/help.e2e.test.ts.snap

@@ -31,3 +31,35 @@ Options:
   -h, --help                 Show help                                 [boolean]
 "
 `;
+
+exports[`CLI help > should print help with help command 1`] = `
+"Code PushUp CLI
+
+Commands:
+  code-pushup               Shortcut for running collect followed by upload
+                                                                       [default]
+  code-pushup autorun       Shortcut for running collect followed by upload
+  code-pushup collect       Run Plugins and collect results
+  code-pushup upload        Upload report results to the portal
+  code-pushup print-config  Print config
+
+Options:
+      --progress             Show progress bar in stdout.
+                                                       [boolean] [default: true]
+      --verbose              When true creates more verbose output. This is help
+                             ful when debugging.      [boolean] [default: false]
+      --config               Path the the config file, e.g. code-pushup.config.j
+                             s       [string] [default: \\"code-pushup.config.js\\"]
+      --persist.outputDir    Directory for the produced reports         [string]
+      --persist.filename     Filename for the produced reports.         [string]
+      --persist.format       Format of the report output. e.g. \`md\`, \`json\`
+                                                                         [array]
+      --upload.organization  Organization slug from portal              [string]
+      --upload.project       Project slug from portal                   [string]
+      --upload.server        URL to your portal server                  [string]
+      --upload.apiKey        API key for the portal server              [string]
+      --onlyPlugins          List of plugins to run. If not set all plugins are
+                             run.                          [array] [default: []]
+  -h, --help                 Show help                                 [boolean]
+"
+`;

e2e/cli-e2e/tests/collect.e2e.test.ts

@@ -1,34 +1,25 @@
-import { afterEach, beforeEach, vi } from 'vitest';
 import { PluginReport, Report, reportSchema } from '@code-pushup/models';
 import { executeProcess, readJsonFile, readTextFile } from '@code-pushup/utils';
-import { cleanFolderPutGitKeep } from '../mocks/fs.mock';
 
 describe('CLI collect', () => {
   const exampleCategoryTitle = 'Code style';
   const exampleAuditTitle = 'Require `const` declarations for variables';
 
+  /* eslint-disable @typescript-eslint/no-unused-vars */
   const omitVariableData = ({
     date,
     duration,
     version,
     ...report
   }: Report | PluginReport) => report;
+  /* eslint-enable @typescript-eslint/no-unused-vars */
 
   const omitVariableReportData = (report: Report) =>
     omitVariableData({
       ...report,
       plugins: report.plugins.map(omitVariableData) as PluginReport[],
     });
 
-  beforeEach(async () => {
-    vi.clearAllMocks();
-    cleanFolderPutGitKeep();
-  });
-
-  afterEach(() => {
-    cleanFolderPutGitKeep();
-  });
-
   it('should run ESLint plugin and create report.json', async () => {
     const { code, stderr } = await executeProcess({
       command: 'code-pushup',
@@ -43,7 +34,7 @@ describe('CLI collect', () => {
 
     expect(() => reportSchema.parse(report)).not.toThrow();
     expect(omitVariableReportData(report as Report)).toMatchSnapshot();
-  }, 120000);
+  });
 
   it('should create report.md', async () => {
     const { code, stderr } = await executeProcess({
@@ -60,7 +51,7 @@ describe('CLI collect', () => {
     expect(md).toContain('# Code PushUp Report');
     expect(md).toContain(exampleCategoryTitle);
     expect(md).toContain(exampleAuditTitle);
-  }, 120000);
+  });
 
   it('should print report summary to stdout', async () => {
     const { code, stdout, stderr } = await executeProcess({
@@ -76,5 +67,5 @@ describe('CLI collect', () => {
     expect(stdout).not.toContain('Generated reports');
     expect(stdout).toContain(exampleCategoryTitle);
     expect(stdout).toContain(exampleAuditTitle);
-  }, 120000);
+  });
 });

e2e/cli-e2e/tests/help.e2e.test.ts

@@ -9,7 +9,15 @@ describe('CLI help', () => {
     expect(code).toBe(0);
     expect(stderr).toBe('');
     expect(stdout).toMatchSnapshot();
-  }, 120000);
+  });
 
-  // @TODO 'should print help with help command'
+  it('should print help with help command', async () => {
+    const { code, stdout, stderr } = await executeProcess({
+      command: 'code-pushup',
+      args: ['help'],
+    });
+    expect(code).toBe(0);
+    expect(stderr).toBe('');
+    expect(stdout).toMatchSnapshot();
+  });
 });

e2e/cli-e2e/tests/print-config.e2e.test.ts

@@ -1,11 +1,10 @@
+import { join } from 'node:path';
 import { expect } from 'vitest';
-import {
-  PERSIST_FILENAME,
-  PERSIST_FORMAT,
-  PERSIST_OUTPUT_DIR,
-} from '@code-pushup/models';
 import { executeProcess } from '@code-pushup/utils';
-import { configFile, extensions } from '../mocks/utils';
+
+const extensions = ['js', 'mjs', 'ts'] as const;
+export const configFilePath = (ext: (typeof extensions)[number]) =>
+  join(process.cwd(), `e2e/cli-e2e/mocks/fixtures/code-pushup.config.${ext}`);
 
 describe('print-config', () => {
   it.each(extensions)(
@@ -17,7 +16,10 @@ describe('print-config', () => {
           'print-config',
           '--verbose',
           '--no-progress',
-          `--config=${configFile(ext)}`,
+          `--config=${configFilePath(ext)}`,
+          '--persist.outputDir=output-dir',
+          '--persist.format=md',
+          `--persist.filename=${ext}-report`,
         ],
       });
 
@@ -28,9 +30,9 @@ describe('print-config', () => {
           config: expect.stringContaining(`code-pushup.config.${ext}`),
           // filled by command options
           persist: {
-            outputDir: PERSIST_OUTPUT_DIR,
-            filename: PERSIST_FILENAME,
-            format: PERSIST_FORMAT,
+            outputDir: 'output-dir',
+            filename: `${ext}-report`,
+            format: ['md'],
           },
           upload: {
             organization: 'code-pushup',
@@ -45,35 +47,10 @@ describe('print-config', () => {
               title: 'ChromeDevTools Lighthouse',
             }),
           ]),
-          // @TODO add test data to config file
           categories: expect.any(Array),
           onlyPlugins: [],
         }),
       );
     },
-    120000,
   );
-
-  it('should load .ts config file and overload it with arguments', async () => {
-    const { code, stderr, stdout } = await executeProcess({
-      command: 'code-pushup',
-      args: [
-        'print-config',
-        '--verbose',
-        '--no-progress',
-        `--config=${configFile('ts')}`,
-        '--persist.outputDir=my-dir',
-        '--persist.format=md',
-        '--persist.filename=my-report',
-      ],
-    });
-
-    expect(JSON.parse(stdout)?.persist).toEqual(
-      expect.objectContaining({
-        outputDir: 'my-dir',
-        format: ['md'],
-        filename: 'my-report',
-      }),
-    );
-  }, 120000);
 });

e2e/cli-e2e/tsconfig.test.json

@@ -7,9 +7,7 @@
   "include": [
     "vite.config.e2e.ts",
     "tests/**/*.e2e.test.ts",
-    "tests/**/*.e2e.test.tsx",
-    "tests/**/*.e2e.test.js",
-    "tests/**/*.e2e.test.jsx",
-    "tests/**/*.d.ts"
+    "tests/**/*.d.ts",
+    "mocks/**/*.ts"
   ]
 }

e2e/cli-e2e/vite.config.e2e.ts

@@ -6,13 +6,14 @@ export default defineConfig({
   cacheDir: '../../node_modules/.vite/cli-e2e',
   plugins: [nxViteTsPaths()],
   test: {
-    testTimeout: 20000,
+    testTimeout: 120_000,
     globals: true,
     cache: {
       dir: '../../node_modules/.vitest',
     },
     environment: 'node',
     include: ['tests/**/*.e2e.test.{js,mjs,cjs,ts,mts,cts,jsx,tsx}'],
     globalSetup: ['global-setup.e2e.ts'],
+    setupFiles: ['../../testing-utils/src/lib/setup/reset.mocks.ts'],
   },
 });

global-setup.e2e.ts

@@ -1,18 +1,17 @@
 import { execSync } from 'child_process';
 import { setup as globalSetup } from './global-setup';
+import { tearDownTestFolder } from './testing-utils/src';
 import startLocalRegistry from './tools/scripts/start-local-registry';
 import stopLocalRegistry from './tools/scripts/stop-local-registry';
 
 export async function setup() {
   await globalSetup();
-
   await startLocalRegistry();
-
   execSync('npm install -g @code-pushup/cli@e2e');
 }
 
 export async function teardown() {
   stopLocalRegistry();
-
   execSync('npm uninstall -g @code-pushup/cli');
+  await tearDownTestFolder('tmp');
 }

testing-utils/src/index.ts

@@ -1,4 +1,5 @@
 export * from './lib/constants';
+export * from './lib/setup/test-folder.setup';
 export * from './lib/utils/execute-process-helper.mock';
 
 // static mocks