Allow multiple API tokens and app urls to be specified (#499)

This hasn't been fully tested yet, but putting it up for review so it can potentially be merged while I'm on holiday if needed.
alwaysmeticulous · Sep 4, 2024 · f3281ad · f3281ad
2 parents b84d1fd + 829dde1
commit f3281ad
Show file tree

Hide file tree

Showing 17 changed files with 692 additions and 285 deletions.
diff --git a/cloud-compute/action.yml b/cloud-compute/action.yml
@@ -2,20 +2,39 @@ name: "Meticulous - Report Diffs (Cloud Compute)"
 description: "Run automated Meticulous tests in Meticulous' cloud."
 inputs:
   api-token:
-    description: "Meticulous API token"
-    required: true
+    description: "Meticulous API token. This must be provided if `projects_yaml` is not set."
+    required: false
   github-token:
     description: "GITHUB_TOKEN or a repo scoped PAT."
     required: true
     default: ${{ github.token }}
   app-url:
     description: |
       The URL to execute the tests against. This URL should serve the code from the current commit (e.g. a localhost URL served up by a local server).
-    required: true
+      This must be provided if `projects_yaml` is not set.
+    required: false
   head-sha:
     description: |
       Override the head commit SHA that we are analysing instead of using the one inferred automatically by Meticulous. This normally should not be set, but is useful in some scenarios.
     required: false
+  projects_yaml:
+    description: |
+      YAML string that defines the projects to run. This must be provided if `api-token` and `app-url` are not set.
+      This is useful when executing tests for multiple projects in a single job.
+      If `skip` is set to `true`, Meticulous will not run tests for that project.
+      Schema:
+      ```yaml
+      projects_yaml: |
+        app:
+          api-token: {{ secrets.METICULOUS_APP_API_TOKEN }}
+          app-url: "http://localhost:3000"
+          skip: false (optional, default is false)
+        admin:
+          api-token: {{ secrets.METICULOUS_ADMIN_API_TOKEN }}
+          app-url: "http://localhost:4000"
+          skip: false (optional, default is false)
+      ```
+    required: false
 
 outputs: {}
 runs:

diff --git a/out/cloud-compute.entrypoint.js b/out/cloud-compute.entrypoint.js
diff --git a/package.json b/package.json
@@ -44,19 +44,22 @@
     "test": "jest"
   },
   "dependencies": {
+    "//": "Upgrading `replay-orchestrator-launcher`? Consider bumping the environment version `LOGICAL_ENVIRONMENT_VERSION` in `constants.ts` if the new version includes visible changes.",
     "@actions/core": "^1.10.0",
     "@actions/github": "^5.1.1",
     "@alwaysmeticulous/client": "^2.147.1",
     "@alwaysmeticulous/common": "^2.146.1",
     "@alwaysmeticulous/remote-replay-launcher": "^2.147.1",
-    "//": "Upgrading `replay-orchestrator-launcher`? Consider bumping the environment version `LOGICAL_ENVIRONMENT_VERSION` in `constants.ts` if the new version includes visible changes.",
     "@alwaysmeticulous/replay-orchestrator-launcher": "^2.147.1",
     "@alwaysmeticulous/sentry": "^2.146.1",
     "@sentry/node": "^7.107.0",
+    "joi": "^17.13.3",
     "lodash.debounce": "^4.0.8",
     "loglevel": "^1.8.1",
+    "loglevel-plugin-prefix": "^0.8.4",
     "luxon": "^3.4.3",
-    "retry": "^0.13.1"
+    "retry": "^0.13.1",
+    "yaml": "^2.5.0"
   },
   "devDependencies": {
     "@alwaysmeticulous/api": "^2.144.0",
@@ -66,8 +69,8 @@
     "@types/jest": "^27.0.3",
     "@types/lodash.debounce": "^4.0.9",
     "@types/luxon": "^3.3.2",
-    "@types/retry": "^0.12.5",
     "@types/node": "^20.11.28",
+    "@types/retry": "^0.12.5",
     "@typescript-eslint/eslint-plugin": "^5.62.0",
     "@typescript-eslint/parser": "^5.62.0",
     "eslint": "^8.56.0",
@@ -78,8 +81,8 @@
     "prettier": "^2.8.7",
     "rimraf": "^5.0.7",
     "ts-jest": "^27.1.1",
-    "typescript": "^4.9.5",
-    "yaml": "^2.4.1"
+    "ts-node": "^10.9.2",
+    "typescript": "^4.9.5"
   },
   "resolutions": {
     "//": "https://nvd.nist.gov/vuln/detail/CVE-2022-46175",

diff --git a/src/actions/cloud-compute/cloud-compute.ts b/src/actions/cloud-compute/cloud-compute.ts
@@ -1,33 +1,12 @@
 import { setFailed } from "@actions/core";
-import { context } from "@actions/github";
-import { IN_PROGRESS_TEST_RUN_STATUS, TestRun } from "@alwaysmeticulous/client";
-import { defer, METICULOUS_LOGGER_NAME } from "@alwaysmeticulous/common";
-import { executeRemoteTestRun } from "@alwaysmeticulous/remote-replay-launcher";
 import { initSentry } from "@alwaysmeticulous/sentry";
-import log from "loglevel";
-import { Duration } from "luxon";
-import { throwIfCannotConnectToOrigin } from "../../common/check-connection";
-import { tryTriggerTestsWorkflowOnBase } from "../../common/ensure-base-exists.utils";
-import { shortCommitSha } from "../../common/environment.utils";
-import {
-  getBaseAndHeadCommitShas,
-  getHeadCommitShaFromRepo,
-} from "../../common/get-base-and-head-commit-shas";
-import { getCodeChangeEvent } from "../../common/get-code-change-event";
-import { isDebugPullRequestRun } from "../../common/is-debug-pr-run";
-import { initLogger, setLogLevel, shortSha } from "../../common/logger.utils";
-import { getOctokitOrFail } from "../../common/octokit";
-import { updateStatusComment } from "../../common/update-status-comment";
-import { getCloudComputeBaseTestRun } from "./get-cloud-compute-base-test-run";
+import { getHeadCommitShaFromRepo } from "../../common/get-base-and-head-commit-shas";
+import { initLogger } from "../../common/logger.utils";
 import { getInCloudActionInputs } from "./get-inputs";
-
-const DEBUG_MODE_KEEP_TUNNEL_OPEN_DURAION = Duration.fromObject({
-  minutes: 45,
-});
+import { runOneTestRun } from "./run-test-run";
 
 export const runMeticulousTestsCloudComputeAction = async (): Promise<void> => {
-  initLogger();
-
+  const logger = initLogger();
   // Init Sentry without sampling traces on the action run.
   // Children processes, (test run executions) will use
   // the global sample rate.
@@ -42,190 +21,74 @@ export const runMeticulousTestsCloudComputeAction = async (): Promise<void> => {
     op: "report-diffs-action.runMeticulousTestsActionInCloud",
   });
 
-  if (+(process.env["RUNNER_DEBUG"] ?? "0")) {
-    setLogLevel("trace");
-  }
-
+  let failureMessage = "";
   const {
-    apiToken,
-    githubToken,
-    appUrl,
+    projectTargets,
     headSha: headShaFromInput,
+    githubToken,
   } = getInCloudActionInputs();
-  const { payload } = context;
-  const event = getCodeChangeEvent(context.eventName, payload);
-  const { owner, repo } = context.repo;
-  const isDebugPRRun = isDebugPullRequestRun(event);
-  const octokit = getOctokitOrFail(githubToken);
-  const logger = log.getLogger(METICULOUS_LOGGER_NAME);
-
-  if (event == null) {
-    logger.warn(
-      `Running report-diffs-action is only supported for 'push', \
-      'pull_request' and 'workflow_dispatch' events, but was triggered \
-      on a '${context.eventName}' event. Skipping execution.`
-    );
-    return;
-  }
 
   // Compute the HEAD commit SHA to use when creating a test run.
   // In a PR workflow this will by default be process.env.GITHUB_SHA (the temporary merge commit) or
   // sometimes the head commit of the PR.
   // Users can also explicitly provide the head commit SHA to use as input. This is useful when the action is not
   // run with the code checked out.
   // Our backend is responsible for computing the correct BASE commit to create the test run for.
-  const head = headShaFromInput || getHeadCommitShaFromRepo();
+  const headSha = headShaFromInput || getHeadCommitShaFromRepo();
 
-  // Compute the base commit SHA to compare to for the HEAD commit.
-  // This will usually be the merge base of the PR head and base commit. In some cases it can be an older main branch commit,
-  // for example when running in a monorepo setup.
-  const { baseCommitSha, baseTestRun } = await getCloudComputeBaseTestRun({
-    apiToken,
-    headCommitSha: head,
-  });
+  const skippedTargets = projectTargets.filter((target) => target.skip);
+  const projectTargetsToRun = projectTargets.filter((target) => !target.skip);
 
-  let shaToCompareAgainst: string | null = null;
-  if (baseTestRun != null) {
-    shaToCompareAgainst = baseCommitSha;
-    logger.info(
-      `Tests already exist for commit ${baseCommitSha} (${baseTestRun.id})`
-    );
-  } else {
-    // We compute and use the base SHA from the code change event rather than the `baseCommitSha` computed above
-    // as `tryTriggerTestsWorkflowOnBase` can only trigger workflow for the HEAD `main` branch commit.
-    // `baseCommitSha` can be an older commit in a monorepo setup (in cases where we selectively run tests for a specific package).
-    // In such cases we won't be able to trigger a workflow for the base commit SHA provided by the backend.
-    // We will instead trigger test run for a newer base which is the base commit SHA from the code change event and
-    // will use that as the base to compare against. This is safe as `codeChangeBase` is guaranteed to be the same
-    // or newer commit to `baseCommitSha`.
-    const { base: codeChangeBase } = await getBaseAndHeadCommitShas(event, {
-      useDeploymentUrl: false,
-    });
-    if (codeChangeBase) {
-      const { baseTestRunExists } = await tryTriggerTestsWorkflowOnBase({
-        logger,
-        event,
-        base: codeChangeBase,
-        context,
-        octokit,
-      });
-
-      if (baseTestRunExists) {
-        shaToCompareAgainst = codeChangeBase;
-      }
-    }
-  }
+  // Single test run execution is a special case where we run a single test run with the "default" name.
+  // This will be the case when the user provides `app-url` and `api-token` inputs directly.
+  // This is used to simplify some of the logging and error handling.
+  const isSingleTestRunExecution =
+    projectTargets.length === 1 && projectTargets[0].name === "default";
 
-  if (shaToCompareAgainst != null) {
+  // Log skipped targets, if any
+  if (skippedTargets) {
+    const skippedTargetNames = skippedTargets.map((target) => target.name);
     logger.info(
-      `Comparing visual snapshots for the commit ${shortSha(
-        head
-      )}, against ${shortSha(shaToCompareAgainst)}`
+      `Skipping test runs for the following targets: ${skippedTargetNames.join(
+        ", "
+      )}`
     );
-  } else {
-    logger.info(`Generating visual snapshots for commit ${shortSha(head)}`);
   }
 
-  try {
-    await throwIfCannotConnectToOrigin(appUrl);
-
-    const onTunnelCreated = ({
-      url,
-      basicAuthUser,
-      basicAuthPassword,
-    }: {
-      url: string;
-      basicAuthUser: string;
-      basicAuthPassword: string;
-    }) => {
-      logger.info(
-        `Secure tunnel to ${appUrl} created: ${url}, user: ${basicAuthUser}, password: ${basicAuthPassword}`
-      );
-
-      if (isDebugPRRun) {
-        updateStatusComment({
-          octokit,
-          event,
-          owner,
-          repo,
-          body: `🤖 Meticulous is running in debug mode. Secure tunnel to ${appUrl} created: ${url} user: \`${basicAuthUser}\` password: \`${basicAuthPassword}\`.\n\n
-Tunnel will be live for up to ${DEBUG_MODE_KEEP_TUNNEL_OPEN_DURAION.toHuman()}. Cancel the workflow run to close the tunnel early.`,
-          testSuiteId: "__meticulous_debug__",
-          shortHeadSha: shortCommitSha(head),
-          createIfDoesNotExist: true,
-        }).catch((err) => {
-          logger.error(err);
-        });
+  (
+    await Promise.allSettled(
+      projectTargetsToRun.map((target) =>
+        runOneTestRun({
+          testRunId: target.name,
+          apiToken: target.apiToken,
+          appUrl: target.appUrl,
+          githubToken,
+          headSha,
+          isSingleTestRunExecution,
+        })
+      )
+    )
+  ).forEach((result, index) => {
+    if (result.status === "rejected") {
+      const message =
+        result.reason instanceof Error
+          ? result.reason.message
+          : `${result.reason}`;
+
+      if (!isSingleTestRunExecution) {
+        failureMessage += `Test run ${projectTargetsToRun[index].name} failed: ${message}\n`;
+      } else {
+        failureMessage = message;
       }
-    };
-
-    const keepTunnelOpenPromise = isDebugPRRun ? defer<void>() : null;
-    let keepTunnelOpenTimeout: NodeJS.Timeout | null = null;
-
-    let lastSeenNumberOfCompletedTestCases = 0;
-
-    const onProgressUpdate = (testRun: TestRun) => {
-      if (
-        !IN_PROGRESS_TEST_RUN_STATUS.includes(testRun.status) &&
-        keepTunnelOpenPromise &&
-        !keepTunnelOpenTimeout
-      ) {
-        logger.info(
-          `Test run execution completed. Keeping tunnel open for ${DEBUG_MODE_KEEP_TUNNEL_OPEN_DURAION.toHuman()}`
-        );
-        keepTunnelOpenTimeout = setTimeout(() => {
-          keepTunnelOpenPromise.resolve();
-        }, DEBUG_MODE_KEEP_TUNNEL_OPEN_DURAION.as("milliseconds"));
-      }
-
-      const numTestCases = testRun.configData.testCases?.length || 0;
-      const completedTestCases = testRun.resultData?.results?.length || 0;
-
-      if (
-        completedTestCases != lastSeenNumberOfCompletedTestCases &&
-        numTestCases
-      ) {
-        logger.info(
-          `Executed ${completedTestCases}/${numTestCases} test cases`
-        );
-        lastSeenNumberOfCompletedTestCases = completedTestCases;
-      }
-    };
-
-    const onTestRunCreated = (testRun: TestRun) => {
-      logger.info(`Test run created: ${testRun.url}`);
-    };
-
-    // We use MERGE_COMMIT_SHA as the deployment is created for the merge commit.
-
-    await executeRemoteTestRun({
-      apiToken,
-      appUrl,
-      commitSha: head,
-      environment: "github-actions",
-      onTunnelCreated,
-      onTestRunCreated,
-      onProgressUpdate,
-      ...(keepTunnelOpenPromise
-        ? { keepTunnelOpenPromise: keepTunnelOpenPromise.promise }
-        : {}),
-    });
-
-    transaction.setStatus("ok");
-    transaction.finish();
-
-    await sentryHub.getClient()?.close(5_000);
-
-    process.exit(0);
-  } catch (error) {
-    const message = error instanceof Error ? error.message : `${error}`;
-    setFailed(message);
-
+    }
+  });
+  if (failureMessage) {
+    setFailed(failureMessage);
     transaction.setStatus("unknown_error");
-    transaction.finish();
-
-    await sentryHub.getClient()?.close(5_000);
-
-    process.exit(1);
+  } else {
+    transaction.setStatus("ok");
   }
+  transaction.finish();
+  await sentryHub.getClient()?.close(5_000);
+  process.exit(failureMessage ? 1 : 0);
 };
diff --git a/src/actions/cloud-compute/consts.ts b/src/actions/cloud-compute/consts.ts
@@ -0,0 +1,5 @@
+import { Duration } from "luxon";
+
+export const DEBUG_MODE_KEEP_TUNNEL_OPEN_DURATION = Duration.fromObject({
+  minutes: 45,
+});