From ae58b62bf53ca4540b6784d7efc318d005067e21 Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 1 Feb 2024 11:43:56 -0600 Subject: [PATCH 01/64] [DOP-4334]: Add support for nextGenDeploy --- .../src/helpers/dependency-helpers.ts | 10 ++----- src/commands/src/shared/next-gen-deploy.ts | 29 +++++++++++++++---- src/job/jobHandler.ts | 2 +- src/job/manifestJobHandler.ts | 2 +- src/job/productionJobHandler.ts | 13 ++++++++- src/job/stagingJobHandler.ts | 2 +- 6 files changed, 41 insertions(+), 17 deletions(-) diff --git a/src/commands/src/helpers/dependency-helpers.ts b/src/commands/src/helpers/dependency-helpers.ts index 3e2f20ae6..c41c45a54 100644 --- a/src/commands/src/helpers/dependency-helpers.ts +++ b/src/commands/src/helpers/dependency-helpers.ts @@ -93,16 +93,13 @@ export async function downloadBuildDependencies( return commands; } +export const checkRedirects = async () => existsAsync(path.join(process.cwd(), 'config/redirects')); + export async function prepareBuild(repoName: string, projectName: string, baseUrl: string, directory?: string) { const repoDir = getRepoDir(repoName, directory); // doing these in parallel - const commandPromises = [ - getCommitHash(repoDir), - getCommitBranch(repoDir), - getPatchId(repoDir), - existsAsync(path.join(process.cwd(), 'config/redirects')), - ]; + const commandPromises = [getCommitHash(repoDir), getCommitBranch(repoDir), getPatchId(repoDir)]; try { const dependencies = await Promise.all(commandPromises); @@ -118,7 +115,6 @@ export async function prepareBuild(repoName: string, projectName: string, baseUr commitHash: dependencies[0] as string, commitBranch: dependencies[1] as string, patchId: dependencies[2] as string | undefined, - hasRedirects: dependencies[3] as boolean, bundlePath: `${repoDir}/bundle.zip`, repoDir, }; diff --git a/src/commands/src/shared/next-gen-deploy.ts b/src/commands/src/shared/next-gen-deploy.ts index 5d86baf73..5bcd52c27 100644 --- a/src/commands/src/shared/next-gen-deploy.ts +++ b/src/commands/src/shared/next-gen-deploy.ts @@ -2,29 +2,46 @@ import { CommandExecutorResponse, CommandExecutorResponseStatus } from '../../.. import { executeAndPipeCommands, executeCliCommand } from '../helpers'; interface NextGenDeployParams { - mutPrefix: string; - gitBranch: string; + branchName: string; hasConfigRedirects: boolean; - bucket: string; - url: string; + mutPrefix?: string | null; + bucket?: string; + url?: string; } /* This is still in development - use with caution */ /* Logs here for future debugging purposes */ export async function nextGenDeploy({ mutPrefix, - gitBranch, + branchName, hasConfigRedirects, bucket, url, }: NextGenDeployParams): Promise { try { - if (hasConfigRedirects && (gitBranch === 'main' || gitBranch === 'master')) { + if (hasConfigRedirects && (branchName === 'main' || branchName === 'master')) { // equivalent to: mut-redirects config/redirects -o public/.htaccess await executeCliCommand({ command: 'mut-redirects', args: ['config/redirects', '-o', 'public/.htaccess'] }); console.log(`COMMAND: mut-redirects config/redirects -o public/.htaccess`); } + if (!bucket) { + console.log(`nextGenStage has failed. Variable for S3 bucket address was undefined.`); + return { + status: CommandExecutorResponseStatus.failed, + output: 'Failed in nextGenStage: No value present for S3 bucket', + error: 'ERROR: No value present for S3 bucket.', + }; + } + if (!url) { + console.log(`nextGenStage has failed. Variable for URL address was undefined.`); + return { + status: CommandExecutorResponseStatus.failed, + output: 'Failed in nextGenStage: No value present for target url.', + error: 'ERROR: No value present for URL.', + }; + } + // equivalent to: yes | mut-publish public ${BUCKET} --prefix=${MUT_PREFIX} --deploy --deployed-url-prefix=${URL} --json --all-subdirectories ${ARGS}; const { outputText } = await executeAndPipeCommands( { command: 'yes' }, diff --git a/src/job/jobHandler.ts b/src/job/jobHandler.ts index 73a50a1c3..1d8d13de7 100644 --- a/src/job/jobHandler.ts +++ b/src/job/jobHandler.ts @@ -619,7 +619,7 @@ export abstract class JobHandler { } @throwIfJobInterupted() - protected async deployGeneric(): Promise { + protected async deployWithMakefiles(): Promise { this.prepDeployCommands(); await this._logger.save(this.currJob._id, `${this._config.get('stage').padEnd(15)}Pushing to ${this.name}`); diff --git a/src/job/manifestJobHandler.ts b/src/job/manifestJobHandler.ts index d5c07a4f4..b5198053b 100644 --- a/src/job/manifestJobHandler.ts +++ b/src/job/manifestJobHandler.ts @@ -117,7 +117,7 @@ export class ManifestJobHandler extends JobHandler { async deploy(): Promise { try { - const resp = await this.deployGeneric(); // runs prepDeployCommands + const resp = await this.deployWithMakefiles(); // runs prepDeployCommands await this.logger.save(this.currJob._id, `(generate manifest) Manifest generation details:\n\n${resp?.output}`); return resp; } catch (errResult) { diff --git a/src/job/productionJobHandler.ts b/src/job/productionJobHandler.ts index b92cb3911..0cdb87fa1 100644 --- a/src/job/productionJobHandler.ts +++ b/src/job/productionJobHandler.ts @@ -13,6 +13,9 @@ import { IRepoConnector } from '../services/repo'; import { getDirectory, JobHandler } from './jobHandler'; import { IJobValidator } from './jobValidator'; import { joinUrlAndPrefix } from './manifestJobHandler'; +import { MONOREPO_NAME } from '../monorepo/utils/monorepo-constants'; +import { nextGenDeploy } from '../commands'; +import { checkRedirects } from '../commands/src/helpers/dependency-helpers'; export class ProductionJobHandler extends JobHandler { constructor( @@ -189,7 +192,15 @@ export class ProductionJobHandler extends JobHandler { } async deploy(): Promise { - const resp = await this.deployGeneric(); + if (process.env.FEATURE_FLAG_MONOREPO_PATH === 'true' && this.currJob.payload.repoName === MONOREPO_NAME) { + const { mutPrefix, branchName } = this.currJob.payload; + const { bucket, url } = await this.getEnvironmentVariables(); + const hasConfigRedirects = await checkRedirects(); + + await nextGenDeploy({ mutPrefix, bucket, url, branchName, hasConfigRedirects }); + } else { + } + const resp = await this.deployWithMakefiles(); try { if (resp?.output) { const makefileOutput = resp.output.replace(/\r/g, '').split(/\n/); diff --git a/src/job/stagingJobHandler.ts b/src/job/stagingJobHandler.ts index cb9eaae9b..e2b2242cc 100644 --- a/src/job/stagingJobHandler.ts +++ b/src/job/stagingJobHandler.ts @@ -97,7 +97,7 @@ export class StagingJobHandler extends JobHandler { resp = await nextGenStage({ job: this.currJob, bucket, url }); await this.logger.save(this.currJob._id, `${'(stage)'.padEnd(15)} ${resp.output}`); } else { - resp = await this.deployGeneric(); + resp = await this.deployWithMakefiles(); } const summary = ''; if (resp.output?.includes('Summary')) { From 3f3ab0b22aa3c928c3f881be0f46807bf5f36916 Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 1 Feb 2024 13:07:03 -0600 Subject: [PATCH 02/64] [DOP-4334]: Remove some unused stuff --- src/job/jobHandler.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/job/jobHandler.ts b/src/job/jobHandler.ts index 1d8d13de7..a999b1a57 100644 --- a/src/job/jobHandler.ts +++ b/src/job/jobHandler.ts @@ -1,5 +1,4 @@ -import path from 'path'; -import axios, { AxiosResponse } from 'axios'; +import axios from 'axios'; import { Payload, Job, JobStatus } from '../entities/job'; import { JobRepository } from '../repositories/jobRepository'; import { RepoBranchesRepository } from '../repositories/repoBranchesRepository'; From 0689a7c3520c8f6121373d9f964bfeb9f54ee330 Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 1 Feb 2024 15:06:34 -0600 Subject: [PATCH 03/64] [DOP-4334]: Add new endpoint for test deploys --- api/controllers/v2/test-deploy.ts | 8 ++++++ .../constructs/api/webhook-api-construct.ts | 28 +++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 api/controllers/v2/test-deploy.ts diff --git a/api/controllers/v2/test-deploy.ts b/api/controllers/v2/test-deploy.ts new file mode 100644 index 000000000..b662a4dab --- /dev/null +++ b/api/controllers/v2/test-deploy.ts @@ -0,0 +1,8 @@ +import { APIGatewayEvent, APIGatewayProxyResult } from 'aws-lambda'; + +export async function handleTestDeployRequest(event: APIGatewayEvent): Promise { + return { + statusCode: 500, + body: 'not complete', + }; +} diff --git a/cdk-infra/lib/constructs/api/webhook-api-construct.ts b/cdk-infra/lib/constructs/api/webhook-api-construct.ts index 52c71b95f..ed8dbab16 100644 --- a/cdk-infra/lib/constructs/api/webhook-api-construct.ts +++ b/cdk-infra/lib/constructs/api/webhook-api-construct.ts @@ -109,6 +109,14 @@ export class WebhookApiConstruct extends Construct { bundling, timeout, }); + const testDeployLambda = new NodejsFunction(this, 'testDeployLambda', { + entry: `${HANDLERS_PATH}/test-deploy.ts`, + runtime, + handler: 'handleTestDeployRequest', + environment, + bundling, + timeout, + }); // generic handler for the root endpoint const rootEndpointLambda = new Function(this, 'RootEndpointLambda', { @@ -125,6 +133,20 @@ export class WebhookApiConstruct extends Construct { proxy: false, }); + const usagePlan = restApi.addUsagePlan('cacheUpdaterUsagePlan', { + name: 'defaultPlan', + apiStages: [ + { + api: restApi, + stage: restApi.deploymentStage, + }, + ], + }); + + const apiKey = restApi.addApiKey('cacheUpdaterApiKey'); + + usagePlan.addApiKey(apiKey); + const webhookEndpoint = restApi.root.addResource('webhook'); const slackEndpoint = webhookEndpoint.addResource('slack'); @@ -132,6 +154,7 @@ export class WebhookApiConstruct extends Construct { const githubEndpoint = webhookEndpoint.addResource('githubEndpoint'); const localEndpoint = webhookEndpoint.addResource('local'); const snootyEndpoint = webhookEndpoint.addResource('snooty'); + const testDeployEndpoint = webhookEndpoint.addResource('test-deploy'); // Shared /githubEndpoint/trigger endpoint const githubEndpointTrigger = githubEndpoint.addResource('trigger'); @@ -174,6 +197,11 @@ export class WebhookApiConstruct extends Construct { .addResource('complete', { defaultCorsPreflightOptions }) .addMethod('POST', new LambdaIntegration(snootyBuildCompleteLambda)); + testDeployEndpoint + .addResource('trigger') + .addResource('build', { defaultCorsPreflightOptions }) + .addMethod('POST', new LambdaIntegration(testDeployLambda), { apiKeyRequired: true }); + // grant permission for lambdas to enqueue messages to the jobs queue jobsQueue.grantSendMessages(slackTriggerLambda); jobsQueue.grantSendMessages(githubTriggerLambda); From 42f04ec351d567b17fdaa2e40c359ebed13342db Mon Sep 17 00:00:00 2001 From: branberry Date: Fri, 9 Feb 2024 14:54:19 -0600 Subject: [PATCH 04/64] [DOP-4334]: Update next-gen-deploy --- .vscode/launch.json | 1 - Dockerfile.local | 2 +- .../persistence/src/services/connector/index.ts | 5 ++++- src/commands/src/helpers/dependency-helpers.ts | 9 ++++++++- src/commands/src/scripts/local-build/index.ts | 6 +++--- .../src/scripts/local-build/utils/create-job.ts | 14 ++++++++++---- .../src/scripts/local-build/utils/get-args.ts | 15 +++++++++++++++ .../src/scripts/local-build/utils/get-env-vars.ts | 1 + src/commands/src/shared/persistence-module.ts | 2 +- 9 files changed, 43 insertions(+), 12 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 929db2703..c34da13e5 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -9,7 +9,6 @@ "type": "node", "request": "attach", "port": 9229, - "address": "localhost", "localRoot": "${workspaceFolder}", "remoteRoot": "/home/docsworker-xlarge", "autoAttachChildProcesses": false, diff --git a/Dockerfile.local b/Dockerfile.local index 4382caf4d..37fa16fc0 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -61,7 +61,7 @@ RUN git clone -b @dop/redoc-cli@${REDOC_CLI_VERSION} --depth 1 https://github.co # Install dependencies for Redoc CLI && cd redoc/ \ && npm ci --prefix cli/ --omit=dev - + FROM initial as persistence RUN mkdir -p modules/persistence && chmod 755 modules/persistence diff --git a/modules/persistence/src/services/connector/index.ts b/modules/persistence/src/services/connector/index.ts index e06c7890f..feaa37f8e 100644 --- a/modules/persistence/src/services/connector/index.ts +++ b/modules/persistence/src/services/connector/index.ts @@ -65,7 +65,10 @@ export const bulkWrite = async (operations: mongodb.AnyBulkWriteOperation[], col if (!operations || !operations.length) { return; } - return dbSession.collection(collection).bulkWrite(operations, { ordered: false }); + const result = await dbSession + .collection(collection) + .bulkWrite(operations, { ordered: false, maxTimeMS: 10000000 }); + return result; } catch (error) { console.error(`Error at bulk write time for ${collection}: ${error}`); throw error; diff --git a/src/commands/src/helpers/dependency-helpers.ts b/src/commands/src/helpers/dependency-helpers.ts index c41c45a54..bfe906e31 100644 --- a/src/commands/src/helpers/dependency-helpers.ts +++ b/src/commands/src/helpers/dependency-helpers.ts @@ -49,10 +49,11 @@ export async function downloadBuildDependencies( directory?: string ) { const commands: string[] = []; + const repoDir = getRepoDir(repoName, directory); await Promise.all( buildDependencies.map(async (dependencyInfo) => { - const repoDir = getRepoDir(repoName, directory); const targetDir = dependencyInfo.targetDir ?? repoDir; + let options = {}; if (targetDir != repoDir) { options = { cwd: repoDir }; @@ -90,6 +91,12 @@ export async function downloadBuildDependencies( ); }) ); + + try { + await executeCliCommand({ command: 'ls', args: ['source/driver-examples'], options: { cwd: repoDir } }); + } catch (e) { + console.error(e); + } return commands; } diff --git a/src/commands/src/scripts/local-build/index.ts b/src/commands/src/scripts/local-build/index.ts index fc5b2c846..b7e4881b4 100644 --- a/src/commands/src/scripts/local-build/index.ts +++ b/src/commands/src/scripts/local-build/index.ts @@ -43,9 +43,9 @@ async function runDockerContainer(env: Record) { } async function main() { - const { repoName, repoOwner, directory, branchName } = getArgs(); + const { repoName, repoOwner, directory, branchName, jobType } = getArgs(); - const env = await getWorkerEnv('stg'); + const env = await getWorkerEnv('dotcomstg'); const octokitClient = getOctokitClient(env.GITHUB_BOT_PASSWORD); const commitPromise = octokitClient.request('GET /repos/{owner}/{repo}/commits/{ref}', { @@ -71,7 +71,7 @@ async function main() { const docsetsRepository = new DocsetsRepository(db, c, consoleLogger); const project = (await docsetsRepository.getProjectByRepoName(repoName, directory)) as string; - const job = createLocalJob({ commit: commit.data, branchName, repoName, repoOwner, project, directory }); + const job = createLocalJob({ commit: commit.data, branchName, repoName, repoOwner, project, directory, jobType }); console.log('inserting job into queue collection'); const { insertedId: jobId } = await collection.insertOne(job); diff --git a/src/commands/src/scripts/local-build/utils/create-job.ts b/src/commands/src/scripts/local-build/utils/create-job.ts index 2a06538a5..8a7707539 100644 --- a/src/commands/src/scripts/local-build/utils/create-job.ts +++ b/src/commands/src/scripts/local-build/utils/create-job.ts @@ -1,12 +1,17 @@ import { EnhancedJob, JobStatus } from '../../../../../entities/job'; import { CommitGetResponse } from './types'; +export const JOB_TYPES = ['githubPush', 'productionDeploy', 'manifestGeneration', 'regression'] as const; +export type JobType = (typeof JOB_TYPES)[number]; + +export const isValidJobType = (val: string): val is JobType => JOB_TYPES.includes(val as JobType); interface Props { branchName: string; repoOwner: string; repoName: string; commit: CommitGetResponse; project: string; + jobType: JobType; directory?: string; } @@ -17,11 +22,12 @@ export function createLocalJob({ commit, project, directory, + jobType, }: Props): Omit { return { isLocal: true, title: `${repoOwner}/${repoName}`, - user: commit.author?.name ?? '', + user: commit.author?.name ?? 'branberry', email: commit.author?.email ?? '', status: JobStatus.inQueue, createdTime: new Date(), @@ -32,9 +38,9 @@ export function createLocalJob({ result: null, pathPrefix: `${project}/docsworker-xlarge/${branchName}`, payload: { - jobType: 'githubPush', - source: 'github', - action: 'push', + jobType: jobType, + source: 'local', + action: 'debug', repoName, branchName, isFork: repoOwner !== '10gen' && repoOwner !== 'mongodb', diff --git a/src/commands/src/scripts/local-build/utils/get-args.ts b/src/commands/src/scripts/local-build/utils/get-args.ts index ac840ac28..e630d2831 100644 --- a/src/commands/src/scripts/local-build/utils/get-args.ts +++ b/src/commands/src/scripts/local-build/utils/get-args.ts @@ -1,3 +1,5 @@ +import { isValidJobType } from './create-job'; + export function getArgs() { const helpIdx = process.argv.findIndex((str) => str === '--help' || str === '-h'); @@ -10,6 +12,7 @@ export function getArgs() { --repo-name, -n (required) The name of the repo e.g. docs-java or cloud-docs. --directory, -d (optional) The project directory path for a monorepo project. A slash is not needed at the beginning. For example, cloud-docs in the monorepo would just be cloud-docs for the argument. --branch-name, -b (optional) The branch name we want to parse. If not provided, the value 'master' is used by default. + --job-type -t (optional) The job type. Can be githubPush (uses the StagingJobHandler), productionDeploy (uses the ProductionJobHandler), manifestGeneration (uses the ManifestJobHandler), and regression (uses the RegressionJobHandler). Default value is githubPush. `); process.exit(0); @@ -20,6 +23,7 @@ export function getArgs() { // optional args const directoryIdx = process.argv.findIndex((str) => str === '--directory' || str === '-d'); const branchNameIdx = process.argv.findIndex((str) => str === '--branch-name' || str === '-b'); + const jobTypeIdx = process.argv.findIndex((str) => str === '--job-type' || str === '-t'); if (repoOwnerIdx === -1) throw new Error( @@ -48,15 +52,26 @@ export function getArgs() { if (process.argv[branchNameIdx + 1].startsWith('-')) throw new Error(`Please provide a valid branch name value. Value provided: ${process.argv[branchNameIdx + 1]}`); + if (jobTypeIdx + 1 === process.argv.length) throw new Error('Please provide a value for the branch name flag'); + if (process.argv[jobTypeIdx + 1].startsWith('-')) + throw new Error(`Please provide a valid branch name value. Value provided: ${process.argv[jobTypeIdx + 1]}`); + const repoOwner = process.argv[repoOwnerIdx + 1]; const repoName = process.argv[repoNameIdx + 1]; const directory = directoryIdx !== -1 ? process.argv[directoryIdx + 1] : undefined; const branchName = branchNameIdx !== -1 ? process.argv[branchNameIdx + 1] : 'master'; + const jobType = jobTypeIdx !== -1 ? process.argv[jobTypeIdx + 1] : 'githubPush'; + + if (!isValidJobType(jobType)) + throw new Error( + `Error! Invalid option provided for jobType. Value provided: ${jobType}. Please run the command with the --help flag to view the valid options for this parameter.` + ); return { repoOwner, repoName, directory, branchName, + jobType, }; } diff --git a/src/commands/src/scripts/local-build/utils/get-env-vars.ts b/src/commands/src/scripts/local-build/utils/get-env-vars.ts index 37d401ec9..651d0c9da 100644 --- a/src/commands/src/scripts/local-build/utils/get-env-vars.ts +++ b/src/commands/src/scripts/local-build/utils/get-env-vars.ts @@ -160,6 +160,7 @@ export async function getWorkerEnv(env: AutoBuilderEnv): Promise Date: Mon, 12 Feb 2024 13:42:05 -0600 Subject: [PATCH 05/64] [DOP-4334]: Push to preprd --- .github/workflows/deploy-stg-ecs.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy-stg-ecs.yml b/.github/workflows/deploy-stg-ecs.yml index c60c38296..d8ff99405 100644 --- a/.github/workflows/deploy-stg-ecs.yml +++ b/.github/workflows/deploy-stg-ecs.yml @@ -1,9 +1,9 @@ on: push: branches: - - "main" - - "integration" - - "DOP-4306" + - 'main' + - 'integration' + - 'DOP-4334' concurrency: group: environment-stg-${{ github.ref }} cancel-in-progress: true From 4d6cf2b837b40c909e7d9333ed6b4ae0ee4d3767 Mon Sep 17 00:00:00 2001 From: branberry Date: Mon, 12 Feb 2024 14:59:52 -0600 Subject: [PATCH 06/64] [DOP-4334]: Add if else statement --- src/job/productionJobHandler.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/job/productionJobHandler.ts b/src/job/productionJobHandler.ts index abfd5e248..af937787e 100644 --- a/src/job/productionJobHandler.ts +++ b/src/job/productionJobHandler.ts @@ -189,15 +189,16 @@ export class ProductionJobHandler extends JobHandler { } async deploy(): Promise { + let resp; if (process.env.FEATURE_FLAG_MONOREPO_PATH === 'true' && this.currJob.payload.repoName === MONOREPO_NAME) { const { mutPrefix, branchName } = this.currJob.payload; const { bucket, url } = await this.getEnvironmentVariables(); const hasConfigRedirects = await checkRedirects(); - await nextGenDeploy({ mutPrefix, bucket, url, branchName, hasConfigRedirects }); + resp = await nextGenDeploy({ mutPrefix, bucket, url, branchName, hasConfigRedirects }); } else { + resp = await this.deployWithMakefiles(); } - const resp = await this.deployWithMakefiles(); try { if (resp?.output) { const makefileOutput = resp.output.replace(/\r/g, '').split(/\n/); From c2383f923bdf610d1d4492cab20ed461ed598704 Mon Sep 17 00:00:00 2001 From: branberry Date: Mon, 12 Feb 2024 17:30:27 -0600 Subject: [PATCH 07/64] [DOP-4334]: Update logging --- src/commands/src/shared/next-gen-deploy.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/commands/src/shared/next-gen-deploy.ts b/src/commands/src/shared/next-gen-deploy.ts index 5bcd52c27..a025820ea 100644 --- a/src/commands/src/shared/next-gen-deploy.ts +++ b/src/commands/src/shared/next-gen-deploy.ts @@ -19,7 +19,7 @@ export async function nextGenDeploy({ url, }: NextGenDeployParams): Promise { try { - if (hasConfigRedirects && (branchName === 'main' || branchName === 'master')) { + if (hasConfigRedirects && (branchName === 'main' || branchName === 'master' || branchName === 'current')) { // equivalent to: mut-redirects config/redirects -o public/.htaccess await executeCliCommand({ command: 'mut-redirects', args: ['config/redirects', '-o', 'public/.htaccess'] }); console.log(`COMMAND: mut-redirects config/redirects -o public/.htaccess`); @@ -62,13 +62,14 @@ export async function nextGenDeploy({ }, } ); - console.log( - `COMMAND: yes | mut-publish public ${bucket} --prefix=${mutPrefix} --deploy --deployed-url-prefix=${url} --json --all-subdirectories --dry-run` - ); - console.log(`${outputText}\n Hosted at ${url}/${mutPrefix}`); + const output = `COMMAND: yes | mut-publish public ${bucket} --prefix=${mutPrefix} --deploy --deployed-url-prefix=${url} --json --all-subdirectories --dry-run + \n${outputText}\n Hosted at ${url}/${mutPrefix} + `; + + console.log(output); return { status: CommandExecutorResponseStatus.success, - output: outputText, + output, error: '', }; } catch (error) { From 55d21d10fbfc539b631c598405004c247497cf0b Mon Sep 17 00:00:00 2001 From: branberry Date: Wed, 14 Feb 2024 12:47:34 -0600 Subject: [PATCH 08/64] [DOP-4334]: Some minor clean up --- src/job/jobHandler.ts | 12 +++++------- src/job/productionJobHandler.ts | 18 +++--------------- 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/src/job/jobHandler.ts b/src/job/jobHandler.ts index 0129ab0bf..a5122dc98 100644 --- a/src/job/jobHandler.ts +++ b/src/job/jobHandler.ts @@ -149,13 +149,11 @@ export abstract class JobHandler { @throwIfJobInterupted() private async constructPrefix(): Promise { const server_user = this._config.get('GATSBY_PARSER_USER'); - const pathPrefix = await this.getPathPrefix(); - // TODO: Can empty string check be removed? - if (pathPrefix || pathPrefix === '') { - this.currJob.payload.pathPrefix = pathPrefix; - const mutPrefix = pathPrefix.split(`/${server_user}`)[0]; - this.currJob.payload.mutPrefix = mutPrefix; - } + const pathPrefix = this.getPathPrefix(); + + this.currJob.payload.pathPrefix = pathPrefix; + const mutPrefix = pathPrefix.split(`/${server_user}`)[0]; + this.currJob.payload.mutPrefix = mutPrefix; } @throwIfJobInterupted() diff --git a/src/job/productionJobHandler.ts b/src/job/productionJobHandler.ts index af937787e..accb46b57 100644 --- a/src/job/productionJobHandler.ts +++ b/src/job/productionJobHandler.ts @@ -144,22 +144,10 @@ export class ProductionJobHandler extends JobHandler { } getPathPrefix(): string { - try { - if (this.currJob.payload.prefix && this.currJob.payload.prefix === '') { - return this.currJob.payload.urlSlug ?? ''; - } - if (this.currJob.payload.urlSlug) { - if (this.currJob.payload.urlSlug === '') { - return this.currJob.payload.prefix; - } else { - return `${this.currJob.payload.prefix}/${this.currJob.payload.urlSlug}`; - } - } - return this.currJob.payload.prefix; - } catch (error) { - this.logger.save(this.currJob._id, error).then(); - throw new InvalidJobError(error.message); + if (this.currJob.payload.urlSlug) { + return `${this.currJob.payload.prefix}/${this.currJob.payload.urlSlug}`; } + return this.currJob.payload.prefix; } private async purgePublishedContent(makefileOutput: Array): Promise { From b044f1b517da33adb306712013b3a3be4d28dfe0 Mon Sep 17 00:00:00 2001 From: branberry Date: Wed, 14 Feb 2024 12:48:07 -0600 Subject: [PATCH 09/64] [DOP-4334]: Add error text to output as well --- src/commands/src/shared/next-gen-deploy.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/commands/src/shared/next-gen-deploy.ts b/src/commands/src/shared/next-gen-deploy.ts index a025820ea..0d7657c4b 100644 --- a/src/commands/src/shared/next-gen-deploy.ts +++ b/src/commands/src/shared/next-gen-deploy.ts @@ -43,7 +43,7 @@ export async function nextGenDeploy({ } // equivalent to: yes | mut-publish public ${BUCKET} --prefix=${MUT_PREFIX} --deploy --deployed-url-prefix=${URL} --json --all-subdirectories ${ARGS}; - const { outputText } = await executeAndPipeCommands( + const { outputText, errorText } = await executeAndPipeCommands( { command: 'yes' }, { command: 'mut-publish', @@ -63,7 +63,7 @@ export async function nextGenDeploy({ } ); const output = `COMMAND: yes | mut-publish public ${bucket} --prefix=${mutPrefix} --deploy --deployed-url-prefix=${url} --json --all-subdirectories --dry-run - \n${outputText}\n Hosted at ${url}/${mutPrefix} + \n${outputText} \n ${errorText} \n Hosted at ${url}/${mutPrefix} `; console.log(output); From 95bbe24596dfe979220ed37a0fd67258edc616bc Mon Sep 17 00:00:00 2001 From: branberry Date: Wed, 14 Feb 2024 13:03:12 -0600 Subject: [PATCH 10/64] [DOP-4334]: Remove dryrun --- src/commands/src/shared/next-gen-deploy.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/commands/src/shared/next-gen-deploy.ts b/src/commands/src/shared/next-gen-deploy.ts index 0d7657c4b..dd82a385a 100644 --- a/src/commands/src/shared/next-gen-deploy.ts +++ b/src/commands/src/shared/next-gen-deploy.ts @@ -55,7 +55,6 @@ export async function nextGenDeploy({ `--deployed-url-prefix=${url}`, '--json', '--all-subdirectories', - '--dry-run', ], options: { cwd: `${process.cwd()}/snooty`, From 8b77681db55fc4be4e9e8561ca210dfa1696fadc Mon Sep 17 00:00:00 2001 From: branberry Date: Wed, 14 Feb 2024 16:37:05 -0600 Subject: [PATCH 11/64] [DOP-4334]: Add repoName and repoDir to redirects --- src/commands/src/helpers/dependency-helpers.ts | 3 ++- src/job/productionJobHandler.ts | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/commands/src/helpers/dependency-helpers.ts b/src/commands/src/helpers/dependency-helpers.ts index bfe906e31..b45e7ce1a 100644 --- a/src/commands/src/helpers/dependency-helpers.ts +++ b/src/commands/src/helpers/dependency-helpers.ts @@ -100,7 +100,8 @@ export async function downloadBuildDependencies( return commands; } -export const checkRedirects = async () => existsAsync(path.join(process.cwd(), 'config/redirects')); +export const checkRedirects = async (repoName: string, directory?: string) => + existsAsync(path.join(process.cwd(), getRepoDir(repoName, directory), 'config/redirects')); export async function prepareBuild(repoName: string, projectName: string, baseUrl: string, directory?: string) { const repoDir = getRepoDir(repoName, directory); diff --git a/src/job/productionJobHandler.ts b/src/job/productionJobHandler.ts index accb46b57..660469b7a 100644 --- a/src/job/productionJobHandler.ts +++ b/src/job/productionJobHandler.ts @@ -180,8 +180,10 @@ export class ProductionJobHandler extends JobHandler { let resp; if (process.env.FEATURE_FLAG_MONOREPO_PATH === 'true' && this.currJob.payload.repoName === MONOREPO_NAME) { const { mutPrefix, branchName } = this.currJob.payload; + await this.logger.save(this.currJob._id, `${'(prod)'.padEnd(15)} Begin Deploy without makefiles`); + const { bucket, url } = await this.getEnvironmentVariables(); - const hasConfigRedirects = await checkRedirects(); + const hasConfigRedirects = await checkRedirects(this.currJob.payload.repoName, this.currJob.payload.directory); resp = await nextGenDeploy({ mutPrefix, bucket, url, branchName, hasConfigRedirects }); } else { From d26a7c6e5df7fb384785a5e2184a0ec758b62b95 Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 15 Feb 2024 09:11:43 -0600 Subject: [PATCH 12/64] [DOP-4334]: Nextgenbuild --- src/commands/src/shared/next-gen-deploy.ts | 2 +- src/job/jobHandler.ts | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/commands/src/shared/next-gen-deploy.ts b/src/commands/src/shared/next-gen-deploy.ts index dd82a385a..61ea7c9f1 100644 --- a/src/commands/src/shared/next-gen-deploy.ts +++ b/src/commands/src/shared/next-gen-deploy.ts @@ -62,7 +62,7 @@ export async function nextGenDeploy({ } ); const output = `COMMAND: yes | mut-publish public ${bucket} --prefix=${mutPrefix} --deploy --deployed-url-prefix=${url} --json --all-subdirectories --dry-run - \n${outputText} \n ${errorText} \n Hosted at ${url}/${mutPrefix} + \n${outputText} \n ${errorText} \n Hosted at ${url}${mutPrefix} `; console.log(output); diff --git a/src/job/jobHandler.ts b/src/job/jobHandler.ts index a5122dc98..24a153095 100644 --- a/src/job/jobHandler.ts +++ b/src/job/jobHandler.ts @@ -550,10 +550,14 @@ export abstract class JobHandler { await this.cloneRepo(this._config.get('repo_dir')); this._logger.save(job._id, 'Cloned Repo'); - await this.commitCheck(); + this._logger.save(job._id, 'Checked Commit'); await this.pullRepo(); + + this._logger.save(job._id, 'Next gen build'); + await this.prepNextGenBuild(); + this._logger.save(job._id, 'Pulled Repo'); await this.setEnvironmentVariables(); this.logger.save(job._id, 'Prepared Environment variables'); From 86719073433a3cb33f975ae1236f5269654a624a Mon Sep 17 00:00:00 2001 From: branberry Date: Tue, 20 Feb 2024 08:30:53 -0600 Subject: [PATCH 13/64] [DOP-4334]: Use dotcomstg --- .github/workflows/clean-feature-branch.yml | 14 +++++++------- .github/workflows/deploy-feature-branch.yml | 6 +++--- .github/workflows/update-feature-branch.yml | 10 +++++----- .../lib/constructs/worker/worker-construct.ts | 2 +- src/commands/src/shared/next-gen-deploy.ts | 2 +- src/job/productionJobHandler.ts | 9 +++++++-- 6 files changed, 24 insertions(+), 19 deletions(-) diff --git a/.github/workflows/clean-feature-branch.yml b/.github/workflows/clean-feature-branch.yml index 2e0f9fec4..86c4503e7 100644 --- a/.github/workflows/clean-feature-branch.yml +++ b/.github/workflows/clean-feature-branch.yml @@ -6,10 +6,10 @@ on: - closed name: Cleanup Feature Branch Infrastructure -jobs: +jobs: destroy: runs-on: ubuntu-latest - + steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v1 @@ -25,11 +25,11 @@ jobs: run: | cd cdk-infra/ npm ci - npm run cdk destroy -- -c customFeatureName=enhancedApp-stg-${{github.head_ref}} --force \ - auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-worker \ - auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-queues \ - auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhooks + npm run cdk destroy -- -c customFeatureName=enhancedApp-dotcomstg-${{github.head_ref}} --force \ + auto-builder-stack-enhancedApp-dotcomstg-${{github.head_ref}}-worker \ + auto-builder-stack-enhancedApp-dotcomstg-${{github.head_ref}}-queues \ + auto-builder-stack-enhancedApp-dotcomstg-${{github.head_ref}}-webhooks - name: Delete cache run: gh cache delete ${{github.head_ref}} - env: + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/deploy-feature-branch.yml b/.github/workflows/deploy-feature-branch.yml index 57e992671..f291ed940 100644 --- a/.github/workflows/deploy-feature-branch.yml +++ b/.github/workflows/deploy-feature-branch.yml @@ -26,9 +26,9 @@ jobs: npm ci cd cdk-infra/ npm ci - npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-queues - npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-worker - npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhooks --outputs-file outputs.json + npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg-${{github.head_ref}} auto-builder-stack-enhancedApp-dotcomstg-${{github.head_ref}}-queues + npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg-${{github.head_ref}} auto-builder-stack-enhancedApp-dotcomstg-${{github.head_ref}}-worker + npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg-${{github.head_ref}} auto-builder-stack-enhancedApp-dotcomstg-${{github.head_ref}}-webhooks --outputs-file outputs.json - name: Get Webhook URL uses: mongodb/docs-worker-actions/comment-pr@main env: diff --git a/.github/workflows/update-feature-branch.yml b/.github/workflows/update-feature-branch.yml index b896e751d..18cf34a27 100644 --- a/.github/workflows/update-feature-branch.yml +++ b/.github/workflows/update-feature-branch.yml @@ -6,7 +6,7 @@ on: - synchronize concurrency: - group: environment-stg-feature-${{ github.ref }} + group: environment-dotcomstg-feature-${{ github.ref }} cancel-in-progress: true name: Update Feature Branch Infrastructure jobs: @@ -73,8 +73,8 @@ jobs: if: steps.filter.outputs.webhooks == 'true' run: | cd cdk-infra/ - npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \ - auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-webhooks + npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg-${{github.head_ref}} \ + auto-builder-stack-enhancedApp-dotcomstg-${{github.head_ref}}-webhooks build-worker: needs: prep-build runs-on: ubuntu-latest @@ -109,5 +109,5 @@ jobs: if: steps.filter.outputs.worker == 'true' run: | cd cdk-infra/ - npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=stg -c customFeatureName=enhancedApp-stg-${{github.head_ref}} \ - auto-builder-stack-enhancedApp-stg-${{github.head_ref}}-worker + npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg-${{github.head_ref}} \ + auto-builder-stack-enhancedApp-dotcomstg-${{github.head_ref}}-worker diff --git a/cdk-infra/lib/constructs/worker/worker-construct.ts b/cdk-infra/lib/constructs/worker/worker-construct.ts index 8c00607dc..d412d3975 100644 --- a/cdk-infra/lib/constructs/worker/worker-construct.ts +++ b/cdk-infra/lib/constructs/worker/worker-construct.ts @@ -12,7 +12,7 @@ import { LogGroup } from 'aws-cdk-lib/aws-logs'; import { IQueue } from 'aws-cdk-lib/aws-sqs'; import { Construct } from 'constructs'; import path from 'path'; -import { getEnv, isEnhanced } from '../../../utils/env'; +import { getEnv } from '../../../utils/env'; interface WorkerConstructProps { dockerEnvironment: Record; diff --git a/src/commands/src/shared/next-gen-deploy.ts b/src/commands/src/shared/next-gen-deploy.ts index 61ea7c9f1..09dc685bb 100644 --- a/src/commands/src/shared/next-gen-deploy.ts +++ b/src/commands/src/shared/next-gen-deploy.ts @@ -61,7 +61,7 @@ export async function nextGenDeploy({ }, } ); - const output = `COMMAND: yes | mut-publish public ${bucket} --prefix=${mutPrefix} --deploy --deployed-url-prefix=${url} --json --all-subdirectories --dry-run + const output = `COMMAND: yes | mut-publish public ${bucket} --prefix=${mutPrefix} --deploy --deployed-url-prefix=${url} --json --all-subdirectories \n${outputText} \n ${errorText} \n Hosted at ${url}${mutPrefix} `; diff --git a/src/job/productionJobHandler.ts b/src/job/productionJobHandler.ts index 660469b7a..cbddf6000 100644 --- a/src/job/productionJobHandler.ts +++ b/src/job/productionJobHandler.ts @@ -1,6 +1,5 @@ import { IConfig } from 'config'; import type { Job } from '../entities/job'; -import { InvalidJobError } from '../errors/errors'; import { DocsetsRepository } from '../repositories/docsetsRepository'; import { JobRepository } from '../repositories/jobRepository'; import { RepoBranchesRepository } from '../repositories/repoBranchesRepository'; @@ -182,10 +181,16 @@ export class ProductionJobHandler extends JobHandler { const { mutPrefix, branchName } = this.currJob.payload; await this.logger.save(this.currJob._id, `${'(prod)'.padEnd(15)} Begin Deploy without makefiles`); + // using this as a way to test deploy with feature branches in dotcomstg (preprod) + const finalMutPrefix = + process.env.IS_FEATURE_BRANCH === 'true' && process.env.FEATURE_NAME + ? `${mutPrefix}${process.env.FEATURE_NAME}` + : mutPrefix; + const { bucket, url } = await this.getEnvironmentVariables(); const hasConfigRedirects = await checkRedirects(this.currJob.payload.repoName, this.currJob.payload.directory); - resp = await nextGenDeploy({ mutPrefix, bucket, url, branchName, hasConfigRedirects }); + resp = await nextGenDeploy({ mutPrefix: finalMutPrefix, bucket, url, branchName, hasConfigRedirects }); } else { resp = await this.deployWithMakefiles(); } From 3403c8d7485e5bacedac5d2dccb0784bb63c9c40 Mon Sep 17 00:00:00 2001 From: branberry Date: Tue, 20 Feb 2024 10:23:06 -0600 Subject: [PATCH 14/64] [DOP-4334]: Update comment pr action --- .github/workflows/deploy-feature-branch.yml | 3 ++- api/controllers/v2/test-deploy.ts | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-feature-branch.yml b/.github/workflows/deploy-feature-branch.yml index f291ed940..365159e8e 100644 --- a/.github/workflows/deploy-feature-branch.yml +++ b/.github/workflows/deploy-feature-branch.yml @@ -30,9 +30,10 @@ jobs: npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg-${{github.head_ref}} auto-builder-stack-enhancedApp-dotcomstg-${{github.head_ref}}-worker npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg-${{github.head_ref}} auto-builder-stack-enhancedApp-dotcomstg-${{github.head_ref}}-webhooks --outputs-file outputs.json - name: Get Webhook URL - uses: mongodb/docs-worker-actions/comment-pr@main + uses: mongodb/docs-worker-actions/comment-pr@DOP-4334 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + STAGE: dotcomstg - name: Cache root node_modules id: cache-root uses: actions/cache@v3 diff --git a/api/controllers/v2/test-deploy.ts b/api/controllers/v2/test-deploy.ts index b662a4dab..c14503804 100644 --- a/api/controllers/v2/test-deploy.ts +++ b/api/controllers/v2/test-deploy.ts @@ -1,6 +1,8 @@ import { APIGatewayEvent, APIGatewayProxyResult } from 'aws-lambda'; export async function handleTestDeployRequest(event: APIGatewayEvent): Promise { + const jobsQueueUrl = process.env.JOBS_QUEUE_URL; + return { statusCode: 500, body: 'not complete', From a2866ce1e8ca1d7b6ab51fe1d971f6d7d26c6f8d Mon Sep 17 00:00:00 2001 From: branberry Date: Tue, 20 Feb 2024 12:34:13 -0600 Subject: [PATCH 15/64] [DOP-4334]: Add test deploy endpoint --- api/controllers/v2/test-deploy.ts | 6 +- src/commands/src/scripts/test-deploy/index.ts | 3 + .../test-deploy/utils/get-test-deploy-args.ts | 63 +++++++++++++++++++ src/services/commandExecutor.ts | 2 +- 4 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 src/commands/src/scripts/test-deploy/index.ts create mode 100644 src/commands/src/scripts/test-deploy/utils/get-test-deploy-args.ts diff --git a/api/controllers/v2/test-deploy.ts b/api/controllers/v2/test-deploy.ts index c14503804..d2c36e100 100644 --- a/api/controllers/v2/test-deploy.ts +++ b/api/controllers/v2/test-deploy.ts @@ -1,8 +1,12 @@ import { APIGatewayEvent, APIGatewayProxyResult } from 'aws-lambda'; +import * as mongodb from 'mongodb'; export async function handleTestDeployRequest(event: APIGatewayEvent): Promise { - const jobsQueueUrl = process.env.JOBS_QUEUE_URL; + const { DB_NAME, JOBS_QUEUE_URL, MONGO_ATLAS_URL } = process.env; + const client = new mongodb.MongoClient(MONGO_ATLAS_URL); + await client.connect(); + const db = client.db(DB_NAME); return { statusCode: 500, body: 'not complete', diff --git a/src/commands/src/scripts/test-deploy/index.ts b/src/commands/src/scripts/test-deploy/index.ts new file mode 100644 index 000000000..24eb6074d --- /dev/null +++ b/src/commands/src/scripts/test-deploy/index.ts @@ -0,0 +1,3 @@ +export async function main(): Promise { + return; +} diff --git a/src/commands/src/scripts/test-deploy/utils/get-test-deploy-args.ts b/src/commands/src/scripts/test-deploy/utils/get-test-deploy-args.ts new file mode 100644 index 000000000..dc9af1162 --- /dev/null +++ b/src/commands/src/scripts/test-deploy/utils/get-test-deploy-args.ts @@ -0,0 +1,63 @@ +export function getTestDeployArgs() { + const helpIdx = process.argv.findIndex((str) => str === '--help' || str === '-h'); + + if (helpIdx !== -1) { + console.log(` + This command builds and runs the Autobuilder for local debugging. + + Flags: + --repo-owner, -o (required) The owner of the repo. Typically this is 'mongodb' or '10gen'. This should be your username for a fork. + --repo-name, -n (required) The name of the repo e.g. docs-java or cloud-docs. + --directory, -d (optional) The project directory path for a monorepo project. A slash is not needed at the beginning. For example, cloud-docs in the monorepo would just be cloud-docs for the argument. + --branch-name, -b (optional) The branch name we want to parse. If not provided, the value 'master' is used by default. + --job-type -t (optional) The job type. Can be githubPush (uses the StagingJobHandler), productionDeploy (uses the ProductionJobHandler), manifestGeneration (uses the ManifestJobHandler), and regression (uses the RegressionJobHandler). Default value is githubPush. + `); + + process.exit(0); + } + const repoOwnerIdx = process.argv.findIndex((str) => str === '--repo-owner' || str === '-o'); + const repoNameIdx = process.argv.findIndex((str) => str === '--repo-name' || str === '-n'); + + // optional args + const directoryIdx = process.argv.findIndex((str) => str === '--directory' || str === '-d'); + const branchNameIdx = process.argv.findIndex((str) => str === '--branch-name' || str === '-b'); + + if (repoOwnerIdx === -1) + throw new Error( + 'Please provide a repo owner. The flag to provide this is --repo-owner, or you can use -o as well.' + ); + + if (repoNameIdx === -1) + throw new Error('Please provide a repo name. The flag to provide this is --repo-name, or you can use -n as well.'); + + if (repoOwnerIdx + 1 === process.argv.length) throw new Error('Please provide a value for the repo owner flag'); + if (process.argv[repoOwnerIdx + 1].startsWith('-')) + throw new Error(`Please provide a valid repo owner value. Value provided: ${process.argv[repoOwnerIdx + 1]}`); + + if (repoNameIdx + 1 === process.argv.length) throw new Error('Please provide a value for the repo name flag'); + if (process.argv[repoNameIdx + 1].startsWith('-')) + throw new Error(`Please provide a valid repo name value. Value provided: ${process.argv[repoNameIdx + 1]}`); + + if (directoryIdx + 1 === process.argv.length) + throw new Error('Please provide a value for the monorepo directory flag'); + if (process.argv[directoryIdx + 1].startsWith('-')) + throw new Error( + `Please provide a valid monorepo directory value. Value provided: ${process.argv[directoryIdx + 1]}` + ); + + if (branchNameIdx + 1 === process.argv.length) throw new Error('Please provide a value for the branch name flag'); + if (process.argv[branchNameIdx + 1].startsWith('-')) + throw new Error(`Please provide a valid branch name value. Value provided: ${process.argv[branchNameIdx + 1]}`); + + const repoOwner = process.argv[repoOwnerIdx + 1]; + const repoName = process.argv[repoNameIdx + 1]; + const directory = directoryIdx !== -1 ? process.argv[directoryIdx + 1] : undefined; + const branchName = branchNameIdx !== -1 ? process.argv[branchNameIdx + 1] : 'master'; + + return { + repoOwner, + repoName, + directory, + branchName, + }; +} diff --git a/src/services/commandExecutor.ts b/src/services/commandExecutor.ts index 9dc2fe04d..9193147be 100644 --- a/src/services/commandExecutor.ts +++ b/src/services/commandExecutor.ts @@ -58,7 +58,7 @@ export class ShellCommandExecutor implements ICommandExecutor { if (cwd) { options.cwd = cwd; } - const { stdout, stderr } = await exec(commands.join(' && '), options); + const { stdout, stderr } = await exec(commands.join(' && '), { ...options, maxBuffer }); resp.output = stdout.trim(); resp.error = stderr; From 40e118c7dc5a43bb6a8cd87a850833615d0e5897 Mon Sep 17 00:00:00 2001 From: branberry Date: Wed, 21 Feb 2024 09:42:56 -0600 Subject: [PATCH 16/64] [DOP-4334]: Start adding logic to create test job --- api/controllers/v2/test-deploy.ts | 38 +++++++++++++++ api/handlers/jobs.ts | 79 ++++++++++++++++++++++++++++++- src/entities/job.ts | 20 +++++--- src/job/jobHandler.ts | 1 - src/services/commandExecutor.ts | 2 +- 5 files changed, 130 insertions(+), 10 deletions(-) diff --git a/api/controllers/v2/test-deploy.ts b/api/controllers/v2/test-deploy.ts index d2c36e100..09a536417 100644 --- a/api/controllers/v2/test-deploy.ts +++ b/api/controllers/v2/test-deploy.ts @@ -1,12 +1,50 @@ import { APIGatewayEvent, APIGatewayProxyResult } from 'aws-lambda'; import * as mongodb from 'mongodb'; +import c from 'config'; +import { RepoEntitlementsRepository } from '../../../src/repositories/repoEntitlementsRepository'; +import { ConsoleLogger } from '../../../src/services/logger'; +import { buildEntitledBranchList, isUserEntitled } from '../../handlers/slack'; +import { RepoBranchesRepository } from '../../../src/repositories/repoBranchesRepository'; +interface TestDeployRequest { + githubUsername: string; + repoName: string; + repoOwner: string; + branchName?: string; + directory?: string; +} export async function handleTestDeployRequest(event: APIGatewayEvent): Promise { const { DB_NAME, JOBS_QUEUE_URL, MONGO_ATLAS_URL } = process.env; + if (!event.body) { + return { + statusCode: 400, + body: 'Event body is undefined', + }; + } + const { githubUsername } = JSON.parse(event.body) as TestDeployRequest; + + if (!MONGO_ATLAS_URL) { + console.error('MONGO_ATLAS_URL is not defined!'); + return { statusCode: 500, body: 'internal server error' }; + } + const client = new mongodb.MongoClient(MONGO_ATLAS_URL); await client.connect(); const db = client.db(DB_NAME); + + const consoleLogger = new ConsoleLogger(); + const repoEntitlementRepository = new RepoEntitlementsRepository(db, c, consoleLogger); + const repoBranchesRepository = new RepoBranchesRepository(db, c, consoleLogger); + + const entitlement = await repoEntitlementRepository.getRepoEntitlementsByGithubUsername(githubUsername); + if (!isUserEntitled(entitlement)) { + return { + statusCode: 401, + body: 'User is not entitled!', + }; + } + return { statusCode: 500, body: 'not complete', diff --git a/api/handlers/jobs.ts b/api/handlers/jobs.ts index 8aedd6ffd..e5f19dad1 100644 --- a/api/handlers/jobs.ts +++ b/api/handlers/jobs.ts @@ -7,7 +7,7 @@ import { JobRepository } from '../../src/repositories/jobRepository'; import { GithubCommenter } from '../../src/services/github'; import { SlackConnector } from '../../src/services/slack'; import { RepoEntitlementsRepository } from '../../src/repositories/repoEntitlementsRepository'; -import { EnhancedJob, Job, JobStatus, Payload } from '../../src/entities/job'; +import { EnhancedJob, EnhancedPayload, Job, JobStatus, Payload } from '../../src/entities/job'; // Although data in payload should always be present, it's not guaranteed from // external callers @@ -280,3 +280,80 @@ function getPreviewUrl(payload: Payload | undefined, env: string): string | unde const possibleStagingSuffix = env === 'stg' ? 'stg' : ''; return `https://preview-mongodb${githubUsernameNoHyphens}${possibleStagingSuffix}.gatsbyjs.io/${project}/${branchName}/`; } + +interface CreateJobProps { + payload: EnhancedPayload; + jobTitle: string; + jobUserName: string; + jobUserEmail?: string; +} + +interface CreatePayloadProps { + jobType: string; + repoOwner: string; + repoName: string; + branchName: string; + newHead: string; + project: string; + prefix: string; + urlSlug; + source: string; + action: string; + aliased: boolean; + primaryAlias: boolean; + stable: boolean; + directory?: string; +} + +export function createPayload({ + jobType, + repoOwner, + repoName, + branchName, + newHead, + project, + prefix, + urlSlug, + source, + action, + aliased = false, + primaryAlias = false, + stable = false, + directory, +}: CreatePayloadProps): EnhancedPayload { + return { + jobType, + source, + action, + repoName, + branchName, + project, + prefix, + aliased, + urlSlug, + isFork: true, + repoOwner, + url: 'https://github.com/' + repoOwner + '/' + repoName, + newHead, + primaryAlias, + stable, + directory, + }; +} + +export function createJob({ jobTitle, jobUserName, payload, jobUserEmail }: CreateJobProps): Omit { + return { + title: jobTitle, + user: jobUserName, + email: jobUserEmail, + status: JobStatus.inQueue, + createdTime: new Date(), + startTime: null, + endTime: null, + priority: 1, + error: {}, + result: null, + payload, + logs: [], + }; +} diff --git a/src/entities/job.ts b/src/entities/job.ts index eaceaba59..b6c8a78af 100644 --- a/src/entities/job.ts +++ b/src/entities/job.ts @@ -75,9 +75,9 @@ export type EnhancedPayload = { pathPrefix?: string | null; mutPrefix?: string | null; aliased?: boolean | null; - primaryAlias?: string | null; + primaryAlias?: boolean | null; stable?: boolean | null; - repoBranches?: any; + repoBranches?: unknown; regression?: boolean | null; urlSlug?: string | null; prefix: string; @@ -112,15 +112,21 @@ export type Job = { shouldGenerateSearchManifest: boolean; }; -export type EnhancedJob = { +/** + * NOTE: This is just the Job type, but we kept them separate + * to minimize any risk of interfering with existing code. This should be + * re-examined at some point so that we can replace the Job type with this one + * to reduce confusion. + */ +export interface EnhancedJob { _id: string; payload: EnhancedPayload; createdTime: Date; endTime: Date | null | undefined; - error?: any; + error?: unknown; logs: string[] | null | undefined; priority: number | null | undefined; - result?: any; + result?: unknown; startTime: Date | null; status: JobStatus | null; title: string; @@ -131,8 +137,8 @@ export type EnhancedJob = { buildCommands?: string[]; deployCommands?: string[]; invalidationStatusURL?: string | null; - email: string | null; // probably can be removed + email?: string | null; // probably can be removed comMessage?: string[] | null; purgedUrls?: string[] | null; shouldGenerateSearchManifest?: boolean; -}; +} diff --git a/src/job/jobHandler.ts b/src/job/jobHandler.ts index c4ec90268..fdb18fa65 100644 --- a/src/job/jobHandler.ts +++ b/src/job/jobHandler.ts @@ -202,7 +202,6 @@ export abstract class JobHandler { try { await this._repoConnector.pullRepo(this.currJob); } catch (error) { - await error; throw error; } } diff --git a/src/services/commandExecutor.ts b/src/services/commandExecutor.ts index 9193147be..9dc2fe04d 100644 --- a/src/services/commandExecutor.ts +++ b/src/services/commandExecutor.ts @@ -58,7 +58,7 @@ export class ShellCommandExecutor implements ICommandExecutor { if (cwd) { options.cwd = cwd; } - const { stdout, stderr } = await exec(commands.join(' && '), { ...options, maxBuffer }); + const { stdout, stderr } = await exec(commands.join(' && '), options); resp.output = stdout.trim(); resp.error = stderr; From 9ca45d59477ecece89c60661be58cbd2f74e50ee Mon Sep 17 00:00:00 2001 From: branberry Date: Wed, 21 Feb 2024 09:46:37 -0600 Subject: [PATCH 17/64] [DOP-4334]: Cache node_modules first --- .github/workflows/deploy-feature-branch.yml | 22 +++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy-feature-branch.yml b/.github/workflows/deploy-feature-branch.yml index 365159e8e..2a59dc5b0 100644 --- a/.github/workflows/deploy-feature-branch.yml +++ b/.github/workflows/deploy-feature-branch.yml @@ -21,6 +21,20 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 + - name: Install node_modules + run: | + npm ci + cd cdk-infra/ + npm ci + - name: Cache root node_modules + id: cache-root + uses: actions/cache@v3 + with: + path: | + node_modules + cdk-infra/node_modules + key: ${{ github.head_ref }} + - name: Deploy Feature Branch Infrastructure run: | npm ci @@ -34,11 +48,3 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} STAGE: dotcomstg - - name: Cache root node_modules - id: cache-root - uses: actions/cache@v3 - with: - path: | - node_modules - cdk-infra/node_modules - key: ${{ github.head_ref }} From 75f3d6baa34d2f0cccc8c41de9784aeb8f243c3a Mon Sep 17 00:00:00 2001 From: branberry Date: Wed, 21 Feb 2024 10:20:27 -0600 Subject: [PATCH 18/64] [DOP-4334]: Add more logging to find where stuff stops working --- api/handlers/jobs.ts | 2 +- src/job/productionJobHandler.ts | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/api/handlers/jobs.ts b/api/handlers/jobs.ts index e5f19dad1..d2e67a54c 100644 --- a/api/handlers/jobs.ts +++ b/api/handlers/jobs.ts @@ -293,7 +293,6 @@ interface CreatePayloadProps { repoOwner: string; repoName: string; branchName: string; - newHead: string; project: string; prefix: string; urlSlug; @@ -303,6 +302,7 @@ interface CreatePayloadProps { primaryAlias: boolean; stable: boolean; directory?: string; + newHead?: string; } export function createPayload({ diff --git a/src/job/productionJobHandler.ts b/src/job/productionJobHandler.ts index cbddf6000..b1905ad22 100644 --- a/src/job/productionJobHandler.ts +++ b/src/job/productionJobHandler.ts @@ -179,7 +179,7 @@ export class ProductionJobHandler extends JobHandler { let resp; if (process.env.FEATURE_FLAG_MONOREPO_PATH === 'true' && this.currJob.payload.repoName === MONOREPO_NAME) { const { mutPrefix, branchName } = this.currJob.payload; - await this.logger.save(this.currJob._id, `${'(prod)'.padEnd(15)} Begin Deploy without makefiles`); + await this.logger.save(this.currJob._id, `${'(prod-monorepo)'.padEnd(15)} Begin Deploy without makefiles`); // using this as a way to test deploy with feature branches in dotcomstg (preprod) const finalMutPrefix = @@ -188,8 +188,10 @@ export class ProductionJobHandler extends JobHandler { : mutPrefix; const { bucket, url } = await this.getEnvironmentVariables(); - const hasConfigRedirects = await checkRedirects(this.currJob.payload.repoName, this.currJob.payload.directory); + await this.logger.save(this.currJob._id, `${'(prod-monorepo)'.padEnd(15)} Check redirects`); + const hasConfigRedirects = await checkRedirects(this.currJob.payload.repoName, this.currJob.payload.directory); + await this.logger.save(this.currJob._id, `${'(prod-monorepo)'.padEnd(15)} Redirects checked`); resp = await nextGenDeploy({ mutPrefix: finalMutPrefix, bucket, url, branchName, hasConfigRedirects }); } else { resp = await this.deployWithMakefiles(); From 14ec5f5d702d826c566a33d9929f1b4fa2935291 Mon Sep 17 00:00:00 2001 From: branberry Date: Wed, 21 Feb 2024 10:20:49 -0600 Subject: [PATCH 19/64] [DOP-4334]: Add more logging to find where stuff stops working --- src/job/productionJobHandler.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/job/productionJobHandler.ts b/src/job/productionJobHandler.ts index b1905ad22..d251263cc 100644 --- a/src/job/productionJobHandler.ts +++ b/src/job/productionJobHandler.ts @@ -192,6 +192,7 @@ export class ProductionJobHandler extends JobHandler { await this.logger.save(this.currJob._id, `${'(prod-monorepo)'.padEnd(15)} Check redirects`); const hasConfigRedirects = await checkRedirects(this.currJob.payload.repoName, this.currJob.payload.directory); await this.logger.save(this.currJob._id, `${'(prod-monorepo)'.padEnd(15)} Redirects checked`); + console.log('Generic log to see that it actually logs stuff in CloudWatch (which it should)'); resp = await nextGenDeploy({ mutPrefix: finalMutPrefix, bucket, url, branchName, hasConfigRedirects }); } else { resp = await this.deployWithMakefiles(); From 3f2175ba77fccbe7b4e7e767dff74abfb12445c5 Mon Sep 17 00:00:00 2001 From: branberry Date: Wed, 21 Feb 2024 13:30:10 -0600 Subject: [PATCH 20/64] [DOP-4334]: Convert github push to production deploy for testing purposes --- .github/workflows/deploy-stg-ecs.yml | 2 +- api/controllers/v2/github.ts | 2 +- src/commands/src/shared/next-gen-deploy.ts | 2 ++ src/job/productionJobHandler.ts | 5 ++++- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy-stg-ecs.yml b/.github/workflows/deploy-stg-ecs.yml index d8ff99405..029132eb1 100644 --- a/.github/workflows/deploy-stg-ecs.yml +++ b/.github/workflows/deploy-stg-ecs.yml @@ -3,7 +3,7 @@ on: branches: - 'main' - 'integration' - - 'DOP-4334' + # - 'DOP-4334' concurrency: group: environment-stg-${{ github.ref }} cancel-in-progress: true diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index a761e742b..adfb4a53e 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -42,7 +42,7 @@ async function prepGithubPushPayload( error: {}, result: null, payload: { - jobType: 'githubPush', + jobType: 'productionDeploy', source: 'github', action: 'push', repoName: githubEvent.repository.name, diff --git a/src/commands/src/shared/next-gen-deploy.ts b/src/commands/src/shared/next-gen-deploy.ts index 09dc685bb..11a529dc9 100644 --- a/src/commands/src/shared/next-gen-deploy.ts +++ b/src/commands/src/shared/next-gen-deploy.ts @@ -18,6 +18,8 @@ export async function nextGenDeploy({ bucket, url, }: NextGenDeployParams): Promise { + console.log('In next gen deploy'); + console.log('branchName', branchName); try { if (hasConfigRedirects && (branchName === 'main' || branchName === 'master' || branchName === 'current')) { // equivalent to: mut-redirects config/redirects -o public/.htaccess diff --git a/src/job/productionJobHandler.ts b/src/job/productionJobHandler.ts index d251263cc..abd954f91 100644 --- a/src/job/productionJobHandler.ts +++ b/src/job/productionJobHandler.ts @@ -191,7 +191,10 @@ export class ProductionJobHandler extends JobHandler { await this.logger.save(this.currJob._id, `${'(prod-monorepo)'.padEnd(15)} Check redirects`); const hasConfigRedirects = await checkRedirects(this.currJob.payload.repoName, this.currJob.payload.directory); - await this.logger.save(this.currJob._id, `${'(prod-monorepo)'.padEnd(15)} Redirects checked`); + await this.logger.save( + this.currJob._id, + `${'(prod-monorepo)'.padEnd(15)} Redirects checked ${hasConfigRedirects}` + ); console.log('Generic log to see that it actually logs stuff in CloudWatch (which it should)'); resp = await nextGenDeploy({ mutPrefix: finalMutPrefix, bucket, url, branchName, hasConfigRedirects }); } else { From c5e6b7a8578cf31b335abea8dff643ba4050c7b6 Mon Sep 17 00:00:00 2001 From: branberry Date: Wed, 21 Feb 2024 13:37:11 -0600 Subject: [PATCH 21/64] [DOP-4334]: Add dummy workflow to test --- .github/workflows/remove-this.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/workflows/remove-this.yml diff --git a/.github/workflows/remove-this.yml b/.github/workflows/remove-this.yml new file mode 100644 index 000000000..4148a7f61 --- /dev/null +++ b/.github/workflows/remove-this.yml @@ -0,0 +1,18 @@ +on: + pull_request: + branches: + - main + types: + - opened + - reopened +name: Initial Feature Branch Deploy +jobs: + deploy: + permissions: write-all + runs-on: ubuntu-latest + steps: + - name: Get Webhook URL + uses: mongodb/docs-worker-actions/comment-pr@DOP-4334 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + STAGE: dotcomstg From bbee0e749d4d746b54eeb9ce43b3b89a14197698 Mon Sep 17 00:00:00 2001 From: branberry Date: Wed, 21 Feb 2024 13:42:13 -0600 Subject: [PATCH 22/64] [DOP-4334]: Use restore --- .github/workflows/deploy-feature-branch.yml | 2 +- .github/workflows/update-feature-branch.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy-feature-branch.yml b/.github/workflows/deploy-feature-branch.yml index 2a59dc5b0..97662846a 100644 --- a/.github/workflows/deploy-feature-branch.yml +++ b/.github/workflows/deploy-feature-branch.yml @@ -28,7 +28,7 @@ jobs: npm ci - name: Cache root node_modules id: cache-root - uses: actions/cache@v3 + uses: actions/cache/save@v4 with: path: | node_modules diff --git a/.github/workflows/update-feature-branch.yml b/.github/workflows/update-feature-branch.yml index 18cf34a27..af6803126 100644 --- a/.github/workflows/update-feature-branch.yml +++ b/.github/workflows/update-feature-branch.yml @@ -34,7 +34,7 @@ jobs: npm ci - name: Cache root node_modules id: cache-root - uses: actions/cache@v3 + uses: actions/cache/save@v4 with: path: | node_modules @@ -54,7 +54,7 @@ jobs: - uses: actions/setup-node@v4 with: node-version: '18.x' - - uses: actions/cache/restore@v3 + - uses: actions/cache/restore@v4 id: cache-restore with: path: | From 91e5f7d8c106400474a8bb0c2b235d8507505c02 Mon Sep 17 00:00:00 2001 From: branberry Date: Wed, 21 Feb 2024 13:49:16 -0600 Subject: [PATCH 23/64] [DOP-4334]: Update restore --- .github/workflows/remove-this.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/remove-this.yml b/.github/workflows/remove-this.yml index 4148a7f61..3405a27f4 100644 --- a/.github/workflows/remove-this.yml +++ b/.github/workflows/remove-this.yml @@ -5,7 +5,7 @@ on: types: - opened - reopened -name: Initial Feature Branch Deploy +name: Dummy test jobs: deploy: permissions: write-all From 890f59068afc1e1ba98e60405ead99bbc588ae5f Mon Sep 17 00:00:00 2001 From: branberry Date: Wed, 21 Feb 2024 14:04:50 -0600 Subject: [PATCH 24/64] [DOP-4334]: Update dummy test --- .github/workflows/remove-this.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/remove-this.yml b/.github/workflows/remove-this.yml index 3405a27f4..373aad401 100644 --- a/.github/workflows/remove-this.yml +++ b/.github/workflows/remove-this.yml @@ -3,8 +3,7 @@ on: branches: - main types: - - opened - - reopened + - synchronize name: Dummy test jobs: deploy: From 8821b38944442ea115c6c2158b14d6c5fea39c77 Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 22 Feb 2024 09:35:30 -0600 Subject: [PATCH 25/64] [DOP-4334]: Update job validator to run production deploy if not 10gen or mongodb --- src/job/jobValidator.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/job/jobValidator.ts b/src/job/jobValidator.ts index 9434bdb48..99c0f69e5 100644 --- a/src/job/jobValidator.ts +++ b/src/job/jobValidator.ts @@ -36,7 +36,11 @@ export class JobValidator implements IJobValidator { const entitlementToFind = `${job.payload.repoOwner}/${job.payload.repoName}${ job.payload.repoName === MONOREPO_NAME ? `/${job.payload.directory}` : `` }`; - if (!entitlementsObject?.repos?.includes(entitlementToFind)) { + if ( + !entitlementsObject?.repos?.includes(entitlementToFind) && + job.payload.repoOwner !== '10gen' && + job.payload.repoOwner !== 'mongodb' + ) { throw new AuthorizationError(`${job.user} is not entitled for repo ${entitlementToFind}`); } } From f6a51d6bf4c01ea0cd0917152956aa86df4bad04 Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 22 Feb 2024 10:28:12 -0600 Subject: [PATCH 26/64] [DOP-4334]: Use setup buildx action --- .github/workflows/update-feature-branch.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/update-feature-branch.yml b/.github/workflows/update-feature-branch.yml index af6803126..931326084 100644 --- a/.github/workflows/update-feature-branch.yml +++ b/.github/workflows/update-feature-branch.yml @@ -44,6 +44,10 @@ jobs: needs: prep-build runs-on: ubuntu-latest steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + version: v0.12.0 - uses: actions/checkout@v4 - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 From 1b890d1a6f86282f9aca4f9e1aefa178090692a7 Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 22 Feb 2024 10:34:40 -0600 Subject: [PATCH 27/64] [DOP-4334]: Add buildkit flag --- cdk-infra/lib/constructs/worker/worker-construct.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/cdk-infra/lib/constructs/worker/worker-construct.ts b/cdk-infra/lib/constructs/worker/worker-construct.ts index d412d3975..658a94895 100644 --- a/cdk-infra/lib/constructs/worker/worker-construct.ts +++ b/cdk-infra/lib/constructs/worker/worker-construct.ts @@ -68,6 +68,7 @@ export class WorkerConstruct extends Construct { buildArgs: { NPM_BASE_64_AUTH: dockerEnvironment.NPM_BASE_64_AUTH, NPM_EMAIL: dockerEnvironment.NPM_EMAIL, + DOCKER_BUILDKIT: '1', }, }; From 47e088f432919decb6bbca95f38e91822178eedf Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 22 Feb 2024 11:37:44 -0600 Subject: [PATCH 28/64] [DOP-4334]: Update logic --- src/job/jobValidator.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/job/jobValidator.ts b/src/job/jobValidator.ts index 99c0f69e5..5ffbdef15 100644 --- a/src/job/jobValidator.ts +++ b/src/job/jobValidator.ts @@ -38,8 +38,7 @@ export class JobValidator implements IJobValidator { }`; if ( !entitlementsObject?.repos?.includes(entitlementToFind) && - job.payload.repoOwner !== '10gen' && - job.payload.repoOwner !== 'mongodb' + (job.payload.repoOwner === '10gen' || job.payload.repoOwner === 'mongodb') ) { throw new AuthorizationError(`${job.user} is not entitled for repo ${entitlementToFind}`); } From 2d34eee341b981104e6a4ca38e3078456ffd46a5 Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 22 Feb 2024 12:07:13 -0600 Subject: [PATCH 29/64] [DOP-4334]: Add caching to docker builds hopefully --- .github/workflows/update-feature-branch.yml | 15 +++++++++++---- .../lib/constructs/worker/worker-construct.ts | 2 +- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/update-feature-branch.yml b/.github/workflows/update-feature-branch.yml index 931326084..8436deb9b 100644 --- a/.github/workflows/update-feature-branch.yml +++ b/.github/workflows/update-feature-branch.yml @@ -44,10 +44,6 @@ jobs: needs: prep-build runs-on: ubuntu-latest steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - version: v0.12.0 - uses: actions/checkout@v4 - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 @@ -83,6 +79,8 @@ jobs: needs: prep-build runs-on: ubuntu-latest steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - uses: actions/checkout@v4 - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 @@ -109,6 +107,15 @@ jobs: - 'cdk-infra/lib/constructs/worker/**' - 'Dockerfile' - 'modules/**' + - uses: docker/build-push-action@v5 + if: steps.filter.outputs.worker == 'true' + name: build Docker image + with: + context: . + tags: autobuilder/enhanced:latest + build-args: NPM_EMAIL=${{ secrets.NPM_EMAIL }},NPM_BASE_64_AUTH=${{ secrets.NPM_BASE64_AUTH }} + cache-from: type=gha,ref=autobuilder/enhanced:latest + cache-to: type=gha,mode=max - name: Update Worker Stack if: steps.filter.outputs.worker == 'true' run: | diff --git a/cdk-infra/lib/constructs/worker/worker-construct.ts b/cdk-infra/lib/constructs/worker/worker-construct.ts index 658a94895..69ea77b4d 100644 --- a/cdk-infra/lib/constructs/worker/worker-construct.ts +++ b/cdk-infra/lib/constructs/worker/worker-construct.ts @@ -94,7 +94,7 @@ export class WorkerConstruct extends Construct { taskRole.addToPolicy(updateTaskProtectionPolicy); taskDefinition.addContainer('workerImage', { - image: ContainerImage.fromAsset(path.join(__dirname, '../../../../'), containerProps), + image: ContainerImage.fromTarball(`${process.cwd()}/image.tar.gz`), environment: dockerEnvironment, command: ['node', '--enable-source-maps', 'enhanced/enhancedApp.js'], logging: LogDrivers.awsLogs({ From 727d7b3b81ce91ac45b0ca76ff8e9eb6449e49f1 Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 22 Feb 2024 12:15:16 -0600 Subject: [PATCH 30/64] [DOP-4334]: Add to env var and use localbuild image for basis to cache --- .github/workflows/update-feature-branch.yml | 3 + Dockerfile | 128 ++++++++++---------- 2 files changed, 68 insertions(+), 63 deletions(-) diff --git a/.github/workflows/update-feature-branch.yml b/.github/workflows/update-feature-branch.yml index 8436deb9b..cedc8f04e 100644 --- a/.github/workflows/update-feature-branch.yml +++ b/.github/workflows/update-feature-branch.yml @@ -116,6 +116,9 @@ jobs: build-args: NPM_EMAIL=${{ secrets.NPM_EMAIL }},NPM_BASE_64_AUTH=${{ secrets.NPM_BASE64_AUTH }} cache-from: type=gha,ref=autobuilder/enhanced:latest cache-to: type=gha,mode=max + env: + NPM_BASE_64_AUTH: ${{ secrets.NPM_BASE64_AUTH }} + NPM_EMAIL: ${{ secrets.NPM_EMAIL }} - name: Update Worker Stack if: steps.filter.outputs.worker == 'true' run: | diff --git a/Dockerfile b/Dockerfile index 6f7dede54..0fbfa5152 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,41 +1,21 @@ -# Build the Typescript app -FROM node:18.16.0-alpine as ts-compiler -WORKDIR /home/docsworker-xlarge -COPY config config/ -COPY package*.json ./ -COPY tsconfig*.json ./ -RUN npm ci --legacy-peer-deps -COPY . ./ -RUN npm run build - -# install persistence module -RUN cd ./modules/persistence \ - && npm ci --legacy-peer-deps \ - && npm run build - -# Build modules -# OAS Page Builder -RUN cd ./modules/oas-page-builder \ - && npm ci --legacy-peer-deps \ - && npm run build - # where repo work will happen -FROM ubuntu:20.04 -ARG WORK_DIRECTORY=/home/docsworker-xlarge -ARG SNOOTY_PARSER_VERSION=0.16.0 -ARG SNOOTY_FRONTEND_VERSION=0.16.1 -ARG MUT_VERSION=0.11.1 +FROM ubuntu:20.04 as initial +# ####################### +ARG NPM_BASE_64_AUTH +ARG NPM_EMAIL +ARG SNOOTY_PARSER_VERSION=0.15.2 +ARG SNOOTY_FRONTEND_VERSION=0.15.7 +ARG MUT_VERSION=0.10.7 ARG REDOC_CLI_VERSION=1.2.3 ARG NPM_BASE_64_AUTH ARG NPM_EMAIL -ENV DEBIAN_FRONTEND=noninteractive +ARG WORK_DIRECTORY=/home/docsworker-xlarge +ENV DEBIAN_FRONTEND=noninteractive +WORKDIR ${WORK_DIRECTORY} # helper libraries for docs builds -RUN apt-get update && apt-get install -y vim git unzip rsync curl - - -ENV PATH="${PATH}:/opt/snooty:/opt/mut:/home/docsworker-xlarge/.local/bin" +RUN apt-get update && apt-get install -y vim git unzip zip rsync # get node 18 # https://gist.github.com/RinatMullayanov/89687a102e696b1d4cab @@ -44,9 +24,6 @@ RUN curl --location https://deb.nodesource.com/setup_18.x | bash - RUN apt-get install --yes nodejs RUN apt-get install --yes build-essential -# use npm 8.* -RUN npm install -g npm@8 - # install snooty parser RUN curl -L -o snooty-parser.zip https://github.com/mongodb/snooty-parser/releases/download/v${SNOOTY_PARSER_VERSION}/snooty-v${SNOOTY_PARSER_VERSION}-linux_x86_64.zip \ && unzip -d /opt/ snooty-parser.zip @@ -55,56 +32,81 @@ RUN curl -L -o snooty-parser.zip https://github.com/mongodb/snooty-parser/releas RUN curl -L -o mut.zip https://github.com/mongodb/mut/releases/download/v${MUT_VERSION}/mut-v${MUT_VERSION}-linux_x86_64.zip \ && unzip -d /opt/ mut.zip + +ENV PATH="${PATH}:/opt/snooty:/opt/mut:/${WORK_DIRECTORY}/.local/bin" + # setup user and root directory RUN useradd -ms /bin/bash docsworker-xlarge RUN chmod 755 -R ${WORK_DIRECTORY} RUN chown -Rv docsworker-xlarge ${WORK_DIRECTORY} USER docsworker-xlarge -WORKDIR ${WORK_DIRECTORY} - -# get shared.mk -RUN curl https://raw.githubusercontent.com/mongodb/docs-worker-pool/meta/makefiles/shared.mk -o shared.mk - # install snooty frontend and docs-tools RUN git clone -b v${SNOOTY_FRONTEND_VERSION} --depth 1 https://github.com/mongodb/snooty.git \ && cd snooty \ - && npm ci --legacy-peer-deps --omit=dev \ - && git clone --depth 1 https://github.com/mongodb/docs-tools.git \ - && mkdir -p ./static/images \ - && mv ./docs-tools/themes/mongodb/static ./static/docs-tools \ - && mv ./docs-tools/themes/guides/static/images/bg-accent.svg ./static/docs-tools/images/bg-accent.svg + && npm ci --legacy-peer-deps --omit=dev + +RUN curl https://raw.githubusercontent.com/mongodb/docs-worker-pool/meta/makefiles/shared.mk -o shared.mk + -# install redoc fork RUN git clone -b @dop/redoc-cli@${REDOC_CLI_VERSION} --depth 1 https://github.com/mongodb-forks/redoc.git redoc \ # Install dependencies for Redoc CLI && cd redoc/ \ && npm ci --prefix cli/ --omit=dev -COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/package*.json ./ -COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/config config/ -COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/build ./ -RUN npm install +FROM initial as persistence -# Persistence module copy -# Create directory and add permissions to allow node module installation RUN mkdir -p modules/persistence && chmod 755 modules/persistence -COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/modules/persistence/package*.json ./modules/persistence/ -COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/modules/persistence/dist ./modules/persistence/ -ENV PERSISTENCE_MODULE_PATH=${WORK_DIRECTORY}/modules/persistence/index.js -RUN cd ./modules/persistence/ && ls && npm ci --legacy-peer-deps +COPY modules/persistence/package*.json ./modules/persistence/ +RUN cd ./modules/persistence \ + && npm ci --legacy-peer-deps +# Build persistence module -# OAS Page Builder module copy -# Create directory and add permissions to allow node module installation -RUN mkdir -p modules/oas-page-builder && chmod 755 modules/oas-page-builder -COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/modules/oas-page-builder/package*.json ./modules/oas-page-builder/ -COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/modules/oas-page-builder/dist ./modules/oas-page-builder/ -RUN cd ./modules/oas-page-builder/ && npm ci --legacy-peer-deps +COPY --chown=docsworker-xlarge modules/persistence/tsconfig*.json ./modules/persistence +COPY --chown=docsworker-xlarge modules/persistence/src ./modules/persistence/src/ +COPY --chown=docsworker-xlarge modules/persistence/index.ts ./modules/persistence -# Needed for OAS Page Builder module in shared.mk -ENV REDOC_PATH=${WORK_DIRECTORY}/redoc/cli/index.js +RUN cd ./modules/persistence \ + && npm run build:esbuild + +FROM initial as oas + +RUN mkdir -p modules/oas-page-builder && chmod 755 modules/oas-page-builder +COPY modules/oas-page-builder/package*.json ./modules/oas-page-builder/ +RUN cd ./modules/oas-page-builder \ + && npm ci --legacy-peer-deps +# Build modules +# OAS Page Builder +COPY --chown=docsworker-xlarge modules/oas-page-builder/tsconfig*.json ./modules/oas-page-builder +COPY --chown=docsworker-xlarge modules/oas-page-builder/src ./modules/oas-page-builder/src/ +COPY --chown=docsworker-xlarge modules/oas-page-builder/index.ts ./modules/oas-page-builder + +RUN cd ./modules/oas-page-builder \ + && npm run build:esbuild + +FROM initial as root + +COPY --from=persistence --chown=docsworker-xlarge ${WORK_DIRECTORY}/modules/persistence/dist/ ./modules/persistence +COPY --from=oas --chown=docsworker-xlarge ${WORK_DIRECTORY}/modules/oas-page-builder/dist/ ./modules/oas-page-builder + +# Root project build +COPY package*.json ./ +RUN npm ci --legacy-peer-deps + + +COPY tsconfig*.json ./ +COPY config config/ +COPY api api/ +COPY src src/ + +RUN npm run build:esbuild + +ENV PERSISTENCE_MODULE_PATH=${WORK_DIRECTORY}/modules/persistence/index.js ENV OAS_MODULE_PATH=${WORK_DIRECTORY}/modules/oas-page-builder/index.js +ENV REDOC_PATH=${WORK_DIRECTORY}/redoc/cli/index.js +RUN mkdir -p modules/persistence && chmod 755 modules/persistence RUN mkdir repos && chmod 755 repos + EXPOSE 3000 CMD ["node", "app.js"] From 68969eae8419c1e139c5595e416e4fba5b568385 Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 22 Feb 2024 12:29:25 -0600 Subject: [PATCH 31/64] [DOP-4334]: Update npmrc? --- Dockerfile | 3 +-- Dockerfile.local | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7b84e16bb..ddbfae7d7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,6 @@ ARG SNOOTY_PARSER_VERSION=0.15.2 ARG SNOOTY_FRONTEND_VERSION=0.15.7 ARG MUT_VERSION=0.10.7 ARG REDOC_CLI_VERSION=1.2.3 -ARG NPM_BASE_64_AUTH ARG NPM_EMAIL ARG WORK_DIRECTORY=/home/docsworker-xlarge @@ -22,7 +21,7 @@ RUN apt-get install --yes curl RUN curl --location https://deb.nodesource.com/setup_18.x | bash - RUN apt-get install --yes nodejs RUN apt-get install --yes build-essential - +RUN echo //artifactory.corp.mongodb.com/artifactory/api/npm/:_auth=${NPM_BASE_64_AUTH} >> ~/.npmrc # install snooty parser RUN curl -L -o snooty-parser.zip https://github.com/mongodb/snooty-parser/releases/download/v${SNOOTY_PARSER_VERSION}/snooty-v${SNOOTY_PARSER_VERSION}-linux_x86_64.zip \ && unzip -d /opt/ snooty-parser.zip diff --git a/Dockerfile.local b/Dockerfile.local index 37fa16fc0..09213b1cd 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -5,7 +5,6 @@ ARG SNOOTY_PARSER_VERSION=0.15.2 ARG SNOOTY_FRONTEND_VERSION=0.15.7 ARG MUT_VERSION=0.10.7 ARG REDOC_CLI_VERSION=1.2.3 -ARG NPM_BASE_64_AUTH ARG NPM_EMAIL ARG WORK_DIRECTORY=/home/docsworker-xlarge From e2c8b4f2b82cca42aa46b73fa559981daa924356 Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 22 Feb 2024 12:39:03 -0600 Subject: [PATCH 32/64] [DOP-4334]: Add email and auth to env --- Dockerfile.local | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile.local b/Dockerfile.local index 09213b1cd..b899d8854 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -8,6 +8,9 @@ ARG REDOC_CLI_VERSION=1.2.3 ARG NPM_EMAIL ARG WORK_DIRECTORY=/home/docsworker-xlarge +ENV NPM_BASE_64_AUTH=${NPM_BASE_64_AUTH} +ENV NPM_EMAIL=${NPM_EMAIL} + ENV DEBIAN_FRONTEND=noninteractive WORKDIR ${WORK_DIRECTORY} From f88b3140c42f297cb507228a02b6d33e9e7a5c5f Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 22 Feb 2024 12:44:44 -0600 Subject: [PATCH 33/64] [DOP-4334]: Update npmrc? --- Dockerfile | 5 +++++ Dockerfile.local | 3 --- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index ddbfae7d7..40347ad50 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,10 @@ ARG REDOC_CLI_VERSION=1.2.3 ARG NPM_EMAIL ARG WORK_DIRECTORY=/home/docsworker-xlarge +ENV NPM_BASE_64_AUTH=${NPM_BASE_64_AUTH} +ENV NPM_EMAIL=${NPM_EMAIL} ENV DEBIAN_FRONTEND=noninteractive + WORKDIR ${WORK_DIRECTORY} # helper libraries for docs builds @@ -21,6 +24,8 @@ RUN apt-get install --yes curl RUN curl --location https://deb.nodesource.com/setup_18.x | bash - RUN apt-get install --yes nodejs RUN apt-get install --yes build-essential +RUN echo email=${NPM_EMAIL} >> ~/.npmrc +RUN echo _auth=${NPM_BASE_64_AUTH} >> ~/.npmrc RUN echo //artifactory.corp.mongodb.com/artifactory/api/npm/:_auth=${NPM_BASE_64_AUTH} >> ~/.npmrc # install snooty parser RUN curl -L -o snooty-parser.zip https://github.com/mongodb/snooty-parser/releases/download/v${SNOOTY_PARSER_VERSION}/snooty-v${SNOOTY_PARSER_VERSION}-linux_x86_64.zip \ diff --git a/Dockerfile.local b/Dockerfile.local index b899d8854..09213b1cd 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -8,9 +8,6 @@ ARG REDOC_CLI_VERSION=1.2.3 ARG NPM_EMAIL ARG WORK_DIRECTORY=/home/docsworker-xlarge -ENV NPM_BASE_64_AUTH=${NPM_BASE_64_AUTH} -ENV NPM_EMAIL=${NPM_EMAIL} - ENV DEBIAN_FRONTEND=noninteractive WORKDIR ${WORK_DIRECTORY} From 03f6b83a130fd8bc8405c3dfe52c9f9792c0b8b8 Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 22 Feb 2024 12:47:53 -0600 Subject: [PATCH 34/64] [DOP-4334]: Remove registry --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 40347ad50..687b807d1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,6 @@ RUN apt-get install --yes nodejs RUN apt-get install --yes build-essential RUN echo email=${NPM_EMAIL} >> ~/.npmrc RUN echo _auth=${NPM_BASE_64_AUTH} >> ~/.npmrc -RUN echo //artifactory.corp.mongodb.com/artifactory/api/npm/:_auth=${NPM_BASE_64_AUTH} >> ~/.npmrc # install snooty parser RUN curl -L -o snooty-parser.zip https://github.com/mongodb/snooty-parser/releases/download/v${SNOOTY_PARSER_VERSION}/snooty-v${SNOOTY_PARSER_VERSION}-linux_x86_64.zip \ && unzip -d /opt/ snooty-parser.zip From d077d5e676b4d26daf6e48f2d05739150aec103c Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 22 Feb 2024 12:51:08 -0600 Subject: [PATCH 35/64] [DOP-4334]: Remove any reference to npmrc --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 687b807d1..6dbe3b3f0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,8 +24,7 @@ RUN apt-get install --yes curl RUN curl --location https://deb.nodesource.com/setup_18.x | bash - RUN apt-get install --yes nodejs RUN apt-get install --yes build-essential -RUN echo email=${NPM_EMAIL} >> ~/.npmrc -RUN echo _auth=${NPM_BASE_64_AUTH} >> ~/.npmrc + # install snooty parser RUN curl -L -o snooty-parser.zip https://github.com/mongodb/snooty-parser/releases/download/v${SNOOTY_PARSER_VERSION}/snooty-v${SNOOTY_PARSER_VERSION}-linux_x86_64.zip \ && unzip -d /opt/ snooty-parser.zip From 7f6648c3882c004e70da8b40905ae9e17e010807 Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 22 Feb 2024 12:54:36 -0600 Subject: [PATCH 36/64] [DOP-4334]: Use correct versions --- Dockerfile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6dbe3b3f0..1b1cda1fc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,15 @@ # where repo work will happen FROM ubuntu:20.04 as initial -ARG NPM_BASE_64_AUTH -ARG NPM_EMAIL -ARG SNOOTY_PARSER_VERSION=0.15.2 -ARG SNOOTY_FRONTEND_VERSION=0.15.7 -ARG MUT_VERSION=0.10.7 +ARG SNOOTY_PARSER_VERSION=0.16.1 +ARG SNOOTY_FRONTEND_VERSION=0.16.2 +ARG MUT_VERSION=0.11.1 ARG REDOC_CLI_VERSION=1.2.3 +ARG NPM_BASE_64_AUTH ARG NPM_EMAIL ARG WORK_DIRECTORY=/home/docsworker-xlarge + + ENV NPM_BASE_64_AUTH=${NPM_BASE_64_AUTH} ENV NPM_EMAIL=${NPM_EMAIL} ENV DEBIAN_FRONTEND=noninteractive From 759b1e10ddaf94376043d76169027404159109e8 Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 22 Feb 2024 15:40:17 -0600 Subject: [PATCH 37/64] [DOP-4334]: No caching --- .github/workflows/remove-this.yml | 17 ----------------- .github/workflows/update-feature-branch.yml | 4 ++-- Dockerfile | 4 +--- 3 files changed, 3 insertions(+), 22 deletions(-) delete mode 100644 .github/workflows/remove-this.yml diff --git a/.github/workflows/remove-this.yml b/.github/workflows/remove-this.yml deleted file mode 100644 index 373aad401..000000000 --- a/.github/workflows/remove-this.yml +++ /dev/null @@ -1,17 +0,0 @@ -on: - pull_request: - branches: - - main - types: - - synchronize -name: Dummy test -jobs: - deploy: - permissions: write-all - runs-on: ubuntu-latest - steps: - - name: Get Webhook URL - uses: mongodb/docs-worker-actions/comment-pr@DOP-4334 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - STAGE: dotcomstg diff --git a/.github/workflows/update-feature-branch.yml b/.github/workflows/update-feature-branch.yml index cedc8f04e..e95e1f3e6 100644 --- a/.github/workflows/update-feature-branch.yml +++ b/.github/workflows/update-feature-branch.yml @@ -114,8 +114,8 @@ jobs: context: . tags: autobuilder/enhanced:latest build-args: NPM_EMAIL=${{ secrets.NPM_EMAIL }},NPM_BASE_64_AUTH=${{ secrets.NPM_BASE64_AUTH }} - cache-from: type=gha,ref=autobuilder/enhanced:latest - cache-to: type=gha,mode=max + # cache-from: type=gha,ref=autobuilder/enhanced:latest + # cache-to: type=gha,mode=max env: NPM_BASE_64_AUTH: ${{ secrets.NPM_BASE64_AUTH }} NPM_EMAIL: ${{ secrets.NPM_EMAIL }} diff --git a/Dockerfile b/Dockerfile index 1b1cda1fc..cc4d24b71 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ # where repo work will happen FROM ubuntu:20.04 as initial + ARG SNOOTY_PARSER_VERSION=0.16.1 ARG SNOOTY_FRONTEND_VERSION=0.16.2 ARG MUT_VERSION=0.11.1 @@ -9,9 +10,6 @@ ARG NPM_EMAIL ARG WORK_DIRECTORY=/home/docsworker-xlarge - -ENV NPM_BASE_64_AUTH=${NPM_BASE_64_AUTH} -ENV NPM_EMAIL=${NPM_EMAIL} ENV DEBIAN_FRONTEND=noninteractive WORKDIR ${WORK_DIRECTORY} From 405bc50d4b19ff3465a22b0814cd9eaed328ed68 Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 22 Feb 2024 16:46:32 -0600 Subject: [PATCH 38/64] [DOP-4334]: Comment out to test --- .github/workflows/update-feature-branch.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/update-feature-branch.yml b/.github/workflows/update-feature-branch.yml index e95e1f3e6..c76aae0aa 100644 --- a/.github/workflows/update-feature-branch.yml +++ b/.github/workflows/update-feature-branch.yml @@ -79,8 +79,8 @@ jobs: needs: prep-build runs-on: ubuntu-latest steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v3 - uses: actions/checkout@v4 - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 From cda783591fba9d22ba82f57ddaf0e074af6fe0a1 Mon Sep 17 00:00:00 2001 From: branberry Date: Fri, 23 Feb 2024 14:48:45 -0600 Subject: [PATCH 39/64] [DOP-4334]: Remove branch --- .github/workflows/deploy-stg-ecs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/deploy-stg-ecs.yml b/.github/workflows/deploy-stg-ecs.yml index 029132eb1..d122336ff 100644 --- a/.github/workflows/deploy-stg-ecs.yml +++ b/.github/workflows/deploy-stg-ecs.yml @@ -3,7 +3,6 @@ on: branches: - 'main' - 'integration' - # - 'DOP-4334' concurrency: group: environment-stg-${{ github.ref }} cancel-in-progress: true From b044903016f0a937f72a6c8b1a8719af5448c515 Mon Sep 17 00:00:00 2001 From: branberry Date: Fri, 23 Feb 2024 15:09:02 -0600 Subject: [PATCH 40/64] [DOP-4334]: Add env back --- .github/workflows/update-feature-branch.yml | 7 ++----- Dockerfile | 2 ++ 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/update-feature-branch.yml b/.github/workflows/update-feature-branch.yml index c76aae0aa..124cf8051 100644 --- a/.github/workflows/update-feature-branch.yml +++ b/.github/workflows/update-feature-branch.yml @@ -114,11 +114,8 @@ jobs: context: . tags: autobuilder/enhanced:latest build-args: NPM_EMAIL=${{ secrets.NPM_EMAIL }},NPM_BASE_64_AUTH=${{ secrets.NPM_BASE64_AUTH }} - # cache-from: type=gha,ref=autobuilder/enhanced:latest - # cache-to: type=gha,mode=max - env: - NPM_BASE_64_AUTH: ${{ secrets.NPM_BASE64_AUTH }} - NPM_EMAIL: ${{ secrets.NPM_EMAIL }} + cache-from: type=gha,ref=autobuilder/enhanced:latest + cache-to: type=gha,mode=max - name: Update Worker Stack if: steps.filter.outputs.worker == 'true' run: | diff --git a/Dockerfile b/Dockerfile index f0f272bbd..de20915bc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,8 @@ ARG NPM_BASE_64_AUTH ARG NPM_EMAIL ARG WORK_DIRECTORY=/home/docsworker-xlarge +ENV NPM_BASE_64_AUTH=${NPM_BASE_64_AUTH} +ENV NPM_EMAIL=${NPM_EMAIL} ENV DEBIAN_FRONTEND=noninteractive From 503a85716da5270b9b1e3511bc1257a42acb89a1 Mon Sep 17 00:00:00 2001 From: branberry Date: Fri, 23 Feb 2024 15:15:36 -0600 Subject: [PATCH 41/64] [DOP-4334]: Add setup buildkit action --- .github/workflows/update-feature-branch.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/update-feature-branch.yml b/.github/workflows/update-feature-branch.yml index 124cf8051..8436deb9b 100644 --- a/.github/workflows/update-feature-branch.yml +++ b/.github/workflows/update-feature-branch.yml @@ -79,8 +79,8 @@ jobs: needs: prep-build runs-on: ubuntu-latest steps: - # - name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - uses: actions/checkout@v4 - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 From 65ffee105fe9a2ad8622b05a4b1ca64a3e1d2d37 Mon Sep 17 00:00:00 2001 From: branberry Date: Fri, 23 Feb 2024 15:25:53 -0600 Subject: [PATCH 42/64] [DOP-4334]: Push down setup node? --- .github/workflows/update-feature-branch.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/update-feature-branch.yml b/.github/workflows/update-feature-branch.yml index 8436deb9b..5f5571805 100644 --- a/.github/workflows/update-feature-branch.yml +++ b/.github/workflows/update-feature-branch.yml @@ -88,9 +88,7 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 - - uses: actions/setup-node@v4 - with: - node-version: '18.x' + - uses: actions/cache/restore@v3 id: cache-restore with: @@ -116,6 +114,9 @@ jobs: build-args: NPM_EMAIL=${{ secrets.NPM_EMAIL }},NPM_BASE_64_AUTH=${{ secrets.NPM_BASE64_AUTH }} cache-from: type=gha,ref=autobuilder/enhanced:latest cache-to: type=gha,mode=max + - uses: actions/setup-node@v4 + with: + node-version: '18.x' - name: Update Worker Stack if: steps.filter.outputs.worker == 'true' run: | From 17c3bb5fe5486dff987414c5707a75b1438cf0ad Mon Sep 17 00:00:00 2001 From: branberry Date: Fri, 23 Feb 2024 15:47:51 -0600 Subject: [PATCH 43/64] [DOP-4334]: Use build args properly --- .github/workflows/update-feature-branch.yml | 11 ++++++----- cdk-infra/lib/constructs/worker/worker-construct.ts | 8 -------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/.github/workflows/update-feature-branch.yml b/.github/workflows/update-feature-branch.yml index 5f5571805..f37f5fc51 100644 --- a/.github/workflows/update-feature-branch.yml +++ b/.github/workflows/update-feature-branch.yml @@ -88,7 +88,9 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 - + - uses: actions/setup-node@v4 + with: + node-version: '18.x' - uses: actions/cache/restore@v3 id: cache-restore with: @@ -111,12 +113,11 @@ jobs: with: context: . tags: autobuilder/enhanced:latest - build-args: NPM_EMAIL=${{ secrets.NPM_EMAIL }},NPM_BASE_64_AUTH=${{ secrets.NPM_BASE64_AUTH }} + build-args: | + NPM_EMAIL=${{ secrets.NPM_EMAIL }} + NPM_BASE_64_AUTH=${{ secrets.NPM_BASE64_AUTH }} cache-from: type=gha,ref=autobuilder/enhanced:latest cache-to: type=gha,mode=max - - uses: actions/setup-node@v4 - with: - node-version: '18.x' - name: Update Worker Stack if: steps.filter.outputs.worker == 'true' run: | diff --git a/cdk-infra/lib/constructs/worker/worker-construct.ts b/cdk-infra/lib/constructs/worker/worker-construct.ts index 69ea77b4d..0b355af09 100644 --- a/cdk-infra/lib/constructs/worker/worker-construct.ts +++ b/cdk-infra/lib/constructs/worker/worker-construct.ts @@ -64,14 +64,6 @@ export class WorkerConstruct extends Construct { executionRole.addToPolicy(executionRoleSsmPolicy); - const containerProps: AssetImageProps = { - buildArgs: { - NPM_BASE_64_AUTH: dockerEnvironment.NPM_BASE_64_AUTH, - NPM_EMAIL: dockerEnvironment.NPM_EMAIL, - DOCKER_BUILDKIT: '1', - }, - }; - const taskDefLogGroup = new LogGroup(this, 'workerLogGroup'); const taskDefinition = new FargateTaskDefinition(this, 'workerTaskDefinition', { cpu: 4096, From 83610b07c0bd5693e243576986ca57007561ce4f Mon Sep 17 00:00:00 2001 From: branberry Date: Fri, 23 Feb 2024 17:49:42 -0600 Subject: [PATCH 44/64] [DOP-4334]: Use outputs --- .github/workflows/update-feature-branch.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/update-feature-branch.yml b/.github/workflows/update-feature-branch.yml index f37f5fc51..0e30f44b5 100644 --- a/.github/workflows/update-feature-branch.yml +++ b/.github/workflows/update-feature-branch.yml @@ -118,6 +118,7 @@ jobs: NPM_BASE_64_AUTH=${{ secrets.NPM_BASE64_AUTH }} cache-from: type=gha,ref=autobuilder/enhanced:latest cache-to: type=gha,mode=max + outputs: type=tar,dest=cdk-infra/image.tar.gz - name: Update Worker Stack if: steps.filter.outputs.worker == 'true' run: | From e7967ab3c135799c7060ded8aa353a18b2a79bec Mon Sep 17 00:00:00 2001 From: branberry Date: Fri, 23 Feb 2024 18:46:09 -0600 Subject: [PATCH 45/64] [DOP-4334]: Use cdk again --- .github/workflows/update-feature-branch.yml | 24 +++++++++---------- .../lib/constructs/worker/worker-construct.ts | 11 ++++++++- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/.github/workflows/update-feature-branch.yml b/.github/workflows/update-feature-branch.yml index 0e30f44b5..4e558ba71 100644 --- a/.github/workflows/update-feature-branch.yml +++ b/.github/workflows/update-feature-branch.yml @@ -107,18 +107,18 @@ jobs: - 'cdk-infra/lib/constructs/worker/**' - 'Dockerfile' - 'modules/**' - - uses: docker/build-push-action@v5 - if: steps.filter.outputs.worker == 'true' - name: build Docker image - with: - context: . - tags: autobuilder/enhanced:latest - build-args: | - NPM_EMAIL=${{ secrets.NPM_EMAIL }} - NPM_BASE_64_AUTH=${{ secrets.NPM_BASE64_AUTH }} - cache-from: type=gha,ref=autobuilder/enhanced:latest - cache-to: type=gha,mode=max - outputs: type=tar,dest=cdk-infra/image.tar.gz + # - uses: docker/build-push-action@v5 + # if: steps.filter.outputs.worker == 'true' + # name: build Docker image + # with: + # context: . + # tags: autobuilder/enhanced:latest + # build-args: | + # NPM_EMAIL=${{ secrets.NPM_EMAIL }} + # NPM_BASE_64_AUTH=${{ secrets.NPM_BASE64_AUTH }} + # cache-from: type=gha,ref=autobuilder/enhanced:latest + # cache-to: type=gha,mode=max + # outputs: type=tar,dest=cdk-infra/image.tar.gz - name: Update Worker Stack if: steps.filter.outputs.worker == 'true' run: | diff --git a/cdk-infra/lib/constructs/worker/worker-construct.ts b/cdk-infra/lib/constructs/worker/worker-construct.ts index 0b355af09..f04c40d51 100644 --- a/cdk-infra/lib/constructs/worker/worker-construct.ts +++ b/cdk-infra/lib/constructs/worker/worker-construct.ts @@ -64,6 +64,15 @@ export class WorkerConstruct extends Construct { executionRole.addToPolicy(executionRoleSsmPolicy); + const containerProps: AssetImageProps = { + buildArgs: { + NPM_BASE_64_AUTH: dockerEnvironment.NPM_BASE_64_AUTH, + NPM_EMAIL: dockerEnvironment.NPM_EMAIL, + }, + cacheTo: { type: 'gha', params: { mode: 'max' } }, + cacheFrom: [{ type: 'gha' }], + }; + const taskDefLogGroup = new LogGroup(this, 'workerLogGroup'); const taskDefinition = new FargateTaskDefinition(this, 'workerTaskDefinition', { cpu: 4096, @@ -86,7 +95,7 @@ export class WorkerConstruct extends Construct { taskRole.addToPolicy(updateTaskProtectionPolicy); taskDefinition.addContainer('workerImage', { - image: ContainerImage.fromTarball(`${process.cwd()}/image.tar.gz`), + image: ContainerImage.fromAsset(path.join(__dirname, '../../../../'), containerProps), environment: dockerEnvironment, command: ['node', '--enable-source-maps', 'enhanced/enhancedApp.js'], logging: LogDrivers.awsLogs({ From 3c9041194690ac12749ddd78104f959cf29d100c Mon Sep 17 00:00:00 2001 From: branberry Date: Mon, 26 Feb 2024 09:04:10 -0600 Subject: [PATCH 46/64] [DOP-4334]: hotswap --- .github/workflows/update-feature-branch.yml | 4 ++-- Dockerfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/update-feature-branch.yml b/.github/workflows/update-feature-branch.yml index 4e558ba71..193f17ea7 100644 --- a/.github/workflows/update-feature-branch.yml +++ b/.github/workflows/update-feature-branch.yml @@ -73,7 +73,7 @@ jobs: if: steps.filter.outputs.webhooks == 'true' run: | cd cdk-infra/ - npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg-${{github.head_ref}} \ + npm run deploy:feature:stack -- --hotswap -c isFeatureBranch=true -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg-${{github.head_ref}} \ auto-builder-stack-enhancedApp-dotcomstg-${{github.head_ref}}-webhooks build-worker: needs: prep-build @@ -123,5 +123,5 @@ jobs: if: steps.filter.outputs.worker == 'true' run: | cd cdk-infra/ - npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg-${{github.head_ref}} \ + npm run deploy:feature:stack -- --hotswap -c isFeatureBranch=true -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg-${{github.head_ref}} \ auto-builder-stack-enhancedApp-dotcomstg-${{github.head_ref}}-worker diff --git a/Dockerfile b/Dockerfile index de20915bc..775330bc0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ ARG WORK_DIRECTORY=/home/docsworker-xlarge ENV NPM_BASE_64_AUTH=${NPM_BASE_64_AUTH} ENV NPM_EMAIL=${NPM_EMAIL} - +ENV DOCKER_BUILDKIT=1 ENV DEBIAN_FRONTEND=noninteractive WORKDIR ${WORK_DIRECTORY} From 400f9d1e7c3dfb858915382175ba6b9fe80c2462 Mon Sep 17 00:00:00 2001 From: branberry Date: Mon, 26 Feb 2024 12:19:15 -0600 Subject: [PATCH 47/64] [DOP-4334]: Update aws cdk --- .../lib/constructs/worker/worker-construct.ts | 5 +- cdk-infra/package-lock.json | 89 +++++++++++-------- cdk-infra/package.json | 2 +- src/job/jobHandler.ts | 1 + src/job/jobValidator.ts | 1 + 5 files changed, 56 insertions(+), 42 deletions(-) diff --git a/cdk-infra/lib/constructs/worker/worker-construct.ts b/cdk-infra/lib/constructs/worker/worker-construct.ts index f04c40d51..90d17b9f9 100644 --- a/cdk-infra/lib/constructs/worker/worker-construct.ts +++ b/cdk-infra/lib/constructs/worker/worker-construct.ts @@ -69,8 +69,8 @@ export class WorkerConstruct extends Construct { NPM_BASE_64_AUTH: dockerEnvironment.NPM_BASE_64_AUTH, NPM_EMAIL: dockerEnvironment.NPM_EMAIL, }, - cacheTo: { type: 'gha', params: { mode: 'max' } }, - cacheFrom: [{ type: 'gha' }], + cacheFrom: [{ type: 'ecr' }], + cacheTo: { type: 'ecr', params: { mode: 'max' } }, }; const taskDefLogGroup = new LogGroup(this, 'workerLogGroup'); @@ -97,6 +97,7 @@ export class WorkerConstruct extends Construct { taskDefinition.addContainer('workerImage', { image: ContainerImage.fromAsset(path.join(__dirname, '../../../../'), containerProps), environment: dockerEnvironment, + command: ['node', '--enable-source-maps', 'enhanced/enhancedApp.js'], logging: LogDrivers.awsLogs({ streamPrefix: 'autobuilderworker', diff --git a/cdk-infra/package-lock.json b/cdk-infra/package-lock.json index d2462491b..faac3c7b3 100644 --- a/cdk-infra/package-lock.json +++ b/cdk-infra/package-lock.json @@ -9,7 +9,7 @@ "version": "0.1.0", "dependencies": { "@aws-sdk/client-ssm": "^3.342.0", - "aws-cdk-lib": "2.73.0", + "aws-cdk-lib": "2.130.0", "constructs": "^10.0.0", "source-map-support": "^0.5.21" }, @@ -44,19 +44,19 @@ } }, "node_modules/@aws-cdk/asset-awscli-v1": { - "version": "2.2.138", - "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.138.tgz", - "integrity": "sha512-1GPAkidoyOFPhOZbkaxnclNBSBxxcN8wejpSMCixifbPvPFMvJXjMd19Eq4WDPeDDHUEjuJU9tEtvzq3OFE7Gw==" + "version": "2.2.202", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.202.tgz", + "integrity": "sha512-JqlF0D4+EVugnG5dAsNZMqhu3HW7ehOXm5SDMxMbXNDMdsF0pxtQKNHRl52z1U9igsHmaFpUgSGjbhAJ+0JONg==" }, "node_modules/@aws-cdk/asset-kubectl-v20": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@aws-cdk/asset-kubectl-v20/-/asset-kubectl-v20-2.1.1.tgz", - "integrity": "sha512-U1ntiX8XiMRRRH5J1IdC+1t5CE89015cwyt5U63Cpk0GnMlN5+h9WsWMlKlPXZR4rdq/m806JRlBMRpBUB2Dhw==" + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-kubectl-v20/-/asset-kubectl-v20-2.1.2.tgz", + "integrity": "sha512-3M2tELJOxQv0apCIiuKQ4pAbncz9GuLwnKFqxifWfe77wuMxyTRPmxssYHs42ePqzap1LT6GDcPygGs+hHstLg==" }, - "node_modules/@aws-cdk/asset-node-proxy-agent-v5": { - "version": "2.0.114", - "resolved": "https://registry.npmjs.org/@aws-cdk/asset-node-proxy-agent-v5/-/asset-node-proxy-agent-v5-2.0.114.tgz", - "integrity": "sha512-xXSptpTYIlxQyTpVud2N6/vzHaAsUSWtDrPxdEZoIJESuZJZKLP69PhELwG/NsD6WtxpWYa6LO6s2qvJhRUjew==" + "node_modules/@aws-cdk/asset-node-proxy-agent-v6": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-node-proxy-agent-v6/-/asset-node-proxy-agent-v6-2.0.1.tgz", + "integrity": "sha512-DDt4SLdLOwWCjGtltH4VCST7hpOI5DzieuhGZsBpZ+AgJdSI2GCjklCXm0GCTwJG/SolkL5dtQXyUKgg9luBDg==" }, "node_modules/@aws-crypto/crc32": { "version": "3.0.0", @@ -2838,9 +2838,9 @@ } }, "node_modules/aws-cdk-lib": { - "version": "2.73.0", - "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.73.0.tgz", - "integrity": "sha512-r9CUe3R7EThr9U0Eb7kQCK4Ee34TDeMH+bonvGD9rNRRTYDauvAgNCsx4DZYYksPrXLRzWjzVbuXAHaDDzWt+A==", + "version": "2.130.0", + "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.130.0.tgz", + "integrity": "sha512-yK7ibePipdjlI4AFM94fwwtsCkmpWJ0JFZTMPahahC/3Pxe/BA/nnI/4Namvl5QPxW5QlU0xQYU7cywioq3RQg==", "bundleDependencies": [ "@balena/dockerignore", "case", @@ -2851,20 +2851,22 @@ "punycode", "semver", "table", - "yaml" + "yaml", + "mime-types" ], "dependencies": { - "@aws-cdk/asset-awscli-v1": "^2.2.97", - "@aws-cdk/asset-kubectl-v20": "^2.1.1", - "@aws-cdk/asset-node-proxy-agent-v5": "^2.0.77", + "@aws-cdk/asset-awscli-v1": "^2.2.202", + "@aws-cdk/asset-kubectl-v20": "^2.1.2", + "@aws-cdk/asset-node-proxy-agent-v6": "^2.0.1", "@balena/dockerignore": "^1.0.2", "case": "1.6.3", - "fs-extra": "^9.1.0", - "ignore": "^5.2.4", + "fs-extra": "^11.2.0", + "ignore": "^5.3.1", "jsonschema": "^1.4.1", + "mime-types": "^2.1.35", "minimatch": "^3.1.2", - "punycode": "^2.3.0", - "semver": "^7.3.8", + "punycode": "^2.3.1", + "semver": "^7.6.0", "table": "^6.8.1", "yaml": "1.10.2" }, @@ -2925,14 +2927,6 @@ "node": ">=8" } }, - "node_modules/aws-cdk-lib/node_modules/at-least-node": { - "version": "1.0.0", - "inBundle": true, - "license": "ISC", - "engines": { - "node": ">= 4.0.0" - } - }, "node_modules/aws-cdk-lib/node_modules/balanced-match": { "version": "1.0.2", "inBundle": true, @@ -2987,26 +2981,25 @@ "license": "MIT" }, "node_modules/aws-cdk-lib/node_modules/fs-extra": { - "version": "9.1.0", + "version": "11.2.0", "inBundle": true, "license": "MIT", "dependencies": { - "at-least-node": "^1.0.0", "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", "universalify": "^2.0.0" }, "engines": { - "node": ">=10" + "node": ">=14.14" } }, "node_modules/aws-cdk-lib/node_modules/graceful-fs": { - "version": "4.2.10", + "version": "4.2.11", "inBundle": true, "license": "ISC" }, "node_modules/aws-cdk-lib/node_modules/ignore": { - "version": "5.2.4", + "version": "5.3.1", "inBundle": true, "license": "MIT", "engines": { @@ -3061,6 +3054,25 @@ "node": ">=10" } }, + "node_modules/aws-cdk-lib/node_modules/mime-db": { + "version": "1.52.0", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/aws-cdk-lib/node_modules/mime-types": { + "version": "2.1.35", + "inBundle": true, + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/aws-cdk-lib/node_modules/minimatch": { "version": "3.1.2", "inBundle": true, @@ -3073,7 +3085,7 @@ } }, "node_modules/aws-cdk-lib/node_modules/punycode": { - "version": "2.3.0", + "version": "2.3.1", "inBundle": true, "license": "MIT", "engines": { @@ -3089,7 +3101,7 @@ } }, "node_modules/aws-cdk-lib/node_modules/semver": { - "version": "7.3.8", + "version": "7.6.0", "inBundle": true, "license": "ISC", "dependencies": { @@ -3158,7 +3170,7 @@ } }, "node_modules/aws-cdk-lib/node_modules/universalify": { - "version": "2.0.0", + "version": "2.0.1", "inBundle": true, "license": "MIT", "engines": { @@ -5216,7 +5228,6 @@ "version": "6.3.0", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", - "dev": true, "bin": { "semver": "bin/semver.js" } diff --git a/cdk-infra/package.json b/cdk-infra/package.json index 71416e9a9..f4159a6ce 100644 --- a/cdk-infra/package.json +++ b/cdk-infra/package.json @@ -31,7 +31,7 @@ }, "dependencies": { "@aws-sdk/client-ssm": "^3.342.0", - "aws-cdk-lib": "2.73.0", + "aws-cdk-lib": "2.130.0", "constructs": "^10.0.0", "source-map-support": "^0.5.21" } diff --git a/src/job/jobHandler.ts b/src/job/jobHandler.ts index 894b611d9..a301b0e89 100644 --- a/src/job/jobHandler.ts +++ b/src/job/jobHandler.ts @@ -265,6 +265,7 @@ export abstract class JobHandler { this.prepStageSpecificNextGenCommands(); this.constructEnvVars(); + return; if (this._currJob.payload.jobType === 'productionDeploy') { this._validator.throwIfNotPublishable(this._currJob); } diff --git a/src/job/jobValidator.ts b/src/job/jobValidator.ts index 5ffbdef15..fa2cef185 100644 --- a/src/job/jobValidator.ts +++ b/src/job/jobValidator.ts @@ -36,6 +36,7 @@ export class JobValidator implements IJobValidator { const entitlementToFind = `${job.payload.repoOwner}/${job.payload.repoName}${ job.payload.repoName === MONOREPO_NAME ? `/${job.payload.directory}` : `` }`; + return; if ( !entitlementsObject?.repos?.includes(entitlementToFind) && (job.payload.repoOwner === '10gen' || job.payload.repoOwner === 'mongodb') From 51c0e728a4a964bd0a8413562cf57a4092f03b05 Mon Sep 17 00:00:00 2001 From: branberry Date: Mon, 26 Feb 2024 12:28:51 -0600 Subject: [PATCH 48/64] [DOP-4334]: Use gha cache --- cdk-infra/lib/constructs/worker/worker-construct.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdk-infra/lib/constructs/worker/worker-construct.ts b/cdk-infra/lib/constructs/worker/worker-construct.ts index 90d17b9f9..7a94cf1e7 100644 --- a/cdk-infra/lib/constructs/worker/worker-construct.ts +++ b/cdk-infra/lib/constructs/worker/worker-construct.ts @@ -69,8 +69,8 @@ export class WorkerConstruct extends Construct { NPM_BASE_64_AUTH: dockerEnvironment.NPM_BASE_64_AUTH, NPM_EMAIL: dockerEnvironment.NPM_EMAIL, }, - cacheFrom: [{ type: 'ecr' }], - cacheTo: { type: 'ecr', params: { mode: 'max' } }, + cacheFrom: [{ type: 'gha' }], + cacheTo: { type: 'gha', params: { mode: 'max' } }, }; const taskDefLogGroup = new LogGroup(this, 'workerLogGroup'); From 3d90bfcf76f28220fa16e35d2e92b728882d6c67 Mon Sep 17 00:00:00 2001 From: branberry Date: Mon, 26 Feb 2024 12:47:21 -0600 Subject: [PATCH 49/64] [DOP-4334]: deploy to preprod --- .github/workflows/deploy-stg-ecs.yml | 5 +++-- .github/workflows/update-feature-branch.yml | 2 ++ cdk-infra/lib/constructs/worker/worker-construct.ts | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-stg-ecs.yml b/.github/workflows/deploy-stg-ecs.yml index 2e0cfb462..d8ff99405 100644 --- a/.github/workflows/deploy-stg-ecs.yml +++ b/.github/workflows/deploy-stg-ecs.yml @@ -1,8 +1,9 @@ on: push: branches: - - "main" - - "integration" + - 'main' + - 'integration' + - 'DOP-4334' concurrency: group: environment-stg-${{ github.ref }} cancel-in-progress: true diff --git a/.github/workflows/update-feature-branch.yml b/.github/workflows/update-feature-branch.yml index 193f17ea7..9373a405e 100644 --- a/.github/workflows/update-feature-branch.yml +++ b/.github/workflows/update-feature-branch.yml @@ -81,6 +81,8 @@ jobs: steps: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + with: + version: latest - uses: actions/checkout@v4 - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 diff --git a/cdk-infra/lib/constructs/worker/worker-construct.ts b/cdk-infra/lib/constructs/worker/worker-construct.ts index 7a94cf1e7..236bd55b6 100644 --- a/cdk-infra/lib/constructs/worker/worker-construct.ts +++ b/cdk-infra/lib/constructs/worker/worker-construct.ts @@ -69,6 +69,7 @@ export class WorkerConstruct extends Construct { NPM_BASE_64_AUTH: dockerEnvironment.NPM_BASE_64_AUTH, NPM_EMAIL: dockerEnvironment.NPM_EMAIL, }, + cacheFrom: [{ type: 'gha' }], cacheTo: { type: 'gha', params: { mode: 'max' } }, }; From dc311ef5e39eec8f857bd8160d0d3a6923ce7a19 Mon Sep 17 00:00:00 2001 From: branberry Date: Mon, 26 Feb 2024 13:39:47 -0600 Subject: [PATCH 50/64] [DOP-4334]: Update ondemand app --- Dockerfile.local | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile.local b/Dockerfile.local index 09213b1cd..187c460c3 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -117,4 +117,4 @@ RUN mkdir repos && chmod 755 repos EXPOSE 3000 -CMD ["node", "--inspect-brk=0.0.0.0", "--enable-source-maps", "dist/entrypoints/onDemandApp.js"] +CMD ["node", "--inspect-brk=0.0.0.0", "--enable-source-maps", "onDemandApp.js"] diff --git a/package.json b/package.json index d78e8845b..9043be8ef 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "clean": "node maintain.js", "build": "tsc", "build:esbuild:cacheUpdater": "esbuild src/cache-updater/index.ts --bundle --platform=node --outdir=./dist/ --allow-overwrite --sourcemap", - "build:esbuild": "esbuild src/onDemandApp.ts --bundle --platform=node --outdir=./dist/entrypoints --allow-overwrite --sourcemap", + "build:esbuild": "esbuild src/onDemandApp.ts --bundle --platform=node --outdir=./ --allow-overwrite --sourcemap", "format": "npm run prettier -- --check", "format:fix": "npm run prettier -- --write", "lint": "eslint --ext .ts .", From 035096de9ecd2850548f07025ea5385c89b833d2 Mon Sep 17 00:00:00 2001 From: branberry Date: Tue, 27 Feb 2024 08:18:15 -0600 Subject: [PATCH 51/64] [DOP-4334]: Up memory --- .gitignore | 4 +++- cdk-infra/package-lock.json | 9 +++++---- cdk-infra/package.json | 2 +- infrastructure/ecs-main/serverless.yml | 4 ++-- src/commands/src/helpers/dependency-helpers.ts | 9 ++++++++- .../src/scripts/local-build/utils/create-job.ts | 2 +- src/job/jobHandler.ts | 12 ++++++++++-- 7 files changed, 30 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index da3053aa9..bb7e1b9e0 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,6 @@ out local.json globalConfig.json .serverless -dist \ No newline at end of file +dist +onDemandApp.js +*.js.map \ No newline at end of file diff --git a/cdk-infra/package-lock.json b/cdk-infra/package-lock.json index faac3c7b3..1e3bc19e1 100644 --- a/cdk-infra/package-lock.json +++ b/cdk-infra/package-lock.json @@ -21,7 +21,7 @@ "@swc/helpers": "^0.5.1", "@types/jest": "^29.4.0", "@types/node": "18.14.6", - "aws-cdk": "2.73.0", + "aws-cdk": "2.130.0", "esbuild": "^0.18.3", "jest": "^29.5.0", "regenerator-runtime": "^0.13.11", @@ -2823,9 +2823,9 @@ } }, "node_modules/aws-cdk": { - "version": "2.73.0", - "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.73.0.tgz", - "integrity": "sha512-4ZnY+OS83goCzv+1sCEpNTNiXWjY6KBzic2RNUObzpHjUskRSwUCtaeiv6OyZ55DZoP0tneAmWIBXHfixJ7iQw==", + "version": "2.130.0", + "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.130.0.tgz", + "integrity": "sha512-MsjGzQ2kZv0FEfXvpW7FTJRnefew0GrYt9M2SMN2Yn45+yjugGl2X8to416kABeFz1OFqW56hq8Y5BiLuFDVLQ==", "dev": true, "bin": { "cdk": "bin/cdk" @@ -5228,6 +5228,7 @@ "version": "6.3.0", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true, "bin": { "semver": "bin/semver.js" } diff --git a/cdk-infra/package.json b/cdk-infra/package.json index f4159a6ce..dc46de026 100644 --- a/cdk-infra/package.json +++ b/cdk-infra/package.json @@ -21,7 +21,7 @@ "@swc/helpers": "^0.5.1", "@types/jest": "^29.4.0", "@types/node": "18.14.6", - "aws-cdk": "2.73.0", + "aws-cdk": "2.130.0", "esbuild": "^0.18.3", "jest": "^29.5.0", "regenerator-runtime": "^0.13.11", diff --git a/infrastructure/ecs-main/serverless.yml b/infrastructure/ecs-main/serverless.yml index 1186632cc..52c5e0342 100644 --- a/infrastructure/ecs-main/serverless.yml +++ b/infrastructure/ecs-main/serverless.yml @@ -46,12 +46,12 @@ custom: stg: '2048' prd: '4096' dotcomstg: '2048' - dotcomprd: '4096' + dotcomprd: '8192' containerMemory: dev: '8192' stg: '8192' prd: '24576' - dotcomstg: '8192' + dotcomstg: '24576' dotcomprd: '24576' desiredCount: diff --git a/src/commands/src/helpers/dependency-helpers.ts b/src/commands/src/helpers/dependency-helpers.ts index e955c2c12..08d2fb935 100644 --- a/src/commands/src/helpers/dependency-helpers.ts +++ b/src/commands/src/helpers/dependency-helpers.ts @@ -104,7 +104,13 @@ export async function downloadBuildDependencies( export const checkRedirects = async (repoName: string, directory?: string) => existsAsync(path.join(process.cwd(), getRepoDir(repoName, directory), 'config/redirects')); -export async function prepareBuild(repoName: string, projectName: string, baseUrl: string, directory?: string) { +export async function prepareBuild( + repoName: string, + projectName: string, + baseUrl: string, + prefix: string, + directory?: string +) { const repoDir = getRepoDir(repoName, directory); // doing these in parallel @@ -116,6 +122,7 @@ export async function prepareBuild(repoName: string, projectName: string, baseUr repoDir, projectName, baseUrl, + prefix, commitHash: dependencies[1] as string | undefined, patchId: dependencies[2] as string | undefined, }); diff --git a/src/commands/src/scripts/local-build/utils/create-job.ts b/src/commands/src/scripts/local-build/utils/create-job.ts index 8a7707539..16bb0f047 100644 --- a/src/commands/src/scripts/local-build/utils/create-job.ts +++ b/src/commands/src/scripts/local-build/utils/create-job.ts @@ -48,7 +48,7 @@ export function createLocalJob({ url: commit.url, newHead: commit.sha, urlSlug: branchName, - prefix: '', + prefix: 'docs-qa', project: project, pathPrefix: `${project}/docsworker-xlarge/${branchName}`, mutPrefix: project, diff --git a/src/job/jobHandler.ts b/src/job/jobHandler.ts index a301b0e89..cce1e45af 100644 --- a/src/job/jobHandler.ts +++ b/src/job/jobHandler.ts @@ -147,10 +147,12 @@ export abstract class JobHandler { } @throwIfJobInterupted() - private async constructPrefix(): Promise { + private constructPrefix(): void { const server_user = this._config.get('GATSBY_PARSER_USER'); const pathPrefix = this.getPathPrefix(); + if (!pathPrefix) return; + this.currJob.payload.pathPrefix = pathPrefix; const mutPrefix = pathPrefix.split(`/${server_user}`)[0]; this.currJob.payload.mutPrefix = mutPrefix; @@ -572,7 +574,13 @@ export abstract class JobHandler { } const baseUrl = docset?.url?.[env] || 'https://mongodbcom-cdn.website.staging.corp.mongodb.com'; - const { patchId } = await prepareBuild(job.payload.repoName, job.payload.project, baseUrl, job.payload.directory); + const { patchId } = await prepareBuild( + job.payload.repoName, + job.payload.project, + baseUrl, + job.payload.prefix, + job.payload.directory + ); // Set patchId on payload for use in nextGenStage this._currJob.payload.patchId = patchId; From f2e94f54224b4763d715c3f427aa0c40560bfb4a Mon Sep 17 00:00:00 2001 From: branberry Date: Tue, 27 Feb 2024 10:32:06 -0600 Subject: [PATCH 52/64] [DOP-4334]: Up memory --- infrastructure/ecs-main/serverless.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/ecs-main/serverless.yml b/infrastructure/ecs-main/serverless.yml index 52c5e0342..46c73751a 100644 --- a/infrastructure/ecs-main/serverless.yml +++ b/infrastructure/ecs-main/serverless.yml @@ -45,8 +45,8 @@ custom: dev: '2048' stg: '2048' prd: '4096' - dotcomstg: '2048' - dotcomprd: '8192' + dotcomstg: '4096' + dotcomprd: '4096' containerMemory: dev: '8192' stg: '8192' From e45ff6dfb8172010ccc1fdd58c37235eaeed8783 Mon Sep 17 00:00:00 2001 From: branberry Date: Tue, 27 Feb 2024 14:38:13 -0600 Subject: [PATCH 53/64] [DOP-4334]: Add cwd to mut-redirects --- src/commands/src/shared/next-gen-deploy.ts | 9 ++++++++- src/job/productionJobHandler.ts | 5 ++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/commands/src/shared/next-gen-deploy.ts b/src/commands/src/shared/next-gen-deploy.ts index 11a529dc9..8ab09ab9b 100644 --- a/src/commands/src/shared/next-gen-deploy.ts +++ b/src/commands/src/shared/next-gen-deploy.ts @@ -4,6 +4,7 @@ import { executeAndPipeCommands, executeCliCommand } from '../helpers'; interface NextGenDeployParams { branchName: string; hasConfigRedirects: boolean; + repoDir: string; mutPrefix?: string | null; bucket?: string; url?: string; @@ -15,6 +16,7 @@ export async function nextGenDeploy({ mutPrefix, branchName, hasConfigRedirects, + repoDir, bucket, url, }: NextGenDeployParams): Promise { @@ -23,10 +25,15 @@ export async function nextGenDeploy({ try { if (hasConfigRedirects && (branchName === 'main' || branchName === 'master' || branchName === 'current')) { // equivalent to: mut-redirects config/redirects -o public/.htaccess - await executeCliCommand({ command: 'mut-redirects', args: ['config/redirects', '-o', 'public/.htaccess'] }); + await executeCliCommand({ + command: 'mut-redirects', + args: ['config/redirects', '-o', 'public/.htaccess'], + options: { cwd: repoDir }, + }); console.log(`COMMAND: mut-redirects config/redirects -o public/.htaccess`); } + console.log('REDIRECTS COMPLETE!'); if (!bucket) { console.log(`nextGenStage has failed. Variable for S3 bucket address was undefined.`); return { diff --git a/src/job/productionJobHandler.ts b/src/job/productionJobHandler.ts index abd954f91..db5344a30 100644 --- a/src/job/productionJobHandler.ts +++ b/src/job/productionJobHandler.ts @@ -15,6 +15,7 @@ import { joinUrlAndPrefix } from './manifestJobHandler'; import { MONOREPO_NAME } from '../monorepo/utils/monorepo-constants'; import { nextGenDeploy } from '../commands'; import { checkRedirects } from '../commands/src/helpers/dependency-helpers'; +import path from 'path'; export class ProductionJobHandler extends JobHandler { constructor( @@ -196,7 +197,9 @@ export class ProductionJobHandler extends JobHandler { `${'(prod-monorepo)'.padEnd(15)} Redirects checked ${hasConfigRedirects}` ); console.log('Generic log to see that it actually logs stuff in CloudWatch (which it should)'); - resp = await nextGenDeploy({ mutPrefix: finalMutPrefix, bucket, url, branchName, hasConfigRedirects }); + const repoDir = path.resolve(process.cwd(), `repos/${getDirectory(this.currJob)}`); + + resp = await nextGenDeploy({ mutPrefix: finalMutPrefix, bucket, url, branchName, hasConfigRedirects, repoDir }); } else { resp = await this.deployWithMakefiles(); } From 92e2bdb01d112514076afee26b525fc69b608bb2 Mon Sep 17 00:00:00 2001 From: branberry Date: Tue, 27 Feb 2024 16:51:21 -0600 Subject: [PATCH 54/64] [DOP-4334]: Add on closed --- src/commands/src/helpers/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/src/helpers/index.ts b/src/commands/src/helpers/index.ts index ba79ca626..488661ccc 100644 --- a/src/commands/src/helpers/index.ts +++ b/src/commands/src/helpers/index.ts @@ -110,7 +110,7 @@ export async function executeAndPipeCommands( reject(new ExecuteCommandError('The second command failed', 1, err)); }); - cmdTo.on('exit', (exitCode) => { + cmdTo.on('close', (exitCode) => { // previous command errored out, return so we don't // accidentally resolve if the second command somehow still // exits without error From 1184ac039fa1ee4bbfcfa4bcefb9de9ba4003947 Mon Sep 17 00:00:00 2001 From: branberry Date: Wed, 28 Feb 2024 11:29:08 -0600 Subject: [PATCH 55/64] [DOP-4334]: Add metrics to commands --- Dockerfile.local | 2 +- src/commands/src/shared/next-gen-deploy.ts | 4 +++- src/commands/src/shared/next-gen-html.ts | 4 ++-- src/commands/src/shared/next-gen-parse.ts | 4 ++-- src/commands/src/shared/persistence-module.ts | 4 +++- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/Dockerfile.local b/Dockerfile.local index 187c460c3..3a74911b5 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -20,7 +20,7 @@ RUN apt-get install --yes curl RUN curl --location https://deb.nodesource.com/setup_18.x | bash - RUN apt-get install --yes nodejs RUN apt-get install --yes build-essential -RUN apt-get install --yes python3-pip libxml2-dev libxslt-dev python-dev pkg-config +RUN apt-get install --yes python3-pip libxml2-dev libxslt-dev python-dev pkg-config time RUN python3 -m pip install poetry diff --git a/src/commands/src/shared/next-gen-deploy.ts b/src/commands/src/shared/next-gen-deploy.ts index 8ab09ab9b..781c6da65 100644 --- a/src/commands/src/shared/next-gen-deploy.ts +++ b/src/commands/src/shared/next-gen-deploy.ts @@ -55,8 +55,10 @@ export async function nextGenDeploy({ const { outputText, errorText } = await executeAndPipeCommands( { command: 'yes' }, { - command: 'mut-publish', + command: 'time', args: [ + '-v', + 'mut-publish', 'public', bucket, `--prefix=${mutPrefix}`, diff --git a/src/commands/src/shared/next-gen-html.ts b/src/commands/src/shared/next-gen-html.ts index fc790f41b..b8b56a77c 100644 --- a/src/commands/src/shared/next-gen-html.ts +++ b/src/commands/src/shared/next-gen-html.ts @@ -3,8 +3,8 @@ import { CliCommandResponse, executeCliCommand } from '../helpers'; export async function nextGenHtml(): Promise { try { const result = await executeCliCommand({ - command: 'npm', - args: ['run', 'build'], + command: 'time', + args: ['-v', 'npm', 'run', 'build'], options: { cwd: `${process.cwd()}/snooty` }, }); return result; diff --git a/src/commands/src/shared/next-gen-parse.ts b/src/commands/src/shared/next-gen-parse.ts index 70bba49d7..763d36e0b 100644 --- a/src/commands/src/shared/next-gen-parse.ts +++ b/src/commands/src/shared/next-gen-parse.ts @@ -31,8 +31,8 @@ export async function nextGenParse({ job, patchId, isProd }: NextGenParseParams) } try { const result = await executeCliCommand({ - command: 'snooty', - args: commandArgs, + command: 'time', + args: ['-v', 'snooty', ...commandArgs], options: { cwd: repoDir }, }); return result; diff --git a/src/commands/src/shared/persistence-module.ts b/src/commands/src/shared/persistence-module.ts index 8264f77b3..9712277bc 100644 --- a/src/commands/src/shared/persistence-module.ts +++ b/src/commands/src/shared/persistence-module.ts @@ -10,6 +10,8 @@ export async function persistenceModule({ job }: PersistenceModuleParams): Promi const bundlePath = `${repoDir}/bundle.zip`; const args = [ + '-v', + 'node', `${process.cwd()}/modules/persistence/index.js`, '--unhandled-rejections=strict', '--path', @@ -25,7 +27,7 @@ export async function persistenceModule({ job }: PersistenceModuleParams): Promi try { const result = await executeCliCommand({ - command: 'node', + command: 'time', args, }); From f52fff1861b8116b35937887b7e64b40f4a95802 Mon Sep 17 00:00:00 2001 From: branberry Date: Wed, 28 Feb 2024 13:29:58 -0600 Subject: [PATCH 56/64] [DOP-4334]: Add time --- Dockerfile | 2 +- infrastructure/ecs-main/serverless.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 775330bc0..324337f38 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ ENV DEBIAN_FRONTEND=noninteractive WORKDIR ${WORK_DIRECTORY} # helper libraries for docs builds -RUN apt-get update && apt-get install -y vim git unzip zip rsync +RUN apt-get update && apt-get install -y vim git unzip zip rsync time # get node 18 # https://gist.github.com/RinatMullayanov/89687a102e696b1d4cab diff --git a/infrastructure/ecs-main/serverless.yml b/infrastructure/ecs-main/serverless.yml index 46c73751a..a91a4d7cd 100644 --- a/infrastructure/ecs-main/serverless.yml +++ b/infrastructure/ecs-main/serverless.yml @@ -45,14 +45,14 @@ custom: dev: '2048' stg: '2048' prd: '4096' - dotcomstg: '4096' - dotcomprd: '4096' + dotcomstg: '8192' + dotcomprd: '8192' containerMemory: dev: '8192' stg: '8192' prd: '24576' - dotcomstg: '24576' - dotcomprd: '24576' + dotcomstg: '32768' + dotcomprd: '32768' desiredCount: dev: '4' From b9456fd1a03d4045525c9d02276b00f6c2658c57 Mon Sep 17 00:00:00 2001 From: branberry Date: Wed, 28 Feb 2024 14:01:32 -0600 Subject: [PATCH 57/64] [DOP-4334]: Increase ephemeral storage --- infrastructure/ecs-main/ecs_service.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/infrastructure/ecs-main/ecs_service.yml b/infrastructure/ecs-main/ecs_service.yml index 2fdad6e1d..9e4f4ce33 100644 --- a/infrastructure/ecs-main/ecs_service.yml +++ b/infrastructure/ecs-main/ecs_service.yml @@ -29,6 +29,7 @@ Resources: Image: ${self:custom.ecs.imageUrl} Cpu: ${self:custom.ecs.containerCpu.${self:provider.stage}} Memory: ${self:custom.ecs.containerMemory.${self:provider.stage}} + SizeInGiB: 100 Environment: - Name: STAGE Value: ${self:provider.stage} From a52f2e6414252c838aae9d132ee088ca4b203736 Mon Sep 17 00:00:00 2001 From: branberry Date: Wed, 28 Feb 2024 14:29:24 -0600 Subject: [PATCH 58/64] [DOP-4334]: Increase ephemeral storage --- infrastructure/ecs-main/ecs_service.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/ecs-main/ecs_service.yml b/infrastructure/ecs-main/ecs_service.yml index 9e4f4ce33..3eb90c26d 100644 --- a/infrastructure/ecs-main/ecs_service.yml +++ b/infrastructure/ecs-main/ecs_service.yml @@ -29,7 +29,7 @@ Resources: Image: ${self:custom.ecs.imageUrl} Cpu: ${self:custom.ecs.containerCpu.${self:provider.stage}} Memory: ${self:custom.ecs.containerMemory.${self:provider.stage}} - SizeInGiB: 100 + SizeInGiB: 100 Environment: - Name: STAGE Value: ${self:provider.stage} From f5dabf5805e5a48d52d30665edc78091b2ba1733 Mon Sep 17 00:00:00 2001 From: branberry Date: Wed, 28 Feb 2024 14:47:11 -0600 Subject: [PATCH 59/64] [DOP-4334]: Increase ephemeral storage --- cdk-infra/lib/constructs/worker/worker-construct.ts | 1 + infrastructure/ecs-main/ecs_service.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cdk-infra/lib/constructs/worker/worker-construct.ts b/cdk-infra/lib/constructs/worker/worker-construct.ts index 236bd55b6..e077ab06f 100644 --- a/cdk-infra/lib/constructs/worker/worker-construct.ts +++ b/cdk-infra/lib/constructs/worker/worker-construct.ts @@ -80,6 +80,7 @@ export class WorkerConstruct extends Construct { memoryLimitMiB: 8192, taskRole, executionRole, + ephemeralStorageGiB, }); const updateTaskProtectionPolicy = new PolicyStatement({ diff --git a/infrastructure/ecs-main/ecs_service.yml b/infrastructure/ecs-main/ecs_service.yml index 3eb90c26d..39816b924 100644 --- a/infrastructure/ecs-main/ecs_service.yml +++ b/infrastructure/ecs-main/ecs_service.yml @@ -24,12 +24,12 @@ Resources: Memory: ${self:custom.ecs.containerMemory.${self:provider.stage}} ExecutionRoleArn: !Ref ExecutionRole TaskRoleArn: !Ref TaskRole + EphemeralStorage: 100 ContainerDefinitions: - Name: ${self:service}-${self:provider.stage} Image: ${self:custom.ecs.imageUrl} Cpu: ${self:custom.ecs.containerCpu.${self:provider.stage}} Memory: ${self:custom.ecs.containerMemory.${self:provider.stage}} - SizeInGiB: 100 Environment: - Name: STAGE Value: ${self:provider.stage} From d664a82440db527e29edc502db2769f75cd295ce Mon Sep 17 00:00:00 2001 From: branberry Date: Wed, 28 Feb 2024 15:09:42 -0600 Subject: [PATCH 60/64] [DOP-4334]: DOuble memory again... --- infrastructure/ecs-main/serverless.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/ecs-main/serverless.yml b/infrastructure/ecs-main/serverless.yml index a91a4d7cd..79c4803bf 100644 --- a/infrastructure/ecs-main/serverless.yml +++ b/infrastructure/ecs-main/serverless.yml @@ -51,8 +51,8 @@ custom: dev: '8192' stg: '8192' prd: '24576' - dotcomstg: '32768' - dotcomprd: '32768' + dotcomstg: '65536' + dotcomprd: '65536' desiredCount: dev: '4' From abcede18b871c412a8b32855fbc4c9761816f9a3 Mon Sep 17 00:00:00 2001 From: branberry Date: Wed, 28 Feb 2024 15:35:55 -0600 Subject: [PATCH 61/64] [DOP-4334]: Double cpu --- cdk-infra/lib/constructs/worker/worker-construct.ts | 1 - infrastructure/ecs-main/serverless.yml | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cdk-infra/lib/constructs/worker/worker-construct.ts b/cdk-infra/lib/constructs/worker/worker-construct.ts index e077ab06f..236bd55b6 100644 --- a/cdk-infra/lib/constructs/worker/worker-construct.ts +++ b/cdk-infra/lib/constructs/worker/worker-construct.ts @@ -80,7 +80,6 @@ export class WorkerConstruct extends Construct { memoryLimitMiB: 8192, taskRole, executionRole, - ephemeralStorageGiB, }); const updateTaskProtectionPolicy = new PolicyStatement({ diff --git a/infrastructure/ecs-main/serverless.yml b/infrastructure/ecs-main/serverless.yml index 79c4803bf..615a219cc 100644 --- a/infrastructure/ecs-main/serverless.yml +++ b/infrastructure/ecs-main/serverless.yml @@ -45,8 +45,8 @@ custom: dev: '2048' stg: '2048' prd: '4096' - dotcomstg: '8192' - dotcomprd: '8192' + dotcomstg: '16384' + dotcomprd: '16384' containerMemory: dev: '8192' stg: '8192' From 6faabe99a8cd1e86bc7d0dc564c02dc5d7cbbb32 Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 29 Feb 2024 09:24:05 -0600 Subject: [PATCH 62/64] [DOP-4334]: undo hotswap --- .github/workflows/deploy-stg-ecs.yml | 1 - .github/workflows/update-feature-branch.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/deploy-stg-ecs.yml b/.github/workflows/deploy-stg-ecs.yml index d8ff99405..d122336ff 100644 --- a/.github/workflows/deploy-stg-ecs.yml +++ b/.github/workflows/deploy-stg-ecs.yml @@ -3,7 +3,6 @@ on: branches: - 'main' - 'integration' - - 'DOP-4334' concurrency: group: environment-stg-${{ github.ref }} cancel-in-progress: true diff --git a/.github/workflows/update-feature-branch.yml b/.github/workflows/update-feature-branch.yml index 9373a405e..cfd7a29d6 100644 --- a/.github/workflows/update-feature-branch.yml +++ b/.github/workflows/update-feature-branch.yml @@ -125,5 +125,5 @@ jobs: if: steps.filter.outputs.worker == 'true' run: | cd cdk-infra/ - npm run deploy:feature:stack -- --hotswap -c isFeatureBranch=true -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg-${{github.head_ref}} \ + npm run deploy:feature:stack -- -c isFeatureBranch=true -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg-${{github.head_ref}} \ auto-builder-stack-enhancedApp-dotcomstg-${{github.head_ref}}-worker From 9f08902a2d0b1ac32b83c4371d2f5b2dc79d0c90 Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 29 Feb 2024 09:44:46 -0600 Subject: [PATCH 63/64] [DOP-4334]: Add new build commands --- .../lib/constructs/api/webhook-api-construct.ts | 12 ++++++++++++ cdk-infra/lib/constructs/worker/worker-construct.ts | 3 ++- package.json | 4 +++- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/cdk-infra/lib/constructs/api/webhook-api-construct.ts b/cdk-infra/lib/constructs/api/webhook-api-construct.ts index ed8dbab16..c27ef7305 100644 --- a/cdk-infra/lib/constructs/api/webhook-api-construct.ts +++ b/cdk-infra/lib/constructs/api/webhook-api-construct.ts @@ -10,6 +10,8 @@ import { getFeatureName } from '../../../utils/env'; const HANDLERS_PATH = path.join(__dirname, '/../../../../api/controllers/v2'); +const memorySize = 1024; + const bundling: BundlingOptions = { sourceMap: true, minify: true, @@ -46,6 +48,7 @@ export class WebhookApiConstruct extends Construct { environment, bundling, timeout, + memorySize, }); const slackDisplayRepoLambda = new NodejsFunction(this, 'slackDisplayRepoLambda', { @@ -55,6 +58,7 @@ export class WebhookApiConstruct extends Construct { environment, bundling, timeout, + memorySize, }); const dochubTriggerUpsertLambda = new NodejsFunction(this, 'dochubTriggerUpsertLambda', { @@ -63,6 +67,7 @@ export class WebhookApiConstruct extends Construct { handler: 'UpsertEdgeDictionaryItem', environment, timeout, + memorySize, }); const githubTriggerLambda = new NodejsFunction(this, 'githubTriggerLambda', { @@ -72,6 +77,7 @@ export class WebhookApiConstruct extends Construct { bundling, environment, timeout, + memorySize, }); const githubDeleteArtifactsLambda = new NodejsFunction(this, 'githubDeleteArtifactsLambda', { @@ -81,6 +87,7 @@ export class WebhookApiConstruct extends Construct { bundling, environment, timeout, + memorySize, }); const triggerLocalBuildLambda = new NodejsFunction(this, 'triggerLocalBuildLambda', { @@ -90,6 +97,7 @@ export class WebhookApiConstruct extends Construct { environment, bundling, timeout, + memorySize, }); const handleJobsLambda = new NodejsFunction(this, 'handleJobsLambda', { @@ -99,6 +107,7 @@ export class WebhookApiConstruct extends Construct { environment, bundling, timeout, + memorySize, }); const snootyBuildCompleteLambda = new NodejsFunction(this, 'snootyBuildCompleteLambda', { @@ -108,6 +117,7 @@ export class WebhookApiConstruct extends Construct { environment, bundling, timeout, + memorySize, }); const testDeployLambda = new NodejsFunction(this, 'testDeployLambda', { entry: `${HANDLERS_PATH}/test-deploy.ts`, @@ -116,6 +126,7 @@ export class WebhookApiConstruct extends Construct { environment, bundling, timeout, + memorySize, }); // generic handler for the root endpoint @@ -124,6 +135,7 @@ export class WebhookApiConstruct extends Construct { runtime, handler: 'RootEndpointLambda', timeout, + memorySize, }); const apiName = `webhookHandlers-${getFeatureName()}`; diff --git a/cdk-infra/lib/constructs/worker/worker-construct.ts b/cdk-infra/lib/constructs/worker/worker-construct.ts index 236bd55b6..956cb2adc 100644 --- a/cdk-infra/lib/constructs/worker/worker-construct.ts +++ b/cdk-infra/lib/constructs/worker/worker-construct.ts @@ -13,6 +13,7 @@ import { IQueue } from 'aws-cdk-lib/aws-sqs'; import { Construct } from 'constructs'; import path from 'path'; import { getEnv } from '../../../utils/env'; +import { Platform } from 'aws-cdk-lib/aws-ecr-assets'; interface WorkerConstructProps { dockerEnvironment: Record; @@ -69,9 +70,9 @@ export class WorkerConstruct extends Construct { NPM_BASE_64_AUTH: dockerEnvironment.NPM_BASE_64_AUTH, NPM_EMAIL: dockerEnvironment.NPM_EMAIL, }, - cacheFrom: [{ type: 'gha' }], cacheTo: { type: 'gha', params: { mode: 'max' } }, + platform: Platform.LINUX_AMD64, }; const taskDefLogGroup = new LogGroup(this, 'workerLogGroup'); diff --git a/package.json b/package.json index 9043be8ef..663754b7c 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,9 @@ "clean": "node maintain.js", "build": "tsc", "build:esbuild:cacheUpdater": "esbuild src/cache-updater/index.ts --bundle --platform=node --outdir=./dist/ --allow-overwrite --sourcemap", - "build:esbuild": "esbuild src/onDemandApp.ts --bundle --platform=node --outdir=./ --allow-overwrite --sourcemap", + "build:esbuild": "npm run build:esbuild:enhanced && npm run build:esbuild:onDemand", + "build:esbuild:onDemand": "esbuild src/onDemandApp.ts --bundle --platform=node --outdir=./ --allow-overwrite --sourcemap", + "build:esbuild:enhanced": "esbuild src/enhanced/enhancedApp.ts --bundle --platform=node --outdir=./enhanced --allow-overwrite --sourcemap", "format": "npm run prettier -- --check", "format:fix": "npm run prettier -- --write", "lint": "eslint --ext .ts .", From 7fcd3629fd0f5f165ed8869d886cd4a1b8d06263 Mon Sep 17 00:00:00 2001 From: branberry Date: Thu, 29 Feb 2024 12:01:23 -0600 Subject: [PATCH 64/64] [DOP-4334]: Update memory --- cdk-infra/lib/constructs/worker/worker-construct.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdk-infra/lib/constructs/worker/worker-construct.ts b/cdk-infra/lib/constructs/worker/worker-construct.ts index 956cb2adc..fa59671ad 100644 --- a/cdk-infra/lib/constructs/worker/worker-construct.ts +++ b/cdk-infra/lib/constructs/worker/worker-construct.ts @@ -77,8 +77,8 @@ export class WorkerConstruct extends Construct { const taskDefLogGroup = new LogGroup(this, 'workerLogGroup'); const taskDefinition = new FargateTaskDefinition(this, 'workerTaskDefinition', { - cpu: 4096, - memoryLimitMiB: 8192, + cpu: 16384, + memoryLimitMiB: 65536, taskRole, executionRole, });