fix(fern-bot): Fix serverless.yml duplication (#1566)

Co-authored-by: dsinghvi <[email protected]>

.github/workflows/deploy-fern-bot-dev.yml

@@ -4,9 +4,6 @@ on:
   push:
     branches:
       - main
-    paths:
-      - "servers/fern-bot/**"
-      - ".github/workflows/deploy-fern-bot-dev.yml"
 
 env:
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}

servers/fern-bot/package.json

@@ -26,27 +26,30 @@
     "@fern-fern/paged-generators-sdk": "0.0.5706",
     "@octokit/openapi-types": "^22.1.0",
     "@slack/web-api": "^6.9.0",
+    "adm-zip": "^0.5.16",
     "cohere-ai": "^7.9.5",
     "execa": "^5.1.1",
     "fern-api": "^0.21.0",
     "find-up": "^7.0.0",
     "js-yaml": "^4.1.0",
     "json-2-csv": "^5.5.1",
     "octokit": "^3.2.0",
+    "semver": "^7.6.2",
     "serverless-step-functions": "^3.21.0",
     "simple-git": "^3.24.0",
     "tmp-promise": "^3.0.3",
     "url-join": "^5.0.0",
-    "zod": "^3.22.4",
-    "semver": "^7.6.2"
+    "zod": "^3.22.4"
   },
   "devDependencies": {
     "@fern-platform/configs": "workspace:*",
     "@octokit/types": "^13.4.1",
     "@serverless/typescript": "^3.0.0",
+    "@types/adm-zip": "^0.5.5",
     "@types/aws-lambda": "^8.10.71",
     "@types/js-yaml": "^4.0.9",
     "@types/node": "^18.7.18",
+    "@types/semver": "^7.5.8",
     "@types/url-join": "4.0.1",
     "esbuild": "0.20.2",
     "json-schema-to-ts": "^1.5.0",
@@ -55,8 +58,7 @@
     "ts-node": "^10.4.0",
     "tsconfig-paths": "^3.9.0",
     "typescript": "^4.1.3",
-    "vitest": "^1.5.0",
-    "@types/semver": "^7.5.8"
+    "vitest": "^1.5.0"
   },
   "license": "MIT"
 }

servers/fern-bot/serverless.yml

@@ -243,9 +243,9 @@ stepFunctions:
                       ProcessorConfig:
                         Mode: DISTRIBUTED
                         ExecutionType: STANDARD
-                      StartAt: HandleRepoForGenerator
+                      StartAt: HandleFDRRepoForGenerator
                       States:
-                        HandleRepoForGenerator:
+                        HandleFDRRepoForGenerator:
                           Type: Task
                           Resource: "arn:aws:states:::lambda:invoke"
                           Parameters:

servers/fern-bot/src/libs/buf.ts

@@ -1,4 +1,5 @@
-import { createLoggingExecutable, LoggingExecutable } from "@utils/createLoggingExecutable";
+import { createLoggingExecutable } from "@utils/createLoggingExecutable";
+import execa from "execa";
 import tmp from "tmp-promise";
 import urlJoin from "url-join";
 
@@ -30,8 +31,10 @@ export declare namespace Buf {
     }
 }
 
+export type CLI = (args?: string[]) => execa.ExecaChildProcess<string>;
+
 export class Buf {
-    private cli: LoggingExecutable | undefined;
+    private cli: CLI | undefined;
 
     public async curl({ request }: { request: Buf.CurlRequest }): Promise<Buf.CurlResponse> {
         const cli = await this.getOrInstall();
@@ -46,7 +49,7 @@ export class Buf {
         };
     }
 
-    private async getOrInstall(): Promise<LoggingExecutable> {
+    private async getOrInstall(): Promise<CLI> {
         if (this.cli) {
             return this.cli;
         }
@@ -63,24 +66,29 @@ export class Buf {
         return this.cli;
     }
 
-    private async install(): Promise<LoggingExecutable> {
-        // Lambdas can only write into temporary directories.
-        const tmpDirPath = (await tmp.dir()).path;
+    private async install(): Promise<CLI> {
+        // Running the commands on Lambdas is a bit odd...specifically you can only write to tmp on a lambda
+        // so here we make sure the CLI is bundled via the `external` block in serverless.yml
+        // and then execute the command directly via node_modules, with the home and cache set to /tmp.
+        const tmpDir = await tmp.dir();
+        const tmpDirPath = tmpDir.path;
         process.env.NPM_CONFIG_CACHE = `${tmpDirPath}/.npm`;
         process.env.HOME = tmpDirPath;
 
-        const npm = createLoggingExecutable("npm", {
-            cwd: process.cwd(),
-        });
-        console.debug(`Installing ${BUF_NPM_PACKAGE} ...`);
-        await npm(["install", "-f", "-g", `${BUF_NPM_PACKAGE}@${BUF_VERSION}`]);
-
-        const cli = this.createBufExecutable();
-        const version = await cli(["--version"]);
-        console.debug(`Successfully installed ${BUF_NPM_PACKAGE} version ${version.stdout}`);
+        // Update config to allow `npm install` to work from within the `fern upgrade` command
+        process.env.NPM_CONFIG_PREFIX = tmpDirPath;
+        // Re-install the CLI to ensure it's at the correct path, given the updated config
+        const install = await execa("npm", ["install", "-g", `${BUF_NPM_PACKAGE}@${BUF_VERSION}`]);
+        if (install.exitCode === 0) {
+            console.log(`Successfully installed ${BUF_NPM_PACKAGE}`);
+        } else {
+            const message = `Failed to install buf \n${install.stdout}\n${install.stderr}`;
+            console.log(message);
+            throw new Error(message);
+        }
 
-        this.cli = cli;
-        return cli;
+        this.cli = this.createBufExecutable();
+        return this.cli;
     }
 
     private getArgsForCurlRequest(request: Buf.CurlRequest): string[] {
@@ -104,10 +112,9 @@ export class Buf {
         return urlJoin(baseUrl, endpoint);
     }
 
-    private createBufExecutable(): LoggingExecutable {
-        return createLoggingExecutable("buf", {
-            cwd: process.cwd(),
-            reject: false, // We want to capture stderr without throwing.
-        });
+    private createBufExecutable(): CLI {
+        return (args) => {
+            return execa("npx", ["buf", ...(args ?? [])]);
+        };
     }
 }

servers/fern-bot/src/utils/fetchAndUnzip.ts

@@ -1,10 +1,10 @@
+import AdmZip from "adm-zip";
 import { createWriteStream } from "fs";
 import { unlink } from "fs/promises";
 import path from "path";
 import { pipeline } from "stream";
 import tmp from "tmp-promise";
 import { promisify } from "util";
-import { loggingExeca } from "./loggingExeca";
 
 export interface FetchAndUnzipRequest {
     destination: string;
@@ -21,14 +21,29 @@ export async function fetchAndUnzip(request: FetchAndUnzipRequest): Promise<void
     });
 
     console.debug(`Unzipping source from ${destinationPath} to ${request.destination}`);
-    await loggingExeca("unzip", ["-o", destinationPath, "-d", request.destination], {
-        doNotPipeOutput: true,
-    });
+    await unzipFile(destinationPath, request.destination);
 
     console.debug(`Removing ${destinationPath}`);
     await unlink(destinationPath);
 }
 
+async function unzipFile(sourcePath: string, destinationPath: string): Promise<void> {
+    return new Promise((resolve, reject) => {
+        try {
+            const zip = new AdmZip(sourcePath);
+
+            // Extract all files
+            zip.extractAllTo(destinationPath, true);
+
+            console.log(`Successfully extracted ${sourcePath} to ${destinationPath}`);
+            resolve();
+        } catch (error) {
+            console.error(`Error unzipping file: ${error}`);
+            reject(error);
+        }
+    });
+}
+
 async function downloadFile({
     sourceUrl,
     destinationPath,