Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Attempt local asset bundling before using Docker. #320

Open
wants to merge 10 commits into
base: main
Choose a base branch
from
43 changes: 43 additions & 0 deletions lib/layer/index.ts
dfkunstler marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import * as cdk from "aws-cdk-lib";
import * as lambda from "aws-cdk-lib/aws-lambda";
import * as s3assets from "aws-cdk-lib/aws-s3-assets";
import * as lpath from "path";
import { execSync } from "child_process";
import { Construct } from "constructs";

interface LayerProps {
Expand All @@ -26,6 +28,47 @@ export class Layer extends Construct {
const layerAsset = new s3assets.Asset(this, "LayerAsset", {
path,
bundling: {
local: {
/* implements a local method of bundling that does not depend on Docker. Local
bundling is preferred over DIND for performance and security reasons.
see https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.ILocalBundling.html and
https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_s3_assets-readme.html#asset-bundling */
tryBundle(outputDir: string, options: cdk.BundlingOptions) {
let canRunLocal = false;
let python = props.runtime.name;

try {
// check if pip is available locally
const testCommand = `${python} -m pip -V`
console.log(`Checking for pip: ${testCommand}`)
// without the stdio arg no output is printed to console
execSync(testCommand, { stdio: 'inherit' });
// no exception means command executed successfully
canRunLocal = true;
} catch {
// execSync throws Error in case return value of child process is non-zero.
// Actual output should be printed to the console.
console.warn(`Unable to do local bundling! ${python} with pip must be on path.`);
}

if (canRunLocal) {
const command = `${python} -m pip install -r ${lpath.posix.join(path, "requirements.txt")} -t ${outputDir} ${autoUpgrade ? '-U' : ''}`;
try {
console.debug(`Local bundling: ${command}`);
// this is where the work gets done
execSync(command, { stdio: 'inherit' });
return true;
} catch (ex) {
// execSync throws Error in case return value of child process
// is non-zero. It'll be printed to the console because of the
// stdio argument.
console.log(`Local bundling attempt failed: ${ex}`)
}
}
// if we get here then Docker will be used as configured below
return false;
}
},
image: runtime.bundlingImage,
platform: architecture.dockerPlatform,
command: [
Expand Down
64 changes: 63 additions & 1 deletion lib/shared/shared-asset-bundler.ts
massi-ang marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,15 @@ import {
BundlingOutput,
DockerImage,
aws_s3_assets,
BundlingOptions
} from "aws-cdk-lib";
import { Code, S3Code } from "aws-cdk-lib/aws-lambda";
import { Asset } from "aws-cdk-lib/aws-s3-assets";
import { md5hash } from "aws-cdk-lib/core/lib/helpers-internal";
import { Construct } from "constructs";
import * as path from "path";
import * as fs from "fs";
import { execSync } from "child_process";

function calculateHash(paths: string[]): string {
return paths.reduce((mh, p) => {
Expand All @@ -33,6 +35,8 @@ function calculateHash(paths: string[]): string {
export class SharedAssetBundler extends Construct {
private readonly sharedAssets: string[];
private readonly WORKING_PATH = "/asset-input/";
private readonly container_image: DockerImage;
dfkunstler marked this conversation as resolved.
Show resolved Hide resolved
private useLocalBundler: boolean = false;
/**
* Instantiate a new SharedAssetBundler. You then invoke `bundleWithAsset(pathToAsset)` to
* bundle your asset code with the common code.
Expand All @@ -47,17 +51,75 @@ export class SharedAssetBundler extends Construct {
constructor(scope: Construct, id: string, sharedAssets: string[]) {
super(scope, id);
this.sharedAssets = sharedAssets;
// Check if we can do local bundling
if (!this.localBundlerTest()) {
// if not, then build Apline from local definition
dfkunstler marked this conversation as resolved.
Show resolved Hide resolved
this.container_image = DockerImage.fromBuild(path.posix.join(__dirname, "alpine-zip"));
dfkunstler marked this conversation as resolved.
Show resolved Hide resolved
} else {
// if yes, then don't build the container. https://hub.docker.com/_/scratch/
this.container_image = DockerImage.fromRegistry("scratch");
}
}

/**
* Check if possible to use local bundling instead of Docker. Sets this.useLocalBundler to
* true if local environment supports bundling. See below in method bundleWithAsset(...).
*/
private localBundlerTest(): boolean {
const command = "zip -v";
console.log(`Checking for zip: ${command}`);
// check if zip is available locally
try {
// without stdio option command output does not appear in console
execSync(command, {stdio: 'inherit'});
// no exception means command executed successfully
this.useLocalBundler = true;
} catch {
// execSync throws Error in case return value of child process
// is non-zero. Actual output should be printed to the console.
console.warn("Unable to do local bundling! Is zip installed?");
}
return this.useLocalBundler;
}

bundleWithAsset(assetPath: string): Asset {
console.log(`Bundling asset ${assetPath}`);

// necessary for access from anonymous class
const runLocal = this.useLocalBundler;

const asset = new aws_s3_assets.Asset(
this,
md5hash(assetPath).slice(0, 6),
{
path: assetPath,
bundling: {
image: DockerImage.fromBuild(path.posix.join(__dirname, "alpine-zip")),
local: {
/* implements a local method of bundling that does not depend on Docker. Local
bundling is preferred over DIND for performance and security reasons.
see https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.ILocalBundling.html and
https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_s3_assets-readme.html#asset-bundling */
tryBundle(outputDir: string, options: BundlingOptions) {
if (runLocal) {
const command = `zip -r ${path.posix.join(outputDir, "asset.zip")} ${assetPath}`;
try {
console.debug(`Local bundling: ${command}`);
// this is where the work gets done
execSync(command, {stdio: 'inherit'});
// no exception means command executed successfully
return true;
} catch (ex) {
// execSync throws Error in case return value of child process
// is non-zero. It'll be printed to the console because of the
// stdio argument.
console.log(`local bundling attempt failed: ${ex}`)
}
}
// if we get here then Docker will be used as configured below
return false;
}
},
image: this.container_image,
command: ["zip", "-r", path.posix.join("/asset-output", "asset.zip"), "."],
volumes: this.sharedAssets.map((f) => ({
containerPath: path.posix.join(this.WORKING_PATH, path.basename(f)),
Expand Down