From 2d59c8132b8938b8189a1ba90a7afe09157c3ce4 Mon Sep 17 00:00:00 2001 From: Chris K Wensel Date: Wed, 9 Aug 2023 18:06:16 -0700 Subject: [PATCH] push down verbosity level to kernel command, improve logging around vpc lookup --- .../java/clusterless/process/ProcessExec.java | 7 +++-- .../substrate/aws/resources/Vpcs.java | 30 +++++++++++++++++++ .../batch/ComputeResourceConstruct.java | 29 +++++++----------- .../substrate/aws/CommonCommand.java | 4 +++ .../substrate/aws/cdk/CDKProcessExec.java | 9 +++--- .../aws/cdk/bootstrap/Bootstrap.java | 2 +- .../substrate/aws/cdk/lifecycle/Deploy.java | 5 +--- .../substrate/aws/cdk/lifecycle/Destroy.java | 2 +- .../substrate/aws/cdk/lifecycle/Diff.java | 2 +- .../substrate/aws/cdk/lifecycle/Import.java | 2 +- .../substrate/aws/cdk/lifecycle/Verify.java | 2 +- 11 files changed, 60 insertions(+), 34 deletions(-) diff --git a/clusterless-main-common/src/main/java/clusterless/process/ProcessExec.java b/clusterless-main-common/src/main/java/clusterless/process/ProcessExec.java index 5d96df15..94ec8d30 100644 --- a/clusterless-main-common/src/main/java/clusterless/process/ProcessExec.java +++ b/clusterless-main-common/src/main/java/clusterless/process/ProcessExec.java @@ -22,17 +22,20 @@ public abstract class ProcessExec { private static final Logger LOG = LogManager.getLogger(ProcessExec.class); protected Supplier dryRun = () -> false; protected Supplier retry = () -> false; + protected Supplier verbosity = () -> 0; public ProcessExec() { } - public ProcessExec(Supplier dryRun) { + public ProcessExec(Supplier dryRun, Supplier verbosity) { this.dryRun = dryRun; + this.verbosity = verbosity; } - public ProcessExec(Supplier dryRun, Supplier retry) { + public ProcessExec(Supplier dryRun, Supplier retry, Supplier verbosity) { this.dryRun = dryRun; this.retry = retry; + this.verbosity = verbosity; } public boolean dryRun() { diff --git a/clusterless-substrate-aws-construct-common/src/main/java/clusterless/substrate/aws/resources/Vpcs.java b/clusterless-substrate-aws-construct-common/src/main/java/clusterless/substrate/aws/resources/Vpcs.java index ee5fb16c..50687917 100644 --- a/clusterless-substrate-aws-construct-common/src/main/java/clusterless/substrate/aws/resources/Vpcs.java +++ b/clusterless-substrate-aws-construct-common/src/main/java/clusterless/substrate/aws/resources/Vpcs.java @@ -9,17 +9,47 @@ package clusterless.substrate.aws.resources; import clusterless.naming.Ref; +import clusterless.substrate.aws.managed.ManagedComponentContext; import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import software.amazon.awscdk.Fn; +import software.amazon.awscdk.services.ec2.IVpc; +import software.amazon.awscdk.services.ec2.Vpc; +import software.amazon.awscdk.services.ec2.VpcLookupOptions; import software.constructs.Construct; import java.util.Objects; public class Vpcs { + private static final Logger LOG = LoggerFactory.getLogger(Vpcs.class); public static final String COMMON_VPC = "CommonVpc"; public static final String VPC = "vpc"; + public static @NotNull IVpc lookupVpc(Construct scope, @NotNull ManagedComponentContext context) { + // need to be careful where we look up region, as it may not be set yet + return lookupVpc(scope, context.deployable().placement().region()); + } + + public static @NotNull IVpc lookupVpc(Construct scope, String region) { + String vpcName = Vpcs.bootstrapVPCName(scope); + LOG.info("looking up vpc: {}, in region: {}", vpcName, region); + + IVpc vpcLookup = Vpc.fromLookup( + scope, + "VpcLookup", + VpcLookupOptions.builder() + .region(region) + .vpcName(vpcName) + .build() + ); + + LOG.info("found vpc id: {}", vpcLookup.getVpcId()); + + return vpcLookup; + } + public static String bootstrapVPCName(Construct scope) { Objects.requireNonNull(scope, "scope is null"); diff --git a/clusterless-substrate-aws-construct-core/src/main/java/clusterless/substrate/aws/resource/batch/ComputeResourceConstruct.java b/clusterless-substrate-aws-construct-core/src/main/java/clusterless/substrate/aws/resource/batch/ComputeResourceConstruct.java index df2e5694..ec386b1e 100644 --- a/clusterless-substrate-aws-construct-core/src/main/java/clusterless/substrate/aws/resource/batch/ComputeResourceConstruct.java +++ b/clusterless-substrate-aws-construct-core/src/main/java/clusterless/substrate/aws/resource/batch/ComputeResourceConstruct.java @@ -16,8 +16,6 @@ import org.jetbrains.annotations.NotNull; import software.amazon.awscdk.services.batch.alpha.FargateComputeEnvironment; import software.amazon.awscdk.services.batch.alpha.IManagedComputeEnvironment; -import software.amazon.awscdk.services.ec2.Vpc; -import software.amazon.awscdk.services.ec2.VpcLookupOptions; import software.constructs.Construct; /** @@ -32,26 +30,19 @@ public ComputeResourceConstruct(@NotNull ManagedComponentContext context, @NotNu String name = Resources.regionallyUniqueProjectName(this, model().computeEnvironmentName()); computeEnvironment = constructWithinHandler(() -> - FargateComputeEnvironment.Builder.create(this, id(model().computeEnvironmentName())) - .computeEnvironmentName(name) // globally unique - .replaceComputeEnvironment(false) - .maxvCpus(4096) - .spot(false) + { + return FargateComputeEnvironment.Builder.create(this, id(model().computeEnvironmentName())) + .computeEnvironmentName(name) // globally unique + .replaceComputeEnvironment(false) + .maxvCpus(4096) + .spot(false) // .terminateOnUpdate(false) // .updateTimeout() // .vpcSubnets() - .vpc( - Vpc.fromLookup( - this, - "VpcLookup", - VpcLookupOptions.builder() - .region(context.managedProject().getRegion()) - .vpcName(Vpcs.bootstrapVPCName(this)) - .build() - ) - ) - .enabled(true) - .build() + .vpc(Vpcs.lookupVpc(this, context)) + .enabled(true) + .build(); + } ); TagsUtil.applyTags(computeEnvironment, model().tags()); diff --git a/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/CommonCommand.java b/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/CommonCommand.java index 9d5c7155..6bc4c70e 100644 --- a/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/CommonCommand.java +++ b/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/CommonCommand.java @@ -23,4 +23,8 @@ protected CommonConfig getCommonConfig() { protected AwsConfig getProviderConfig() { return kernel.configurations().get(Provider.NAME); } + + protected int verbosityLevel() { + return kernel.verbosity().level(); + } } diff --git a/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/CDKProcessExec.java b/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/CDKProcessExec.java index feadfceb..1564470e 100644 --- a/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/CDKProcessExec.java +++ b/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/CDKProcessExec.java @@ -82,12 +82,12 @@ public class CDKProcessExec extends ProcessExec { public CDKProcessExec() { } - public CDKProcessExec(CommonCommandOptions commandOptions) { - super(commandOptions::dryRun); + public CDKProcessExec(CommonCommandOptions commandOptions, Supplier verbosity) { + super(commandOptions::dryRun, verbosity); } - public CDKProcessExec(Supplier dryRun, Supplier retry) { - super(dryRun, retry); + public CDKProcessExec(Supplier dryRun, Supplier retry, Supplier verbosity) { + super(dryRun, retry, verbosity); } public String cdk() { @@ -137,6 +137,7 @@ public Integer executeLifecycleProcess(@NotNull CommonConfig commonConfig, @NotN kernelArgs.addAll(Lists.list(OrderedSafeMaps.of("--exclude-arc", commandOptions.excludeArcNames().isEmpty() ? null : String.join(",", commandOptions.excludeArcNames())))); kernelArgs.addAll(Lists.list(OrderedSafeMaps.of("--only-resource", commandOptions.onlyResourceNames().isEmpty() ? null : String.join(",", commandOptions.onlyResourceNames())))); kernelArgs.addAll(Lists.list(OrderedSafeMaps.of("--exclude-all-tags", commandOptions.excludeAllTags().map(b -> Boolean.toString(b)).orElse(null)))); + kernelArgs.addAll(SafeList.of(verbosity.get() == 0 ? null : String.format("-%s", "v".repeat(verbosity.get())))); return executeCDKApp(commonConfig, awsConfig, cdkCommand, cdkCommandArgs, "synth", kernelArgs); } diff --git a/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/bootstrap/Bootstrap.java b/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/bootstrap/Bootstrap.java index 04d43f7e..b8c02084 100644 --- a/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/bootstrap/Bootstrap.java +++ b/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/bootstrap/Bootstrap.java @@ -55,7 +55,7 @@ public class Bootstrap extends BaseCDKCommand implements Callable { @CommandLine.Mixin BootstrapCommandOptions commandOptions = new BootstrapCommandOptions(); @CommandLine.Mixin - CDKProcessExec processExec = new CDKProcessExec(commandOptions::dryRun, commandOptions::retry); + CDKProcessExec processExec = new CDKProcessExec(commandOptions::dryRun, commandOptions::retry, this::verbosityLevel); @Override public Integer call() throws Exception { diff --git a/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/lifecycle/Deploy.java b/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/lifecycle/Deploy.java index 86df7091..322d2b4d 100644 --- a/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/lifecycle/Deploy.java +++ b/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/lifecycle/Deploy.java @@ -12,8 +12,6 @@ import clusterless.substrate.aws.cdk.BaseCDKCommand; import clusterless.substrate.aws.cdk.CDKCommand; import clusterless.substrate.aws.cdk.CDKProcessExec; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import picocli.CommandLine; import java.util.concurrent.Callable; @@ -22,11 +20,10 @@ name = "deploy" ) public class Deploy extends BaseCDKCommand implements Callable { - private static final Logger LOG = LogManager.getLogger(Deploy.class); @CommandLine.Mixin DeployCommandOptions commandOptions = new DeployCommandOptions(); @CommandLine.Mixin - CDKProcessExec processExec = new CDKProcessExec(commandOptions); + CDKProcessExec processExec = new CDKProcessExec(commandOptions, this::verbosityLevel); @Override public Integer call() throws Exception { diff --git a/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/lifecycle/Destroy.java b/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/lifecycle/Destroy.java index 3245f870..a3c23cbc 100644 --- a/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/lifecycle/Destroy.java +++ b/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/lifecycle/Destroy.java @@ -24,7 +24,7 @@ public class Destroy extends BaseCDKCommand implements Callable { @CommandLine.Mixin DestroyCommandOptions commandOptions = new DestroyCommandOptions(); @CommandLine.Mixin - CDKProcessExec processExec = new CDKProcessExec(commandOptions::dryRun, commandOptions::retry); + CDKProcessExec processExec = new CDKProcessExec(commandOptions::dryRun, commandOptions::retry, this::verbosityLevel); @Override public Integer call() throws Exception { diff --git a/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/lifecycle/Diff.java b/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/lifecycle/Diff.java index 8c558079..4ae7a2e1 100644 --- a/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/lifecycle/Diff.java +++ b/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/lifecycle/Diff.java @@ -23,7 +23,7 @@ public class Diff extends BaseCDKCommand implements Callable { @CommandLine.Mixin DiffCommandOptions commandOptions = new DiffCommandOptions(); @CommandLine.Mixin - CDKProcessExec processExec = new CDKProcessExec(commandOptions); + CDKProcessExec processExec = new CDKProcessExec(commandOptions, this::verbosityLevel); @Override public Integer call() throws Exception { diff --git a/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/lifecycle/Import.java b/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/lifecycle/Import.java index b5bb9e64..31f7fbd0 100644 --- a/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/lifecycle/Import.java +++ b/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/lifecycle/Import.java @@ -26,7 +26,7 @@ public class Import extends BaseCDKCommand implements Callable { @CommandLine.Mixin ImportCommandOptions commandOptions = new ImportCommandOptions(); @CommandLine.Mixin - CDKProcessExec processExec = new CDKProcessExec(commandOptions); + CDKProcessExec processExec = new CDKProcessExec(commandOptions, this::verbosityLevel); @Override public Integer call() throws Exception { diff --git a/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/lifecycle/Verify.java b/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/lifecycle/Verify.java index 8ebfcfff..2d738e90 100644 --- a/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/lifecycle/Verify.java +++ b/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/lifecycle/Verify.java @@ -23,7 +23,7 @@ public class Verify extends BaseCDKCommand implements Callable { @CommandLine.Mixin VerifyCommandOptions commandOptions = new VerifyCommandOptions(); @CommandLine.Mixin - CDKProcessExec processExec = new CDKProcessExec(commandOptions); + CDKProcessExec processExec = new CDKProcessExec(commandOptions, this::verbosityLevel); public Verify() { }