add retry option on bootstrap destroy, increase retries by one, enabl…

…e retry in scenarios
ClusterlessHQ · Aug 4, 2023 · eeef832 · eeef832
1 parent 3bc76e8
commit eeef832
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 2 deletions.
diff --git a/clusterless-main-common/src/main/java/clusterless/command/BootstrapCommandOptions.java b/clusterless-main-common/src/main/java/clusterless/command/BootstrapCommandOptions.java
@@ -44,6 +44,11 @@ public class BootstrapCommandOptions extends CommonCommandOptions {
     )
     boolean synth = false;
 
+    @CommandLine.Option(
+            names = "--retry",
+            description = "retry the destroy operation")
+    private boolean retry = false;
+
     @SuppressWarnings("OptionalUsedAsFieldOrParameterType")
     @CommandLine.Option(
             names = "--approve",
@@ -77,4 +82,9 @@ public boolean synth() {
     public Optional<Boolean> approve() {
         return approve;
     }
+
+    public boolean retry() {
+        // only enable retry if destroy is enabled
+        return destroy && retry;
+    }
 }
diff --git a/clusterless-main-common/src/main/java/clusterless/process/ProcessExec.java b/clusterless-main-common/src/main/java/clusterless/process/ProcessExec.java
@@ -55,6 +55,12 @@ protected int executeProcess(Map<String, String> environment, List<String> args)
                 LOG.warn("got exit code: {}, for command: {}, retrying in 15 seconds", exitCode, args);
                 Thread.sleep(15 * 1000);
                 exitCode = process(environment, args);
+
+                if (retry() && exitCode != 0) {
+                    LOG.warn("got exit code: {}, for command: {}, retrying in 30 seconds", exitCode, args);
+                    Thread.sleep(30 * 1000);
+                    exitCode = process(environment, args);
+                }
             }
 
             return exitCode;

diff --git a/...ain/java/clusterless/scenario/conductor/worker/cli/exec/ClusterlessBootstrapExecutor.java b/...ain/java/clusterless/scenario/conductor/worker/cli/exec/ClusterlessBootstrapExecutor.java
@@ -35,7 +35,8 @@ protected void createCommand(List<String> args) {
                 "--stage",
                 placement.get("stage"),
                 "-v",
-                destroy ? "--destroy" : null
+                destroy ? "--destroy" : null,
+                destroy ? "--retry" : null
         ));
     }
 

diff --git a/...substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/bootstrap/Bootstrap.java b/...substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/bootstrap/Bootstrap.java
@@ -54,7 +54,7 @@ public class Bootstrap extends CDKCommand implements Callable<Integer> {
     @CommandLine.Mixin
     BootstrapCommandOptions commandOptions = new BootstrapCommandOptions();
     @CommandLine.Mixin
-    CDKProcessExec processExec = new CDKProcessExec(commandOptions);
+    CDKProcessExec processExec = new CDKProcessExec(commandOptions::dryRun, commandOptions::retry);
 
     @Override
     public Integer call() throws Exception {