From eeef83288b7ee82dd3c58bd0f994e600211a762c Mon Sep 17 00:00:00 2001
From: Chris K Wensel <chris@wensel.net>
Date: Fri, 4 Aug 2023 10:10:56 -0700
Subject: [PATCH] add retry option on bootstrap destroy, increase retries by
 one, enable retry in scenarios

---
 .../clusterless/command/BootstrapCommandOptions.java   | 10 ++++++++++
 .../src/main/java/clusterless/process/ProcessExec.java |  6 ++++++
 .../worker/cli/exec/ClusterlessBootstrapExecutor.java  |  3 ++-
 .../substrate/aws/cdk/bootstrap/Bootstrap.java         |  2 +-
 4 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/clusterless-main-common/src/main/java/clusterless/command/BootstrapCommandOptions.java b/clusterless-main-common/src/main/java/clusterless/command/BootstrapCommandOptions.java
index c6a3c67c..e260b3e9 100644
--- a/clusterless-main-common/src/main/java/clusterless/command/BootstrapCommandOptions.java
+++ b/clusterless-main-common/src/main/java/clusterless/command/BootstrapCommandOptions.java
@@ -44,6 +44,11 @@ public class BootstrapCommandOptions extends CommonCommandOptions {
     )
     boolean synth = false;
 
+    @CommandLine.Option(
+            names = "--retry",
+            description = "retry the destroy operation")
+    private boolean retry = false;
+
     @SuppressWarnings("OptionalUsedAsFieldOrParameterType")
     @CommandLine.Option(
             names = "--approve",
@@ -77,4 +82,9 @@ public boolean synth() {
     public Optional<Boolean> approve() {
         return approve;
     }
+
+    public boolean retry() {
+        // only enable retry if destroy is enabled
+        return destroy && retry;
+    }
 }
diff --git a/clusterless-main-common/src/main/java/clusterless/process/ProcessExec.java b/clusterless-main-common/src/main/java/clusterless/process/ProcessExec.java
index 4a06ae3b..5d96df15 100644
--- a/clusterless-main-common/src/main/java/clusterless/process/ProcessExec.java
+++ b/clusterless-main-common/src/main/java/clusterless/process/ProcessExec.java
@@ -55,6 +55,12 @@ protected int executeProcess(Map<String, String> environment, List<String> args)
                 LOG.warn("got exit code: {}, for command: {}, retrying in 15 seconds", exitCode, args);
                 Thread.sleep(15 * 1000);
                 exitCode = process(environment, args);
+
+                if (retry() && exitCode != 0) {
+                    LOG.warn("got exit code: {}, for command: {}, retrying in 30 seconds", exitCode, args);
+                    Thread.sleep(30 * 1000);
+                    exitCode = process(environment, args);
+                }
             }
 
             return exitCode;
diff --git a/clusterless-scenario/src/main/java/clusterless/scenario/conductor/worker/cli/exec/ClusterlessBootstrapExecutor.java b/clusterless-scenario/src/main/java/clusterless/scenario/conductor/worker/cli/exec/ClusterlessBootstrapExecutor.java
index 3cde73b4..f5117fee 100644
--- a/clusterless-scenario/src/main/java/clusterless/scenario/conductor/worker/cli/exec/ClusterlessBootstrapExecutor.java
+++ b/clusterless-scenario/src/main/java/clusterless/scenario/conductor/worker/cli/exec/ClusterlessBootstrapExecutor.java
@@ -35,7 +35,8 @@ protected void createCommand(List<String> args) {
                 "--stage",
                 placement.get("stage"),
                 "-v",
-                destroy ? "--destroy" : null
+                destroy ? "--destroy" : null,
+                destroy ? "--retry" : null
         ));
     }
 
diff --git a/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/bootstrap/Bootstrap.java b/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/bootstrap/Bootstrap.java
index 1bb64011..6c4c6e6f 100644
--- a/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/bootstrap/Bootstrap.java
+++ b/clusterless-substrate-aws-kernel/src/main/java/clusterless/substrate/aws/cdk/bootstrap/Bootstrap.java
@@ -54,7 +54,7 @@ public class Bootstrap extends CDKCommand implements Callable<Integer> {
     @CommandLine.Mixin
     BootstrapCommandOptions commandOptions = new BootstrapCommandOptions();
     @CommandLine.Mixin
-    CDKProcessExec processExec = new CDKProcessExec(commandOptions);
+    CDKProcessExec processExec = new CDKProcessExec(commandOptions::dryRun, commandOptions::retry);
 
     @Override
     public Integer call() throws Exception {