From 1c159d8f34c651732661287f43fcd4de226c186a Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 24 Sep 2024 20:28:29 +0000
Subject: [PATCH 01/30] added a cleanup_experment funciton to ci_utils that
 cleans local archrive files

---
 ci/Jenkinsfile               |  1 +
 ci/scripts/check_ci.sh       |  1 +
 ci/scripts/utils/ci_utils.sh | 13 +++++++++++++
 3 files changed, 15 insertions(+)

diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index 2654adba29..66ab489aa5 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -218,6 +218,7 @@ pipeline {
                                                 sh(script: " rm -f ${error_file}")
                                                 try {
                                                     sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${system}")
+                                                    sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh clean_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}")
                                                 } catch (Exception error_experment) {
                                                     sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}")
                                                     ws(CUSTOM_WORKSPACE) {
diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index 825d8f5e8b..710a4492de 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -168,6 +168,7 @@ for pr in ${pr_list}; do
     fi
     if [[ "${rocoto_state}" == "DONE" ]]; then
       #Remove Experment cases that completed successfully
+      "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh" cleanup_experiment "${pslot_dir}"
       rm -Rf "${pslot_dir}"
       rm -Rf "${pr_dir}/RUNTESTS/COMROOT/${pslot}"
       rm -f "${output_ci_single}"
diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 2a51467d38..4c56c84876 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -155,3 +155,16 @@ function publish_logs() {
     fi
     echo "${URL}"
 }
+
+function cleanup_experiment() {
+    # cleanup_experiment function
+    # This function takes a directory path of an EXPDIR and
+    # gets HOMEDIR value from config.base to
+    # clean up the archived files on disk.
+
+    local PSLOT_PATH="$1"
+    local pslot=$(basename "${PSLOT_PATH}")
+    local HOMEDIR=$(grep 'export HOMEDIR=' "${PSLOT_PATH}/config.base" | cut -d'=' -f2 | tr -d '[:space:]')
+    local ARCHIVEDIR="${HOMEDIR}/archive"
+    rm -rf "${ARCHIVEDIR}/${pslot}"
+    
\ No newline at end of file

From 463655a5ff872060435435dd05fdb7e4928c121c Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 25 Sep 2024 20:38:33 +0000
Subject: [PATCH 02/30] moved rm for EXPDIR into cleanup_expermiment function

---
 ci/scripts/check_ci.sh       | 2 --
 ci/scripts/utils/ci_utils.sh | 3 +++
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index 710a4492de..a89a661042 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -169,8 +169,6 @@ for pr in ${pr_list}; do
     if [[ "${rocoto_state}" == "DONE" ]]; then
       #Remove Experment cases that completed successfully
       "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh" cleanup_experiment "${pslot_dir}"
-      rm -Rf "${pslot_dir}"
-      rm -Rf "${pr_dir}/RUNTESTS/COMROOT/${pslot}"
       rm -f "${output_ci_single}"
       # echo "\`\`\`" > "${output_ci_single}"
       DATE=$(date +'%D %r')
diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 4c56c84876..adfdf09096 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -166,5 +166,8 @@ function cleanup_experiment() {
     local pslot=$(basename "${PSLOT_PATH}")
     local HOMEDIR=$(grep 'export HOMEDIR=' "${PSLOT_PATH}/config.base" | cut -d'=' -f2 | tr -d '[:space:]')
     local ARCHIVEDIR="${HOMEDIR}/archive"
+
     rm -rf "${ARCHIVEDIR}/${pslot}"
+    rm -Rf "${PSLOT_PATH}"
+    rm -Rf "${PSLOT_PATH}/../COMROOT/${pslot}"
     
\ No newline at end of file

From 40c96b250374418f34db00a09ba30084eac14a27 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 26 Sep 2024 14:37:48 +0000
Subject: [PATCH 03/30] fixed bracket sytax error in ci_utils and missnamed
 function call in Jenkins file to cleanup experiment

---
 ci/Jenkinsfile               | 2 +-
 ci/scripts/utils/ci_utils.sh | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index 66ab489aa5..5aac2d51bc 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -218,7 +218,7 @@ pipeline {
                                                 sh(script: " rm -f ${error_file}")
                                                 try {
                                                     sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${system}")
-                                                    sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh clean_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}")
+                                                    sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}")
                                                 } catch (Exception error_experment) {
                                                     sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}")
                                                     ws(CUSTOM_WORKSPACE) {
diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index adfdf09096..65bad793d2 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -170,4 +170,5 @@ function cleanup_experiment() {
     rm -rf "${ARCHIVEDIR}/${pslot}"
     rm -Rf "${PSLOT_PATH}"
     rm -Rf "${PSLOT_PATH}/../COMROOT/${pslot}"
+}
     
\ No newline at end of file

From 54ce216c249b2ada57742e76408685356add3ad5 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 26 Sep 2024 14:48:49 +0000
Subject: [PATCH 04/30] fixed SC2155 in bash utils for cleanup

---
 ci/scripts/utils/ci_utils.sh | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 65bad793d2..52b9449e21 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -162,10 +162,15 @@ function cleanup_experiment() {
     # gets HOMEDIR value from config.base to
     # clean up the archived files on disk.
 
-    local PSLOT_PATH="$1"
-    local pslot=$(basename "${PSLOT_PATH}")
-    local HOMEDIR=$(grep 'export HOMEDIR=' "${PSLOT_PATH}/config.base" | cut -d'=' -f2 | tr -d '[:space:]')
-    local ARCHIVEDIR="${HOMEDIR}/archive"
+    local PSLOT_PATH
+    local pslot
+    local HOMEDIR
+    local ARCHIVEDIR
+
+    PSLOT_PATH="$1"
+    pslot=$(basename "${PSLOT_PATH}")
+    HOMEDIR=$(grep 'export HOMEDIR=' "${PSLOT_PATH}/config.base" | cut -d'=' -f2 | tr -d '[:space:]')
+    ARCHIVEDIR="${HOMEDIR}/archive"
 
     rm -rf "${ARCHIVEDIR}/${pslot}"
     rm -Rf "${PSLOT_PATH}"

From cacde8cf98034f043eb3aacc0163df3dec76f351 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 26 Sep 2024 14:54:34 +0000
Subject: [PATCH 05/30] fixed SC2115 for var expation in path for clean
 experiment util

---
 ci/scripts/utils/ci_utils.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 52b9449e21..f9a7cbf019 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -172,7 +172,7 @@ function cleanup_experiment() {
     HOMEDIR=$(grep 'export HOMEDIR=' "${PSLOT_PATH}/config.base" | cut -d'=' -f2 | tr -d '[:space:]')
     ARCHIVEDIR="${HOMEDIR}/archive"
 
-    rm -rf "${ARCHIVEDIR}/${pslot}"
+    rm -Rf "${ARCHIVEDIR:?}/${pslot}"
     rm -Rf "${PSLOT_PATH}"
     rm -Rf "${PSLOT_PATH}/../COMROOT/${pslot}"
 }

From c8a10a9d12dd1c400991ef97832881ca08fa53e4 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 26 Sep 2024 14:57:06 +0000
Subject: [PATCH 06/30] needed the OR truch for var assigment via actions in
 bash

---
 ci/scripts/utils/ci_utils.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index f9a7cbf019..0ad5ff7e5b 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -169,7 +169,7 @@ function cleanup_experiment() {
 
     PSLOT_PATH="$1"
     pslot=$(basename "${PSLOT_PATH}")
-    HOMEDIR=$(grep 'export HOMEDIR=' "${PSLOT_PATH}/config.base" | cut -d'=' -f2 | tr -d '[:space:]')
+    HOMEDIR=$(grep 'export HOMEDIR=' "${PSLOT_PATH}/config.base" | cut -d'=' -f2 | tr -d '[:space:]' || true) || true
     ARCHIVEDIR="${HOMEDIR}/archive"
 
     rm -Rf "${ARCHIVEDIR:?}/${pslot}"

From 7c93f4b64b60860a259fb5c74743e1462fbc0215 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 27 Sep 2024 14:24:47 +0000
Subject: [PATCH 07/30] fixed the evaluation of HOMDIR so that internal
 strification of env vars work in cleanup_experiment

---
 ci/scripts/utils/ci_utils.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 0ad5ff7e5b..4a0ece7f4d 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -169,8 +169,8 @@ function cleanup_experiment() {
 
     PSLOT_PATH="$1"
     pslot=$(basename "${PSLOT_PATH}")
-    HOMEDIR=$(grep 'export HOMEDIR=' "${PSLOT_PATH}/config.base" | cut -d'=' -f2 | tr -d '[:space:]' || true) || true
-    ARCHIVEDIR="${HOMEDIR}/archive"
+    HOMEDIR=$(grep 'export HOMEDIR=' "${PSLOT_PATH}/config.base" | cut -d'=' -f2 | tr -d '[:space:]"' || true) || true
+    eval HOMEDIR="${HOMEDIR}"
 
     rm -Rf "${ARCHIVEDIR:?}/${pslot}"
     rm -Rf "${PSLOT_PATH}"

From bfcbba5728aef9ae33df28a53bb0e4d82e91af21 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 27 Sep 2024 14:27:40 +0000
Subject: [PATCH 08/30] still needed archive tagged back onto HOMEDIR

---
 ci/scripts/utils/ci_utils.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 4a0ece7f4d..1149553413 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -171,6 +171,7 @@ function cleanup_experiment() {
     pslot=$(basename "${PSLOT_PATH}")
     HOMEDIR=$(grep 'export HOMEDIR=' "${PSLOT_PATH}/config.base" | cut -d'=' -f2 | tr -d '[:space:]"' || true) || true
     eval HOMEDIR="${HOMEDIR}"
+    ARCHIVEDIR="${HOMEDIR}/archive"
 
     rm -Rf "${ARCHIVEDIR:?}/${pslot}"
     rm -Rf "${PSLOT_PATH}"

From 49b4489e85bc56ed5161720ca5c4b85b6093a8e8 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 27 Sep 2024 14:33:10 +0000
Subject: [PATCH 09/30] new linee at aend of ci_utils

---
 ci/scripts/utils/ci_utils.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 1149553413..7fb201ac3d 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -177,4 +177,3 @@ function cleanup_experiment() {
     rm -Rf "${PSLOT_PATH}"
     rm -Rf "${PSLOT_PATH}/../COMROOT/${pslot}"
 }
-    
\ No newline at end of file

From d8af194d940f8cef8e0b366a6bb6490ecf3fce77 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 23 Oct 2024 14:10:30 +0000
Subject: [PATCH 10/30] removed eval in cleanup_experiment and added envsubst
 at end of pipe instead

---
 ci/scripts/utils/ci_utils.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 7fb201ac3d..11f638f194 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -169,8 +169,7 @@ function cleanup_experiment() {
 
     PSLOT_PATH="$1"
     pslot=$(basename "${PSLOT_PATH}")
-    HOMEDIR=$(grep 'export HOMEDIR=' "${PSLOT_PATH}/config.base" | cut -d'=' -f2 | tr -d '[:space:]"' || true) || true
-    eval HOMEDIR="${HOMEDIR}"
+    HOMEDIR=$(grep 'export HOMEDIR=' "${PSLOT_PATH}/config.base" | cut -d'=' -f2 | tr -d '[:space:]"' | envsubst || true) || true
     ARCHIVEDIR="${HOMEDIR}/archive"
 
     rm -Rf "${ARCHIVEDIR:?}/${pslot}"

From d54bd8d4898dcc0faa63fb98dd475d7a65e293d5 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 24 Oct 2024 14:24:50 +0000
Subject: [PATCH 11/30] updated archival directory to ARCDIR from config.base
 in cleanup_experiment

---
 ci/scripts/utils/ci_utils.sh | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 11f638f194..fbdf769ddd 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -157,22 +157,20 @@ function publish_logs() {
 }
 
 function cleanup_experiment() {
-    # cleanup_experiment function
-    # This function takes a directory path of an EXPDIR and
-    # gets HOMEDIR value from config.base to
-    # clean up the archived files on disk.
+    # cleanup_experiment function removes:
+    #   1 The directory path of an EXPDIR (given as an argument)
+    #   2 The adjadjacent COMROOT directory (uses basename of EXPDIR for pslot)
+    #   3 The ARCDIR directory (uses ARCDIR from config.base file in EXPDIR)
 
-    local PSLOT_PATH
+    local EXPDIR="$1"
     local pslot
-    local HOMEDIR
-    local ARCHIVEDIR
+    local ARCDIR
 
     PSLOT_PATH="$1"
     pslot=$(basename "${PSLOT_PATH}")
-    HOMEDIR=$(grep 'export HOMEDIR=' "${PSLOT_PATH}/config.base" | cut -d'=' -f2 | tr -d '[:space:]"' | envsubst || true) || true
-    ARCHIVEDIR="${HOMEDIR}/archive"
+    ARCDIR=$(grep 'export ARCDIR=' "${PSLOT_PATH}/config.base" | cut -d'=' -f2 | tr -d '[:space:]"' | envsubst || true) || true
 
-    rm -Rf "${ARCHIVEDIR:?}/${pslot}"
+    rm -Rf "${ARCDIR:?}"
     rm -Rf "${PSLOT_PATH}"
     rm -Rf "${PSLOT_PATH}/../COMROOT/${pslot}"
 }

From 85f63698027dc982e97edc2e02c9124f787024d9 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 24 Oct 2024 14:28:40 +0000
Subject: [PATCH 12/30] forgot to clean up EXPDIR replacement in
 cleanup_experment

---
 ci/scripts/utils/ci_utils.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index fbdf769ddd..b9513fb8ff 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -167,10 +167,10 @@ function cleanup_experiment() {
     local ARCDIR
 
     PSLOT_PATH="$1"
-    pslot=$(basename "${PSLOT_PATH}")
-    ARCDIR=$(grep 'export ARCDIR=' "${PSLOT_PATH}/config.base" | cut -d'=' -f2 | tr -d '[:space:]"' | envsubst || true) || true
+    pslot=$(basename "${EXPDIR}")
+    ARCDIR=$(grep 'export ARCDIR=' "${EXPDIR}/config.base" | cut -d'=' -f2 | tr -d '[:space:]"' | envsubst || true) || true
 
     rm -Rf "${ARCDIR:?}"
-    rm -Rf "${PSLOT_PATH}"
-    rm -Rf "${PSLOT_PATH}/../COMROOT/${pslot}"
+    rm -Rf "${EXPDIR}"
+    rm -Rf "${EXPDIR}/../COMROOT/${pslot}"
 }

From 6a09b3680efaa7475a5e815ab5d3878dba723f89 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 24 Oct 2024 14:29:40 +0000
Subject: [PATCH 13/30] forgot to clean up EXPDIR replacement in
 cleanup_experment still

---
 ci/scripts/utils/ci_utils.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index b9513fb8ff..2dc4bff79b 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -166,7 +166,7 @@ function cleanup_experiment() {
     local pslot
     local ARCDIR
 
-    PSLOT_PATH="$1"
+    EXPDIR="$1"
     pslot=$(basename "${EXPDIR}")
     ARCDIR=$(grep 'export ARCDIR=' "${EXPDIR}/config.base" | cut -d'=' -f2 | tr -d '[:space:]"' | envsubst || true) || true
 

From 32b8cd0cb79500e6e07b7e5bb36910221c1faf1e Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Fri, 25 Oct 2024 09:39:08 -0400
Subject: [PATCH 14/30] Update ci/scripts/utils/ci_utils.sh
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

of course, duh ◔_◔

Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com>
---
 ci/scripts/utils/ci_utils.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 2dc4bff79b..4c300dfed1 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -171,6 +171,6 @@ function cleanup_experiment() {
     ARCDIR=$(grep 'export ARCDIR=' "${EXPDIR}/config.base" | cut -d'=' -f2 | tr -d '[:space:]"' | envsubst || true) || true
 
     rm -Rf "${ARCDIR:?}"
-    rm -Rf "${EXPDIR}"
     rm -Rf "${EXPDIR}/../COMROOT/${pslot}"
+    rm -Rf "${EXPDIR}"
 }

From e01d9d360e5d7cba4cc2968c48f751aa02b1fc0c Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 5 Nov 2024 21:29:23 +0000
Subject: [PATCH 15/30] added get_config_var util to simplify cleanup_experment

---
 ci/scripts/utils/ci_utils.sh       | 15 ++++++----
 ci/scripts/utils/get_config_var.py | 44 ++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+), 6 deletions(-)
 create mode 100755 ci/scripts/utils/get_config_var.py

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 4c300dfed1..d3b3e0870f 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -137,7 +137,6 @@ function publish_logs() {
     local PR_header="$1"
     local dir_path="$2"
     local file="$3"
-
     local full_paths=""
     while IFS= read -r line; do
         full_path="${dir_path}/${line}"
@@ -157,20 +156,24 @@ function publish_logs() {
 }
 
 function cleanup_experiment() {
-    # cleanup_experiment function removes:
-    #   1 The directory path of an EXPDIR (given as an argument)
-    #   2 The adjadjacent COMROOT directory (uses basename of EXPDIR for pslot)
-    #   3 The ARCDIR directory (uses ARCDIR from config.base file in EXPDIR)
 
     local EXPDIR="$1"
     local pslot
     local ARCDIR
+    local ATARDIR
 
     EXPDIR="$1"
     pslot=$(basename "${EXPDIR}")
-    ARCDIR=$(grep 'export ARCDIR=' "${EXPDIR}/config.base" | cut -d'=' -f2 | tr -d '[:space:]"' | envsubst || true) || true
+
+    # Use the Python utility to get the required variables
+    read -r ATARDIR < <("${HOMEgfs}/ci/scripts/utils/get_config_var.py" ATARDIR "${EXPDIR}")
+    read -r ARCDIR < <("${HOMEgfs}/ci/scripts/utils/get_config_var.py" ARCDIR "${EXPDIR}")
+
+    echo -e "Cleaning up:\t\n${EXPDIR}\t\n${EXPDIR}/../COMROOT/${pslot}\t\n${ARCDIR}\t\n${ATARDIR}"
+    exit 0
 
     rm -Rf "${ARCDIR:?}"
+    rm -Rf "${ATARDIR:?}"
     rm -Rf "${EXPDIR}/../COMROOT/${pslot}"
     rm -Rf "${EXPDIR}"
 }
diff --git a/ci/scripts/utils/get_config_var.py b/ci/scripts/utils/get_config_var.py
new file mode 100755
index 0000000000..e55dc7b399
--- /dev/null
+++ b/ci/scripts/utils/get_config_var.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import argparse
+from wxflow import Configuration
+
+def get_config_vars(var_names, config_path):
+    """
+    GET_CONFIG_VARS Get configuration variables from a config file or directory.
+    
+    Parameters:
+    var_names (list of str): The names of the configuration variables to retrieve.
+    config_path (str): The path to the configuration file or directory.
+    
+    Returns:
+    list of str: The values of the specified configuration variables.
+    """
+    if os.path.isfile(config_path):
+        config_dir = os.path.dirname(config_path)
+        config_file = os.path.basename(config_path)
+    elif os.path.isdir(config_path):
+        config_dir = config_path
+        config_file = 'config.base'
+    config = Configuration(config_dir)
+    config_data = config.parse_config(config_file)
+    return [config_data[var_name] for var_name in var_names]
+
+if __name__ == "__main__":
+    """
+    Main entry point for the script.
+    Parses command-line arguments and retrieves the specified configuration variables.
+    """
+    parser = argparse.ArgumentParser(description="Get configuration variables from a config file or directory.")
+    parser.add_argument("var_names", nargs='+', help="The names of the configuration variables to retrieve.")
+    parser.add_argument("config_path", help="The path to the configuration file or directory.")
+    
+    args = parser.parse_args()
+
+    var_names = args.var_names
+    config_path = args.config_path
+
+    values = get_config_vars(var_names, config_path)
+    print(" ".join(values))

From 3d5e8518381c2b98894d995385c39c19501d1dc3 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 5 Nov 2024 21:32:13 +0000
Subject: [PATCH 16/30] removed debug outputs in cleanup_experiment

---
 ci/scripts/utils/ci_utils.sh | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index d3b3e0870f..07d2b13d60 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -160,20 +160,16 @@ function cleanup_experiment() {
     local EXPDIR="$1"
     local pslot
     local ARCDIR
-    local ATARDIR
+    local ATADIR
 
     EXPDIR="$1"
     pslot=$(basename "${EXPDIR}")
 
     # Use the Python utility to get the required variables
-    read -r ATARDIR < <("${HOMEgfs}/ci/scripts/utils/get_config_var.py" ATARDIR "${EXPDIR}")
-    read -r ARCDIR < <("${HOMEgfs}/ci/scripts/utils/get_config_var.py" ARCDIR "${EXPDIR}")
-
-    echo -e "Cleaning up:\t\n${EXPDIR}\t\n${EXPDIR}/../COMROOT/${pslot}\t\n${ARCDIR}\t\n${ATARDIR}"
-    exit 0
+    read -r ARCDIR ATADIR < <("${HOMEgfs}/ci/scripts/utils/get_config_var.py" ARCDIR ATADIR "${EXPDIR}")
 
     rm -Rf "${ARCDIR:?}"
-    rm -Rf "${ATARDIR:?}"
+    rm -Rf "${ATADIR:?}"
     rm -Rf "${EXPDIR}/../COMROOT/${pslot}"
     rm -Rf "${EXPDIR}"
 }

From 162474a9e3e2192ad44c9e04b5b43f4e3cf5363e Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 5 Nov 2024 22:04:30 +0000
Subject: [PATCH 17/30] added next step to use get_config_var to remove
 previous DATAROOT data when startig new experimentsa

---
 ci/Jenkinsfile               | 3 +++
 ci/scripts/utils/ci_utils.sh | 3 ++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index 5aac2d51bc..e94bb646d2 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -217,6 +217,9 @@ pipeline {
                                                 def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs"
                                                 sh(script: " rm -f ${error_file}")
                                                 try {
+                                                    // Remove any data in that might still be in the DATAROOT directory from previous runs
+                                                    DATAROOT=sh(script: "${HOMEgfs}/ci/scripts/utils/get_config_var DATAROOT ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}", returnStdout: true).trim()
+                                                    sh(script: "rm -Rf ${DATAROOT}")
                                                     sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${system}")
                                                     sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}")
                                                 } catch (Exception error_experment) {
diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 07d2b13d60..27b84d53e9 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -166,8 +166,9 @@ function cleanup_experiment() {
     pslot=$(basename "${EXPDIR}")
 
     # Use the Python utility to get the required variables
-    read -r ARCDIR ATADIR < <("${HOMEgfs}/ci/scripts/utils/get_config_var.py" ARCDIR ATADIR "${EXPDIR}")
+    read -r ARCDIR ATADIR DATAROOT < <("${HOMEgfs}/ci/scripts/utils/get_config_var.py" ARCDIR ATADIR DATAROOT "${EXPDIR}")
 
+    rm -Rf "${DATAROOT:?}"
     rm -Rf "${ARCDIR:?}"
     rm -Rf "${ATADIR:?}"
     rm -Rf "${EXPDIR}/../COMROOT/${pslot}"

From c6ad4ec2aaa0427e7243f0d4340d54276bc564eb Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 5 Nov 2024 22:44:31 +0000
Subject: [PATCH 18/30] updated Jenkinsfile to use clean experiment and to also
 clean DATADIR in advance of running a experiment for the case that it had
 already ran

---
 ci/Jenkinsfile | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index 6a2e064be0..4477ce4391 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -221,7 +221,11 @@ pipeline {
                                                 def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp"
                                                 def build_system = yaml_case.experiment.system
                                                 try {
+                                                    // Remove any data in that might still be in the DATAROOT directory from previous runs
+                                                    DATAROOT=sh(script: "${HOMEgfs}/ci/scripts/utils/get_config_var DATAROOT ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}", returnStdout: true).trim()
+                                                    sh(script: "rm -Rf ${DATAROOT}")
                                                     sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${build_system}")
+                                                    sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}")
                                                 } catch (Exception error_experment) {
                                                     sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}")
                                                     ws(CUSTOM_WORKSPACE) {

From bef32e7302f3097b9abe6a04ef5ab260f7f79b8b Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 5 Nov 2024 23:09:25 +0000
Subject: [PATCH 19/30] updating correct ATARDIR variable in cleaup

---
 ci/scripts/utils/ci_utils.sh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 27b84d53e9..463e690b94 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -160,17 +160,18 @@ function cleanup_experiment() {
     local EXPDIR="$1"
     local pslot
     local ARCDIR
-    local ATADIR
+    local ATARDIR
+    local DATAROOT
 
     EXPDIR="$1"
     pslot=$(basename "${EXPDIR}")
 
     # Use the Python utility to get the required variables
-    read -r ARCDIR ATADIR DATAROOT < <("${HOMEgfs}/ci/scripts/utils/get_config_var.py" ARCDIR ATADIR DATAROOT "${EXPDIR}")
+    read -r ARCDIR ATARDIR DATAROOT < <("${HOMEgfs}/ci/scripts/utils/get_config_var.py" ARCDIR ATARDIR DATAROOT "${EXPDIR}")
 
     rm -Rf "${DATAROOT:?}"
     rm -Rf "${ARCDIR:?}"
-    rm -Rf "${ATADIR:?}"
+    rm -Rf "${ATARDIR:?}"
     rm -Rf "${EXPDIR}/../COMROOT/${pslot}"
     rm -Rf "${EXPDIR}"
 }

From 02642fc560cfa5b1b2a19c74c120160abc50467f Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 5 Nov 2024 23:12:24 +0000
Subject: [PATCH 20/30] added true for shell norms

---
 ci/scripts/utils/ci_utils.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 463e690b94..d9c01939a2 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -167,7 +167,7 @@ function cleanup_experiment() {
     pslot=$(basename "${EXPDIR}")
 
     # Use the Python utility to get the required variables
-    read -r ARCDIR ATARDIR DATAROOT < <("${HOMEgfs}/ci/scripts/utils/get_config_var.py" ARCDIR ATARDIR DATAROOT "${EXPDIR}")
+    read -r ARCDIR ATARDIR DATAROOT < <("${HOMEgfs}/ci/scripts/utils/get_config_var.py" ARCDIR ATARDIR DATAROOT "${EXPDIR}") || true
 
     rm -Rf "${DATAROOT:?}"
     rm -Rf "${ARCDIR:?}"

From 62ec3f6f4256551f565c544cbf1601a865b3b7cc Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Wed, 6 Nov 2024 15:08:33 +0000
Subject: [PATCH 21/30] Update ci/scripts/utils/ci_utils.sh

Of course, love it.

Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com>
---
 ci/scripts/utils/ci_utils.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index d9c01939a2..6caae4a6f0 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -167,11 +167,11 @@ function cleanup_experiment() {
     pslot=$(basename "${EXPDIR}")
 
     # Use the Python utility to get the required variables
-    read -r ARCDIR ATARDIR DATAROOT < <("${HOMEgfs}/ci/scripts/utils/get_config_var.py" ARCDIR ATARDIR DATAROOT "${EXPDIR}") || true
+    read -r ARCDIR ATARDIR DATAROOT COMROOT < <("${HOMEgfs}/ci/scripts/utils/get_config_var.py" ARCDIR ATARDIR DATAROOT COMROOT "${EXPDIR}") || true
 
     rm -Rf "${DATAROOT:?}"
     rm -Rf "${ARCDIR:?}"
     rm -Rf "${ATARDIR:?}"
-    rm -Rf "${EXPDIR}/../COMROOT/${pslot}"
+    rm -Rf "${COMROOT:?}"
     rm -Rf "${EXPDIR}"
 }

From 7e5b4fb7220e9173772e66206154d684f7ea7088 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Wed, 6 Nov 2024 10:24:46 -0500
Subject: [PATCH 22/30] Update ci/scripts/utils/ci_utils.sh

oh sure, yes. Notice also we do the DATAROOT remove independent of the cleanup_experiment before it is ran again in Jenkins.

Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com>
---
 ci/scripts/utils/ci_utils.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 6caae4a6f0..3d3741389d 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -162,6 +162,7 @@ function cleanup_experiment() {
     local ARCDIR
     local ATARDIR
     local DATAROOT
+    local COMROOT
 
     EXPDIR="$1"
     pslot=$(basename "${EXPDIR}")

From bf3b80fc4ea68a4090d97589cd28ee17fb2a8430 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Wed, 6 Nov 2024 12:25:32 -0500
Subject: [PATCH 23/30] Update ci/scripts/utils/get_config_var.py

Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com>
---
 ci/scripts/utils/get_config_var.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ci/scripts/utils/get_config_var.py b/ci/scripts/utils/get_config_var.py
index e55dc7b399..1f4b6b1bc1 100755
--- a/ci/scripts/utils/get_config_var.py
+++ b/ci/scripts/utils/get_config_var.py
@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 
 import os
-import sys
 import argparse
 from wxflow import Configuration
 

From cd4e27b32dfc7b7a914c4d77a7410fa7300cb87e Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Wed, 6 Nov 2024 12:25:52 -0500
Subject: [PATCH 24/30] Update ci/scripts/utils/get_config_var.py

que?

Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com>
---
 ci/scripts/utils/get_config_var.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ci/scripts/utils/get_config_var.py b/ci/scripts/utils/get_config_var.py
index 1f4b6b1bc1..e062795e01 100755
--- a/ci/scripts/utils/get_config_var.py
+++ b/ci/scripts/utils/get_config_var.py
@@ -4,6 +4,7 @@
 import argparse
 from wxflow import Configuration
 
+
 def get_config_vars(var_names, config_path):
     """
     GET_CONFIG_VARS Get configuration variables from a config file or directory.

From 5ea8d3d42f8b8bbd6668675e7fbe6e0121336065 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Wed, 6 Nov 2024 12:26:04 -0500
Subject: [PATCH 25/30] Update ci/scripts/utils/get_config_var.py

que que

Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com>
---
 ci/scripts/utils/get_config_var.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ci/scripts/utils/get_config_var.py b/ci/scripts/utils/get_config_var.py
index e062795e01..25b47c3134 100755
--- a/ci/scripts/utils/get_config_var.py
+++ b/ci/scripts/utils/get_config_var.py
@@ -8,7 +8,6 @@
 def get_config_vars(var_names, config_path):
     """
     GET_CONFIG_VARS Get configuration variables from a config file or directory.
-    
     Parameters:
     var_names (list of str): The names of the configuration variables to retrieve.
     config_path (str): The path to the configuration file or directory.

From 99b93220bf96367e0a45a238d8dd0dd9588bfaf7 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Wed, 6 Nov 2024 12:26:17 -0500
Subject: [PATCH 26/30] Update ci/scripts/utils/get_config_var.py

que que que

Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com>
---
 ci/scripts/utils/get_config_var.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ci/scripts/utils/get_config_var.py b/ci/scripts/utils/get_config_var.py
index 25b47c3134..ba92709d1b 100755
--- a/ci/scripts/utils/get_config_var.py
+++ b/ci/scripts/utils/get_config_var.py
@@ -11,7 +11,6 @@ def get_config_vars(var_names, config_path):
     Parameters:
     var_names (list of str): The names of the configuration variables to retrieve.
     config_path (str): The path to the configuration file or directory.
-    
     Returns:
     list of str: The values of the specified configuration variables.
     """

From f56be3370c995467230a1ce9a7bc8b7fb1d09fbc Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Wed, 6 Nov 2024 12:26:33 -0500
Subject: [PATCH 27/30] Update ci/scripts/utils/get_config_var.py

que que que que

Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com>
---
 ci/scripts/utils/get_config_var.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ci/scripts/utils/get_config_var.py b/ci/scripts/utils/get_config_var.py
index ba92709d1b..3e63fcf376 100755
--- a/ci/scripts/utils/get_config_var.py
+++ b/ci/scripts/utils/get_config_var.py
@@ -24,6 +24,7 @@ def get_config_vars(var_names, config_path):
     config_data = config.parse_config(config_file)
     return [config_data[var_name] for var_name in var_names]
 
+
 if __name__ == "__main__":
     """
     Main entry point for the script.

From 9a381b43ec296cc17308d1104dddcd08594175cf Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 6 Nov 2024 18:54:37 +0000
Subject: [PATCH 28/30] fixed name of get_config_var in pipeline (left off the
 py extention)

---
 ci/Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index 4477ce4391..38b4eb0016 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -222,7 +222,7 @@ pipeline {
                                                 def build_system = yaml_case.experiment.system
                                                 try {
                                                     // Remove any data in that might still be in the DATAROOT directory from previous runs
-                                                    DATAROOT=sh(script: "${HOMEgfs}/ci/scripts/utils/get_config_var DATAROOT ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}", returnStdout: true).trim()
+                                                    DATAROOT=sh(script: "${HOMEgfs}/ci/scripts/utils/get_config_var.py DATAROOT ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}", returnStdout: true).trim()
                                                     sh(script: "rm -Rf ${DATAROOT}")
                                                     sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${build_system}")
                                                     sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}")

From 788dd8fb58331bc560fe1f24a17f1267b1058b07 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 6 Nov 2024 20:30:42 +0000
Subject: [PATCH 29/30] DATAROOT is not a config.base value but is construced
 from STMP in tasks.py

---
 ci/Jenkinsfile               | 4 ++--
 ci/scripts/utils/ci_utils.sh | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index 38b4eb0016..e78486f5be 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -222,8 +222,8 @@ pipeline {
                                                 def build_system = yaml_case.experiment.system
                                                 try {
                                                     // Remove any data in that might still be in the DATAROOT directory from previous runs
-                                                    DATAROOT=sh(script: "${HOMEgfs}/ci/scripts/utils/get_config_var.py DATAROOT ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}", returnStdout: true).trim()
-                                                    sh(script: "rm -Rf ${DATAROOT}")
+                                                    STMP=sh(script: "${HOMEgfs}/ci/scripts/utils/get_config_var.py STMP ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}", returnStdout: true).trim()
+                                                    sh(script: "rm -Rf ${STMP}/RUNDIR/${pslot}")
                                                     sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${build_system}")
                                                     sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}")
                                                 } catch (Exception error_experment) {
diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 3d3741389d..03a71f90ec 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -168,9 +168,9 @@ function cleanup_experiment() {
     pslot=$(basename "${EXPDIR}")
 
     # Use the Python utility to get the required variables
-    read -r ARCDIR ATARDIR DATAROOT COMROOT < <("${HOMEgfs}/ci/scripts/utils/get_config_var.py" ARCDIR ATARDIR DATAROOT COMROOT "${EXPDIR}") || true
+    read -r ARCDIR ATARDIR STMP COMROOT < <("${HOMEgfs}/ci/scripts/utils/get_config_var.py" ARCDIR ATARDIR STMP COMROOT "${EXPDIR}") || true
 
-    rm -Rf "${DATAROOT:?}"
+    rm -Rf "${STMP}/RUNDIRS/${pslot}"
     rm -Rf "${ARCDIR:?}"
     rm -Rf "${ATARDIR:?}"
     rm -Rf "${COMROOT:?}"

From 7d3b8bfe359e06b93be11bea5ed51ecd35e8ca31 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <Terry.McGuinness@noaa.gov>
Date: Fri, 8 Nov 2024 11:43:31 -0600
Subject: [PATCH 30/30] updated Jenkins file with correct RUNDIRS string (had
 RUNDIR as a bug)

---
 ci/Jenkinsfile               | 2 +-
 ci/scripts/utils/ci_utils.sh | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
index e78486f5be..8b987a16fa 100644
--- a/ci/Jenkinsfile
+++ b/ci/Jenkinsfile
@@ -223,7 +223,7 @@ pipeline {
                                                 try {
                                                     // Remove any data in that might still be in the DATAROOT directory from previous runs
                                                     STMP=sh(script: "${HOMEgfs}/ci/scripts/utils/get_config_var.py STMP ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}", returnStdout: true).trim()
-                                                    sh(script: "rm -Rf ${STMP}/RUNDIR/${pslot}")
+                                                    sh(script: "rm -Rf ${STMP}/RUNDIRS/${pslot}")
                                                     sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${build_system}")
                                                     sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}")
                                                 } catch (Exception error_experment) {
diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index 03a71f90ec..64988986fe 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -161,7 +161,6 @@ function cleanup_experiment() {
     local pslot
     local ARCDIR
     local ATARDIR
-    local DATAROOT
     local COMROOT
 
     EXPDIR="$1"