feat: allow configuring of object storage files deletion for warehous…

…e destinations (#1896)
rudderlabs · Feb 12, 2025 · 2547b0c · 2547b0c
1 parent b9bb684
commit 2547b0c
Show file tree

Hide file tree

Showing 33 changed files with 161 additions and 0 deletions.
diff --git a/src/configurations/destinations/azure_datalake/db-config.json b/src/configurations/destinations/azure_datalake/db-config.json
@@ -34,6 +34,7 @@
         "syncFrequency",
         "syncStartAt",
         "underscoreDivideNumbers",
+        "cleanupObjectStorageFiles",
         "allowUsersContextTraits"
       ],
       "android": [

diff --git a/src/configurations/destinations/azure_datalake/schema.json b/src/configurations/destinations/azure_datalake/schema.json
@@ -40,6 +40,10 @@
       "syncStartAt": {
         "type": "string"
       },
+      "cleanupObjectStorageFiles": {
+        "type": "boolean",
+        "default": false
+      },
       "oneTrustCookieCategories": {
         "type": "object",
         "properties": {

diff --git a/src/configurations/destinations/azure_datalake/ui-config.json b/src/configurations/destinations/azure_datalake/ui-config.json
@@ -84,6 +84,12 @@
           "default": false,
           "footerNote": "Use this to Grant limited access to Azure Storage resources using shared access signatures (SAS)"
         },
+        {
+          "type": "checkbox",
+          "label": "Enable for cleanup of object storage files (deletion) after successful sync",
+          "value": "cleanupObjectStorageFiles",
+          "default": false
+        },
         {
           "type": "singleSelect",
           "label": "Sync Frequency",

diff --git a/src/configurations/destinations/azure_synapse/db-config.json b/src/configurations/destinations/azure_synapse/db-config.json
@@ -52,6 +52,7 @@
         "skipTracksTable",
         "skipUsersTable",
         "underscoreDivideNumbers",
+        "cleanupObjectStorageFiles",
         "allowUsersContextTraits"
       ],
       "android": [

diff --git a/src/configurations/destinations/azure_synapse/schema.json b/src/configurations/destinations/azure_synapse/schema.json
@@ -927,6 +927,12 @@
           "required": ["useRudderStorage"]
         },
         "then": {
+          "properties": {
+            "cleanupObjectStorageFiles": {
+              "type": "boolean",
+              "default": false
+            }
+          },
           "required": ["bucketProvider"]
         }
       },

diff --git a/src/configurations/destinations/azure_synapse/ui-config.json b/src/configurations/destinations/azure_synapse/ui-config.json
@@ -551,6 +551,16 @@
           "label": "Use SSL for connection",
           "value": "useSSL",
           "default": true
+        },
+        {
+          "type": "checkbox",
+          "label": "Enable for cleanup of object storage files (deletion) after successful sync",
+          "value": "cleanupObjectStorageFiles",
+          "default": false,
+          "preRequisiteField": {
+            "name": "useRudderStorage",
+            "selectedValue": false
+          }
         }
       ]
     },

diff --git a/src/configurations/destinations/bq/db-config.json b/src/configurations/destinations/bq/db-config.json
@@ -37,6 +37,7 @@
         "underscoreDivideNumbers",
         "allowUsersContextTraits",
         "partitionColumn",
+        "cleanupObjectStorageFiles",
         "partitionType"
       ],
       "android": [

diff --git a/src/configurations/destinations/bq/schema.json b/src/configurations/destinations/bq/schema.json
@@ -66,6 +66,10 @@
           }
         }
       },
+      "cleanupObjectStorageFiles": {
+        "type": "boolean",
+        "default": false
+      },
       "jsonPaths": {
         "type": "string",
         "pattern": "(^env[.].*)|.*"

diff --git a/src/configurations/destinations/bq/ui-config.json b/src/configurations/destinations/bq/ui-config.json
@@ -67,6 +67,12 @@
           "footerNote": "Create a service account in your GCP Project for RudderStack with roles of 'BigQuery User'",
           "secret": true
         },
+        {
+          "type": "checkbox",
+          "label": "Enable for cleanup of object storage files (deletion) after successful sync",
+          "value": "cleanupObjectStorageFiles",
+          "default": false
+        },
         {
           "type": "singleSelect",
           "label": "Sync Frequency",

diff --git a/src/configurations/destinations/clickhouse/db-config.json b/src/configurations/destinations/clickhouse/db-config.json
@@ -56,6 +56,7 @@
         "excludeWindow",
         "useRudderStorage",
         "underscoreDivideNumbers",
+        "cleanupObjectStorageFiles",
         "allowUsersContextTraits"
       ],
       "android": [

diff --git a/src/configurations/destinations/clickhouse/schema.json b/src/configurations/destinations/clickhouse/schema.json
@@ -938,6 +938,12 @@
           "required": ["useRudderStorage"]
         },
         "then": {
+          "properties": {
+            "cleanupObjectStorageFiles": {
+              "type": "boolean",
+              "default": false
+            }
+          },
           "required": ["bucketProvider"]
         }
       },

diff --git a/src/configurations/destinations/clickhouse/ui-config.json b/src/configurations/destinations/clickhouse/ui-config.json
@@ -559,6 +559,16 @@
           "label": "Use SSL for connection",
           "value": "useSSL",
           "default": true
+        },
+        {
+          "type": "checkbox",
+          "label": "Enable for cleanup of object storage files (deletion) after successful sync",
+          "value": "cleanupObjectStorageFiles",
+          "default": false,
+          "preRequisiteField": {
+            "name": "useRudderStorage",
+            "selectedValue": false
+          }
         }
       ]
     },

diff --git a/src/configurations/destinations/deltalake/db-config.json b/src/configurations/destinations/deltalake/db-config.json
@@ -53,6 +53,7 @@
         "externalLocation",
         "useRudderStorage",
         "underscoreDivideNumbers",
+        "cleanupObjectStorageFiles",
         "allowUsersContextTraits"
       ],
       "android": [

diff --git a/src/configurations/destinations/deltalake/schema.json b/src/configurations/destinations/deltalake/schema.json
@@ -922,6 +922,12 @@
           "required": ["useRudderStorage"]
         },
         "then": {
+          "properties": {
+            "cleanupObjectStorageFiles": {
+              "type": "boolean",
+              "default": false
+            }
+          },
           "required": ["bucketProvider"]
         }
       },

diff --git a/src/configurations/destinations/deltalake/ui-config.json b/src/configurations/destinations/deltalake/ui-config.json
@@ -526,6 +526,16 @@
           "required": true,
           "secret": true,
           "footerNote": "Create a service account in your GCP Project for RudderStack with roles of 'storage.objectCreator'"
+        },
+        {
+          "type": "checkbox",
+          "label": "Enable for cleanup of object storage files (deletion) after successful sync",
+          "value": "cleanupObjectStorageFiles",
+          "default": false,
+          "preRequisiteField": {
+            "name": "useRudderStorage",
+            "selectedValue": false
+          }
         }
       ]
     },

diff --git a/src/configurations/destinations/gcs_datalake/db-config.json b/src/configurations/destinations/gcs_datalake/db-config.json
@@ -46,6 +46,7 @@
         "skipUsersTable",
         "syncStartAt",
         "underscoreDivideNumbers",
+        "cleanupObjectStorageFiles",
         "allowUsersContextTraits"
       ],
       "android": [

diff --git a/src/configurations/destinations/gcs_datalake/schema.json b/src/configurations/destinations/gcs_datalake/schema.json
@@ -24,6 +24,10 @@
         "type": "boolean",
         "default": true
       },
+      "cleanupObjectStorageFiles": {
+        "type": "boolean",
+        "default": false
+      },
       "tableSuffix": {
         "type": "string",
         "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|.*"

diff --git a/src/configurations/destinations/gcs_datalake/ui-config.json b/src/configurations/destinations/gcs_datalake/ui-config.json
@@ -46,6 +46,12 @@
           "required": false,
           "placeholder": "e.g: TABLE_SUFFIX_PATH"
         },
+        {
+          "type": "checkbox",
+          "label": "Enable for cleanup of object storage files (deletion) after successful sync",
+          "value": "cleanupObjectStorageFiles",
+          "default": false
+        },
         {
           "type": "singleSelect",
           "label": "Choose time window layout",

diff --git a/src/configurations/destinations/mssql/db-config.json b/src/configurations/destinations/mssql/db-config.json
@@ -65,6 +65,7 @@
         "excludeWindow",
         "useRudderStorage",
         "underscoreDivideNumbers",
+        "cleanupObjectStorageFiles",
         "allowUsersContextTraits"
       ],
       "android": [

diff --git a/src/configurations/destinations/mssql/schema.json b/src/configurations/destinations/mssql/schema.json
@@ -920,6 +920,10 @@
               "type": "string",
               "enum": ["S3", "GCS", "AZURE_BLOB", "MINIO"],
               "default": "MINIO"
+            },
+            "cleanupObjectStorageFiles": {
+              "type": "boolean",
+              "default": false
             }
           },
           "required": ["bucketProvider"]

diff --git a/src/configurations/destinations/mssql/ui-config.json b/src/configurations/destinations/mssql/ui-config.json
@@ -551,6 +551,16 @@
           "label": "Use SSL for connection",
           "value": "useSSL",
           "default": true
+        },
+        {
+          "type": "checkbox",
+          "label": "Enable for cleanup of object storage files (deletion) after successful sync",
+          "value": "cleanupObjectStorageFiles",
+          "default": false,
+          "preRequisiteField": {
+            "name": "useRudderStorage",
+            "selectedValue": false
+          }
         }
       ]
     },

diff --git a/src/configurations/destinations/postgres/db-config.json b/src/configurations/destinations/postgres/db-config.json
@@ -74,6 +74,7 @@
         "clientCert",
         "serverCA",
         "underscoreDivideNumbers",
+        "cleanupObjectStorageFiles",
         "allowUsersContextTraits"
       ],
       "android": [

diff --git a/src/configurations/destinations/postgres/schema.json b/src/configurations/destinations/postgres/schema.json
@@ -966,6 +966,12 @@
           "required": ["useRudderStorage"]
         },
         "then": {
+          "properties": {
+            "cleanupObjectStorageFiles": {
+              "type": "boolean",
+              "default": false
+            }
+          },
           "required": ["bucketProvider"]
         }
       },

diff --git a/src/configurations/destinations/postgres/ui-config.json b/src/configurations/destinations/postgres/ui-config.json
@@ -643,6 +643,16 @@
           "label": "Use SSL for connection",
           "value": "useSSL",
           "default": true
+        },
+        {
+          "type": "checkbox",
+          "label": "Enable for cleanup of object storage files (deletion) after successful sync",
+          "value": "cleanupObjectStorageFiles",
+          "default": false,
+          "preRequisiteField": {
+            "name": "useRudderStorage",
+            "selectedValue": false
+          }
         }
       ]
     },

diff --git a/src/configurations/destinations/rs/db-config.json b/src/configurations/destinations/rs/db-config.json
@@ -68,6 +68,7 @@
         "jsonPaths",
         "useRudderStorage",
         "underscoreDivideNumbers",
+        "cleanupObjectStorageFiles",
         "allowUsersContextTraits"
       ],
       "web": [

diff --git a/src/configurations/destinations/rs/schema.json b/src/configurations/destinations/rs/schema.json
@@ -1014,6 +1014,10 @@
             "enableSSE": {
               "type": "boolean",
               "default": false
+            },
+            "cleanupObjectStorageFiles": {
+              "type": "boolean",
+              "default": false
             }
           },
           "required": ["bucketName"],

diff --git a/src/configurations/destinations/rs/ui-config.json b/src/configurations/destinations/rs/ui-config.json
@@ -426,6 +426,16 @@
             "name": "useRudderStorage",
             "selectedValue": false
           }
+        },
+        {
+          "type": "checkbox",
+          "label": "Enable for cleanup of object storage files (deletion) after successful sync",
+          "value": "cleanupObjectStorageFiles",
+          "default": false,
+          "preRequisiteField": {
+            "name": "useRudderStorage",
+            "selectedValue": false
+          }
         }
       ]
     },

diff --git a/src/configurations/destinations/s3_datalake/db-config.json b/src/configurations/destinations/s3_datalake/db-config.json
@@ -52,6 +52,7 @@
         "timeWindowLayout",
         "enableSSE",
         "underscoreDivideNumbers",
+        "cleanupObjectStorageFiles",
         "allowUsersContextTraits"
       ],
       "android": [

diff --git a/src/configurations/destinations/s3_datalake/schema.json b/src/configurations/destinations/s3_datalake/schema.json
@@ -44,6 +44,10 @@
       "syncStartAt": {
         "type": "string"
       },
+      "cleanupObjectStorageFiles": {
+        "type": "boolean",
+        "default": false
+      },
       "oneTrustCookieCategories": {
         "type": "object",
         "properties": {

diff --git a/src/configurations/destinations/s3_datalake/ui-config.json b/src/configurations/destinations/s3_datalake/ui-config.json
@@ -111,6 +111,16 @@
           "value": "enableSSE",
           "default": false
         },
+        {
+          "type": "checkbox",
+          "label": "Enable for cleanup of object storage files (deletion) after successful sync",
+          "value": "cleanupObjectStorageFiles",
+          "default": false,
+          "preRequisiteField": {
+            "name": "useRudderStorage",
+            "selectedValue": false
+          }
+        },
         {
           "type": "singleSelect",
           "label": "Sync Frequency",

diff --git a/src/configurations/destinations/snowflake/db-config.json b/src/configurations/destinations/snowflake/db-config.json
@@ -68,6 +68,7 @@
         "privateKey",
         "privateKeyPassphrase",
         "underscoreDivideNumbers",
+        "cleanupObjectStorageFiles",
         "allowUsersContextTraits"
       ],
       "android": [

diff --git a/src/configurations/destinations/snowflake/schema.json b/src/configurations/destinations/snowflake/schema.json
@@ -975,6 +975,10 @@
             "prefix": {
               "type": "string",
               "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$"
+            },
+            "cleanupObjectStorageFiles": {
+              "type": "boolean",
+              "default": false
             }
           },
           "required": ["cloudProvider"]

diff --git a/src/configurations/destinations/snowflake/ui-config.json b/src/configurations/destinations/snowflake/ui-config.json
@@ -574,6 +574,16 @@
           "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|.*",
           "required": true,
           "footerNote": "Create a service account in your GCP Project for RudderStack with roles of 'storage.objectCreator'"
+        },
+        {
+          "type": "checkbox",
+          "label": "Enable for cleanup of object storage files (deletion) after successful sync",
+          "value": "cleanupObjectStorageFiles",
+          "default": false,
+          "preRequisiteField": {
+            "name": "useRudderStorage",
+            "selectedValue": false
+          }
         }
       ]
     },