Delete processed files (#2590)

- added an event to delete processed RawCosvFile after 7 days
- refactored deleting and backing up unexpected keys in storage

.run/save-cloud [startMinioService].run.xml → .run/save-cloud [startMinio].run.xml

@@ -1,5 +1,5 @@
 <component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="save-cloud [startMinioService]" type="GradleRunConfiguration" factoryName="Gradle">
+  <configuration default="false" name="save-cloud [startMinio]" type="GradleRunConfiguration" factoryName="Gradle">
     <ExternalSystemSettings>
       <option name="executionName" />
       <option name="externalProjectPath" value="$PROJECT_DIR$" />
@@ -10,7 +10,7 @@
       </option>
       <option name="taskNames">
         <list>
-          <option value="startMinioService" />
+          <option value="startMinio" />
         </list>
       </option>
       <option name="vmOptions" />

.run/save-cloud [startMysqlService].run.xml → .run/save-cloud [startMysqlDb].run.xml

@@ -1,5 +1,5 @@
 <component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="save-cloud [startMysqlService]" type="GradleRunConfiguration" factoryName="Gradle">
+  <configuration default="false" name="save-cloud [startMysqlDb]" type="GradleRunConfiguration" factoryName="Gradle">
     <ExternalSystemSettings>
       <option name="executionName" />
       <option name="externalProjectPath" value="$PROJECT_DIR$" />
@@ -10,7 +10,7 @@
       </option>
       <option name="taskNames">
         <list>
-          <option value="startMysqlService" />
+          <option value="startMysqlDb" />
         </list>
       </option>
       <option name="vmOptions" />

db/v-2/events/event_scheduler.xml

@@ -35,4 +35,13 @@
         </sql>
     </changeSet>
 
+    <changeSet id="event-delete-raw-cosv-file" author="nulls" context="prod">
+        <sql>
+            CREATE EVENT delete_processed_raw_cosv_file
+            ON SCHEDULE EVERY 1 DAY
+            STARTS TIMESTAMP(CURRENT_DATE + INTERVAL 1 HOUR)
+            DO DELETE FROM raw_cosv_file WHERE status = 'PROCESSED' AND DATEDIFF(NOW(), update_date) >= 7;
+        </sql>
+    </changeSet>
+
 </databaseChangeLog>

save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/entities/cosv/RawCosvFileDto.kt

@@ -1,6 +1,7 @@
 package com.saveourtool.save.entities.cosv
 
 import com.saveourtool.save.entities.DtoWithId
+import kotlinx.datetime.LocalDateTime
 import kotlinx.serialization.Serializable
 
 /**
@@ -10,8 +11,9 @@ import kotlinx.serialization.Serializable
  * @property userName
  * @property organizationName
  * @property status
- * @property id
  * @property statusMessage
+ * @property updateDate
+ * @property id
  */
 @Serializable
 data class RawCosvFileDto(
@@ -20,5 +22,6 @@ data class RawCosvFileDto(
     val organizationName: String,
     val status: RawCosvFileStatus = RawCosvFileStatus.UPLOADED,
     val statusMessage: String? = null,
+    val updateDate: LocalDateTime? = null,
     override val id: Long? = null,
 ) : DtoWithId()

save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/entities/cosv/RawCosvFile.kt

@@ -4,12 +4,15 @@ import com.saveourtool.save.entities.Organization
 import com.saveourtool.save.entities.User
 import com.saveourtool.save.spring.entity.BaseEntityWithDtoWithId
 import com.saveourtool.save.spring.entity.IBaseEntityWithDate
+
 import java.time.LocalDateTime
 import javax.persistence.Entity
 import javax.persistence.Enumerated
 import javax.persistence.JoinColumn
 import javax.persistence.ManyToOne
 
+import kotlinx.datetime.toKotlinLocalDateTime
+
 /**
  * Entity for table `raw_cosv_file`
  *
@@ -33,16 +36,17 @@ class RawCosvFile(
     var organization: Organization,
     @Enumerated
     var status: RawCosvFileStatus,
+    var statusMessage: String? = null,
     override var createDate: LocalDateTime? = null,
     override var updateDate: LocalDateTime? = null,
-    var statusMessage: String? = null,
 ) : BaseEntityWithDtoWithId<RawCosvFileDto>(), IBaseEntityWithDate {
     override fun toDto(): RawCosvFileDto = RawCosvFileDto(
         fileName = fileName,
         userName = user.name,
         organizationName = organization.name,
         status = status,
         statusMessage = statusMessage,
+        updateDate = requiredUpdateDate().toKotlinLocalDateTime(),
         id = requiredId(),
     )
 

save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/storage/BackupStorageUtils.kt

@@ -5,6 +5,7 @@
 package com.saveourtool.save.storage
 
 import com.saveourtool.save.s3.S3Operations
+import com.saveourtool.save.storage.key.AbstractS3KeyDatabaseManager
 import com.saveourtool.save.utils.ListCompletableFuture
 import com.saveourtool.save.utils.debug
 import com.saveourtool.save.utils.getLogger
@@ -40,6 +41,22 @@ fun S3Operations.backupUnexpectedKeys(
         }
     }
 
+/**
+ * Back up unexpected s3 key (according to [AbstractS3KeyDatabaseManager]) which are detected S3 storage (by [AbstractS3KeyDatabaseManager])
+ *
+ * @param storageName
+ * @param s3KeyManager
+ * @return [CompletableFuture] without body
+ */
+fun S3Operations.backupUnexpectedKeys(
+    storageName: String,
+    s3KeyManager: AbstractS3KeyDatabaseManager<*, *, *>,
+): CompletableFuture<Unit> = backupUnexpectedKeys(
+    storageName = storageName,
+    commonPrefix = s3KeyManager.commonPrefix,
+    s3KeyValidator = s3KeyManager.asS3KeyValidator(),
+)
+
 /**
  * Delete unexpected s3 key (according to [s3KeyValidator]) which are detected S3 storage (by common prefix [commonPrefix])
  *
@@ -61,6 +78,27 @@ fun S3Operations.deleteUnexpectedKeys(
         }
     }
 
+/**
+ * Delete unexpected s3 key (according to [AbstractS3KeyDatabaseManager]) which are detected S3 storage (by [AbstractS3KeyDatabaseManager])
+ *
+ * @param storageName
+ * @param s3KeyManager
+ * @return [CompletableFuture] without body
+ */
+fun S3Operations.deleteUnexpectedKeys(
+    storageName: String,
+    s3KeyManager: AbstractS3KeyDatabaseManager<*, *, *>,
+): CompletableFuture<Unit> = deleteUnexpectedKeys(
+    storageName = storageName,
+    commonPrefix = s3KeyManager.commonPrefix,
+    s3KeyValidator = s3KeyManager.asS3KeyValidator(),
+)
+
+private fun AbstractS3KeyDatabaseManager<*, *, *>.asS3KeyValidator(): (String) -> Boolean = { s3Key ->
+    val id = s3Key.removePrefix(commonPrefix).toLong()
+    findKeyByEntityId(id) == null
+}
+
 private fun S3Operations.doBackupUnexpectedKeys(
     storageName: String,
     commonPrefix: String,

save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/storage/ReactiveStorageWithDatabase.kt

@@ -21,10 +21,7 @@ open class ReactiveStorageWithDatabase<K : Any, E : BaseEntity, M : AbstractS3Ke
     override fun doInit(underlying: DefaultStorageProjectReactor<K>): Mono<Unit> = Mono.fromFuture {
         s3Operations.backupUnexpectedKeys(
             storageName = "${this::class.simpleName}",
-            commonPrefix = s3KeyManager.commonPrefix,
-        ) { s3Key ->
-            val id = s3Key.removePrefix(s3KeyManager.commonPrefix).toLong()
-            s3KeyManager.findKeyByEntityId(id)?.let { true } ?: false
-        }
+            s3KeyManager = s3KeyManager,
+        )
     }.publishOn(s3Operations.scheduler)
 }

save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/storage/SuspendingStorageWithDatabase.kt

@@ -2,7 +2,6 @@ package com.saveourtool.save.storage
 
 import com.saveourtool.save.s3.S3Operations
 import com.saveourtool.save.spring.entity.BaseEntity
-import com.saveourtool.save.spring.repository.BaseEntityRepository
 import com.saveourtool.save.storage.key.AbstractS3KeyDatabaseManager
 import kotlinx.coroutines.future.asDeferred
 import kotlinx.coroutines.withContext
@@ -12,12 +11,10 @@ import kotlinx.coroutines.withContext
  *
  * @property s3Operations interface to operate with S3 storage
  * @property s3KeyManager [AbstractS3KeyDatabaseManager] manager for S3 keys using database
- * @property repository repository for [E] which is entity for [K]
  */
-open class SuspendingStorageWithDatabase<K : Any, E : BaseEntity, R : BaseEntityRepository<E>, M : AbstractS3KeyDatabaseManager<K, E, R>>(
+open class SuspendingStorageWithDatabase<K : Any, E : BaseEntity, M : AbstractS3KeyDatabaseManager<K, E, *>>(
     private val s3Operations: S3Operations,
     override val s3KeyManager: M,
-    private val repository: R,
 ) : AbstractSuspendingStorage<K>(s3Operations) {
     /**
      * Init method to back up unexpected ids which are detected in storage,but missed in database
@@ -26,11 +23,8 @@ open class SuspendingStorageWithDatabase<K : Any, E : BaseEntity, R : BaseEntity
         withContext(s3Operations.coroutineDispatcher) {
             s3Operations.backupUnexpectedKeys(
                 storageName = "${this::class.simpleName}",
-                commonPrefix = s3KeyManager.commonPrefix,
-            ) { s3Key ->
-                val id = s3Key.removePrefix(s3KeyManager.commonPrefix).toLong()
-                repository.findById(id).isEmpty
-            }
+                s3KeyManager = s3KeyManager,
+            )
                 .asDeferred()
                 .await()
         }

save-cosv/src/main/kotlin/com/saveourtool/save/cosv/storage/CosvKey.kt

@@ -5,8 +5,10 @@ import kotlinx.datetime.LocalDateTime
 /**
  * @property id
  * @property modified
+ * @property isValid
  */
 data class CosvKey(
     val id: String,
     val modified: LocalDateTime,
+    val isValid: Boolean = true,
 )

save-cosv/src/main/kotlin/com/saveourtool/save/cosv/storage/CosvStorage.kt

@@ -3,10 +3,13 @@ package com.saveourtool.save.cosv.storage
 import com.saveourtool.save.s3.S3Operations
 import com.saveourtool.save.s3.S3OperationsProperties
 import com.saveourtool.save.storage.AbstractSimpleReactiveStorage
+import com.saveourtool.save.storage.PATH_DELIMITER
 import com.saveourtool.save.storage.concatS3Key
 import com.saveourtool.save.storage.s3KeyToParts
+import com.saveourtool.save.utils.getCurrentLocalDateTime
 
 import org.springframework.stereotype.Service
+import reactor.core.publisher.Flux
 
 import kotlinx.datetime.LocalDateTime
 
@@ -24,9 +27,16 @@ class CosvStorage(
     s3Operations,
     concatS3Key(s3OperationsPropertiesProvider.s3Storage.prefix, "cosv"),
 ) {
-    override fun doBuildKeyFromSuffix(s3KeySuffix: String): CosvKey {
+    override fun list(): Flux<CosvKey> {
+        // a workaround to support migration
+        return super.list().filter { it.isValid }
+    }
+
+    override fun doBuildKeyFromSuffix(s3KeySuffix: String): CosvKey = if (s3KeySuffix.removeSuffix(PATH_DELIMITER).removePrefix(PATH_DELIMITER).contains(PATH_DELIMITER)) {
         val (id, modified) = s3KeySuffix.s3KeyToParts()
-        return CosvKey(id, LocalDateTime.parse(modified.replace('_', ':')))
+        CosvKey(id, LocalDateTime.parse(modified.replace('_', ':')))
+    } else {
+        CosvKey(s3KeySuffix, getCurrentLocalDateTime(), false)
     }
 
     override fun doBuildS3KeySuffix(key: CosvKey): String = concatS3Key(key.id, key.modified.toString().replace(':', '_'))

save-cosv/src/main/kotlin/com/saveourtool/save/cosv/storage/RawCosvFileStorage.kt

@@ -6,7 +6,9 @@ import com.saveourtool.save.entities.cosv.RawCosvFile
 import com.saveourtool.save.entities.cosv.RawCosvFileDto
 import com.saveourtool.save.entities.cosv.RawCosvFileStatus
 import com.saveourtool.save.s3.S3Operations
+import com.saveourtool.save.storage.DefaultStorageProjectReactor
 import com.saveourtool.save.storage.ReactiveStorageWithDatabase
+import com.saveourtool.save.storage.deleteUnexpectedKeys
 import com.saveourtool.save.utils.blockingToFlux
 import com.saveourtool.save.utils.blockingToMono
 import com.saveourtool.save.utils.switchIfEmptyToNotFound
@@ -22,12 +24,22 @@ typealias OrganizationAndOwner = Pair<Organization, User>
  */
 @Component
 class RawCosvFileStorage(
-    s3Operations: S3Operations,
+    private val s3Operations: S3Operations,
     s3KeyManager: RawCosvFileS3KeyManager,
 ) : ReactiveStorageWithDatabase<RawCosvFileDto, RawCosvFile, RawCosvFileS3KeyManager>(
     s3Operations,
     s3KeyManager,
 ) {
+    /**
+     * Init method to remove deleted (unexpected) ids which are detected in storage, but missed in database
+     */
+    override fun doInit(underlying: DefaultStorageProjectReactor<RawCosvFileDto>): Mono<Unit> = Mono.fromFuture {
+        s3Operations.deleteUnexpectedKeys(
+            storageName = "${this::class.simpleName}",
+            s3KeyManager = s3KeyManager,
+        )
+    }.publishOn(s3Operations.scheduler)
+
     /**
      * @param organizationName
      * @return all [RawCosvFileDto]s which has provided [RawCosvFile.organization]

save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/basic/fileuploader/CosvFileManagerComponent.kt

@@ -177,7 +177,7 @@ val cosvFileManagerComponent: FC<Props> = FC { _ ->
                         }
                         +when (file.status) {
                             RawCosvFileStatus.IN_PROGRESS -> " (in progress)"
-                            RawCosvFileStatus.PROCESSED -> " (processed)"
+                            RawCosvFileStatus.PROCESSED -> " (processed, will be deleted after ${file.updateDate?.date})"
                             RawCosvFileStatus.FAILED -> " (with errors)"
                             else -> " "
                         }