Auto-approve for bulk upload

db/v-2/tables/db.changelog-tables.xml

@@ -58,6 +58,7 @@
     <include file="vulnerability-metadata-project.xml" relativeToChangelogFile="true"/>
     <include file="cosv-file.xml" relativeToChangelogFile="true"/>
     <include file="link-between-cosv-file-and-vulnerability-metadata.xml" relativeToChangelogFile="true"/>
+    <include file="vulnerability-generated-id.xml" relativeToChangelogFile="true"/>
 
     <changeSet id="02-tables" author="frolov">
         <tagDatabase tag="v2.0-tables"/>

db/v-2/tables/link-between-cosv-file-and-vulnerability-metadata.xml

@@ -28,4 +28,8 @@
                                  referencedColumnNames="id"
                                  onDelete="CASCADE"/>
     </changeSet>
+    <changeSet id="vulnerability-metadata-latest-cosv-file-required" author="nulls">
+        <addNotNullConstraint tableName="vulnerability_metadata"
+                              columnName="latest_cosv_file_id" columnDataType="bigint"/>
+    </changeSet>
 </databaseChangeLog>

db/v-2/tables/vulnerability-generated-id.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+                      http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.8.xsd">
+
+    <changeSet id="vulnerability-generated-id-1" author="nulls">
+        <createTable tableName="vulnerability_generated_id">
+            <column name="id" type="bigint" autoIncrement="true">
+                <constraints primaryKey="true" nullable="false"/>
+            </column>
+            <column name="create_date" type="DATETIME(3)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="update_date" type="DATETIME(3)">
+                <constraints nullable="false"/>
+            </column>
+        </createTable>
+    </changeSet>
+
+</databaseChangeLog>

save-backend/src/main/kotlin/com/saveourtool/save/backend/service/vulnerability/VulnerabilityService.kt

@@ -28,9 +28,7 @@ import reactor.kotlin.core.publisher.toMono
 
 import javax.persistence.criteria.*
 
-import kotlin.random.Random
 import kotlinx.datetime.LocalDateTime
-import kotlinx.datetime.toJavaLocalDateTime
 
 /**
  * A service that provides `Vulnerability`
@@ -48,6 +46,7 @@ class VulnerabilityService(
     private val vulnerabilityMetadataRepository: VulnerabilityMetadataRepository,
     private val lnkVulnerabilityMetadataTagRepository: LnkVulnerabilityMetadataTagRepository,
     private val tagRepository: TagRepository,
+    private val vulnerabilityGeneratedIdRepository: VulnerabilityGeneratedIdRepository,
 ) {
     private fun List<VulnerabilityMetadata>.toTagMap() = lnkVulnerabilityMetadataTagRepository.findAllById(this.map { it.requiredId() })
         .map { link -> link.vulnerabilityMetadata to link.tag }
@@ -213,26 +212,7 @@ class VulnerabilityService(
         authentication: Authentication,
     ): Mono<VulnerabilityMetadataDto> = blockingToMono {
         vulnerabilityDto.identifier.ifEmpty {
-            val metadata = vulnerabilityMetadataRepository.saveAndFlush(
-                VulnerabilityMetadata(
-                    identifier = "default-${Random.nextInt()}",
-                    summary = "STUB",
-                    details = "STUB",
-                    severityNum = 0f,
-                    submitted = getCurrentLocalDateTime().toJavaLocalDateTime(),
-                    modified = getCurrentLocalDateTime().toJavaLocalDateTime(),
-                    user = userRepository.getByIdOrNotFound(authentication.userId()),
-                    organization = null,
-                    language = VulnerabilityLanguage.OTHER,
-                    status = VulnerabilityStatus.PENDING_REVIEW,
-                    latestCosvFile = null,
-                )
-            )
-            val newName = "SOTV-${getCurrentLocalDateTime().year}-${metadata.requiredId()}"
-            vulnerabilityMetadataRepository.save(metadata.apply {
-                identifier = newName
-            })
-            newName
+            vulnerabilityGeneratedIdRepository.saveAndFlush(VulnerabilityGeneratedId()).getIdentifier()
         }
     }
         .flatMap { identifier ->

save-backend/src/test/kotlin/com/saveourtool/save/backend/DatabaseTest.kt

@@ -33,6 +33,7 @@ import org.springframework.context.annotation.Import
     MockBean(RawCosvFileRepository::class),
     MockBean(CosvFileRepository::class),
     MockBean(BlockingBridge::class),
+    MockBean(VulnerabilityGeneratedIdRepository::class),
 )
 class DatabaseTest {
     @Autowired

save-backend/src/test/kotlin/com/saveourtool/save/backend/DownloadFilesTest.kt

@@ -78,6 +78,7 @@ import kotlin.io.path.*
     MockBean(RawCosvFileRepository::class),
     MockBean(CosvFileRepository::class),
     MockBean(BlockingBridge::class),
+    MockBean(VulnerabilityGeneratedIdRepository::class),
 )
 class DownloadFilesTest {
     private val organization = Organization.stub(2).apply {

save-backend/src/test/kotlin/com/saveourtool/save/backend/controller/JpaSpecificationTest.kt

@@ -30,6 +30,7 @@ import org.springframework.context.annotation.Import
     MockBean(RawCosvFileRepository::class),
     MockBean(CosvFileRepository::class),
     MockBean(BlockingBridge::class),
+    MockBean(VulnerabilityGeneratedIdRepository::class),
 )
 class JpaSpecificationTest {
     @Autowired

save-backend/src/test/kotlin/com/saveourtool/save/backend/controller/LnkUserOrganizationControllerTest.kt

@@ -43,6 +43,7 @@ import org.springframework.test.web.reactive.server.WebTestClient
     MockBean(RawCosvFileRepository::class),
     MockBean(CosvFileRepository::class),
     MockBean(BlockingBridge::class),
+    MockBean(VulnerabilityGeneratedIdRepository::class),
 )
 @AutoConfigureWebTestClient
 class LnkUserOrganizationControllerTest {

save-backend/src/test/kotlin/com/saveourtool/save/backend/controller/OrganizationControllerTest.kt

@@ -95,6 +95,7 @@ import java.util.concurrent.TimeUnit
     MockBean(RawCosvFileRepository::class),
     MockBean(CosvFileRepository::class),
     MockBean(BlockingBridge::class),
+    MockBean(VulnerabilityGeneratedIdRepository::class),
 )
 @AutoConfigureWebTestClient
 @Suppress("UnsafeCallOnNullableType")

save-backend/src/test/kotlin/com/saveourtool/save/backend/controller/PermissionControllerTest.kt

@@ -52,6 +52,7 @@ import reactor.util.function.Tuples
     MockBean(RawCosvFileRepository::class),
     MockBean(CosvFileRepository::class),
     MockBean(BlockingBridge::class),
+    MockBean(VulnerabilityGeneratedIdRepository::class),
 )
 @AutoConfigureWebTestClient
 class PermissionControllerTest {

save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/entities/cosv/VulnerabilityGeneratedId.kt

@@ -0,0 +1,15 @@
+package com.saveourtool.save.entities.cosv
+
+import com.saveourtool.save.spring.entity.BaseEntityWithDate
+import javax.persistence.Entity
+
+/**
+ * Entity for generated id of vulnerability
+ */
+@Entity
+class VulnerabilityGeneratedId : BaseEntityWithDate() {
+    /**
+     * @return Vulnerability identifier for saved entity
+     */
+    fun getIdentifier(): String = "SOTV-${requiredCreateDate().year}-${requiredId()}"
+}

save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/entities/cosv/VulnerabilityMetadata.kt

@@ -45,16 +45,8 @@ class VulnerabilityMetadata(
     var organization: Organization?,
     @OneToOne
     @JoinColumn(name = "latest_cosv_file_id")
-    var latestCosvFile: CosvFile?,
+    var latestCosvFile: CosvFile,
 ) : BaseEntityWithDto<VulnerabilityMetadataDto>() {
-    /**
-     * @return [latestCosvFile] as not null with validating
-     * @throws IllegalArgumentException when [latestCosvFile] is not set that means entity is created to generate a unique identifier
-     */
-    fun requiredLatestCosvFile(): CosvFile = requireNotNull(latestCosvFile) {
-        "Entity is in internal state: $this"
-    }
-
     override fun toDto(): VulnerabilityMetadataDto = VulnerabilityMetadataDto(
         identifier = identifier,
         summary = summary,

save-cosv/src/main/kotlin/com/saveourtool/save/cosv/repository/VulnerabilityGeneratedIdRepository.kt

@@ -0,0 +1,11 @@
+package com.saveourtool.save.cosv.repository
+
+import com.saveourtool.save.entities.cosv.VulnerabilityGeneratedId
+import com.saveourtool.save.spring.repository.BaseEntityRepository
+import org.springframework.stereotype.Repository
+
+/**
+ * Repository for [VulnerabilityGeneratedId]
+ */
+@Repository
+interface VulnerabilityGeneratedIdRepository : BaseEntityRepository<VulnerabilityGeneratedId>

save-cosv/src/main/kotlin/com/saveourtool/save/cosv/service/CosvService.kt

@@ -21,7 +21,6 @@ import org.springframework.stereotype.Service
 import reactor.core.publisher.Mono
 import reactor.kotlin.core.publisher.toFlux
 
-import kotlinx.serialization.SerializationException
 import kotlinx.serialization.serializer
 
 private typealias ManualCosvSchema = CosvSchema<Unit, Unit, Unit, Unit>
@@ -65,17 +64,25 @@ class CosvService(
     ): Mono<Unit> = rawCosvFileStorage.downloadById(rawCosvFileId)
         .collectToInputStream()
         .flatMap { inputStream ->
-            val cosvListOpt = try {
+            val errorMessage by lazy {
+                "Failed to process raw COSV file with id: $rawCosvFileId"
+            }
+            Mono.fromCallable {
                 cosvProcessor.decode(inputStream)
-            } catch (e: SerializationException) {
-                val errorMessage: () -> String = { "Failed to process raw COSV file with id: $rawCosvFileId" }
-                log.error(e, errorMessage)
-                return@flatMap rawCosvFileStorage.update(rawCosvFileId, RawCosvFileStatus.FAILED, "$errorMessage is due to ${e.message}")
             }
-            cosvListOpt.toFlux()
-                .flatMap { save(it, user, organization) }
+                .flatMapIterable { it }
+                .flatMap { save(it, user, organization, isAutoApprove = true) }
+                .onErrorResume { error ->
+                    val cause = error.firstCauseOrThis()
+                    rawCosvFileStorage.update(rawCosvFileId, RawCosvFileStatus.FAILED, "$errorMessage is due to ${cause.message}")
+                        .then(Mono.error(error))
+                }
                 .collectList()
                 .flatMap { rawCosvFileStorage.update(rawCosvFileId, RawCosvFileStatus.PROCESSED, "Processed as ${it.map(VulnerabilityMetadataDto::identifier)}") }
+                .onErrorResume { error ->
+                    log.error(error) { errorMessage }
+                    Mono.just(Unit)
+                }
         }
 
     /**
@@ -154,10 +161,11 @@ class CosvService(
         cosv: CosvSchema<D, A_E, A_D, A_R_D>,
         user: User,
         organization: Organization?,
+        isAutoApprove: Boolean = false,
     ): Mono<VulnerabilityMetadataDto> = cosvRepository.save(cosv, serializer())
         .flatMap { key ->
             blockingToMono {
-                vulnerabilityMetadataService.createOrUpdate(key, cosv, user, organization).toDto()
+                vulnerabilityMetadataService.createOrUpdate(key, cosv, user, organization, isAutoApprove).toDto()
             }
                 .onErrorResume { error ->
                     log.error(error) {
@@ -175,7 +183,7 @@ class CosvService(
      */
     fun getVulnerabilityExt(identifier: String): Mono<VulnerabilityExt> = blockingToMono { vulnerabilityMetadataService.findByIdentifier(identifier) }
         .flatMap { metadata ->
-            cosvRepository.download(metadata.requiredLatestCosvFile(), serializer<RawCosvSchema>()).blockingMap { content ->
+            cosvRepository.download(metadata.latestCosvFile, serializer<RawCosvSchema>()).blockingMap { content ->
                 VulnerabilityExt(
                     metadata = metadata.toDto(),
                     cosv = content,
@@ -187,5 +195,6 @@ class CosvService(
 
     companion object {
         private val log: Logger = getLogger<CosvService>()
+        private fun Throwable.firstCauseOrThis(): Throwable = generateSequence(this, Throwable::cause).last()
     }
 }

save-cosv/src/main/kotlin/com/saveourtool/save/cosv/service/VulnerabilityMetadataService.kt

@@ -33,6 +33,7 @@ class VulnerabilityMetadataService(
      * @param cosv
      * @param user
      * @param organization
+     * @param isAutoApprove
      * @return updated [VulnerabilityMetadata]
      */
     @Transactional
@@ -41,15 +42,16 @@ class VulnerabilityMetadataService(
         cosv: AnyCosvSchema,
         user: User,
         organization: Organization?,
+        isAutoApprove: Boolean,
     ): VulnerabilityMetadata {
         val metadata = vulnerabilityMetadataRepository.findByIdentifier(cosv.id)
             ?.let { existedMetadata ->
                 validateMetadata(existedMetadata, cosv, user, organization)
                 cosvFileS3KeyManager.setPrevCosvFile(cosvFile, existedMetadata)
-                existedMetadata.updateBy(cosv, cosvFile)
+                existedMetadata.updateBy(cosv, cosvFile, isAutoApprove)
             }
             ?: run {
-                cosv.toNewMetadata(user, organization, cosvFile)
+                cosv.toNewMetadata(user, organization, cosvFile, isAutoApprove)
             }
         return vulnerabilityMetadataRepository.save(metadata)
     }
@@ -67,6 +69,7 @@ class VulnerabilityMetadataService(
             user: User,
             organization: Organization?,
             cosvFile: CosvFile,
+            isAutoApprove: Boolean,
         ) = VulnerabilityMetadata(
             identifier = id,
             summary = summary ?: "Summary not provided",
@@ -77,7 +80,7 @@ class VulnerabilityMetadataService(
             user = user,
             organization = organization,
             language = getLanguage() ?: VulnerabilityLanguage.OTHER,
-            status = VulnerabilityStatus.PENDING_REVIEW,
+            status = isAutoApprove.toVulnerabilityStatus(),
             latestCosvFile = cosvFile,
         )
 
@@ -117,6 +120,7 @@ class VulnerabilityMetadataService(
         private fun VulnerabilityMetadata.updateBy(
             entry: CosvSchema<*, *, *, *>,
             cosvFile: CosvFile,
+            isAutoApprove: Boolean,
         ): VulnerabilityMetadata = apply {
             summary = entry.summary ?: "Summary not provided"
             details = entry.details ?: "Details not provided"
@@ -125,6 +129,9 @@ class VulnerabilityMetadataService(
                 ?.toFloat() ?: 0f
             modified = entry.modified.toJavaLocalDateTime()
             latestCosvFile = cosvFile
+            status = isAutoApprove.toVulnerabilityStatus()
         }
+
+        private fun Boolean.toVulnerabilityStatus() = if (this) VulnerabilityStatus.APPROVED else VulnerabilityStatus.PENDING_REVIEW
     }
 }

save-cosv/src/main/kotlin/com/saveourtool/save/cosv/storage/CosvFileS3KeyManager.kt

@@ -44,7 +44,7 @@ class CosvFileS3KeyManager(
         vulnerabilityMetadata: VulnerabilityMetadata,
     ): CosvFile = repository.save(
         cosvFile.apply {
-            this.prevCosvFile = vulnerabilityMetadata.requiredLatestCosvFile()
+            this.prevCosvFile = vulnerabilityMetadata.latestCosvFile
         }
     )
 }

save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/basic/fileuploader/CosvFileManagerComponent.kt

@@ -142,7 +142,7 @@ val cosvFileManagerComponent: FC<Props> = FC { _ ->
                         type = InputType.checkbox
                         id = "checkbox"
                         checked = file in selectedFiles
-                        disabled = file.status in setOf(RawCosvFileStatus.PROCESSED, RawCosvFileStatus.IN_PROGRESS)
+                        disabled = file.isNotSelectable()
                         onChange = { event ->
                             if (event.target.checked) {
                                 setSelectedFiles { it.plus(file) }
@@ -189,6 +189,7 @@ val cosvFileManagerComponent: FC<Props> = FC { _ ->
             li {
                 className = ClassName("list-group-item p-0 d-flex bg-light")
                 dragAndDropForm {
+                    isDisabled = selectedOrganization.isNullOrEmpty()
                     isMultipleFilesSupported = true
                     tooltipMessage = "Only JSON files"
                     onChangeEventHandler = { files ->
@@ -200,10 +201,15 @@ val cosvFileManagerComponent: FC<Props> = FC { _ ->
             // SUBMIT to process
             li {
                 className = ClassName("list-group-item p-0 d-flex bg-light justify-content-center")
+                buttonBuilder("Select all", isDisabled = availableFiles.isEmpty()) {
+                    setSelectedFiles(availableFiles.filterNot { it.isNotSelectable() })
+                }
                 buttonBuilder("Submit", isDisabled = selectedFiles.isEmpty()) {
                     submitCosvFiles()
                 }
             }
         }
     }
 }
+
+private fun RawCosvFileDto.isNotSelectable() = status in setOf(RawCosvFileStatus.PROCESSED, RawCosvFileStatus.IN_PROGRESS)

save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/basic/fileuploader/FileManagerComponent.kt

@@ -9,7 +9,6 @@ package com.saveourtool.save.frontend.components.basic.fileuploader
 import com.saveourtool.save.domain.ProjectCoordinates
 import com.saveourtool.save.entities.FileDto
 import com.saveourtool.save.frontend.components.inputform.dragAndDropForm
-import com.saveourtool.save.frontend.externals.fontawesome.*
 import com.saveourtool.save.frontend.http.postUploadFile
 import com.saveourtool.save.frontend.utils.*
 import com.saveourtool.save.frontend.utils.noopLoadingHandler
@@ -117,6 +116,7 @@ val fileManagerComponent: FC<FileManagerProps> = FC { props ->
             li {
                 className = ClassName("list-group-item p-0 d-flex bg-light")
                 dragAndDropForm {
+                    isDisabled = false
                     isMultipleFilesSupported = true
                     tooltipMessage = "Regular files/Executable files/ZIP Archives"
                     onChangeEventHandler = { files ->

save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/basic/fileuploader/SandboxFileUploader.kt

@@ -9,7 +9,6 @@ package com.saveourtool.save.frontend.components.basic.fileuploader
 import com.saveourtool.save.domain.*
 import com.saveourtool.save.frontend.components.basic.codeeditor.FileType
 import com.saveourtool.save.frontend.components.inputform.dragAndDropForm
-import com.saveourtool.save.frontend.externals.fontawesome.*
 import com.saveourtool.save.frontend.http.postUploadFile
 import com.saveourtool.save.frontend.utils.*
 import com.saveourtool.save.frontend.utils.noopLoadingHandler
@@ -101,6 +100,7 @@ val sandboxFileUploader: FC<SandboxFileUploaderProps> = FC { props ->
             li {
                 className = ClassName("list-group-item p-0 d-flex bg-light")
                 dragAndDropForm {
+                    isDisabled = false
                     isMultipleFilesSupported = false
                     tooltipMessage = "upload your tested tool and all other needed files"
                     onChangeEventHandler = { files ->