From ebad7c8b3fd39517c7c1d72d7cbdfc1d5e1318d8 Mon Sep 17 00:00:00 2001 From: Alexander Drugakov Date: Mon, 13 Nov 2023 16:30:38 +0300 Subject: [PATCH] Added automatically added tags about affected ecosystems (#2853) ### What's done: - added logic for automatically added tags about affected ecosystems during vulnerabilities uploading. - changed tag length range from [3, 15] to [2, 15]. --- .../backend/service/BackendForCosvService.kt | 7 +++ .../save/backend/service/TagService.kt | 30 +++++++++- .../validation/ValidationErrorMessages.kt | 2 +- .../save/validation/ValidationUtils.kt | 2 +- .../save/backend/service/IBackendService.kt | 11 ++++ .../cosv/controllers/RawCosvFileController.kt | 4 +- .../save/cosv/service/CosvService.kt | 56 +++++++++++++------ .../views/vuln/VulnerabilityTagsComponent.kt | 2 +- 8 files changed, 90 insertions(+), 24 deletions(-) diff --git a/save-backend/src/main/kotlin/com/saveourtool/save/backend/service/BackendForCosvService.kt b/save-backend/src/main/kotlin/com/saveourtool/save/backend/service/BackendForCosvService.kt index bf52e309fe..ff068ecc40 100644 --- a/save-backend/src/main/kotlin/com/saveourtool/save/backend/service/BackendForCosvService.kt +++ b/save-backend/src/main/kotlin/com/saveourtool/save/backend/service/BackendForCosvService.kt @@ -5,6 +5,7 @@ import com.saveourtool.save.backend.security.OrganizationPermissionEvaluator import com.saveourtool.save.backend.security.UserPermissionEvaluator import com.saveourtool.save.entities.Organization import com.saveourtool.save.entities.User +import com.saveourtool.save.entities.cosv.LnkVulnerabilityMetadataTag import com.saveourtool.save.permission.Permission import org.springframework.security.core.Authentication import org.springframework.stereotype.Service @@ -19,6 +20,7 @@ class BackendForCosvService( private val userDetailsService: UserDetailsService, private val userPermissionEvaluator: UserPermissionEvaluator, private val organizationPermissionEvaluator: OrganizationPermissionEvaluator, + private val tagService: TagService, configProperties: ConfigProperties, ) : IBackendService { override val workingDir: Path = configProperties.workingDir @@ -41,4 +43,9 @@ class BackendForCosvService( override fun saveUser(user: User): User = userDetailsService.saveUser(user) override fun saveOrganization(organization: Organization) = organizationService.updateOrganization(organization) + + override fun addVulnerabilityTags( + identifier: String, + tagName: Set + ): List? = tagService.addVulnerabilityTags(identifier, tagName) } diff --git a/save-backend/src/main/kotlin/com/saveourtool/save/backend/service/TagService.kt b/save-backend/src/main/kotlin/com/saveourtool/save/backend/service/TagService.kt index 32e62c620c..0b666bcd76 100644 --- a/save-backend/src/main/kotlin/com/saveourtool/save/backend/service/TagService.kt +++ b/save-backend/src/main/kotlin/com/saveourtool/save/backend/service/TagService.kt @@ -5,8 +5,8 @@ import com.saveourtool.save.cosv.repository.LnkVulnerabilityMetadataTagRepositor import com.saveourtool.save.cosv.repository.VulnerabilityMetadataRepository import com.saveourtool.save.entities.Tag import com.saveourtool.save.entities.cosv.LnkVulnerabilityMetadataTag +import com.saveourtool.save.utils.error import com.saveourtool.save.utils.getLogger -import com.saveourtool.save.utils.info import com.saveourtool.save.utils.orNotFound import com.saveourtool.save.validation.TAG_ERROR_MESSAGE import com.saveourtool.save.validation.isValidTag @@ -37,8 +37,6 @@ class TagService( */ @Transactional fun addVulnerabilityTag(identifier: String, tagName: String): LnkVulnerabilityMetadataTag { - log.info { "Trying to add $tagName to $identifier vulnerability" } - if (!tagName.isValidTag()) { throw ResponseStatusException(HttpStatus.CONFLICT, TAG_ERROR_MESSAGE) } @@ -52,6 +50,32 @@ class TagService( ) } + /** + * @param identifier [com.saveourtool.save.entities.cosv.VulnerabilityMetadata.identifier] + * @param tagNames tags to add + * @return new [LnkVulnerabilityMetadataTag] + */ + @Transactional + fun addVulnerabilityTags(identifier: String, tagNames: Set): List? { + if (tagNames.any { !it.isValidTag() }) { + log.error { TAG_ERROR_MESSAGE } + return null + } + + val metadata = vulnerabilityMetadataRepository.findByIdentifier(identifier) ?: run { + log.error { "Could not find metadata for vulnerability $identifier" } + return null + } + + val links = tagNames.map { + tagRepository.findByName(it) ?: tagRepository.save(Tag(it)) + }.map { + LnkVulnerabilityMetadataTag(metadata, it) + } + + return lnkVulnerabilityMetadataTagRepository.saveAll(links) + } + /** * @param identifier [com.saveourtool.save.entities.cosv.VulnerabilityMetadata.identifier] * @param tagName tag to delete diff --git a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/validation/ValidationErrorMessages.kt b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/validation/ValidationErrorMessages.kt index f10e638c6f..ac3940a05f 100644 --- a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/validation/ValidationErrorMessages.kt +++ b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/validation/ValidationErrorMessages.kt @@ -43,7 +43,7 @@ const val CVE_NAME_ERROR_MESSAGE = "CVE identifier is invalid" /** * Error message that is shown when tag is invalid. */ -const val TAG_ERROR_MESSAGE = "Tag length should be in [3, 15] range, no commas are allowed." +const val TAG_ERROR_MESSAGE = "Tag length should be in [2, 15] range, no commas are allowed." /** * Error message that is shown when severity score vector is invalid. diff --git a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/validation/ValidationUtils.kt b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/validation/ValidationUtils.kt index 1ddfe66420..c92c63df64 100644 --- a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/validation/ValidationUtils.kt +++ b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/validation/ValidationUtils.kt @@ -13,7 +13,7 @@ const val NAMING_MAX_LENGTH = 22 private val namingAllowedSpecialSymbols = setOf('-', '_', '.') @Suppress("MagicNumber") -private val tagLengthRange = 3..15 +private val tagLengthRange = 2..15 /** * Check if name is valid. diff --git a/save-cosv/src/main/kotlin/com/saveourtool/save/backend/service/IBackendService.kt b/save-cosv/src/main/kotlin/com/saveourtool/save/backend/service/IBackendService.kt index 055b883a40..c65124bf8d 100644 --- a/save-cosv/src/main/kotlin/com/saveourtool/save/backend/service/IBackendService.kt +++ b/save-cosv/src/main/kotlin/com/saveourtool/save/backend/service/IBackendService.kt @@ -4,6 +4,7 @@ package com.saveourtool.save.backend.service import com.saveourtool.save.entities.Organization import com.saveourtool.save.entities.User +import com.saveourtool.save.entities.cosv.LnkVulnerabilityMetadataTag import com.saveourtool.save.info.UserPermissions import com.saveourtool.save.permission.Permission import org.jetbrains.annotations.Blocking @@ -63,4 +64,14 @@ interface IBackendService { organizationName: String, permission: Permission, ): Boolean + + /** + * @param identifier [com.saveourtool.save.entities.cosv.VulnerabilityMetadata.identifier] + * @param tagName tag to add + * @return new [LnkVulnerabilityMetadataTag] + */ + fun addVulnerabilityTags( + identifier: String, + tagName: Set + ): List? } diff --git a/save-cosv/src/main/kotlin/com/saveourtool/save/cosv/controllers/RawCosvFileController.kt b/save-cosv/src/main/kotlin/com/saveourtool/save/cosv/controllers/RawCosvFileController.kt index 8db9c92b93..902aaa9ef4 100644 --- a/save-cosv/src/main/kotlin/com/saveourtool/save/cosv/controllers/RawCosvFileController.kt +++ b/save-cosv/src/main/kotlin/com/saveourtool/save/cosv/controllers/RawCosvFileController.kt @@ -278,7 +278,7 @@ class RawCosvFileController( backendService.getOrganizationByName(organizationName) to backendService.getUserByName(authentication.name) } .flatMap { (organization, user) -> - cosvService.process(ids, user, organization) + cosvService.processAndAddTagsAndUpdateRating(ids, user, organization) } .subscribeOn(Schedulers.boundedElastic()) .subscribe() @@ -287,7 +287,7 @@ class RawCosvFileController( /** * @param organizationName * @param authentication - * @return statistics [RawCosvFileStatisticDto] with counts of all, uploaded, processing, failed raw cosv files in [organizationName] + * @return statistics [RawCosvFileStatisticsDto] with counts of all, uploaded, processing, failed raw cosv files in [organizationName] */ @RequiresAuthorizationSourceHeader @GetMapping("/statistics") diff --git a/save-cosv/src/main/kotlin/com/saveourtool/save/cosv/service/CosvService.kt b/save-cosv/src/main/kotlin/com/saveourtool/save/cosv/service/CosvService.kt index 650649e473..c34fb11ae6 100644 --- a/save-cosv/src/main/kotlin/com/saveourtool/save/cosv/service/CosvService.kt +++ b/save-cosv/src/main/kotlin/com/saveourtool/save/cosv/service/CosvService.kt @@ -22,7 +22,6 @@ import reactor.core.publisher.Mono import reactor.core.scheduler.Schedulers import reactor.kotlin.core.publisher.switchIfEmpty import reactor.kotlin.core.publisher.toFlux -import reactor.kotlin.extra.math.sumAll import java.nio.ByteBuffer import javax.annotation.PostConstruct @@ -31,6 +30,8 @@ import kotlin.time.Duration.Companion.minutes import kotlin.time.Duration.Companion.seconds import kotlinx.serialization.serializer +typealias VulnerabilityMetadataDtoList = List + /** * Service for vulnerabilities */ @@ -46,6 +47,8 @@ class CosvService( private val lnkVulnerabilityMetadataTagRepository: LnkVulnerabilityMetadataTagRepository, private val cosvGeneratedIdRepository: CosvGeneratedIdRepository, ) { + private val scheduler = Schedulers.boundedElastic() + /** * Init method to restore all raw cosv files with `in progress` state to process */ @@ -71,7 +74,7 @@ class CosvService( "Storage ${RawCosvFileStorage::class.simpleName} and repository ${CosvRepository::class.simpleName} are not initialized in $initMaxTime" } } - .subscribeOn(Schedulers.boundedElastic()) + .subscribeOn(scheduler) .subscribe() } @@ -90,7 +93,11 @@ class CosvService( .flatMap { doProcess(it.requiredId(), user, organization) } - .updateRating(user, organization) + .collectList() + .map { it.flatten() } + .blockingMap { + vulnerabilityRatingService.addRatingForBulkUpload(user, organization, it.size) + } } } .thenJust(Unit) @@ -102,12 +109,14 @@ class CosvService( fun generateIdentifier(): String = cosvGeneratedIdRepository.saveAndFlush(CosvGeneratedId()).getIdentifier() /** + * Method to process all raw cosv files, add ecosystem tags to new vulnerabilities, update user rating + * * @param rawCosvFileIds * @param user * @param organization * @return empty [Mono] */ - fun process( + fun processAndAddTagsAndUpdateRating( rawCosvFileIds: Collection, user: User, organization: Organization, @@ -118,7 +127,30 @@ class CosvService( doProcess(rawCosvFileId, user, organization) } } - .updateRating(user, organization) + .collectList() + .map { it.flatten() } + .flatMap { metadataList -> + Mono.just(metadataList.size).doOnSuccess { + metadataList.toFlux().flatMap { vulnerabilityMetadataDto -> + getVulnerabilityExt(vulnerabilityMetadataDto.identifier).mapNotNull { vulnerabilityExt -> + vulnerabilityExt.cosv.affected?.mapNotNull { it.`package`?.ecosystem }?.toSet()?.let { + vulnerabilityExt.metadataDto.identifier to it + } + } + } + .collectList() + .blockingMap { identifierToTagsList -> + identifierToTagsList.forEach { (identifier, tags) -> + backendService.addVulnerabilityTags(identifier, tags) + } + } + .subscribeOn(scheduler) + .subscribe() + } + } + .blockingMap { + vulnerabilityRatingService.addRatingForBulkUpload(user, organization, it) + } .map { log.debug { "Finished processing raw COSV files $rawCosvFileIds" @@ -144,7 +176,7 @@ class CosvService( rawCosvFileId: Long, user: User, organization: Organization, - ): Mono = rawCosvFileStorage.downloadById(rawCosvFileId) + ): Mono = rawCosvFileStorage.downloadById(rawCosvFileId) .collectToInputStream() .flatMap { inputStream -> val errorMessage by lazy { @@ -170,22 +202,14 @@ class CosvService( .flatMap { rawCosvFileStorage.deleteById(rawCosvFileId) } - .thenReturn(metadataList.size) + .thenReturn(metadataList) } .onErrorResume { error -> log.error(error) { errorMessage } - Mono.just(0) + Mono.just(emptyList()) } } - private fun Flux.updateRating( - user: User, - organization: Organization, - ): Mono = sumAll() - .blockingMap { - vulnerabilityRatingService.addRatingForBulkUpload(user, organization, it) - } - /** * @param cosvId * @param updater diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/vuln/VulnerabilityTagsComponent.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/vuln/VulnerabilityTagsComponent.kt index a12bccc18f..01309740ad 100644 --- a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/vuln/VulnerabilityTagsComponent.kt +++ b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/vuln/VulnerabilityTagsComponent.kt @@ -46,7 +46,7 @@ val vulnerabilityTagsComponent: FC = FC { props className = ClassName("form-control custom-input $validityClassName") value = newTag placeholder = "Add a new tag...".t() - title = "Tag should not have commas, length should be more than 2 and less than 16.".t() + title = "Tag should not have commas, length should be more than 1 and less than 16.".t() asDynamic()["data-toggle"] = "tooltip" asDynamic()["data-placement"] = "top" onChange = { event -> setNewTag(event.target.value) }