diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/ScannerPermissionCheckHandler.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/ScannerPermissionCheckHandler.kt index 5e747a7583..d776b93b68 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/ScannerPermissionCheckHandler.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/ScannerPermissionCheckHandler.kt @@ -32,13 +32,13 @@ import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.artifact.constant.PROJECT_ID import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder import com.tencent.bkrepo.common.security.exception.PermissionException -import com.tencent.bkrepo.common.security.manager.PermissionManager import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.common.security.permission.PermissionCheckHandler import com.tencent.bkrepo.common.security.permission.Principal import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.analyst.model.SubScanTaskDefinition +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.artifact.pojo.RepositoryId import com.tencent.bkrepo.common.security.permission.PrincipalType import org.springframework.context.annotation.Primary diff --git a/src/backend/archive/biz-archive/src/test/kotlin/com/tencent/bkrepo/archive/core/compress/BDZipManagerTest.kt b/src/backend/archive/biz-archive/src/test/kotlin/com/tencent/bkrepo/archive/core/compress/BDZipManagerTest.kt index f890ce6254..6ca58de3f5 100644 --- a/src/backend/archive/biz-archive/src/test/kotlin/com/tencent/bkrepo/archive/core/compress/BDZipManagerTest.kt +++ b/src/backend/archive/biz-archive/src/test/kotlin/com/tencent/bkrepo/archive/core/compress/BDZipManagerTest.kt @@ -13,7 +13,6 @@ import com.tencent.bkrepo.common.bksync.file.BkSyncDeltaSource.Companion.toBkSyn import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import com.tencent.bkrepo.common.storage.StorageAutoConfiguration import com.tencent.bkrepo.common.storage.core.StorageService -import com.tencent.bkrepo.repository.api.RepositoryClient import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.BeforeEach @@ -44,9 +43,6 @@ class BDZipManagerTest @Autowired constructor( @MockBean lateinit var fileReferenceService: FileReferenceService - @MockBean - lateinit var repositoryClient: RepositoryClient - private val timeout = Duration.ofSeconds(10) @BeforeEach diff --git a/src/backend/auth/api-auth/src/main/kotlin/com/tencent/bkrepo/auth/pojo/enums/ResourceActionMapping.kt b/src/backend/auth/api-auth/src/main/kotlin/com/tencent/bkrepo/auth/pojo/enums/ResourceActionMapping.kt index 7de46b97c3..a6e13fa77d 100644 --- a/src/backend/auth/api-auth/src/main/kotlin/com/tencent/bkrepo/auth/pojo/enums/ResourceActionMapping.kt +++ b/src/backend/auth/api-auth/src/main/kotlin/com/tencent/bkrepo/auth/pojo/enums/ResourceActionMapping.kt @@ -28,26 +28,32 @@ package com.tencent.bkrepo.auth.pojo.enums enum class ResourceActionMapping(val resourceType: String, val actions: List) { - PROJECT_ACTIONS(ResourceType.PROJECT.id(), listOf( + PROJECT_ACTIONS( + ResourceType.PROJECT.id(), listOf( ActionTypeMapping.PROJECT_VIEW.id(), ActionTypeMapping.PROJECT_EDIT.id(), ActionTypeMapping.PROJECT_MANAGE.id(), ActionTypeMapping.REPO_CREATE.id() - )), - REPO_ACTIONS(ResourceType.REPO.id(), - listOf( - ActionTypeMapping.REPO_VIEW.id(), - ActionTypeMapping.REPO_EDIT.id(), - ActionTypeMapping.REPO_MANAGE.id(), - ActionTypeMapping.REPO_DELETE.id(), - ActionTypeMapping.NODE_CREATE.id() - )), - NODE_ACTIONS(ResourceType.NODE.id(), - listOf( - ActionTypeMapping.NODE_DELETE.id(), - ActionTypeMapping.NODE_DOWNLOAD.id(), - ActionTypeMapping.NODE_EDIT.id(), - ActionTypeMapping.NODE_WRITE.id(), - ActionTypeMapping.NODE_VIEW.id() - )); + ) + ), + REPO_ACTIONS( + ResourceType.REPO.id(), + listOf( + ActionTypeMapping.REPO_VIEW.id(), + ActionTypeMapping.REPO_EDIT.id(), + ActionTypeMapping.REPO_MANAGE.id(), + ActionTypeMapping.REPO_DELETE.id(), + ActionTypeMapping.NODE_CREATE.id() + ) + ), + NODE_ACTIONS( + ResourceType.NODE.id(), + listOf( + ActionTypeMapping.NODE_DELETE.id(), + ActionTypeMapping.NODE_DOWNLOAD.id(), + ActionTypeMapping.NODE_EDIT.id(), + ActionTypeMapping.NODE_WRITE.id(), + ActionTypeMapping.NODE_VIEW.id() + ) + ); } diff --git a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/dao/repository/OauthTokenRepository.kt b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/dao/repository/OauthTokenRepository.kt index 58a829231d..d7504e5db9 100644 --- a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/dao/repository/OauthTokenRepository.kt +++ b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/dao/repository/OauthTokenRepository.kt @@ -33,7 +33,6 @@ import org.springframework.stereotype.Repository @Repository interface OauthTokenRepository : MongoRepository { - fun findFirstByAccountIdAndUserId(accountId: String, userId: String): TOauthToken? fun findFirstByAccessToken(accessToken: String): TOauthToken? fun findByUserId(userId: String): List fun findFirstByAccountIdAndRefreshToken(accountId: String, refreshToken: String): TOauthToken? diff --git a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/model/TOauthToken.kt b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/model/TOauthToken.kt index fe7ab7723b..fb514cbd7f 100644 --- a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/model/TOauthToken.kt +++ b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/model/TOauthToken.kt @@ -27,13 +27,28 @@ package com.tencent.bkrepo.auth.model +import com.tencent.bkrepo.auth.model.TOauthToken.Companion.ACCESS_TOKEN_IDX +import com.tencent.bkrepo.auth.model.TOauthToken.Companion.ACCESS_TOKEN_IDX_DEF +import com.tencent.bkrepo.auth.model.TOauthToken.Companion.ACCOUNT_ID_ACCESS_TOKEN_IDX +import com.tencent.bkrepo.auth.model.TOauthToken.Companion.ACCOUNT_ID_ACCESS_TOKEN_IDX_DEF +import com.tencent.bkrepo.auth.model.TOauthToken.Companion.ACCOUNT_ID_USER_ID_IDX +import com.tencent.bkrepo.auth.model.TOauthToken.Companion.ACCOUNT_ID_USER_ID_IDX_DEF +import com.tencent.bkrepo.auth.model.TOauthToken.Companion.USER_IDX +import com.tencent.bkrepo.auth.model.TOauthToken.Companion.USER_IDX_DEF import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.auth.pojo.oauth.IdToken +import org.springframework.data.mongodb.core.index.CompoundIndex +import org.springframework.data.mongodb.core.index.CompoundIndexes import org.springframework.data.mongodb.core.mapping.Document import java.time.Instant @Document("oauth_token") -data class TOauthToken( +@CompoundIndexes( + CompoundIndex(name = ACCESS_TOKEN_IDX, def = ACCESS_TOKEN_IDX_DEF, background = true), + CompoundIndex(name = USER_IDX, def = USER_IDX_DEF, background = true), + CompoundIndex(name = ACCOUNT_ID_ACCESS_TOKEN_IDX, def = ACCOUNT_ID_ACCESS_TOKEN_IDX_DEF, background = true), + CompoundIndex(name = ACCOUNT_ID_USER_ID_IDX, def = ACCOUNT_ID_USER_ID_IDX_DEF, background = true), +)data class TOauthToken( val id: String? = null, var accessToken: String, var refreshToken: String?, @@ -44,4 +59,15 @@ data class TOauthToken( var scope: Set?, var issuedAt: Instant, var idToken: IdToken? -) +) { + companion object { + const val ACCESS_TOKEN_IDX = "access_token" + const val ACCESS_TOKEN_IDX_DEF = "{'accessToken': 1}" + const val USER_IDX = "user_id" + const val USER_IDX_DEF = "{'userId': 1}" + const val ACCOUNT_ID_ACCESS_TOKEN_IDX = "account_id_access_token" + const val ACCOUNT_ID_ACCESS_TOKEN_IDX_DEF = "{'accountId': 1, 'access_token': 1}" + const val ACCOUNT_ID_USER_ID_IDX = "account_id_user_id" + const val ACCOUNT_ID_USER_ID_IDX_DEF = "{'accountId': 1, 'userId': 1}" + } +} diff --git a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/impl/ProxyServiceImpl.kt b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/impl/ProxyServiceImpl.kt index eccaf04b75..db0469f24b 100644 --- a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/impl/ProxyServiceImpl.kt +++ b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/impl/ProxyServiceImpl.kt @@ -27,6 +27,7 @@ package com.tencent.bkrepo.auth.service.impl +import com.tencent.bkrepo.auth.dao.ProxyDao import com.tencent.bkrepo.auth.message.AuthMessageCode import com.tencent.bkrepo.auth.model.TProxy import com.tencent.bkrepo.auth.pojo.enums.PermissionAction @@ -37,15 +38,14 @@ import com.tencent.bkrepo.auth.pojo.proxy.ProxyListOption import com.tencent.bkrepo.auth.pojo.proxy.ProxyStatus import com.tencent.bkrepo.auth.pojo.proxy.ProxyStatusRequest import com.tencent.bkrepo.auth.pojo.proxy.ProxyUpdateRequest -import com.tencent.bkrepo.auth.dao.ProxyDao import com.tencent.bkrepo.auth.service.ProxyService import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.api.util.Preconditions import com.tencent.bkrepo.common.api.util.UrlFormatter +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.mongo.dao.util.Pages -import com.tencent.bkrepo.common.security.manager.PermissionManager import com.tencent.bkrepo.common.security.util.AESUtils import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HttpContextHolder diff --git a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/oauth/OauthAuthorizationServiceImpl.kt b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/oauth/OauthAuthorizationServiceImpl.kt index 700d449b33..4df366536d 100644 --- a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/oauth/OauthAuthorizationServiceImpl.kt +++ b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/oauth/OauthAuthorizationServiceImpl.kt @@ -199,29 +199,19 @@ class OauthAuthorizationServiceImpl( client: TAccount, openId: Boolean ): TOauthToken { - var tOauthToken = oauthTokenRepository.findFirstByAccountIdAndUserId(client.id!!, userId) - val idToken = generateOpenIdToken(client.id, userId, nonce) - if (tOauthToken == null) { - tOauthToken = TOauthToken( - accessToken = idToken.toJwtToken(), - refreshToken = OauthUtils.generateRefreshToken(), - expireSeconds = oauthProperties.expiredDuration.seconds, - type = "Bearer", - accountId = client.id, - userId = userId, - scope = client.scope, - issuedAt = Instant.now(Clock.systemDefaultZone()), - idToken = if (openId) idToken else null - ) - } - if (client.scope != tOauthToken.scope) { - tOauthToken.scope = client.scope!! - } - tOauthToken.userId = userId - tOauthToken.accessToken = idToken.toJwtToken() - tOauthToken.idToken = if (openId) idToken else null - tOauthToken.issuedAt = Instant.now(Clock.systemDefaultZone()) - oauthTokenRepository.save(tOauthToken) + val idToken = generateOpenIdToken(client.id!!, userId, nonce) + val tOauthToken = TOauthToken( + accessToken = idToken.toJwtToken(), + refreshToken = OauthUtils.generateRefreshToken(), + expireSeconds = oauthProperties.expiredDuration.seconds, + type = "Bearer", + accountId = client.id, + userId = userId, + scope = client.scope, + issuedAt = Instant.now(Clock.systemDefaultZone()), + idToken = if (openId) idToken else null, + ) + oauthTokenRepository.insert(tOauthToken) return tOauthToken } @@ -261,17 +251,11 @@ class OauthAuthorizationServiceImpl( } override fun validateToken(accessToken: String): String? { - val token = oauthTokenRepository.findFirstByAccessToken(accessToken) - ?: throw ErrorCodeException(CommonMessageCode.RESOURCE_NOT_FOUND, "access_token[$accessToken]") - if (token.expireSeconds == null) { - return token.userId - } - - val expiredInstant = Instant.ofEpochSecond(token.issuedAt.epochSecond + token.expireSeconds) - if (expiredInstant.isBefore(Instant.now())) { - throw ErrorCodeException(CommonMessageCode.RESOURCE_EXPIRED, "access_token[$accessToken]") - } - return token.userId + val claims = JwtUtils.validateToken( + signingKey = RsaUtils.stringToPrivateKey(cryptoProperties.privateKeyStr2048PKCS8), + token = accessToken + ) + return claims.body.subject } override fun deleteToken(clientId: String, clientSecret: String, accessToken: String) { diff --git a/src/backend/build.gradle.kts b/src/backend/build.gradle.kts index cc4a29f7a9..d1b28482d1 100644 --- a/src/backend/build.gradle.kts +++ b/src/backend/build.gradle.kts @@ -76,6 +76,7 @@ allprojects { dependency("com.playtika.reactivefeign:feign-reactor-spring-cloud-starter:${Versions.ReactiveFeign}") dependency("com.tencent.bk.sdk:crypto-java-sdk:${Versions.CryptoJavaSdk}") dependency("org.apache.tika:tika-core:${Versions.TiKa}") + dependency("com.tencent.bk.sdk:spring-boot-bk-audit-starter:${Versions.Audit}") dependency("com.tencent.devops:devops-schedule-common:${Versions.DevopsBootSNAPSHOT}") dependency("com.tencent.devops:devops-schedule-model:${Versions.DevopsBootSNAPSHOT}") dependency("com.tencent.devops:devops-schedule-server:${Versions.DevopsBootSNAPSHOT}") diff --git a/src/backend/buildSrc/src/main/kotlin/Versions.kt b/src/backend/buildSrc/src/main/kotlin/Versions.kt index 2761fb08a3..7a7671d51c 100644 --- a/src/backend/buildSrc/src/main/kotlin/Versions.kt +++ b/src/backend/buildSrc/src/main/kotlin/Versions.kt @@ -71,4 +71,5 @@ object Versions { const val JavaCpp = "1.5.9" const val Notice = "1.0.0" const val SpringCloudFunction = "3.2.11" + const val Audit = "1.0.8" } diff --git a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/exception/OverloadException.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/exception/OverloadException.kt new file mode 100644 index 0000000000..35150db7d7 --- /dev/null +++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/exception/OverloadException.kt @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.api.exception + +import com.tencent.bkrepo.common.api.constant.HttpStatus +import com.tencent.bkrepo.common.api.message.CommonMessageCode + +/** + * 超过限流配置异常 + */ +class OverloadException( + val resource: String +) : ErrorCodeException(CommonMessageCode.RATE_LIMITER_OVERLOAD, resource, status = HttpStatus.TOO_MANY_REQUESTS) \ No newline at end of file diff --git a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/message/CommonMessageCode.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/message/CommonMessageCode.kt index 84a3d35d8d..280eb767d8 100644 --- a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/message/CommonMessageCode.kt +++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/message/CommonMessageCode.kt @@ -61,6 +61,9 @@ enum class CommonMessageCode(private val key: String) : MessageCode { MEDIA_TYPE_UNACCEPTABLE("system.media-type.unacceptable"), TOO_MANY_REQUESTS("too.many.requests"), PIPELINE_NOT_RUNNING("pipeline.not-running"), + INVALID_CONFIG("system.config.invalid"), + ACQUIRE_LOCK_FAILED("acquire.lock.failed"), + RATE_LIMITER_OVERLOAD("rate.limiter.overload") ; override fun getBusinessCode() = ordinal + 1 diff --git a/src/backend/common/common-api/src/main/resources/i18n/messages_en.properties b/src/backend/common/common-api/src/main/resources/i18n/messages_en.properties index c5923db62d..1dbf17a2bd 100644 --- a/src/backend/common/common-api/src/main/resources/i18n/messages_en.properties +++ b/src/backend/common/common-api/src/main/resources/i18n/messages_en.properties @@ -55,3 +55,6 @@ operation.cross-cluster.not-allowed=Cross location operation is not allowed system.media-type.unacceptable=Unacceptable Media Type too.many.requests=Too Many Requests: {0} pipeline.not-running=Pipeline[{0}] is not running status +system.config.invalid=Config [{0}] is invalid +acquire.lock.failed=acquire lock failed:[{0}] +rate.limiter.overload=resource requests reached rate limit:[{0}] \ No newline at end of file diff --git a/src/backend/common/common-api/src/main/resources/i18n/messages_zh_CN.properties b/src/backend/common/common-api/src/main/resources/i18n/messages_zh_CN.properties index a298ce086a..afda36c8c1 100644 --- a/src/backend/common/common-api/src/main/resources/i18n/messages_zh_CN.properties +++ b/src/backend/common/common-api/src/main/resources/i18n/messages_zh_CN.properties @@ -55,3 +55,6 @@ operation.cross-cluster.not-allowed=不允许跨地点操作 system.media-type.unacceptable=不接受的Media Type too.many.requests=请求过多: {0} pipeline.not-running=流水线[{0}]不是运行状态 +system.config.invalid=配置[{0}]无效 +acquire.lock.failed=获取锁失败: [{0}] +rate.limiter.overload=资源请求量超过限流值: [{0}] \ No newline at end of file diff --git a/src/backend/common/common-api/src/main/resources/i18n/messages_zh_TW.properties b/src/backend/common/common-api/src/main/resources/i18n/messages_zh_TW.properties index 815c32bce1..bbac10f772 100644 --- a/src/backend/common/common-api/src/main/resources/i18n/messages_zh_TW.properties +++ b/src/backend/common/common-api/src/main/resources/i18n/messages_zh_TW.properties @@ -55,3 +55,6 @@ operation.cross-cluster.not-allowed=不允許跨地點操作 system.media-type.unacceptable=不接受的Media Type too.many.requests=請求過多: {0} pipeline.not-running=流水線[{0}]不是運行狀態 +system.config.invalid=配置[{0}]無效 +acquire.lock.failed=獲取鎖失敗: [{0}] +rate.limiter.overload=資源請求量超過限流值: [{0}] \ No newline at end of file diff --git a/src/backend/common/common-artifact/artifact-service/build.gradle.kts b/src/backend/common/common-artifact/artifact-service/build.gradle.kts index 42c9e945d4..5036f200fb 100644 --- a/src/backend/common/common-artifact/artifact-service/build.gradle.kts +++ b/src/backend/common/common-artifact/artifact-service/build.gradle.kts @@ -40,6 +40,7 @@ dependencies { api(project(":common:common-security")) api(project(":common:common-artifact:artifact-api")) api(project(":common:common-storage:storage-service")) + api(project(":common:common-ratelimiter")) api(project(":common:common-stream")) api(project(":common:common-metrics-push")) api(project(":common:common-metadata:metadata-service")) @@ -49,6 +50,7 @@ dependencies { api("io.micrometer:micrometer-registry-prometheus") api("org.influxdb:influxdb-java") api("org.apache.commons:commons-text") + api("com.tencent.bk.sdk:spring-boot-bk-audit-starter") testImplementation("org.mockito.kotlin:mockito-kotlin") testImplementation("io.mockk:mockk") diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/ArtifactAutoConfiguration.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/ArtifactAutoConfiguration.kt index ad38ed91e4..c65ba0c094 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/ArtifactAutoConfiguration.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/ArtifactAutoConfiguration.kt @@ -31,6 +31,7 @@ package com.tencent.bkrepo.common.artifact +import com.tencent.bkrepo.common.artifact.audit.BkAuditConfiguration import com.tencent.bkrepo.common.artifact.cluster.ArtifactClusterConfiguration import com.tencent.bkrepo.common.artifact.cns.CnsConfiguration import com.tencent.bkrepo.common.artifact.event.ArtifactEventConfiguration @@ -70,5 +71,6 @@ import org.springframework.context.annotation.PropertySource ArtifactClusterConfiguration::class, CnsConfiguration::class, ArtifactRouterControllerConfiguration::class, + BkAuditConfiguration::class, ) class ArtifactAutoConfiguration diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/audit/ActionAuditContent.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/audit/ActionAuditContent.kt new file mode 100644 index 0000000000..200de35498 --- /dev/null +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/audit/ActionAuditContent.kt @@ -0,0 +1,91 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.artifact.audit + +import com.tencent.bk.audit.constants.AuditAttributeNames.INSTANCE_ID +import com.tencent.bk.audit.constants.AuditAttributeNames.INSTANCE_NAME + +@Suppress("MaxLineLength") +object ActionAuditContent { + + private const val CONTENT_TEMPLATE = "[{{$INSTANCE_NAME}}]({{$INSTANCE_ID}})" + private const val PROJECT_CODE_CONTENT_TEMPLATE = "[{{@PROJECT_CODE}}]" + private const val REPO_NAME_CONTENT_TEMPLATE = "[{{@REPO_NAME}}]" + const val PROJECT_CODE_TEMPLATE = "@PROJECT_CODE" + const val REPO_NAME_TEMPLATE = "@REPO_NAME" + const val TOKEN_TEMPLATE = "@TOKEN" + const val DATE_TEMPLATE = "@DATE" + const val EXPIRES_DYAS_TEMPLATE = "@EXPIRES_DYAS" + const val NAME_TEMPLATE = "@NAME" + const val NEW_PROJECT_CODE_CONTENT_TEMPLATE = "@NEW_PROJECT_CODE" + const val NEW_REPO_NAME_CONTENT_TEMPLATE = "@NEW_REPO_NAME" + const val VERSION_TEMPLATE = "@VERSION" + + // 项目 + const val PROJECT_CREATE_CONTENT = "create project $CONTENT_TEMPLATE" + const val PROJECT_EDIT_CONTENT = "update project $CONTENT_TEMPLATE" + const val PROJECT_VIEW_CONTENT = "get project $CONTENT_TEMPLATE" + + //仓库 + const val REPO_VIEW_CONTENT = "get repo info $CONTENT_TEMPLATE in project $PROJECT_CODE_CONTENT_TEMPLATE" + const val REPO_QUOTE_VIEW_CONTENT = "get quote of repo $CONTENT_TEMPLATE in project $PROJECT_CODE_CONTENT_TEMPLATE" + const val REPO_QUOTE_EDIT_CONTENT = "update quote of repo $CONTENT_TEMPLATE in project $PROJECT_CODE_CONTENT_TEMPLATE" + const val REPO_EXIST_CHECK_CONTENT = "check repo $CONTENT_TEMPLATE exist in project $PROJECT_CODE_CONTENT_TEMPLATE" + const val REPO_CREATE_CONTENT = "create repo info $CONTENT_TEMPLATE in project $PROJECT_CODE_CONTENT_TEMPLATE" + const val REPO_LIST_CONTENT = "list repos in project $PROJECT_CODE_CONTENT_TEMPLATE" + const val REPO_EDIT_CONTENT = "update repo $CONTENT_TEMPLATE in project $PROJECT_CODE_CONTENT_TEMPLATE" + const val REPO_DELETE_CONTENT = "delete repo $CONTENT_TEMPLATE in project $PROJECT_CODE_CONTENT_TEMPLATE" + const val REPO_REPLICATION_CREATE_CONTENT = "create replication task in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE" + const val REPO_REPLICATION_EXECUTE_CONTENT = "execute replication task [{{@NAME}}] in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE" + const val REPO_PACKAGE_DELETE_CONTENT = "delete package [{{@NAME}}] in repo $CONTENT_TEMPLATE project $PROJECT_CODE_CONTENT_TEMPLATE" + const val REPO_PACKAGE_VERSION_DELETE_CONTENT = "delete version [{{@VERSION}}] of package [{{@NAME}}] in repo $CONTENT_TEMPLATE project $PROJECT_CODE_CONTENT_TEMPLATE" + + + // 节点 + const val NODE_SHARE_CREATE_CONTENT = "create share link for node info $CONTENT_TEMPLATE in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE" + const val NODE_SHARE_DOWNLOAD_CONTENT = "download share node $CONTENT_TEMPLATE with token [{{@TOKEN}}] in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE" + const val NODE_DOWNLOAD_WITH_TOKEN_CONTENT = "download node $CONTENT_TEMPLATE with token [{{@TOKEN}}] in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE" + const val NODE_UPLOAD_WITH_TOKEN_CONTENT = "upload node $CONTENT_TEMPLATE with token [{{@TOKEN}}] in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE" + + const val NODE_VIEW_CONTENT = "get node info $CONTENT_TEMPLATE in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE" + const val NODE_CREATE_CONTENT = "create node $CONTENT_TEMPLATE in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE" + const val NODE_DELETE_CONTENT = "delete node $CONTENT_TEMPLATE in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE" + const val NODE_CLEAN_CONTENT = "clean node $CONTENT_TEMPLATE before [{{@DATE}}] in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE" + const val NODE_EXPIRES_EDIT_CONTENT = "set [{{@EXPIRES_DYAS}}] expire days to node $CONTENT_TEMPLATE in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE" + const val NODE_RENAME_CONTENT = "rename node $CONTENT_TEMPLATE to [{{@NAME}}] in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE" + const val NODE_MOVE_CONTENT = "move node $CONTENT_TEMPLATE from repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE to [{{@NAME}}] in [{{@NEW_PROJECT_CODE}}]|[{{@NEW_REPO_NAME}}]" + const val NODE_COPY_CONTENT = "copy node $CONTENT_TEMPLATE in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE to [{{@NAME}}] in [{{@NEW_PROJECT_CODE}}]|[{{@NEW_REPO_NAME}}]" + const val NODE_RESTORE_CONTENT = "restore node $CONTENT_TEMPLATE with deleted time [{{@DATE}}] in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE" + const val NODE_METADATA_VIEW_CONTENT = "get metadata of node $CONTENT_TEMPLATE in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE" + const val NODE_METADATA_EDIT_CONTENT = "update metadata of node $CONTENT_TEMPLATE in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE" + const val NODE_METADATA_FORBID_CONTENT = "forbid metadata of node $CONTENT_TEMPLATE in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE" + const val NODE_METADATA_DELETE_CONTENT = "delete metadata of node $CONTENT_TEMPLATE in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE" + const val NODE_UPLOAD_CONTENT = "upload node $CONTENT_TEMPLATE in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE" + const val NODE_DOWNLOAD_CONTENT = "download node $CONTENT_TEMPLATE in repo $PROJECT_CODE_CONTENT_TEMPLATE|$REPO_NAME_CONTENT_TEMPLATE" + +} \ No newline at end of file diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/audit/BkAuditConfiguration.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/audit/BkAuditConfiguration.kt new file mode 100644 index 0000000000..df033a8776 --- /dev/null +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/audit/BkAuditConfiguration.kt @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.artifact.audit + +import com.tencent.bk.audit.AuditRequestProvider +import com.tencent.bkrepo.common.artifact.metrics.ArtifactMetricsProperties +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.context.annotation.Primary + +@Configuration +@ConditionalOnProperty(name = ["audit.enabled"], havingValue = "true", matchIfMissing = true) +class BkAuditConfiguration { + @Bean + @Primary + fun bkAuditRequestProvider( + artifactMetricsProperties: ArtifactMetricsProperties + ): AuditRequestProvider { + return BkAuditRequestProvider(artifactMetricsProperties) + } + + @Bean + fun bkAuditPostFilter(): BkAuditPostFilter { + return BkAuditPostFilter() + } +} diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/audit/BkAuditPostFilter.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/audit/BkAuditPostFilter.kt new file mode 100644 index 0000000000..ceced95a67 --- /dev/null +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/audit/BkAuditPostFilter.kt @@ -0,0 +1,65 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.artifact.audit + +import com.tencent.bk.audit.filter.AuditPostFilter +import com.tencent.bk.audit.model.AuditEvent + +class BkAuditPostFilter: AuditPostFilter { + override fun map(auditEvent: AuditEvent): AuditEvent { + auditEvent.scopeType = "project" + return auditEvent + } +} \ No newline at end of file diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/audit/BkAuditRequestProvider.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/audit/BkAuditRequestProvider.kt new file mode 100644 index 0000000000..49f0b58ac3 --- /dev/null +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/audit/BkAuditRequestProvider.kt @@ -0,0 +1,109 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.artifact.audit + +import com.tencent.bk.audit.AuditRequestProvider +import com.tencent.bk.audit.constants.AccessTypeEnum +import com.tencent.bk.audit.constants.UserIdentifyTypeEnum +import com.tencent.bk.audit.exception.AuditException +import com.tencent.bk.audit.model.AuditHttpRequest +import com.tencent.bkrepo.common.artifact.metrics.ArtifactMetricsProperties +import com.tencent.bkrepo.common.artifact.metrics.TransferUserAgent +import com.tencent.bkrepo.common.artifact.util.TransferUserAgentUtil +import com.tencent.bkrepo.common.security.util.SecurityUtils +import com.tencent.bkrepo.common.service.util.HttpContextHolder +import com.tencent.bkrepo.common.service.util.SpringContextUtils +import org.slf4j.LoggerFactory +import org.springframework.web.context.request.RequestContextHolder +import org.springframework.web.context.request.ServletRequestAttributes +import javax.servlet.http.HttpServletRequest + +class BkAuditRequestProvider( + private val artifactMetricsProperties: ArtifactMetricsProperties, +) : AuditRequestProvider { + + override fun getRequest(): AuditHttpRequest { + val httpServletRequest = HttpContextHolder.getRequest() + return AuditHttpRequest(httpServletRequest) + } + + private fun getHttpServletRequest(): HttpServletRequest { + val requestAttributes = RequestContextHolder.getRequestAttributes() + if (requestAttributes == null) { + logger.error("Could not get RequestAttributes from RequestContext!") + throw AuditException("Parse http request error") + } + return (requestAttributes as ServletRequestAttributes).request + } + + override fun getUsername(): String { + return SecurityUtils.getUserId() + } + + override fun getUserIdentifyType(): UserIdentifyTypeEnum { + return UserIdentifyTypeEnum.PERSONAL + } + + override fun getUserIdentifyTenantId(): String? { + val httpServletRequest = getHttpServletRequest() + return httpServletRequest.getHeader(HEADER_USER_IDENTIFY_TENANT_ID) + } + + override fun getAccessType(): AccessTypeEnum { + val agent = TransferUserAgentUtil.getUserAgent( + webPlatformId = artifactMetricsProperties.webPlatformId, + host = artifactMetricsProperties.host, + builderAgentList = artifactMetricsProperties.builderAgentList, + clientAgentList = artifactMetricsProperties.clientAgentList + ) + return when (agent) { + TransferUserAgent.BK_WEB -> AccessTypeEnum.WEB + TransferUserAgent.BUILDER, + TransferUserAgent.BK_CLIENT -> AccessTypeEnum.CONSOLE + TransferUserAgent.OPENAPI -> AccessTypeEnum.API + else -> AccessTypeEnum.OTHER + } + } + + override fun getRequestId(): String? { + return SpringContextUtils.getTraceId() + } + + override fun getClientIp(): String { + return HttpContextHolder.getClientAddress() + } + + override fun getUserAgent(): String? { + return HttpContextHolder.getUserAgent() + } + + companion object { + private const val HEADER_USER_IDENTIFY_TENANT_ID = "X-User-Identify-Tenant-Id" + private val logger = LoggerFactory.getLogger(BkAuditRequestProvider::class.java) + } +} \ No newline at end of file diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/audit/ResourceAndActionConstants.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/audit/ResourceAndActionConstants.kt new file mode 100644 index 0000000000..1db8639fd5 --- /dev/null +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/audit/ResourceAndActionConstants.kt @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.artifact.audit + +const val PROJECT_CREATE_ACTION = "project_create" +const val PROJECT_MANAGE_ACTION = "project_manage" +const val PROJECT_DOWNLOAD_ACTION = "project_download" +const val PROJECT_EDIT_ACTION = "project_edit" +const val PROJECT_VIEW_ACTION = "project_view" +const val REPO_CREATE_ACTION = "repo_create" +const val REPO_MANAGE_ACTION = "repo_manage" +const val REPO_VIEW_ACTION = "repo_view" +const val REPO_DOWNLOAD_ACTION = "repo_download" +const val REPO_EDIT_ACTION = "repo_edit" +const val REPO_DELETE_ACTION = "repo_delete" +const val NODE_CREATE_ACTION = "node_create" +const val NODE_VIEW_ACTION = "node_view" +const val NODE_READ_ACTION = "node_read" +const val NODE_DOWNLOAD_ACTION = "node_download" +const val NODE_EDIT_ACTION = "node_edit" +const val NODE_WRITE_ACTION = "node_write" +const val NODE_DELETE_ACTION = "node_delete" + + +const val SYSTEM_RESOURCE = "system" +const val PROJECT_RESOURCE = "project" +const val REPO_RESOURCE = "repo" +const val NODE_RESOURCE = "node" +const val ENDPOINT_RESOURCE = "endpoint" diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/listener/ArtifactDownloadListener.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/listener/ArtifactDownloadListener.kt index 1268b7f2ef..ccb4888aac 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/listener/ArtifactDownloadListener.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/listener/ArtifactDownloadListener.kt @@ -229,6 +229,7 @@ class ArtifactDownloadListener( data[SHA256] = node.sha256 ?: StringPool.EMPTY data[SHARE_USER_ID] = context.shareUserId data[USER_AGENT] = request?.getHeader(HttpHeaders.USER_AGENT) ?: StringPool.EMPTY + data[SIZE] = node.size return NodeDownloadedEvent( projectId = node.projectId, repoName = node.repoName, @@ -244,5 +245,6 @@ class ArtifactDownloadListener( private const val SHA256 = "sha256" private const val SHARE_USER_ID = "shareUserId" private const val USER_AGENT = "userAgent" + private const val SIZE = "size" } } diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManager.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManager.kt index 3993f58136..78ee7e6b7a 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManager.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManager.kt @@ -27,6 +27,7 @@ package com.tencent.bkrepo.common.artifact.manager +import com.tencent.bk.audit.context.ActionAuditContext import com.tencent.bkrepo.common.api.constant.HttpStatus import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.message.CommonMessageCode @@ -82,6 +83,7 @@ class StorageManager( ): NodeDetail { val affectedCount = storageService.store(request.sha256!!, artifactFile, storageCredentials) try { + ActionAuditContext.current().setInstance(request) return nodeService.createNode(request) } catch (exception: Exception) { if (affectedCount == 1) { diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/permission/ArtifactPermissionCheckHandler.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/permission/ArtifactPermissionCheckHandler.kt index 8a50c29e20..edcfb263e3 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/permission/ArtifactPermissionCheckHandler.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/permission/ArtifactPermissionCheckHandler.kt @@ -34,8 +34,8 @@ package com.tencent.bkrepo.common.artifact.permission import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.artifact.constant.PROJECT_ID import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.security.exception.PermissionException -import com.tencent.bkrepo.common.security.manager.PermissionManager import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.common.security.permission.PermissionCheckHandler import com.tencent.bkrepo.common.security.permission.Principal diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/permission/ArtifactPermissionConfiguration.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/permission/ArtifactPermissionConfiguration.kt index c4233ffb02..7246f7e06b 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/permission/ArtifactPermissionConfiguration.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/permission/ArtifactPermissionConfiguration.kt @@ -31,7 +31,7 @@ package com.tencent.bkrepo.common.artifact.permission -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.security.permission.PermissionCheckHandler import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/AbstractArtifactRepository.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/AbstractArtifactRepository.kt index ec19bf0a05..7de1bbe1d9 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/AbstractArtifactRepository.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/core/AbstractArtifactRepository.kt @@ -27,6 +27,7 @@ package com.tencent.bkrepo.common.artifact.repository.core +import com.tencent.bk.audit.context.ActionAuditContext import com.tencent.bkrepo.common.api.constant.HttpHeaders import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.api.exception.MethodNotAllowedException @@ -141,6 +142,11 @@ abstract class AbstractArtifactRepository : ArtifactRepository { val artifactResponse = this.onDownload(context) ?: throw ArtifactNotFoundException(context.artifactInfo.toString()) val throughput = artifactResourceWriter.write(artifactResponse) + if (artifactResponse.node != null) { + ActionAuditContext.current().setInstance(artifactResponse.node) + } else { + ActionAuditContext.current().setInstance(artifactResponse.nodes) + } this.onDownloadSuccess(context, artifactResponse, throughput) } catch (exception: ArtifactResponseException) { val principal = SecurityUtils.getPrincipal() diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/redirect/CosRedirectService.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/redirect/CosRedirectService.kt index 0bfdd3dd15..41798791ac 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/redirect/CosRedirectService.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/redirect/CosRedirectService.kt @@ -37,7 +37,7 @@ import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.artifact.util.http.HttpHeaderUtils.determineMediaType import com.tencent.bkrepo.common.artifact.util.http.HttpHeaderUtils.encodeDisposition import com.tencent.bkrepo.common.artifact.util.http.HttpRangeUtils -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.common.storage.config.StorageProperties diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/virtual/VirtualRepository.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/virtual/VirtualRepository.kt index ba2a9ab3a7..b65909c9b6 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/virtual/VirtualRepository.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/virtual/VirtualRepository.kt @@ -42,7 +42,7 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactSearchConte import com.tencent.bkrepo.common.artifact.repository.core.AbstractArtifactRepository import com.tencent.bkrepo.common.artifact.repository.core.ArtifactRepository import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/ArtifactResolverConfiguration.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/ArtifactResolverConfiguration.kt index 22d3dbfb65..5bb8d5741b 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/ArtifactResolverConfiguration.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/ArtifactResolverConfiguration.kt @@ -39,6 +39,7 @@ import com.tencent.bkrepo.common.artifact.resolve.path.ResolverMap import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResourceWriter import com.tencent.bkrepo.common.artifact.resolve.response.DefaultArtifactResourceWriter import com.tencent.bkrepo.common.storage.config.StorageProperties +import com.tencent.bkrepo.common.ratelimiter.service.RequestLimitCheckService import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration @@ -84,8 +85,14 @@ class ArtifactResolverConfiguration { @Bean @ConditionalOnMissingBean(ArtifactResourceWriter::class) - fun artifactResourceWriter(storageProperties: StorageProperties): ArtifactResourceWriter { - return DefaultArtifactResourceWriter(storageProperties) + fun artifactResourceWriter( + storageProperties: StorageProperties, + requestLimitCheckService: RequestLimitCheckService + ): ArtifactResourceWriter { + return DefaultArtifactResourceWriter( + storageProperties, + requestLimitCheckService + ) } @Bean diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/ArtifactDataReceiver.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/ArtifactDataReceiver.kt index 1fa20ddcc6..bcf75261e3 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/ArtifactDataReceiver.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/ArtifactDataReceiver.kt @@ -28,6 +28,7 @@ package com.tencent.bkrepo.common.artifact.resolve.file import com.tencent.bkrepo.common.api.constant.retry +import com.tencent.bkrepo.common.api.exception.OverloadException import com.tencent.bkrepo.common.artifact.exception.ArtifactReceiveException import com.tencent.bkrepo.common.artifact.hash.sha256 import com.tencent.bkrepo.common.artifact.metrics.ArtifactMetrics @@ -35,14 +36,15 @@ import com.tencent.bkrepo.common.artifact.metrics.TrafficHandler import com.tencent.bkrepo.common.artifact.stream.DigestCalculateListener import com.tencent.bkrepo.common.artifact.stream.rateLimit import com.tencent.bkrepo.common.artifact.util.http.IOExceptionUtils +import com.tencent.bkrepo.common.ratelimiter.service.RequestLimitCheckService +import com.tencent.bkrepo.common.ratelimiter.stream.CommonRateLimitInputStream +import com.tencent.bkrepo.common.storage.config.MonitorProperties import com.tencent.bkrepo.common.storage.config.ReceiveProperties import com.tencent.bkrepo.common.storage.core.locator.HashFileLocator -import com.tencent.bkrepo.common.storage.config.MonitorProperties import com.tencent.bkrepo.common.storage.monitor.StorageHealthMonitor import com.tencent.bkrepo.common.storage.monitor.Throughput import com.tencent.bkrepo.common.storage.util.createFile import com.tencent.bkrepo.common.storage.util.delete -import org.slf4j.LoggerFactory import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream import java.io.IOException @@ -57,6 +59,7 @@ import java.security.SecureRandom import java.time.Duration import kotlin.math.abs import kotlin.system.measureTimeMillis +import org.slf4j.LoggerFactory /** * artifact数据接收类，作用： @@ -75,6 +78,8 @@ class ArtifactDataReceiver( private val filename: String = generateRandomName(), private val randomPath: Boolean = false, private val originPath: Path = path, + private val requestLimitCheckService: RequestLimitCheckService? = null, + private val contentLength: Long? = null, ) : StorageHealthMonitor.Observer, AutoCloseable { /** @@ -187,9 +192,15 @@ class ArtifactDataReceiver( startTime = System.nanoTime() } try { + requestLimitCheckService?.uploadBandwidthCheck( + length.toLong(), + receiveProperties.circuitBreakerThreshold + ) writeData(chunk, offset, length) } catch (exception: IOException) { handleIOException(exception) + } catch (overloadEx: OverloadException) { + handleOverloadException(overloadEx) } } @@ -203,6 +214,9 @@ class ArtifactDataReceiver( startTime = System.nanoTime() } try { + requestLimitCheckService?.uploadBandwidthCheck( + 1, receiveProperties.circuitBreakerThreshold + ) checkFallback() outputStream.write(b) listener.data(b) @@ -210,6 +224,8 @@ class ArtifactDataReceiver( checkThreshold() } catch (exception: IOException) { handleIOException(exception) + } catch (overloadEx: OverloadException) { + handleOverloadException(overloadEx) } } @@ -222,8 +238,13 @@ class ArtifactDataReceiver( if (startTime == 0L) { startTime = System.nanoTime() } + var rateLimitFlag = false + var exp: Exception? = null try { - val input = source.rateLimit(receiveProperties.rateLimit.toBytes()) + val input = requestLimitCheckService?.bandwidthCheck( + source, receiveProperties.circuitBreakerThreshold, contentLength + ) ?: source.rateLimit(receiveProperties.rateLimit.toBytes()) + rateLimitFlag = input is CommonRateLimitInputStream val buffer = ByteArray(bufferSize) input.use { var bytes = input.read(buffer) @@ -233,7 +254,15 @@ class ArtifactDataReceiver( } } } catch (exception: IOException) { + exp = exception handleIOException(exception) + } catch (overloadEx: OverloadException) { + exp = overloadEx + handleOverloadException(overloadEx) + } finally { + if (rateLimitFlag) { + requestLimitCheckService?.bandwidthFinish(exp) + } } } @@ -322,9 +351,7 @@ class ArtifactDataReceiver( * 处理IO异常 */ private fun handleIOException(exception: IOException) { - finished = true - endTime = System.nanoTime() - close() + finishWithException() if (IOExceptionUtils.isClientBroken(exception)) { throw ArtifactReceiveException(exception.message.orEmpty()) } else { @@ -332,6 +359,20 @@ class ArtifactDataReceiver( } } + /** + * 处理限流请求 + */ + private fun handleOverloadException(exception: OverloadException) { + finishWithException() + throw exception + } + + private fun finishWithException() { + finished = true + endTime = System.nanoTime() + close() + } + /** * 检查是否需要fall back操作 */ diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/ArtifactFileFactory.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/ArtifactFileFactory.kt index d99b4b93c5..90de9a6769 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/ArtifactFileFactory.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/ArtifactFileFactory.kt @@ -37,6 +37,7 @@ import com.tencent.bkrepo.common.artifact.resolve.file.multipart.MultipartArtifa import com.tencent.bkrepo.common.artifact.resolve.file.stream.StreamArtifactFile import com.tencent.bkrepo.common.bksync.BlockChannel import com.tencent.bkrepo.common.storage.config.StorageProperties +import com.tencent.bkrepo.common.ratelimiter.service.RequestLimitCheckService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.monitor.StorageHealthMonitor import com.tencent.bkrepo.common.storage.monitor.StorageHealthMonitorHelper @@ -54,17 +55,20 @@ import java.io.InputStream class ArtifactFileFactory( storageProperties: StorageProperties, storageHealthMonitorHelper: StorageHealthMonitorHelper, + private val limitCheckService: RequestLimitCheckService ) { init { monitorHelper = storageHealthMonitorHelper properties = storageProperties + requestLimitCheckService = limitCheckService } companion object { private lateinit var monitorHelper: StorageHealthMonitorHelper private lateinit var properties: StorageProperties + private lateinit var requestLimitCheckService: RequestLimitCheckService const val ARTIFACT_FILES = "artifact.files" @@ -89,34 +93,49 @@ class ArtifactFileFactory( * 构造分块接收数据的artifact file */ fun buildChunked(): ChunkedArtifactFile { - return ChunkedArtifactFile(getMonitor(), properties, getStorageCredentials()).apply { + return ChunkedArtifactFile( + getMonitor(), properties, getStorageCredentials(), + ).apply { track(this) } } fun buildChunked(storageCredentials: StorageCredentials): ChunkedArtifactFile { - return ChunkedArtifactFile(getMonitor(storageCredentials), properties, storageCredentials).apply { + return ChunkedArtifactFile( + getMonitor(storageCredentials), properties, storageCredentials, + ).apply { track(this) } } fun buildDfsArtifactFile(): RandomAccessArtifactFile { - return RandomAccessArtifactFile(getMonitor(), getStorageCredentials(), properties).apply { + return RandomAccessArtifactFile( + getMonitor(), getStorageCredentials(), properties, + ).apply { + track(this) + } + } + + /** + * 通过输入流构造artifact file, 主要针对上传请求对其做限流操作 + * @param inputStream 输入流 + */ + fun buildWithRateLimiter(inputStream: InputStream, contentLength: Long? = null): ArtifactFile { + return StreamArtifactFile( + inputStream, getMonitor(), properties, getStorageCredentials(), contentLength, + requestLimitCheckService = requestLimitCheckService + ).apply { track(this) } } /** - * 通过输入流构造artifact file + * 通过输入流构造artifact file，服务内部输入流转换成文件使用 * @param inputStream 输入流 */ fun build(inputStream: InputStream, contentLength: Long? = null): ArtifactFile { return StreamArtifactFile( - inputStream, - getMonitor(), - properties, - getStorageCredentials(), - contentLength, + inputStream, getMonitor(), properties, getStorageCredentials(), contentLength ).apply { track(this) } @@ -137,10 +156,8 @@ class ArtifactFileFactory( */ fun build(multipartFile: MultipartFile, storageCredentials: StorageCredentials): ArtifactFile { return MultipartArtifactFile( - multipartFile, - getMonitor(storageCredentials), - properties, - storageCredentials, + multipartFile, getMonitor(storageCredentials), properties, storageCredentials, + requestLimitCheckService = requestLimitCheckService ).apply { track(this) } diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/chunk/RandomAccessArtifactFile.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/chunk/RandomAccessArtifactFile.kt index cbea40ee8b..1855681dcd 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/chunk/RandomAccessArtifactFile.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/chunk/RandomAccessArtifactFile.kt @@ -23,7 +23,7 @@ import java.nio.file.NoSuchFileException class RandomAccessArtifactFile( private val monitor: StorageHealthMonitor, private val storageCredentials: StorageCredentials, - storageProperties: StorageProperties + storageProperties: StorageProperties, ) : ArtifactFile { /** @@ -43,7 +43,9 @@ class RandomAccessArtifactFile( init { val path = storageCredentials.upload.location.toPath() - receiver = ArtifactDataReceiver(storageProperties.receive, storageProperties.monitor, path) + receiver = ArtifactDataReceiver( + storageProperties.receive, storageProperties.monitor, path, + ) monitor.add(receiver) if (!monitor.healthy.get()) { receiver.unhealthy(monitor.getFallbackPath(), monitor.fallBackReason) diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/multipart/MultipartArtifactFile.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/multipart/MultipartArtifactFile.kt index 657a4db3ce..75b207f496 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/multipart/MultipartArtifactFile.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/multipart/MultipartArtifactFile.kt @@ -29,6 +29,7 @@ package com.tencent.bkrepo.common.artifact.resolve.file.multipart import com.tencent.bkrepo.common.artifact.resolve.file.stream.StreamArtifactFile import com.tencent.bkrepo.common.storage.config.StorageProperties +import com.tencent.bkrepo.common.ratelimiter.service.RequestLimitCheckService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.monitor.StorageHealthMonitor import org.springframework.web.multipart.MultipartFile @@ -37,9 +38,11 @@ class MultipartArtifactFile( private val multipartFile: MultipartFile, monitor: StorageHealthMonitor, storageProperties: StorageProperties, - storageCredentials: StorageCredentials + storageCredentials: StorageCredentials, + requestLimitCheckService: RequestLimitCheckService ) : StreamArtifactFile( - multipartFile.inputStream, monitor, storageProperties, storageCredentials, multipartFile.size + multipartFile.inputStream, monitor, storageProperties, storageCredentials, multipartFile.size, + requestLimitCheckService ) { fun getOriginalFilename() = multipartFile.originalFilename.orEmpty() } diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/stream/ArtifactFileMethodArgumentResolver.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/stream/ArtifactFileMethodArgumentResolver.kt index 524fe08496..ed7d4e06ff 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/stream/ArtifactFileMethodArgumentResolver.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/stream/ArtifactFileMethodArgumentResolver.kt @@ -60,6 +60,6 @@ class ArtifactFileMethodArgumentResolver : HandlerMethodArgumentResolver { } private fun resolveOctetStream(request: HttpServletRequest): ArtifactFile { - return ArtifactFileFactory.build(request.inputStream, request.contentLengthLong) + return ArtifactFileFactory.buildWithRateLimiter(request.inputStream, request.contentLengthLong) } } diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/stream/StreamArtifactFile.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/stream/StreamArtifactFile.kt index f08f0c7445..dad470cfbc 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/stream/StreamArtifactFile.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/file/stream/StreamArtifactFile.kt @@ -31,6 +31,7 @@ import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.event.ArtifactReceivedEvent import com.tencent.bkrepo.common.artifact.hash.sha1 import com.tencent.bkrepo.common.artifact.resolve.file.ArtifactDataReceiver +import com.tencent.bkrepo.common.ratelimiter.service.RequestLimitCheckService import com.tencent.bkrepo.common.service.util.SpringContextUtils import com.tencent.bkrepo.common.storage.config.StorageProperties import com.tencent.bkrepo.common.storage.credentials.StorageCredentials @@ -47,7 +48,8 @@ open class StreamArtifactFile( private val monitor: StorageHealthMonitor, private val storageProperties: StorageProperties, private val storageCredentials: StorageCredentials, - private val contentLength: Long? = null + private val contentLength: Long? = null, + private val requestLimitCheckService: RequestLimitCheckService? = null ) : ArtifactFile { /** @@ -83,7 +85,9 @@ open class StreamArtifactFile( storageProperties.receive, storageProperties.monitor, receivePath, - randomPath = !useLocalPath + randomPath = !useLocalPath, + requestLimitCheckService = requestLimitCheckService, + contentLength = contentLength ) if (!storageProperties.receive.resolveLazily) { init() diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/response/AbstractArtifactResourceHandler.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/response/AbstractArtifactResourceHandler.kt index 403c4d16c4..cf7078da6a 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/response/AbstractArtifactResourceHandler.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/response/AbstractArtifactResourceHandler.kt @@ -36,6 +36,9 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHold import com.tencent.bkrepo.common.artifact.stream.rateLimit import com.tencent.bkrepo.common.artifact.util.http.IOExceptionUtils import com.tencent.bkrepo.common.storage.config.StorageProperties +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.ratelimiter.service.RequestLimitCheckService +import com.tencent.bkrepo.common.ratelimiter.stream.CommonRateLimitInputStream import com.tencent.bkrepo.common.storage.monitor.Throughput import com.tencent.bkrepo.common.storage.monitor.measureThroughput import org.slf4j.LoggerFactory @@ -47,7 +50,8 @@ import javax.servlet.http.HttpServletResponse abstract class AbstractArtifactResourceHandler( - private val storageProperties: StorageProperties + private val storageProperties: StorageProperties, + private val requestLimitCheckService: RequestLimitCheckService ) : ArtifactResourceWriter { /** * 获取动态buffer size @@ -88,6 +92,14 @@ abstract class AbstractArtifactResourceHandler( } } + /** + * 当仓库配置下载限速小于等于最低限速时则直接将请求断开, 避免占用过多连接 + */ + protected fun downloadRateLimitCheck(resource: ArtifactResource) { + val applyPermits = resource.getSingleStream().range.length + requestLimitCheckService.postLimitCheck(applyPermits) + } + /** * 将数据流以Range方式写入响应 */ @@ -100,13 +112,23 @@ abstract class AbstractArtifactResourceHandler( if (request.method == HttpMethod.HEAD.name) { return Throughput.EMPTY } + val length = inputStream.range.length + var rateLimitFlag = false + var exp: Exception? = null val recordAbleInputStream = RecordAbleInputStream(inputStream) try { return measureThroughput { - recordAbleInputStream.rateLimit(responseRateLimitWrapper(storageProperties.response.rateLimit)).use { + val stream = requestLimitCheckService.bandwidthCheck( + recordAbleInputStream, storageProperties.response.circuitBreakerThreshold, + length + ) ?: recordAbleInputStream.rateLimit( + responseRateLimitWrapper(storageProperties.response.rateLimit) + ) + rateLimitFlag = stream is CommonRateLimitInputStream + stream.use { it.copyTo( out = response.outputStream, - bufferSize = getBufferSize(inputStream.range.length) + bufferSize = getBufferSize(length) ) } } @@ -116,13 +138,21 @@ abstract class AbstractArtifactResourceHandler( // org.springframework.http.converter.HttpMessageNotWritableException异常，会重定向到/error页面 // 又因为/error页面不存在，最终返回404，所以要对IOException进行包装，在上一层捕捉处理 val message = exception.message.orEmpty() - val status = if (IOExceptionUtils.isClientBroken(exception)){ + val status = if (IOExceptionUtils.isClientBroken(exception)) { HttpStatus.BAD_REQUEST } else { logger.warn("write range stream failed", exception) HttpStatus.INTERNAL_SERVER_ERROR } + exp = exception throw ArtifactResponseException(message, status) + } catch (overloadEx: OverloadException) { + exp = overloadEx + throw overloadEx + } finally { + if (rateLimitFlag) { + requestLimitCheckService.bandwidthFinish(exp) + } } } diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/response/DefaultArtifactResourceWriter.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/response/DefaultArtifactResourceWriter.kt index 12e9fefee9..df9b39d9c0 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/response/DefaultArtifactResourceWriter.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/response/DefaultArtifactResourceWriter.kt @@ -31,6 +31,7 @@ import com.tencent.bkrepo.common.api.constant.HttpHeaders import com.tencent.bkrepo.common.api.constant.HttpStatus import com.tencent.bkrepo.common.api.constant.MediaTypes import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.api.exception.OverloadException import com.tencent.bkrepo.common.artifact.constant.X_CHECKSUM_MD5 import com.tencent.bkrepo.common.artifact.constant.X_CHECKSUM_SHA256 import com.tencent.bkrepo.common.artifact.exception.ArtifactResponseException @@ -43,6 +44,8 @@ import com.tencent.bkrepo.common.artifact.stream.rateLimit import com.tencent.bkrepo.common.artifact.util.http.HttpHeaderUtils.determineMediaType import com.tencent.bkrepo.common.artifact.util.http.HttpHeaderUtils.encodeDisposition import com.tencent.bkrepo.common.artifact.util.http.IOExceptionUtils.isClientBroken +import com.tencent.bkrepo.common.ratelimiter.service.RequestLimitCheckService +import com.tencent.bkrepo.common.ratelimiter.stream.CommonRateLimitInputStream import com.tencent.bkrepo.common.service.otel.util.TraceHeaderUtils import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.common.storage.config.StorageProperties @@ -55,6 +58,7 @@ import java.io.IOException import java.time.LocalDateTime import java.time.ZoneOffset import java.time.format.DateTimeFormatter +import java.util.Locale import java.util.zip.ZipEntry import java.util.zip.ZipOutputStream import javax.servlet.http.HttpServletRequest @@ -64,12 +68,16 @@ import javax.servlet.http.HttpServletResponse * ArtifactResourceWriter默认实现 */ open class DefaultArtifactResourceWriter( - private val storageProperties: StorageProperties -) : AbstractArtifactResourceHandler(storageProperties) { + private val storageProperties: StorageProperties, + private val requestLimitCheckService: RequestLimitCheckService +) : AbstractArtifactResourceHandler( + storageProperties, requestLimitCheckService +) { - @Throws(ArtifactResponseException::class) + @Throws(ArtifactResponseException::class, OverloadException::class) override fun write(resource: ArtifactResource): Throughput { responseRateLimitCheck() + downloadRateLimitCheck(resource) TraceHeaderUtils.setResponseHeader() return if (resource.containsMultiArtifact()) { writeMultiArtifact(resource) @@ -88,7 +96,7 @@ open class DefaultArtifactResourceWriter( val name = resource.getSingleName() val range = resource.getSingleStream().range val cacheControl = resource.node?.metadata?.get(HttpHeaders.CACHE_CONTROL)?.toString() - ?: resource.node?.metadata?.get(HttpHeaders.CACHE_CONTROL.toLowerCase())?.toString() + ?: resource.node?.metadata?.get(HttpHeaders.CACHE_CONTROL.lowercase(Locale.getDefault()))?.toString() ?: StringPool.NO_CACHE response.bufferSize = getBufferSize(range.length) @@ -176,6 +184,8 @@ open class DefaultArtifactResourceWriter( if (request.method == HttpMethod.HEAD.name) { return Throughput.EMPTY } + var rateLimitFlag = false + var exp: Exception? = null try { return measureThroughput { val zipOutput = ZipOutputStream(response.outputStream.buffered()) @@ -184,9 +194,14 @@ open class DefaultArtifactResourceWriter( resource.artifactMap.forEach { (name, inputStream) -> val recordAbleInputStream = RecordAbleInputStream(inputStream) zipOutput.putNextEntry(generateZipEntry(name, inputStream)) - recordAbleInputStream.rateLimit( + val stream = requestLimitCheckService.bandwidthCheck( + recordAbleInputStream, storageProperties.response.circuitBreakerThreshold, + inputStream.range.length + ) ?: recordAbleInputStream.rateLimit( responseRateLimitWrapper(storageProperties.response.rateLimit) - ).use { + ) + rateLimitFlag = stream is CommonRateLimitInputStream + stream.use { it.copyTo( out = zipOutput, bufferSize = getBufferSize(inputStream.range.length) @@ -205,8 +220,15 @@ open class DefaultArtifactResourceWriter( logger.warn("write zip stream failed", exception) HttpStatus.INTERNAL_SERVER_ERROR } + exp = exception throw ArtifactResponseException(message, status) + } catch (overloadEx: OverloadException) { + exp = overloadEx + throw overloadEx } finally { + if (rateLimitFlag) { + requestLimitCheckService.bandwidthFinish(exp) + } resource.artifactMap.values.forEach { it.closeQuietly() } } } diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/MetadataAutoConfiguration.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/MetadataAutoConfiguration.kt index 971602d22e..7a66eb1c49 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/MetadataAutoConfiguration.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/MetadataAutoConfiguration.kt @@ -27,15 +27,33 @@ package com.tencent.bkrepo.common.metadata +import com.tencent.bkrepo.auth.api.ServiceExternalPermissionClient +import com.tencent.bkrepo.auth.api.ServicePermissionClient +import com.tencent.bkrepo.auth.api.ServiceUserClient +import com.tencent.bkrepo.common.api.pojo.ClusterArchitecture +import com.tencent.bkrepo.common.api.pojo.ClusterNodeType import com.tencent.bkrepo.common.artifact.properties.ArtifactEventProperties import com.tencent.bkrepo.common.artifact.properties.RouterControllerProperties +import com.tencent.bkrepo.common.metadata.condition.SyncCondition import com.tencent.bkrepo.common.metadata.config.RepositoryProperties +import com.tencent.bkrepo.common.metadata.permission.EdgePermissionManager +import com.tencent.bkrepo.common.metadata.permission.PermissionManager +import com.tencent.bkrepo.common.metadata.permission.ProxyPermissionManager import com.tencent.bkrepo.common.metadata.properties.OperateProperties import com.tencent.bkrepo.common.metadata.properties.ProjectUsageStatisticsProperties +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.project.ProjectService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService +import com.tencent.bkrepo.common.security.http.core.HttpAuthProperties +import com.tencent.bkrepo.common.security.manager.PrincipalManager +import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties import com.tencent.bkrepo.common.storage.config.StorageProperties +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.context.annotation.Bean import org.springframework.context.annotation.ComponentScan +import org.springframework.context.annotation.Conditional import org.springframework.context.annotation.Configuration @Configuration @@ -49,4 +67,73 @@ import org.springframework.context.annotation.Configuration ArtifactEventProperties::class, RepositoryProperties::class, ) -class MetadataAutoConfiguration +class MetadataAutoConfiguration { + + @Bean + @Suppress("LongParameterList") + @Conditional(SyncCondition::class) + fun permissionManager( + projectService: ProjectService, + repositoryService: RepositoryService, + permissionResource: ServicePermissionClient, + externalPermissionResource: ServiceExternalPermissionClient, + userResource: ServiceUserClient, + nodeService: NodeService, + clusterProperties: ClusterProperties, + httpAuthProperties: HttpAuthProperties, + principalManager: PrincipalManager + ): PermissionManager { + return if (clusterProperties.role == ClusterNodeType.EDGE + && clusterProperties.architecture == ClusterArchitecture.COMMIT_EDGE + && clusterProperties.commitEdge.auth.center + ) { + EdgePermissionManager( + projectService = projectService, + repositoryService = repositoryService, + permissionResource = permissionResource, + externalPermissionResource = externalPermissionResource, + userResource = userResource, + nodeService = nodeService, + clusterProperties = clusterProperties, + httpAuthProperties = httpAuthProperties, + principalManager = principalManager + ) + } else { + PermissionManager( + projectService = projectService, + repositoryService = repositoryService, + permissionResource = permissionResource, + externalPermissionResource = externalPermissionResource, + userResource = userResource, + nodeService = nodeService, + httpAuthProperties = httpAuthProperties, + principalManager = principalManager + ) + } + } + + @Bean + @ConditionalOnMissingBean + @Conditional(SyncCondition::class) + fun proxyPermissionManager( + projectService: ProjectService, + repositoryService: RepositoryService, + permissionResource: ServicePermissionClient, + externalPermissionResource: ServiceExternalPermissionClient, + userResource: ServiceUserClient, + nodeService: NodeService, + httpAuthProperties: HttpAuthProperties, + principalManager: PrincipalManager + ): ProxyPermissionManager { + return ProxyPermissionManager( + projectService = projectService, + repositoryService = repositoryService, + permissionResource = permissionResource, + externalPermissionResource = externalPermissionResource, + userResource = userResource, + nodeService = nodeService, + httpAuthProperties = httpAuthProperties, + principalManager = principalManager + ) + } +} diff --git a/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/manager/edge/EdgePermissionManager.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/permission/EdgePermissionManager.kt similarity index 82% rename from src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/manager/edge/EdgePermissionManager.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/permission/EdgePermissionManager.kt index 5e3940f9a1..d8acce0fbf 100644 --- a/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/manager/edge/EdgePermissionManager.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/permission/EdgePermissionManager.kt @@ -25,39 +25,41 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.common.security.manager.edge +package com.tencent.bkrepo.common.metadata.permission -import com.tencent.bkrepo.auth.api.cluster.ClusterPermissionClient -import com.tencent.bkrepo.auth.api.cluster.ClusterUserClient import com.tencent.bkrepo.auth.api.ServiceExternalPermissionClient import com.tencent.bkrepo.auth.api.ServicePermissionClient import com.tencent.bkrepo.auth.api.ServiceUserClient +import com.tencent.bkrepo.auth.api.cluster.ClusterPermissionClient +import com.tencent.bkrepo.auth.api.cluster.ClusterUserClient import com.tencent.bkrepo.auth.pojo.permission.CheckPermissionRequest +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.project.ProjectService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.security.http.core.HttpAuthProperties -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.security.manager.PrincipalManager import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties import com.tencent.bkrepo.common.service.feign.FeignClientFactory -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.api.ProjectClient -import com.tencent.bkrepo.repository.api.RepositoryClient class EdgePermissionManager( - projectClient: ProjectClient, - repositoryClient: RepositoryClient, + projectService: ProjectService, + repositoryService: RepositoryService, permissionResource: ServicePermissionClient, externalPermissionResource: ServiceExternalPermissionClient, userResource: ServiceUserClient, - nodeClient: NodeClient, + nodeService: NodeService, clusterProperties: ClusterProperties, - httpAuthProperties: HttpAuthProperties + httpAuthProperties: HttpAuthProperties, + principalManager: PrincipalManager ) : PermissionManager( - projectClient, - repositoryClient, + projectService, + repositoryService, permissionResource, externalPermissionResource, userResource, - nodeClient, - httpAuthProperties + nodeService, + httpAuthProperties, + principalManager ) { private val centerPermissionClient: ClusterPermissionClient diff --git a/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/manager/PermissionManager.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/permission/PermissionManager.kt similarity index 92% rename from src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/manager/PermissionManager.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/permission/PermissionManager.kt index 24eb4ee074..36fef6d4b7 100644 --- a/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/manager/PermissionManager.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/permission/PermissionManager.kt @@ -29,7 +29,7 @@ * SOFTWARE. */ -package com.tencent.bkrepo.common.security.manager +package com.tencent.bkrepo.common.metadata.permission import com.google.common.cache.CacheBuilder import com.google.common.cache.CacheLoader @@ -48,20 +48,22 @@ import com.tencent.bkrepo.common.api.constant.MediaTypes import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.common.api.util.readJsonString import com.tencent.bkrepo.common.api.util.toJsonString +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.constant.PIPELINE import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException import com.tencent.bkrepo.common.artifact.path.PathUtils +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.project.ProjectService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.security.exception.AuthenticationException import com.tencent.bkrepo.common.security.exception.PermissionException import com.tencent.bkrepo.common.security.http.core.HttpAuthProperties +import com.tencent.bkrepo.common.security.manager.PrincipalManager import com.tencent.bkrepo.common.security.permission.PrincipalType import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.common.service.util.LocaleMessageUtils -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.api.ProjectClient -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.constant.NODE_DETAIL_LIST_KEY import com.tencent.bkrepo.repository.constant.SYSTEM_USER import com.tencent.bkrepo.repository.pojo.node.NodeDetail @@ -81,13 +83,14 @@ import java.util.concurrent.TimeUnit * 权限管理类 */ open class PermissionManager( - private val projectClient: ProjectClient, - private val repositoryClient: RepositoryClient, + private val projectService: ProjectService, + private val repositoryService: RepositoryService, private val permissionResource: ServicePermissionClient, private val externalPermissionResource: ServiceExternalPermissionClient, private val userResource: ServiceUserClient, - private val nodeClient: NodeClient, - private val httpAuthProperties: HttpAuthProperties + private val nodeService: NodeService, + private val httpAuthProperties: HttpAuthProperties, + private val principalManager: PrincipalManager ) { private val httpClient = @@ -202,25 +205,7 @@ open class PermissionManager( * @param principalType 身份类型 */ open fun checkPrincipal(userId: String, principalType: PrincipalType) { - val platformId = SecurityUtils.getPlatformId() - checkAnonymous(userId, platformId) - - if (principalType == PrincipalType.ADMIN) { - if (!isAdminUser(userId)) { - throw PermissionException() - } - } else if (principalType == PrincipalType.PLATFORM) { - if (userId.isEmpty()) { - logger.warn("platform auth with empty userId[$platformId,$userId]") - } - if (platformId == null && !isAdminUser(userId)) { - throw PermissionException() - } - } else if (principalType == PrincipalType.GENERAL) { - if (userId.isEmpty() || userId == ANONYMOUS_USER) { - throw PermissionException() - } - } + principalManager.checkPrincipal(userId, principalType) } /** @@ -286,7 +271,7 @@ open class PermissionManager( */ open fun queryProjectEnabledStatus(projectId: String): Boolean { return try { - projectClient.isProjectEnabled(projectId).data!! + projectService.isProjectEnabled(projectId) } catch (e: Exception) { true } @@ -296,7 +281,7 @@ open class PermissionManager( * 查询仓库信息 */ open fun queryRepositoryInfo(projectId: String, repoName: String): RepositoryInfo { - return repositoryClient.getRepoInfo(projectId, repoName).data ?: throw RepoNotFoundException(repoName) + return repositoryService.getRepoInfo(projectId, repoName) ?: throw RepoNotFoundException(repoName) } private fun serviceRequestCheck(): Boolean { @@ -326,6 +311,7 @@ open class PermissionManager( anonymous: Boolean = false, userId: String = SecurityUtils.getUserId(), ) { + // 判断是否开启认证 if (!httpAuthProperties.enabled) { return @@ -458,9 +444,8 @@ open class PermissionManager( val nodeDetailList = if (repoName.isNullOrBlank() || paths.isNullOrEmpty()) { null } else if (paths.size == 1) { - val node = nodeClient.getNodeDetail(projectId, repoName, paths.first()).data ?: throw NodeNotFoundException( - paths.first() - ) + val node = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, paths.first())) + ?: throw NodeNotFoundException(paths.first()) listOf(node) } else { queryNodeDetailList(projectId, repoName, paths) @@ -483,8 +468,8 @@ open class PermissionManager( val option = NodeListOption( pageNumber = pageNumber, pageSize = 1000, includeFolder = true, includeMetadata = true, deep = true ) - val records = nodeClient.listNodePage(projectId, repoName, prefix, option).data?.records - if (records.isNullOrEmpty()) { + val records = nodeService.listNodePage(ArtifactInfo(projectId, repoName, prefix), option).records + if (records.isEmpty()) { break } nodeDetailList.addAll(records.filter { paths.contains(it.fullPath) }.map { NodeDetail(it) }) diff --git a/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/manager/proxy/ProxyPermissionManager.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/permission/ProxyPermissionManager.kt similarity index 84% rename from src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/manager/proxy/ProxyPermissionManager.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/permission/ProxyPermissionManager.kt index fca7050222..6fcf13bf86 100644 --- a/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/manager/proxy/ProxyPermissionManager.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/permission/ProxyPermissionManager.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.common.security.manager.proxy +package com.tencent.bkrepo.common.metadata.permission import com.tencent.bkrepo.auth.api.ServiceExternalPermissionClient import com.tencent.bkrepo.auth.api.ServicePermissionClient @@ -35,31 +35,33 @@ import com.tencent.bkrepo.auth.api.proxy.ProxyUserClient import com.tencent.bkrepo.auth.pojo.externalPermission.ExternalPermission import com.tencent.bkrepo.auth.pojo.permission.CheckPermissionRequest import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException +import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.project.ProjectService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.security.http.core.HttpAuthProperties -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.security.manager.PrincipalManager import com.tencent.bkrepo.common.service.proxy.ProxyFeignClientFactory -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.api.ProjectClient -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.api.proxy.ProxyRepositoryClient import com.tencent.bkrepo.repository.pojo.repo.RepositoryInfo class ProxyPermissionManager( - projectClient: ProjectClient, - repositoryClient: RepositoryClient, + projectService: ProjectService, + repositoryService: RepositoryService, permissionResource: ServicePermissionClient, externalPermissionResource: ServiceExternalPermissionClient, userResource: ServiceUserClient, - nodeClient: NodeClient, - httpAuthProperties: HttpAuthProperties + nodeService: NodeService, + httpAuthProperties: HttpAuthProperties, + principalManager: PrincipalManager ) : PermissionManager( - projectClient, - repositoryClient, + projectService, + repositoryService, permissionResource, externalPermissionResource, userResource, - nodeClient, - httpAuthProperties + nodeService, + httpAuthProperties, + principalManager ) { private val proxyPermissionClient: ProxyPermissionClient by lazy { ProxyFeignClientFactory.create("auth") } diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/CommonQueryInterpreter.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/CommonQueryInterpreter.kt index 19f931f166..d0b5df75ce 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/CommonQueryInterpreter.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/CommonQueryInterpreter.kt @@ -28,10 +28,10 @@ package com.tencent.bkrepo.common.metadata.search.common import com.tencent.bkrepo.auth.pojo.enums.PermissionAction +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.query.builder.MongoQueryInterpreter import com.tencent.bkrepo.common.query.interceptor.QueryContext import com.tencent.bkrepo.common.query.model.QueryModel -import com.tencent.bkrepo.common.security.manager.PermissionManager open class CommonQueryInterpreter( private val permissionManager: PermissionManager, diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/RepoNameRuleInterceptor.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/RepoNameRuleInterceptor.kt index 4e3572d0f8..97f1e51a2f 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/RepoNameRuleInterceptor.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/common/RepoNameRuleInterceptor.kt @@ -37,17 +37,18 @@ import com.tencent.bkrepo.common.api.constant.ensureSuffix import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.metadata.condition.SyncCondition +import com.tencent.bkrepo.common.metadata.permission.PermissionManager +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService +import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper.listPermissionPaths import com.tencent.bkrepo.common.query.enums.OperationType import com.tencent.bkrepo.common.query.interceptor.QueryContext import com.tencent.bkrepo.common.query.interceptor.QueryRuleInterceptor import com.tencent.bkrepo.common.query.model.Rule import com.tencent.bkrepo.common.security.exception.PermissionException -import com.tencent.bkrepo.common.security.manager.PermissionManager import com.tencent.bkrepo.common.security.util.SecurityUtils +import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.repository.pojo.node.NodeInfo import com.tencent.bkrepo.repository.pojo.repo.RepoListOption -import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService -import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper.listPermissionPaths import org.slf4j.LoggerFactory import org.springframework.context.annotation.Conditional import org.springframework.data.mongodb.core.query.Criteria @@ -127,7 +128,7 @@ class RepoNameRuleInterceptor( userId = userId, projectId = projectId, option = RepoListOption() - )?.map { it.name }?.filter { repo -> repo !in (value.map { it.toString() }) } + ).map { it.name }.filter { repo -> repo !in (value.map { it.toString() }) } return buildRule(projectId, repoNameList) } @@ -217,7 +218,7 @@ class RepoNameRuleInterceptor( repoName: String, repoPublic: Boolean? = null ): Boolean { - if (SecurityUtils.isServiceRequest()) { + if (HttpContextHolder.getRequestOrNull() == null || SecurityUtils.isServiceRequest()) { return true } return try { diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/node/NodeModelInterceptor.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/node/NodeModelInterceptor.kt index 46e75d4ff7..af8359cf3b 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/node/NodeModelInterceptor.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/node/NodeModelInterceptor.kt @@ -32,14 +32,14 @@ package com.tencent.bkrepo.common.metadata.search.node import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.metadata.model.TNode +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.query.enums.OperationType import com.tencent.bkrepo.common.query.interceptor.QueryContext import com.tencent.bkrepo.common.query.model.QueryModel import com.tencent.bkrepo.common.query.model.Rule -import com.tencent.bkrepo.common.security.manager.PermissionManager import com.tencent.bkrepo.common.security.permission.PrincipalType import com.tencent.bkrepo.common.security.util.SecurityUtils -import com.tencent.bkrepo.common.metadata.model.TNode import com.tencent.bkrepo.common.metadata.search.common.ModelValidateInterceptor import org.slf4j.LoggerFactory diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/node/NodeQueryInterpreter.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/node/NodeQueryInterpreter.kt index 61b46a300e..b0d3e7e8ac 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/node/NodeQueryInterpreter.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/node/NodeQueryInterpreter.kt @@ -32,9 +32,9 @@ package com.tencent.bkrepo.common.metadata.search.node import com.tencent.bkrepo.common.metadata.condition.SyncCondition +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.query.interceptor.QueryContext import com.tencent.bkrepo.common.query.model.QueryModel -import com.tencent.bkrepo.common.security.manager.PermissionManager import com.tencent.bkrepo.common.metadata.search.common.CommonQueryInterpreter import com.tencent.bkrepo.common.metadata.search.common.LocalDatetimeRuleInterceptor import com.tencent.bkrepo.common.metadata.search.common.MetadataRuleInterceptor diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/packages/PackageSearchInterpreter.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/packages/PackageSearchInterpreter.kt index 7854b27b74..3e5b213b8e 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/packages/PackageSearchInterpreter.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/search/packages/PackageSearchInterpreter.kt @@ -31,9 +31,9 @@ package com.tencent.bkrepo.common.metadata.search.packages +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.query.interceptor.QueryContext import com.tencent.bkrepo.common.query.model.QueryModel -import com.tencent.bkrepo.common.security.manager.PermissionManager import com.tencent.bkrepo.common.metadata.search.common.CommonQueryInterpreter import com.tencent.bkrepo.common.metadata.search.common.LocalDatetimeRuleInterceptor import com.tencent.bkrepo.common.metadata.search.common.MetadataRuleInterceptor diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/log/OperateLogConfiguration.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/log/OperateLogConfiguration.kt index 42f5a09d16..605934eba1 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/log/OperateLogConfiguration.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/log/OperateLogConfiguration.kt @@ -11,7 +11,6 @@ import com.tencent.bkrepo.common.metadata.properties.ProjectUsageStatisticsPrope import com.tencent.bkrepo.common.metadata.service.log.impl.CommitEdgeOperateLogServiceImpl import com.tencent.bkrepo.common.metadata.service.log.impl.OperateLogServiceImpl import com.tencent.bkrepo.common.metadata.service.project.ProjectUsageStatisticsService -import com.tencent.bkrepo.common.security.manager.PermissionManager import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty @@ -31,7 +30,6 @@ class OperateLogConfiguration { fun operateLogService( operateProperties: OperateProperties, operateLogDao: OperateLogDao, - permissionManager: PermissionManager, clusterProperties: ClusterProperties ): OperateLogService { return if (clusterProperties.role == ClusterNodeType.EDGE && diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeDeleteOperation.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeDeleteOperation.kt index b9c78e8907..a33f46b09d 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeDeleteOperation.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/NodeDeleteOperation.kt @@ -77,7 +77,11 @@ interface NodeDeleteOperation { * 根据最后访问时间删除[date]之前的历史数据 */ fun deleteBeforeDate( - projectId: String, repoName: String, - date: LocalDateTime, operator: String, path: String + projectId: String, + repoName: String, + date: LocalDateTime, + operator: String, + path: String, + decreaseVolume: Boolean = true ): NodeDeleteResult } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/PipelineNodeService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/PipelineNodeService.kt similarity index 97% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/PipelineNodeService.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/PipelineNodeService.kt index 54ae16fefb..1d67cf8ed7 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/PipelineNodeService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/PipelineNodeService.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.repository.service.node +package com.tencent.bkrepo.common.metadata.service.node import com.tencent.bkrepo.common.artifact.constant.PIPELINE import com.tencent.bkrepo.repository.pojo.node.NodeInfo diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt index d4ee15cc0b..74bae3b2b8 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeDeleteSupport.kt @@ -165,7 +165,8 @@ open class NodeDeleteSupport( repoName: String, date: LocalDateTime, operator: String, - path: String + path: String, + decreaseVolume: Boolean ): NodeDeleteResult { val option = NodeListOption(includeFolder = false, deep = true) val timeCriteria = Criteria().orOperator( @@ -175,7 +176,7 @@ open class NodeDeleteSupport( val criteria = NodeQueryHelper.nodeListCriteria(projectId, repoName, path, option) .andOperator(timeCriteria) val query = Query(criteria) - val nodeDeleteResult = delete(query, operator, criteria, projectId, repoName) + val nodeDeleteResult = delete(query, operator, criteria, projectId, repoName, decreaseVolume = decreaseVolume) publishEvent( buildNodeCleanEvent( projectId, repoName, path, operator, nodeDeleteResult.deletedTime.toString() diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeServiceImpl.kt index f4e2db348b..082c2f3f7d 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/NodeServiceImpl.kt @@ -161,8 +161,9 @@ class NodeServiceImpl( date: LocalDateTime, operator: String, path: String, + decreaseVolume: Boolean ): NodeDeleteResult { - return NodeDeleteSupport(this).deleteBeforeDate(projectId, repoName, date, operator, path) + return NodeDeleteSupport(this).deleteBeforeDate(projectId, repoName, date, operator, path, decreaseVolume) } @Transactional(rollbackFor = [Throwable::class]) diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/PipelineNodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/PipelineNodeServiceImpl.kt similarity index 95% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/PipelineNodeServiceImpl.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/PipelineNodeServiceImpl.kt index 39f7d818b2..d988179197 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/PipelineNodeServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/PipelineNodeServiceImpl.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.repository.service.node.impl +package com.tencent.bkrepo.common.metadata.service.node.impl import com.tencent.bkrepo.auth.api.ServicePipelineClient import com.tencent.bkrepo.common.artifact.api.DefaultArtifactInfo @@ -33,7 +33,7 @@ import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.repository.pojo.node.NodeInfo import com.tencent.bkrepo.repository.pojo.node.NodeListOption import com.tencent.bkrepo.common.metadata.service.node.NodeService -import com.tencent.bkrepo.repository.service.node.PipelineNodeService +import com.tencent.bkrepo.common.metadata.service.node.PipelineNodeService import org.springframework.stereotype.Service @Service diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeDeleteSupport.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeDeleteSupport.kt index a97d04195d..46d83081a5 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeDeleteSupport.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeDeleteSupport.kt @@ -104,11 +104,12 @@ class CenterNodeDeleteSupport( repoName: String, date: LocalDateTime, operator: String, - path: String + path: String, + decreaseVolume: Boolean ): NodeDeleteResult { val clusterName = SecurityUtils.getClusterName() if (clusterName.isNullOrEmpty()) { - return super.deleteBeforeDate(projectId, repoName, date, operator, path) + return super.deleteBeforeDate(projectId, repoName, date, operator, path, decreaseVolume) } var deletedSize = 0L var deletedNum = 0L diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt index 8bb67187ca..d1b7beeba1 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/center/CenterNodeServiceImpl.kt @@ -240,14 +240,16 @@ class CenterNodeServiceImpl( repoName: String, date: LocalDateTime, operator: String, - path: String + path: String, + decreaseVolume: Boolean ): NodeDeleteResult { return CenterNodeDeleteSupport(this, clusterProperties).deleteBeforeDate( projectId, repoName, date, operator, - path + path, + decreaseVolume ) } diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/edge/EdgeNodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/edge/EdgeNodeServiceImpl.kt index d127744651..83fa9ea862 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/edge/EdgeNodeServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/node/impl/edge/EdgeNodeServiceImpl.kt @@ -189,6 +189,7 @@ class EdgeNodeServiceImpl( date: LocalDateTime, operator: String, path: String, + decreaseVolume: Boolean ): NodeDeleteResult { ignoreException( projectId = projectId, @@ -197,7 +198,7 @@ class EdgeNodeServiceImpl( ) { centerNodeClient.deleteNodeLastModifiedBeforeDate(projectId, repoName, path, date, operator) } - return NodeDeleteSupport(this).deleteBeforeDate(projectId, repoName, date, operator, path) + return NodeDeleteSupport(this).deleteBeforeDate(projectId, repoName, date, operator, path, decreaseVolume) } @Transactional(rollbackFor = [Throwable::class]) diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/repo/StorageCredentialService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/repo/StorageCredentialService.kt index 5dc0c22acb..cab5b68379 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/repo/StorageCredentialService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/repo/StorageCredentialService.kt @@ -76,4 +76,10 @@ interface StorageCredentialService { * @param key 要删除的存储凭证key */ fun forceDelete(key: String) + + /** + * 获取存储映射关系 + * key为存储storageKey, value为使用相同后端存储的storageKey + */ + fun getStorageKeyMapping(): Map> } diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/repo/impl/RepositoryServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/repo/impl/RepositoryServiceImpl.kt index 3031d38fc1..1cdbc37fe9 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/repo/impl/RepositoryServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/repo/impl/RepositoryServiceImpl.kt @@ -282,6 +282,7 @@ class RepositoryServiceImpl( quota?.let { Preconditions.checkArgument(it >= (repository.used ?: 0), this::quota.name) repository.quota = it + repository.used = repository.used ?: 0 } val oldConfiguration = repository.configuration.readJsonString() repository.public = public ?: repository.public diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/repo/impl/StorageCredentialServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/repo/impl/StorageCredentialServiceImpl.kt index 075e916df1..30fa55ed0d 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/repo/impl/StorageCredentialServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/repo/impl/StorageCredentialServiceImpl.kt @@ -31,10 +31,12 @@ package com.tencent.bkrepo.common.metadata.service.repo.impl +import com.tencent.bkrepo.common.api.collection.concurrent.ConcurrentHashSet import com.tencent.bkrepo.common.api.exception.BadRequestException import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.exception.NotFoundException import com.tencent.bkrepo.common.api.message.CommonMessageCode +import com.tencent.bkrepo.common.artifact.constant.DEFAULT_STORAGE_KEY import com.tencent.bkrepo.common.metadata.condition.SyncCondition import com.tencent.bkrepo.common.metadata.dao.repo.RepositoryDao import com.tencent.bkrepo.common.metadata.dao.repo.StorageCredentialsDao @@ -44,6 +46,10 @@ import com.tencent.bkrepo.common.metadata.util.StorageCredentialHelper.Companion import com.tencent.bkrepo.common.metadata.util.StorageCredentialHelper.Companion.checkCreateRequest import com.tencent.bkrepo.common.metadata.util.StorageCredentialHelper.Companion.convert import com.tencent.bkrepo.common.storage.config.StorageProperties +import com.tencent.bkrepo.common.storage.credentials.FileSystemCredentials +import com.tencent.bkrepo.common.storage.credentials.HDFSCredentials +import com.tencent.bkrepo.common.storage.credentials.InnerCosCredentials +import com.tencent.bkrepo.common.storage.credentials.S3Credentials import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.repository.message.RepositoryMessageCode import com.tencent.bkrepo.repository.pojo.credendials.StorageCredentialsCreateRequest @@ -51,6 +57,7 @@ import com.tencent.bkrepo.repository.pojo.credendials.StorageCredentialsUpdateRe import org.springframework.context.annotation.Conditional import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional +import java.util.concurrent.ConcurrentHashMap /** * 存储凭证服务实现类 @@ -119,4 +126,45 @@ class StorageCredentialServiceImpl( override fun forceDelete(key: String) { return storageCredentialsDao.deleteById(key) } + + override fun getStorageKeyMapping(): Map> { + val mappingKeys = ConcurrentHashMap>() + val credentials = list() + default() + credentials.forEach { credential1 -> + val keys = mappingKeys.getOrPut(credential1.key ?: DEFAULT_STORAGE_KEY) { ConcurrentHashSet() } + credentials.forEach { credential2 -> + if (credential1.key != credential2.key && sameStorage(credential1, credential2)) { + keys.add(credential2.key ?: DEFAULT_STORAGE_KEY) + } + } + } + return mappingKeys + } + + /** + * 判断使用的后端存储是否相同 + */ + private fun sameStorage(credential1: StorageCredentials, credential2: StorageCredentials): Boolean { + if (credential1::class.java != credential2::class.java) { + return false + } + + if (credential1 is InnerCosCredentials && credential2 is InnerCosCredentials) { + return credential1.bucket == credential2.bucket + } + + if (credential1 is S3Credentials && credential2 is S3Credentials) { + return credential1.bucket == credential2.bucket + } + + if (credential1 is FileSystemCredentials && credential2 is FileSystemCredentials) { + return credential1.path == credential2.path + } + + if (credential1 is HDFSCredentials && credential2 is HDFSCredentials) { + throw RuntimeException("Unsupported credentials type[HDFSCredentials]") + } + + return false + } } diff --git a/src/backend/common/common-ratelimiter/build.gradle.kts b/src/backend/common/common-ratelimiter/build.gradle.kts new file mode 100644 index 0000000000..aca0432d66 --- /dev/null +++ b/src/backend/common/common-ratelimiter/build.gradle.kts @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +dependencies { + api("io.micrometer:micrometer-registry-prometheus") + api(project(":common:common-artifact:artifact-api")) + api(project(":common:common-redis")) + api(project(":common:common-api")) + api(project(":common:common-security")) + api(project(":common:common-mongo")) + testImplementation("org.springframework.boot:spring-boot-starter-test") + testImplementation("de.flapdoodle.embed:de.flapdoodle.embed.mongo") + testImplementation("org.mockito.kotlin:mockito-kotlin") + testImplementation("io.mockk:mockk") + testImplementation(project(":common:common-redis")) + testImplementation("it.ozimov:embedded-redis:${Versions.EmbeddedRedis}") { + exclude("org.slf4j", "slf4j-simple") + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/RateLimiterAutoConfiguration.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/RateLimiterAutoConfiguration.kt new file mode 100644 index 0000000000..37731778cc --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/RateLimiterAutoConfiguration.kt @@ -0,0 +1,265 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter + +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.interceptor.RateLimitHandlerInterceptor +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.repository.RateLimitRepository +import com.tencent.bkrepo.common.ratelimiter.service.RequestLimitCheckService +import com.tencent.bkrepo.common.ratelimiter.service.bandwidth.DownloadBandwidthRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.bandwidth.UploadBandwidthRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.url.UrlRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.url.UrlRepoRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.url.user.UserUrlRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.url.user.UserUrlRepoRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.usage.DownloadUsageRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.usage.UploadUsageRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.usage.user.UserDownloadUsageRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.usage.user.UserUploadUsageRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import io.micrometer.core.instrument.MeterRegistry +import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.context.annotation.Import +import org.springframework.core.Ordered +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.web.servlet.config.annotation.InterceptorRegistry +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer + +@Configuration +@EnableConfigurationProperties(RateLimiterProperties::class) +@ConditionalOnWebApplication +@Import(RateLimitRepository::class) +class RateLimiterAutoConfiguration { + + @Bean + fun rateLimitService(rateLimitRepository: RateLimitRepository): RateLimiterConfigService { + return RateLimiterConfigService(rateLimitRepository) + } + + @Bean + fun rateLimiterMetrics(registry: MeterRegistry): RateLimiterMetrics { + return RateLimiterMetrics(registry) + } + + @Bean(URL_REPO_RATELIMITER_SERVICE) + fun urlRepoRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService + ): UrlRepoRateLimiterService { + return UrlRepoRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService + ) + } + + @Bean(URL_RATELIMITER_SERVICE) + fun urlRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService + ): UrlRateLimiterService { + return UrlRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService + ) + } + + @Bean(USER_URL_REPO_RATELIMITER_SERVICE) + fun userUrlRepoRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService + ): UserUrlRepoRateLimiterService { + return UserUrlRepoRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService + ) + } + + @Bean(UPLOAD_USAGE_RATELIMITER_SERVICE) + fun uploadUsageRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService + ): UploadUsageRateLimiterService { + return UploadUsageRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService + ) + } + + @Bean(DOWNLOAD_USAGE_RATELIMITER_SERVICE) + fun downloadUsageRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService + ): DownloadUsageRateLimiterService { + return DownloadUsageRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService + ) + } + + @Bean(USER_DOWNLOAD_USAGE_RATELIMITER_SERVICE) + fun userDownloadUsageRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService + ): UserDownloadUsageRateLimiterService { + return UserDownloadUsageRateLimiterService( + taskScheduler, rateLimiterProperties, rateLimiterMetrics, redisTemplate, rateLimiterConfigService + ) + } + + @Bean(USER_UPLOAD_USAGE_RATELIMITER_SERVICE) + fun userUploadUsageRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService + ): UserUploadUsageRateLimiterService { + return UserUploadUsageRateLimiterService( + taskScheduler, rateLimiterProperties, rateLimiterMetrics, redisTemplate, rateLimiterConfigService + ) + } + + @Bean(USER_URL_RATELIMITER_SERVICE) + fun userUrlRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService + ): UserUrlRateLimiterService { + return UserUrlRateLimiterService( + taskScheduler, rateLimiterProperties, rateLimiterMetrics, redisTemplate, rateLimiterConfigService + ) + } + + @Bean(UPLOAD_BANDWIDTH_RATELIMITER_ERVICE) + fun uploadBandwidthRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService + ): UploadBandwidthRateLimiterService { + return UploadBandwidthRateLimiterService( + taskScheduler, rateLimiterProperties, rateLimiterMetrics, redisTemplate, rateLimiterConfigService + ) + } + + @Bean(DOWNLOAD_BANDWIDTH_RATELIMITER_SERVICE) + fun downloadBandwidthRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService + ): DownloadBandwidthRateLimiterService { + return DownloadBandwidthRateLimiterService( + taskScheduler, rateLimiterProperties, rateLimiterMetrics, redisTemplate, rateLimiterConfigService + ) + } + + @Bean + fun requestLimitCheckService( + rateLimiterProperties: RateLimiterProperties, + ): RequestLimitCheckService { + return RequestLimitCheckService(rateLimiterProperties) + } + + @Bean + @ConditionalOnWebApplication + fun rateLimitHandlerInterceptorRegister( + requestLimitCheckService: RequestLimitCheckService + ): WebMvcConfigurer { + return object : WebMvcConfigurer { + override fun addInterceptors(registry: InterceptorRegistry) { + registry.addInterceptor( + RateLimitHandlerInterceptor( + requestLimitCheckService = requestLimitCheckService + ) + ) + .excludePathPatterns("/service/**", "/replica/**") + .order(Ordered.LOWEST_PRECEDENCE) + super.addInterceptors(registry) + } + } + } + + + companion object { + const val DOWNLOAD_BANDWIDTH_RATELIMITER_SERVICE = "downloadBandwidthRateLimiterService" + const val UPLOAD_BANDWIDTH_RATELIMITER_ERVICE = "uploadBandwidthRateLimiterService" + const val USER_URL_RATELIMITER_SERVICE = "userUrlRateLimiterService" + const val USER_UPLOAD_USAGE_RATELIMITER_SERVICE = "userUploadUsageRateLimiterService" + const val USER_DOWNLOAD_USAGE_RATELIMITER_SERVICE = "userDownloadUsageRateLimiterService" + const val DOWNLOAD_USAGE_RATELIMITER_SERVICE = "downloadUsageRateLimiterService" + const val UPLOAD_USAGE_RATELIMITER_SERVICE = "uploadUsageRateLimiterService" + const val URL_RATELIMITER_SERVICE = "urlRateLimiterService" + const val URL_REPO_RATELIMITER_SERVICE = "urlRepoRateLimiterService" + const val USER_URL_REPO_RATELIMITER_SERVICE = "userUrlRepoRateLimiterService" + + } + +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedFixedWindowRateLimiter.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedFixedWindowRateLimiter.kt new file mode 100644 index 0000000000..c6322b576e --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedFixedWindowRateLimiter.kt @@ -0,0 +1,79 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import com.tencent.bkrepo.common.ratelimiter.redis.LuaScript +import java.time.Duration +import kotlin.system.measureTimeMillis +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.data.redis.core.script.DefaultRedisScript + +/** + * 分布式固定时间窗口算法实现 + */ +class DistributedFixedWindowRateLimiter( + private val key: String, + private val limit: Long, + private val duration: Duration, + private val redisTemplate: RedisTemplate, +) : RateLimiter { + override fun tryAcquire(permits: Long): Boolean { + try { + var acquireResult = false + val elapsedTime = measureTimeMillis { + val redisScript = DefaultRedisScript(LuaScript.fixWindowRateLimiterScript, Long::class.java) + // 注意， 由于redis expire只支持秒为单位，所以周期最小单位为秒 + val result = redisTemplate.execute( + redisScript, listOf(key), limit.toString(), permits.toString(), duration.seconds.toString() + ) + acquireResult = result == 1L + } + if (logger.isDebugEnabled) { + logger.debug( + "acquire distributed fixed window rateLimiter " + + "elapsed time: $elapsedTime ms, acquireResult: $acquireResult" + ) + } + return acquireResult + } catch (e: Exception) { + logger.warn("${this.javaClass.simpleName} acquire error: ${e.message}") + throw AcquireLockFailedException("distributed lock acquire failed: $e") + } + } + + override fun removeCacheLimit(key: String) { + redisTemplate.delete(key) + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(DistributedFixedWindowRateLimiter::class.java) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedLeakyRateLimiter.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedLeakyRateLimiter.kt new file mode 100644 index 0000000000..ed03b9e806 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedLeakyRateLimiter.kt @@ -0,0 +1,91 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import com.tencent.bkrepo.common.ratelimiter.redis.LuaScript +import kotlin.system.measureTimeMillis +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.data.redis.core.script.DefaultRedisScript + +/** + * 分布式漏桶算法实现 + */ +class DistributedLeakyRateLimiter( + private val key: String, + private val permitsPerSecond: Double, + private val capacity: Long, + private val redisTemplate: RedisTemplate, +) : RateLimiter { + override fun tryAcquire(permits: Long): Boolean { + try { + var acquireResult = false + val elapsedTime = measureTimeMillis { + val redisScript = DefaultRedisScript(LuaScript.leakyRateLimiterScript, List::class.java) + // 时间统一从redis server获取 + // lua脚本中使用命令获取时间指令需要配合replicate_commands()使用，但是由于redis只有在某个特定版本上才支持该指令， + // 所以无法从lua脚本中去获取时间，只能分为多次调用。 + val currentTime = redisTemplate.execute { connection -> + connection.time() + } ?: System.currentTimeMillis() + val currentSeconds = (currentTime / 1000) + val results = redisTemplate.execute( + redisScript, getKeys(key), permitsPerSecond.toString(), + capacity.toString(), permits.toString(), currentSeconds.toString() + ) + acquireResult = results[0] == 1L + } + if (logger.isDebugEnabled) { + logger.debug( + "acquire distributed leaky rateLimiter" + + " elapsed time: $elapsedTime ms, acquireResult: $acquireResult" + ) + } + return acquireResult + } catch (e: Exception) { + logger.warn("${this.javaClass.simpleName} acquire error: ${e.message}") + throw AcquireLockFailedException("distributed lock acquire failed: $e") + } + } + + private fun getKeys(key: String): List { + return listOf(key, "$key.timestamp") + } + + override fun removeCacheLimit(key: String) { + getKeys(key).forEach { + redisTemplate.delete(it) + } + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(DistributedLeakyRateLimiter::class.java) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedSlidingWindowRateLimiter.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedSlidingWindowRateLimiter.kt new file mode 100644 index 0000000000..ef4d460988 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedSlidingWindowRateLimiter.kt @@ -0,0 +1,94 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import com.tencent.bkrepo.common.ratelimiter.redis.LuaScript +import java.time.Duration +import kotlin.system.measureTimeMillis +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.data.redis.core.script.DefaultRedisScript + +/** + * 分布式滑动窗口算法实现 + */ +class DistributedSlidingWindowRateLimiter( + private val key: String, + private val limit: Long, + private val duration: Duration, + private val redisTemplate: RedisTemplate, +) : RateLimiter { + override fun tryAcquire(permits: Long): Boolean { + try { + var acquireResult = false + val elapsedTime = measureTimeMillis { + val redisScript = DefaultRedisScript(LuaScript.slidingWindowRateLimiterScript, List::class.java) + // 时间统一从redis server获取 + // lua脚本中使用命令获取时间指令需要配合replicate_commands()使用，但是由于redis只有在某个特定版本上才支持该指令， + // 所以无法从lua脚本中去获取时间，只能分为多次调用。 + val currentTime = redisTemplate.execute { connection -> + connection.time() + } ?: System.currentTimeMillis() + val currentSeconds = (currentTime / 1000) + val random = (currentTime % 1000) + // 注意， 由于redis expire只支持秒为单位，所以周期最小单位为秒 + val results = redisTemplate.execute( + redisScript, getKeys(key), limit.toString(), (duration.seconds).toString(), + permits.toString(), currentSeconds.toString(), random.toString() + ) + acquireResult = results[0] == 1L + } + if (logger.isDebugEnabled) { + logger.debug( + "acquire distributed sliding window rateLimiter" + + " elapsed time: $elapsedTime ms, acquireResult: $acquireResult" + ) + } + return acquireResult + } catch (e: Exception) { + logger.warn("${this.javaClass.simpleName} acquire error: ${e.message}") + throw AcquireLockFailedException("distributed lock acquire failed: $e") + } + } + + private fun getKeys(key: String): List { + return listOf(key) + } + + override fun removeCacheLimit(key: String) { + getKeys(key).forEach { + redisTemplate.delete(it) + } + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(DistributedSlidingWindowRateLimiter::class.java) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedTokenBucketRateLimiter.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedTokenBucketRateLimiter.kt new file mode 100644 index 0000000000..26dfe401ab --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedTokenBucketRateLimiter.kt @@ -0,0 +1,91 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import com.tencent.bkrepo.common.ratelimiter.redis.LuaScript +import kotlin.system.measureTimeMillis +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.data.redis.core.script.DefaultRedisScript + +/** + * 分布式令牌桶算法实现 + */ +class DistributedTokenBucketRateLimiter( + private val key: String, + private val permitsPerSecond: Double, + private val capacity: Long, + private val redisTemplate: RedisTemplate, +) : RateLimiter { + override fun tryAcquire(permits: Long): Boolean { + try { + var acquireResult: Boolean + val elapsedTime = measureTimeMillis { + val redisScript = DefaultRedisScript(LuaScript.tokenBucketRateLimiterScript, List::class.java) + // 时间统一从redis server获取 + // lua脚本中使用命令获取时间指令需要配合replicate_commands()使用，但是由于redis只有在某个特定版本上才支持该指令， + // 所以无法从lua脚本中去获取时间，只能分为多次调用。 + val currentTime = redisTemplate.execute { connection -> + connection.time() + } ?: System.currentTimeMillis() + val currentSeconds = (currentTime / 1000) + val results = redisTemplate.execute( + redisScript, getKeys(key), permitsPerSecond.toString(), + capacity.toString(), permits.toString(), currentSeconds.toString() + ) + acquireResult = results[0] == 1L + } + if (logger.isDebugEnabled) { + logger.debug( + "acquire distributed token bucket rateLimiter" + + " elapsed time: $elapsedTime ms, acquireResult: $acquireResult" + ) + } + return acquireResult + } catch (e: Exception) { + logger.warn("${this.javaClass.simpleName} acquire error: ${e.message}") + throw AcquireLockFailedException("distributed lock acquire failed: $e") + } + } + + private fun getKeys(key: String): List { + return listOf(key, "$key.timestamp") + } + + override fun removeCacheLimit(key: String) { + getKeys(key).forEach { + redisTemplate.delete(it) + } + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(DistributedTokenBucketRateLimiter::class.java) + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/FixedWindowRateLimiter.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/FixedWindowRateLimiter.kt new file mode 100644 index 0000000000..af71989aea --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/FixedWindowRateLimiter.kt @@ -0,0 +1,75 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.google.common.base.Stopwatch +import com.tencent.bkrepo.common.ratelimiter.constant.TRY_LOCK_TIMEOUT +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import java.time.Duration +import java.util.concurrent.TimeUnit +import java.util.concurrent.locks.Lock +import java.util.concurrent.locks.ReentrantLock + +/** + * 单机固定窗口算法实现 + */ +class FixedWindowRateLimiter( + private val limit: Long, + private val duration: Duration, + private val stopWatch: Stopwatch = Stopwatch.createStarted() +) : RateLimiter { + + private var currentValue: Long = 0 + private val lock: Lock = ReentrantLock() + + override fun tryAcquire(permits: Long): Boolean { + // TODO 当剩余容量少于permit时，会导致一直获取不到 + try { + if (!lock.tryLock(TRY_LOCK_TIMEOUT, TimeUnit.MILLISECONDS)) { + throw AcquireLockFailedException("fix window tryLock wait too long: $TRY_LOCK_TIMEOUT ms") + } + try { + if (stopWatch.elapsed(TimeUnit.MILLISECONDS) > duration.toMillis()) { + currentValue = 0 + stopWatch.reset() + } + if (currentValue + permits > limit) return false + currentValue += permits + return true + } finally { + lock.unlock() + } + } catch (e: InterruptedException) { + throw AcquireLockFailedException("fix window tryLock is interrupted by lock timeout: $e") + } + } + + override fun removeCacheLimit(key: String) { + // 非redis类实现不需要处理 + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/LeakyRateLimiter.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/LeakyRateLimiter.kt new file mode 100644 index 0000000000..902d2d1e30 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/LeakyRateLimiter.kt @@ -0,0 +1,86 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.ratelimiter.constant.TRY_LOCK_TIMEOUT +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import java.util.concurrent.TimeUnit +import java.util.concurrent.locks.Lock +import java.util.concurrent.locks.ReentrantLock + + +/** + * 单机漏桶算法实现 + */ +class LeakyRateLimiter( + private val rate: Double, + private val capacity: Long, +) : RateLimiter { + + // 计算的起始时间 + private var lastLeakTime = System.currentTimeMillis() + private var water: Long = 0 + private val lock: Lock = ReentrantLock() + + + override fun tryAcquire(permits: Long): Boolean { + try { + if (!lock.tryLock(TRY_LOCK_TIMEOUT, TimeUnit.MILLISECONDS)) { + throw AcquireLockFailedException("leaky tryLock wait too long: $TRY_LOCK_TIMEOUT ms") + } + try { + return allow(permits) + } finally { + lock.unlock() + } + } catch (e: InterruptedException) { + throw AcquireLockFailedException("leaky tryLock is interrupted by lock timeout: $e") + } + } + + override fun removeCacheLimit(key: String) { + // 非redis类实现不需要处理 + } + + private fun allow(permits: Long): Boolean { + if (water == 0L) { + lastLeakTime = System.currentTimeMillis() + water += permits + return true + } + val waterLeaked: Double = ((System.currentTimeMillis() - lastLeakTime) / 1000) * rate + val waterLeft = (water - waterLeaked).toLong() + water = Math.max(0, waterLeft) // 漏水 + lastLeakTime = System.currentTimeMillis() + if (water + permits <= capacity) { + water += permits + return true + } + return false + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/RateLimiter.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/RateLimiter.kt new file mode 100644 index 0000000000..c8a654fa11 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/RateLimiter.kt @@ -0,0 +1,36 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +interface RateLimiter { + + fun tryAcquire(permits: Long): Boolean + + fun removeCacheLimit(key: String) + +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/SlidingWindowRateLimiter.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/SlidingWindowRateLimiter.kt new file mode 100644 index 0000000000..ce17cc7f83 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/SlidingWindowRateLimiter.kt @@ -0,0 +1,114 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.ratelimiter.constant.TRY_LOCK_TIMEOUT +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import java.time.Duration +import java.util.concurrent.TimeUnit +import java.util.concurrent.locks.Lock +import java.util.concurrent.locks.ReentrantLock + + +/** + * 单机滑动窗口算法实现 + * -- limit: 窗口时间单位内的阈值 + * -- interval: 窗口大小， + * -- limitUnit 窗口时间单位 + */ +class SlidingWindowRateLimiter( + private val limit: Long, + private val duration: Duration, +) : RateLimiter { + + private val lock: Lock = ReentrantLock() + + /** + * 子窗口个数 + */ + private val subWindowNum = 10 + + /** + * 窗口列表 + */ + private var windowArray = Array(subWindowNum) { WindowInfo() } + private val windowSize = duration.toMillis() * subWindowNum + + override fun tryAcquire(permits: Long): Boolean { + try { + if (!lock.tryLock(TRY_LOCK_TIMEOUT, TimeUnit.MILLISECONDS)) { + throw AcquireLockFailedException("sliding window tryLock wait too long: $TRY_LOCK_TIMEOUT ms") + } + try { + return allow(permits) + } finally { + lock.unlock() + } + } catch (e: InterruptedException) { + throw AcquireLockFailedException("sliding window tryLock is interrupted by lock timeout: $e") + } + } + + override fun removeCacheLimit(key: String) { + // 非redis类实现不需要处理 + } + + private fun allow(permits: Long): Boolean { + val currentTimeMillis = System.currentTimeMillis() + // 1. 计算当前时间窗口 + val currentIndex = (currentTimeMillis % windowSize / (windowSize / subWindowNum)).toInt() + // 2. 更新当前窗口计数 & 重置过期窗口计数 + var sum = 0L + val windowArrayCopy = windowArray.map { it.copy() }.toTypedArray() + for (i in windowArray.indices) { + val windowInfo = windowArray[i] + if (currentTimeMillis - windowInfo.time > windowSize) { + windowInfo.count = 0 + windowInfo.time = currentTimeMillis + } + if (currentIndex == i && windowInfo.count <= limit) { + windowInfo.count += permits + } + sum += windowInfo.count + } + // 3. 当前是否超过限制 + return if (sum <= limit) { + true + } else { + //如果最终sum>limit, windowInfo.count需要减掉permits + windowArray = windowArrayCopy + false + } + } + + data class WindowInfo( + var time: Long = System.currentTimeMillis(), + var count: Long = 0 + ) + +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/TokenBucketRateLimiter.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/TokenBucketRateLimiter.kt new file mode 100644 index 0000000000..6c3d39156b --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/TokenBucketRateLimiter.kt @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.ratelimiter.constant.TRY_LOCK_TIMEOUT +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import java.util.concurrent.TimeUnit + +/** + * 单机令牌桶算法实现 + */ +class TokenBucketRateLimiter( + private val permitsPerSecond: Double, +) : RateLimiter { + + private val guavaRateLimiter = com.google.common.util.concurrent.RateLimiter.create(permitsPerSecond) + + override fun tryAcquire(permits: Long): Boolean { + try { + return guavaRateLimiter.tryAcquire(permits.toInt(), TRY_LOCK_TIMEOUT, TimeUnit.MILLISECONDS) + } catch (e: Exception) { + throw AcquireLockFailedException("lock acquire failed: $e") + } + } + + override fun removeCacheLimit(key: String) { + // 非redis类实现不需要处理 + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/config/RateLimiterProperties.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/config/RateLimiterProperties.kt new file mode 100644 index 0000000000..1136d45645 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/config/RateLimiterProperties.kt @@ -0,0 +1,51 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.config + +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.springframework.boot.context.properties.ConfigurationProperties + +@ConfigurationProperties(prefix = "rate.limiter") +data class RateLimiterProperties( + var enabled: Boolean = false, + var dryRun: Boolean = false, + // 配置规则刷新频率 单位为秒 + var refreshDuration: Long = 10L, + // 本地缓存限流算法实现的最大个数 + var cacheCapacity: Long = 1024L, + // 限流配置 + var rules: List = mutableListOf(), + // 等待时间，单位毫秒 + var latency: Long = 70, + // 重试次数 + var waitRound: Int = 5, + // 针对读流的请求，避免频繁去请求，每次申请固定大小 + var permitsOnce: Long = 1024 * 1024, + // 只对指定url进行从request body解析项目仓库信息 + var specialUrls: List = emptyList() +) diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/constant/Constants.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/constant/Constants.kt new file mode 100644 index 0000000000..6de61b22ca --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/constant/Constants.kt @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.constant + +const val TRY_LOCK_TIMEOUT = 10L +const val KEY_PREFIX = "rateLimiter:" +const val SLEEP_TIME = 10 + +const val TAG_STATUS = "status" +const val TAG_NAME = "name" + +const val RATE_LIMITER_TOTAL_COUNT = "rate.limiter.total.count" +const val RATE_LIMITER_TOTAL_COUNT_DESC = "总请求数" + +const val RATE_LIMITER_PASSED_COUNT = "rate.limiter.passed.count" +const val RATE_LIMITER_PASSED_COUNT_DESC = "通过请求数" + +const val RATE_LIMITER_LIMITED_COUNT = "rate.limiter.limited.count" +const val RATE_LIMITER_LIMITED_COUNT_DESC = "限流请求数" + +const val RATE_LIMITER_EXCEPTION_COUNT = "rate.limiter.exception.count" +const val RATE_LIMITER_EXCEPTION_COUNT_DESC = "异常请求数" diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/enums/Algorithms.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/enums/Algorithms.kt new file mode 100644 index 0000000000..6d4447f9bb --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/enums/Algorithms.kt @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.enums + +/** + * 限流算法类型 + */ +enum class Algorithms { + FIXED_WINDOW, + SLIDING_WINDOW, + LEAKY_BUCKET, + TOKEN_BUCKET; +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/enums/LimitDimension.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/enums/LimitDimension.kt new file mode 100644 index 0000000000..8ffd9890de --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/enums/LimitDimension.kt @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.enums + +/** + * 限流维度： + */ +enum class LimitDimension { + URL, // 针对指定URL限流 + URL_REPO, // 针对访问指定项目/仓库的url进行限流 + UPLOAD_USAGE, // 针对仓库上传总大小进行限流 + DOWNLOAD_USAGE, // 针对仓库下载总大小进行限流 + USER_URL, // 针对指定用户指定请求进行限流 + USER_URL_REPO, // 针对指定用户访问指定项目/仓库的url进行限流 + USER_UPLOAD_USAGE, // 针对指定用户上传总大小进行限流 + USER_DOWNLOAD_USAGE, // 针对指定用户下载总大小进行限流 + UPLOAD_BANDWIDTH, // 针对项目维度上传带宽进行限流 + DOWNLOAD_BANDWIDTH, // 针对项目维度下载带宽进行限流 +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/enums/WorkScope.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/enums/WorkScope.kt new file mode 100644 index 0000000000..53bce5d00a --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/enums/WorkScope.kt @@ -0,0 +1,36 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.enums + +/** + * 配置生效范围， 全局生效或者本地生效 + */ +enum class WorkScope { + LOCAL, + GLOBAL +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/exception/AcquireLockFailedException.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/exception/AcquireLockFailedException.kt new file mode 100644 index 0000000000..0490d90567 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/exception/AcquireLockFailedException.kt @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.exception + +import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.api.message.CommonMessageCode + +/** + * 获取对应执行计划失败 + */ +data class AcquireLockFailedException( + val reason: String +) : ErrorCodeException(CommonMessageCode.ACQUIRE_LOCK_FAILED, reason) + diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/exception/InvalidResourceException.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/exception/InvalidResourceException.kt new file mode 100644 index 0000000000..0255af6676 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/exception/InvalidResourceException.kt @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.exception + +import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.api.message.CommonMessageCode + +/** + * 资源无效异常 + */ +data class InvalidResourceException( + val reason: String +) : ErrorCodeException(CommonMessageCode.INVALID_CONFIG, reason) + diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/MonitorRateLimiterInterceptorAdaptor.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/MonitorRateLimiterInterceptorAdaptor.kt new file mode 100644 index 0000000000..f7681dd10e --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/MonitorRateLimiterInterceptorAdaptor.kt @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.interceptor + +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit + +/** + * 限流相关指标采集拦截器 + */ +class MonitorRateLimiterInterceptorAdaptor( + private val rateLimiterMetrics: RateLimiterMetrics +) : RateLimiterInterceptorAdapter() { + + override fun beforeLimitCheck(resource: String, resourceLimit: ResourceLimit) = Unit + + override fun afterLimitCheck( + resource: String, resourceLimit: ResourceLimit?, + result: Boolean, e: Exception? + ) { + if (resourceLimit == null) return + rateLimiterMetrics.collectMetrics(resource = resource, result = result, e = e) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimitHandlerInterceptor.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimitHandlerInterceptor.kt new file mode 100644 index 0000000000..c3bac2ce38 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimitHandlerInterceptor.kt @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.interceptor + +import com.tencent.bkrepo.common.ratelimiter.service.RequestLimitCheckService +import org.springframework.web.servlet.HandlerInterceptor +import javax.servlet.http.HttpServletRequest +import javax.servlet.http.HttpServletResponse + +/** + * 针对http请求添加限流拦截 + */ +class RateLimitHandlerInterceptor( + private val requestLimitCheckService: RequestLimitCheckService +) : HandlerInterceptor { + override fun preHandle(request: HttpServletRequest, response: HttpServletResponse, handler: Any): Boolean { + requestLimitCheckService.preLimitCheck(request) + return super.preHandle(request, response, handler) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimiterInterceptor.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimiterInterceptor.kt new file mode 100644 index 0000000000..a2e532231e --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimiterInterceptor.kt @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.interceptor + +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit + +/** + * 限流执行拦截器 + */ +interface RateLimiterInterceptor { + + /** + * 限流判断前处理 + */ + fun beforeLimitCheck(resource: String, resourceLimit: ResourceLimit) + + /** + * 限流判断后处理 + */ + fun afterLimitCheck( + resource: String, resourceLimit: ResourceLimit?, + result: Boolean, e: Exception? + ) +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimiterInterceptorAdapter.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimiterInterceptorAdapter.kt new file mode 100644 index 0000000000..a9f61df5df --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimiterInterceptorAdapter.kt @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.interceptor + +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit + +/** + * 限流算法执行前后校验拦截器 + */ +abstract class RateLimiterInterceptorAdapter : RateLimiterInterceptor { + override fun beforeLimitCheck(resource: String, resourceLimit: ResourceLimit) = Unit + + override fun afterLimitCheck( + resource: String, resourceLimit: ResourceLimit?, + result: Boolean, e: Exception? + ) = Unit +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimiterInterceptorChain.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimiterInterceptorChain.kt new file mode 100644 index 0000000000..5165ce3912 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimiterInterceptorChain.kt @@ -0,0 +1,73 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.interceptor + +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit + +/** + * 限流拦截器链 + */ +class RateLimiterInterceptorChain( + private val interceptors: MutableList = mutableListOf() +) { + + fun doBeforeLimitCheck(resource: String, resourceLimit: ResourceLimit) { + this.interceptors.forEach { + it.beforeLimitCheck(resource, resourceLimit) + } + } + + fun doAfterLimitCheck( + resource: String, resourceLimit: ResourceLimit?, + result: Boolean, e: Exception? + ) { + this.interceptors.forEach { + it.afterLimitCheck(resource, resourceLimit, result, e) + } + } + + fun addInterceptor(interceptor: RateLimiterInterceptor) { + this.interceptors.add(interceptor) + } + + fun addInterceptors(interceptors: Collection) { + this.interceptors.addAll(interceptors) + } + + fun clear() { + this.interceptors.clear() + } + + fun isEmpty(): Boolean { + return this.interceptors.isEmpty() + } + + fun size(): Int { + return this.interceptors.size + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/TargetRateLimiterInterceptorAdaptor.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/TargetRateLimiterInterceptorAdaptor.kt new file mode 100644 index 0000000000..8780b6b7c8 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/TargetRateLimiterInterceptorAdaptor.kt @@ -0,0 +1,51 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.interceptor + +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService + +/** + * 执行限流机器判断 + */ +class TargetRateLimiterInterceptorAdaptor( + private val rateLimiterConfigService: RateLimiterConfigService, +) : RateLimiterInterceptorAdapter() { + + override fun beforeLimitCheck(resource: String, resourceLimit: ResourceLimit) { + if (resourceLimit.targets.isNotEmpty() && !resourceLimit.targets.contains(rateLimiterConfigService.host)) { + throw InvalidResourceException("targets not contain ${rateLimiterConfigService.host}") + } + } + + override fun afterLimitCheck( + resource: String, resourceLimit: ResourceLimit?, + result: Boolean, e: Exception? + ) = Unit +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/metrics/MetricType.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/metrics/MetricType.kt new file mode 100644 index 0000000000..7f0513980b --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/metrics/MetricType.kt @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.metrics + +/** + * 指标类型 + */ +enum class MetricType { + TOTAL, // 总请求数量 + PASSED, // 通过数量 + LIMITED, //限流数量 + EXCEPTION, //异常请求数量 +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/metrics/RateLimiterMetrics.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/metrics/RateLimiterMetrics.kt new file mode 100644 index 0000000000..05a56b27e2 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/metrics/RateLimiterMetrics.kt @@ -0,0 +1,97 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.metrics + +import com.tencent.bkrepo.common.ratelimiter.constant.RATE_LIMITER_EXCEPTION_COUNT +import com.tencent.bkrepo.common.ratelimiter.constant.RATE_LIMITER_EXCEPTION_COUNT_DESC +import com.tencent.bkrepo.common.ratelimiter.constant.RATE_LIMITER_LIMITED_COUNT +import com.tencent.bkrepo.common.ratelimiter.constant.RATE_LIMITER_LIMITED_COUNT_DESC +import com.tencent.bkrepo.common.ratelimiter.constant.RATE_LIMITER_PASSED_COUNT +import com.tencent.bkrepo.common.ratelimiter.constant.RATE_LIMITER_PASSED_COUNT_DESC +import com.tencent.bkrepo.common.ratelimiter.constant.RATE_LIMITER_TOTAL_COUNT +import com.tencent.bkrepo.common.ratelimiter.constant.RATE_LIMITER_TOTAL_COUNT_DESC +import com.tencent.bkrepo.common.ratelimiter.constant.TAG_NAME +import com.tencent.bkrepo.common.ratelimiter.constant.TAG_STATUS +import io.micrometer.core.instrument.Counter +import io.micrometer.core.instrument.MeterRegistry + +/** + * 限流指标写入 + */ +class RateLimiterMetrics(private val registry: MeterRegistry) { + + fun collectMetrics( + resource: String, result: Boolean, e: Exception? + ) { + try { + getTotalCounter(resource).increment() + if (result) { + getPassedCounter(resource).increment() + } else { + getLimitedCounter(resource).increment() + } + if (e != null) { + getExceptionCounter(resource).increment() + } + } catch (ignore: Exception) { + } + + } + + private fun getTotalCounter(resource: String): Counter { + return getMetricsCount( + RATE_LIMITER_TOTAL_COUNT, RATE_LIMITER_TOTAL_COUNT_DESC, MetricType.TOTAL.name, resource + ) + } + + private fun getPassedCounter(resource: String): Counter { + return getMetricsCount( + RATE_LIMITER_PASSED_COUNT, RATE_LIMITER_PASSED_COUNT_DESC, MetricType.PASSED.name, resource + ) + } + + private fun getLimitedCounter(resource: String): Counter { + return getMetricsCount( + RATE_LIMITER_LIMITED_COUNT, RATE_LIMITER_LIMITED_COUNT_DESC, MetricType.LIMITED.name, resource + ) + } + + private fun getExceptionCounter(resource: String): Counter { + return getMetricsCount( + RATE_LIMITER_EXCEPTION_COUNT, RATE_LIMITER_EXCEPTION_COUNT_DESC, MetricType.EXCEPTION.name, resource + ) + } + + private fun getMetricsCount(metricsName: String, metricsDes: String, status: String, resource: String): Counter { + return Counter.builder(metricsName) + .description(metricsDes) + .tag(TAG_STATUS, status) + .tag(TAG_NAME, resource) + .register(registry) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/model/RateLimitCreatOrUpdateRequest.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/model/RateLimitCreatOrUpdateRequest.kt new file mode 100644 index 0000000000..4d2b2ddd99 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/model/RateLimitCreatOrUpdateRequest.kt @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.model + +data class RateLimitCreatOrUpdateRequest( + val id: String? = null, + // 算法选择 + var algo: String, + // 资源标识 + var resource: String, + // 限流维度 + var limitDimension: String, + // 限流值 + var limit: Long, + // 限流周期 + var duration: Long, + // 桶容量(令牌桶和漏桶使用) + var capacity: Long? = null, + // 生效范围 + var scope: String, + // 指定机器上运行 + var targets: List? = emptyList(), + // 模块名 + var moduleName: List +) diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/model/TRateLimit.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/model/TRateLimit.kt new file mode 100644 index 0000000000..8b00c843df --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/model/TRateLimit.kt @@ -0,0 +1,71 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.model + +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import org.springframework.data.mongodb.core.index.CompoundIndex +import org.springframework.data.mongodb.core.index.CompoundIndexes +import org.springframework.data.mongodb.core.mapping.Document +import java.time.Duration + +@Document(collection = "rate_limit") +@CompoundIndexes( + CompoundIndex( + name = "resource_limitDimension_idx", + def = "{'resource': 1,'limitDimension': 1}", + background = true + ), + CompoundIndex( + name = "limitDimension_idx", + def = "{'limitDimension': 1}", + background = true + ) +) +data class TRateLimit( + var id: String?, + // 算法选择 + var algo: String = Algorithms.FIXED_WINDOW.name, + // 资源标识 + var resource: String = "/", + // 限流维度 + var limitDimension: String = LimitDimension.URL.name, + // 限流值 + var limit: Long = -1, + // 限流周期 + var duration: Duration = Duration.ofSeconds(1), + // 桶容量(令牌桶和漏桶使用) + var capacity: Long? = null, + // 生效范围 + var scope: String = WorkScope.LOCAL.name, + // 指定机器上运行 + var targets: List = emptyList(), + // 模块名 + var moduleName: List +) diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/redis/LuaScript.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/redis/LuaScript.kt new file mode 100644 index 0000000000..9d5e45975c --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/redis/LuaScript.kt @@ -0,0 +1,68 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.redis + +import org.slf4j.LoggerFactory +import org.springframework.util.StreamUtils +import java.io.IOException +import java.nio.charset.StandardCharsets + +/** + * lua脚本加载 + */ +object LuaScript { + private val logger = LoggerFactory.getLogger(LuaScript::class.java) + private const val FIX_WINDOW_RATE_LIMITER_FILE_PATH = "fix-window-rate-limiter.lua" + private const val TOKEN_BUCKET_RATE_LIMITER_FILE_PATH = "token-bucket-rate-limiter.lua" + private const val SLIDING_WINDOW_RATE_LIMITER_FILE_PATH = "sliding-window-rate-limiter.lua" + private const val LEAKY_RATE_LIMITER_FILE_PATH = "leaky-rate-limiter.lua" + + lateinit var fixWindowRateLimiterScript: String + lateinit var tokenBucketRateLimiterScript: String + lateinit var slidingWindowRateLimiterScript: String + lateinit var leakyRateLimiterScript: String + + init { + val fixWindowInput = Thread.currentThread().contextClassLoader + .getResourceAsStream(FIX_WINDOW_RATE_LIMITER_FILE_PATH) + val tokenBucketInput = Thread.currentThread().contextClassLoader + .getResourceAsStream(TOKEN_BUCKET_RATE_LIMITER_FILE_PATH) + val slidingWindowInput = Thread.currentThread().contextClassLoader + .getResourceAsStream(SLIDING_WINDOW_RATE_LIMITER_FILE_PATH) + val leakyInput = Thread.currentThread().contextClassLoader + .getResourceAsStream(LEAKY_RATE_LIMITER_FILE_PATH) + try { + fixWindowRateLimiterScript = StreamUtils.copyToString(fixWindowInput, StandardCharsets.UTF_8) + tokenBucketRateLimiterScript = StreamUtils.copyToString(tokenBucketInput, StandardCharsets.UTF_8) + slidingWindowRateLimiterScript = StreamUtils.copyToString(slidingWindowInput, StandardCharsets.UTF_8) + leakyRateLimiterScript = StreamUtils.copyToString(leakyInput, StandardCharsets.UTF_8) + } catch (e: IOException) { + logger.error("lua script Initialization failed, $e") + } + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/repository/RateLimitRepository.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/repository/RateLimitRepository.kt new file mode 100644 index 0000000000..25736c10a9 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/repository/RateLimitRepository.kt @@ -0,0 +1,88 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.repository + +import com.tencent.bkrepo.common.mongo.dao.simple.SimpleMongoDao +import com.tencent.bkrepo.common.ratelimiter.model.TRateLimit +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Repository + +@Repository +class RateLimitRepository : SimpleMongoDao() { + + fun existsById(id: String) : Boolean { + return find(Query(TRateLimit::id.isEqualTo(id))).isNotEmpty() + } + + fun existsByResourceAndLimitDimension(resource: String, limitDimension: String) : Boolean { + return exists( + Query( + Criteria.where(TRateLimit::resource.name).isEqualTo(resource) + .and(TRateLimit::limitDimension.name).isEqualTo(limitDimension) + ) + ) + } + + fun findByResourceAndLimitDimension(resource: String, limitDimension: String) : List { + return find( + Query( + Criteria.where(TRateLimit::resource.name).isEqualTo(resource) + .and(TRateLimit::limitDimension.name).isEqualTo(limitDimension) + ) + ) + } + + fun findByModuleNameAndLimitDimension(moduleName: String, limitDimension: String): List { + return find( + Query( + Criteria.where(TRateLimit::moduleName.name).regex("$moduleName") + .and(TRateLimit::limitDimension.name).isEqualTo(limitDimension) + ) + ) + } + + fun findByModuleNameAndLimitDimensionAndResource( + resource: String, + moduleName: List, + limitDimension: String + ): TRateLimit? { + return findOne( + Query( + Criteria.where(TRateLimit::moduleName.name).isEqualTo("$moduleName") + .and(TRateLimit::limitDimension.name).isEqualTo(limitDimension) + .and(TRateLimit::resource.name).isEqualTo(resource) + ) + ) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/PathResourceLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/PathResourceLimitRule.kt new file mode 100644 index 0000000000..4e7fc5add4 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/PathResourceLimitRule.kt @@ -0,0 +1,216 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule + +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.common.PathNode +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.utils.ResourcePathUtils +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import java.util.concurrent.ConcurrentHashMap +import java.util.regex.Pattern + +/** + * path资源规则类 + * pathLengthCheck : 当为false时，不校验resource根据/分割后的字符个数与查出的队友匹配规则的resource按照/分割后的字符个数是否相等 + * 如果是url 类型的路径， 如果配置resource为/project1/, 则其子目录都遵循该规则, 此时pathLengthCheck设置为false + * 如果是project/repo类型的路径，如何配置resource为/project1/,则只有当查询资源为/project1/才能够匹配。 此时pathLengthCheck应该设置为 true + */ +open class PathResourceLimitRule( + private val root: PathNode = PathNode("/"), + private val pathLengthCheck: Boolean = false +) { + + fun isEmpty(): Boolean { + return root.getEdges().isEmpty() && root.getResourceLimit() == null + } + + /** + * 添加资源对应的规则 + */ + fun addPathResourceLimit(resourceLimit: ResourceLimit, limits: List) { + if (!limits.contains(resourceLimit.limitDimension)) { + return + } + val resourcePath = resourceLimit.resource + if (!resourcePath.startsWith("/")) { + throw InvalidResourceException(resourcePath) + } + // TODO 配置/*/blueking/* 能识别/api/node/blueking/generic-local + addPathNode(resourcePath, resourceLimit) + } + + fun addPathResourceLimits(resourceLimits: List, limits: List) { + resourceLimits.forEach { + addPathResourceLimit(it, limits) + } + } + + /** + * 根据资源获取对应规则 + */ + open fun getPathResourceLimit(resource: String): ResourceLimit? { + if (resource.isBlank()) { + return null + } + if (resource == "/") { + return root.getResourceLimit() + } + val pathDirs = ResourcePathUtils.tokenizeResourcePath(resource) + if (pathDirs.isEmpty()) { + logger.warn("config resource path $resource is empty!") + return null + } + return findResourceLimit(pathDirs) + } + + private fun findResourceLimit(pathDirs: List): ResourceLimit? { + var p = root + var currentLimit: ResourceLimit? = null + if (p.getResourceLimit() != null) { + currentLimit = p.getResourceLimit() + } + for (path in pathDirs) { + val children = p.getEdges() + var matchedNode = children[path] + if (matchedNode == null) { + val child = findInChildren(children, path) + if (child != null) { + matchedNode = child + } + } + if (matchedNode == null) { + break + } + p = matchedNode + if (matchedNode.getResourceLimit() != null) { + currentLimit = matchedNode.getResourceLimit() + } + } + if (pathLengthCheck) { + return if (pathLengthCheck(currentLimit, pathDirs.size)) { + currentLimit + } else { + null + } + } + return currentLimit + } + + private fun pathLengthCheck(currentLimit: ResourceLimit?, pathDirSize: Int): Boolean { + val length = if (currentLimit?.resource.isNullOrEmpty()) { + 0 + } else { + ResourcePathUtils.tokenizeResourcePath(currentLimit!!.resource).size + } + if (length == pathDirSize) { + return true + } + return false + } + + /** + * 将资源路径按照/拆分，存对应每级对应规则 + */ + private fun addPathNode( + resourcePath: String, + resourceLimit: ResourceLimit + ) { + if (resourcePath == "/") { + root.setResourceLimit(resourceLimit) + return + } + + val pathDirs = ResourcePathUtils.tokenizeResourcePath(resourcePath) + if (pathDirs.isEmpty()) { + logger.warn("config resource path $resourcePath is empty!") + return + } + + var p = root + pathDirs.forEach { + val children = p.getEdges() + var pathDirPattern = it + var isPattern = false + if (isTemplateVariable(it)) { + pathDirPattern = getPathDirPatten(it) + isPattern = true + } + val newNode = PathNode(pathDirPattern, isPattern) + p = children.putIfAbsent(pathDirPattern, newNode) ?: newNode + } + p.setResourceLimit(resourceLimit) + logger.debug("$resourcePath set limit info $resourceLimit") + } + + private fun findInChildren( + children: ConcurrentHashMap, + path: String, + ): PathNode? { + children.entries.forEach { entry -> + val n = entry.value + if (n.isPattern) { + if (Pattern.matches(n.pathDir, path)) { + return n + } + } + } + return null + } + + /** + * 判断是否是模板 + */ + private fun isTemplateVariable(pathDir: String): Boolean { + return pathDir.startsWith("{") && pathDir.endsWith("}") || + pathDir == "*" || pathDir == "**" || pathDir.contains('*') + } + + /** + * 如果模板自带正则表达式，则格式必须为{(^[a-zA-Z]*$)}, ()内味对应正则表达式 + */ + private fun getPathDirPatten(pathDir: String): String { + val patternBuilder = StringBuilder() + val isRegex = pathDir.contains("{(") && pathDir.contains(")}") + if (isRegex) { + val variablePattern = pathDir.substring(2, pathDir.length - 2) + patternBuilder.append('(') + patternBuilder.append(variablePattern) + patternBuilder.append(')') + } else { + patternBuilder.append(PATH_REGEX) + } + return patternBuilder.toString() + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(PathResourceLimitRule::class.java) + private const val PATH_REGEX = ".*" + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/RateLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/RateLimitRule.kt new file mode 100644 index 0000000000..e570801c3c --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/RateLimitRule.kt @@ -0,0 +1,66 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule + +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResLimitInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit + +/** + * 限流配置规则处理 + */ +interface RateLimitRule { + + /** + * 是否存在相应配置规则 + */ + fun isEmpty(): Boolean + + /** + * 获取资源对应的规则 + * 优先查找resource， 如查不到对应规则，则通过extraResource查找 + * resource一般是特定类型，如特定用户，特定URL，特定项目仓库等 + * extraResource一般是某一类类型，如所有用户、URL模版、所有仓库等 + */ + fun getRateLimitRule(resInfo: ResInfo): ResLimitInfo? + + /** + * 添加限流规则 + */ + fun addRateLimitRule(resourceLimit: ResourceLimit) + + /** + * 批量添加限流规则 + */ + fun addRateLimitRules(resourceLimit: List) + + /** + * 过滤不符合条件的规则 + */ + fun filterResourceLimit(resourceLimit: ResourceLimit) +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/BandwidthResourceLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/BandwidthResourceLimitRule.kt new file mode 100644 index 0000000000..6094dc6f9d --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/BandwidthResourceLimitRule.kt @@ -0,0 +1,63 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.bandwidth + +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.rule.PathResourceLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.PathNode +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit + +/** + * 带宽资源规则类 + * 如果配置了仓库级别限流配置，则仓库下所有请求遵循这个配置，即同一仓库下所有请求共享这个带宽限额； + * 如果配置了项目级别限流配置，则项目下所有请求遵循这个配置，即同一项目下所有请求共享这个带宽限额。 + */ +class BandwidthResourceLimitRule( + root: PathNode = PathNode("/"), + pathLengthCheck: Boolean = true +) : PathResourceLimitRule(root, pathLengthCheck) { + + /** + * 添加资源对应对应的规则 + */ + fun addUrlResourceLimit(resourceLimit: ResourceLimit) { + addPathResourceLimit(resourceLimit, bandwidthDimensionList) + } + + fun addUrlResourceLimits(resourceLimits: List) { + resourceLimits.forEach { + addUrlResourceLimit(it) + } + } + + companion object { + private val bandwidthDimensionList = listOf( + LimitDimension.UPLOAD_BANDWIDTH.name, LimitDimension.DOWNLOAD_BANDWIDTH.name + ) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/DownloadBandwidthRateLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/DownloadBandwidthRateLimitRule.kt new file mode 100644 index 0000000000..fa5428338d --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/DownloadBandwidthRateLimitRule.kt @@ -0,0 +1,55 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.bandwidth + +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +/** + * 下载带宽限流配置规则实现 + */ +class DownloadBandwidthRateLimitRule( + bandwidthLimitRules: BandwidthResourceLimitRule = BandwidthResourceLimitRule() +) : UploadBandwidthRateLimitRule(bandwidthLimitRules) { + + override fun filterResourceLimit(resourceLimit: ResourceLimit) { + if (resourceLimit.limitDimension != LimitDimension.DOWNLOAD_BANDWIDTH.name) { + throw InvalidResourceException(resourceLimit.limitDimension) + } + if (resourceLimit.resource.isBlank()) { + throw InvalidResourceException(resourceLimit.resource) + } + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(DownloadBandwidthRateLimitRule::class.java) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/UploadBandwidthRateLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/UploadBandwidthRateLimitRule.kt new file mode 100644 index 0000000000..8e7b35ca7f --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/UploadBandwidthRateLimitRule.kt @@ -0,0 +1,96 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.bandwidth + +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResLimitInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +/** + * 上传带宽限流配置规则实现 + */ +open class UploadBandwidthRateLimitRule( + private val bandwidthLimitRules: BandwidthResourceLimitRule = BandwidthResourceLimitRule() +) : RateLimitRule { + + override fun isEmpty(): Boolean { + return bandwidthLimitRules.isEmpty() + } + + override fun getRateLimitRule(resInfo: ResInfo): ResLimitInfo? { + var realResource = resInfo.resource + if (realResource.isBlank()) { + return null + } + var ruleLimit = bandwidthLimitRules.getPathResourceLimit(realResource) + if (ruleLimit == null && resInfo.extraResource.isNotEmpty()) { + for (res in resInfo.extraResource) { + ruleLimit = bandwidthLimitRules.getPathResourceLimit(res) + if (ruleLimit != null) { + realResource = res + break + } + } + } + if (ruleLimit == null) return null + return ResLimitInfo(realResource, ruleLimit) + } + + override fun addRateLimitRule(resourceLimit: ResourceLimit) { + filterResourceLimit(resourceLimit) + bandwidthLimitRules.addUrlResourceLimit(resourceLimit) + } + + override fun addRateLimitRules(resourceLimit: List) { + resourceLimit.forEach { + try { + addRateLimitRule(it) + } catch (e: Exception) { + logger.error("add config $it for ${this.javaClass.simpleName} failed, error is ${e.message}") + } + } + } + + override fun filterResourceLimit(resourceLimit: ResourceLimit) { + if (resourceLimit.limitDimension != LimitDimension.UPLOAD_BANDWIDTH.name) { + throw InvalidResourceException(resourceLimit.limitDimension) + } + if (resourceLimit.resource.isBlank()) { + throw InvalidResourceException(resourceLimit.resource) + } + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(UploadBandwidthRateLimitRule::class.java) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/common/PathNode.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/common/PathNode.kt new file mode 100644 index 0000000000..6a05e7c575 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/common/PathNode.kt @@ -0,0 +1,53 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.common + +import java.util.concurrent.ConcurrentHashMap + +/** + * path 对应配置 + */ +class PathNode( + val pathDir: String, + val isPattern: Boolean = false, +) { + private val edges: ConcurrentHashMap = ConcurrentHashMap() + private var resourceLimit: ResourceLimit? = null + + fun getEdges(): ConcurrentHashMap { + return edges + } + + fun setResourceLimit(resourceLimit: ResourceLimit) { + this.resourceLimit = resourceLimit + } + + fun getResourceLimit(): ResourceLimit? { + return this.resourceLimit + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/common/ResInfo.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/common/ResInfo.kt new file mode 100644 index 0000000000..3e3a1014bb --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/common/ResInfo.kt @@ -0,0 +1,37 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.common + +/** + * 资源信息 + */ +data class ResInfo( + val resource: String, + val extraResource: List = emptyList() +) + diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/common/ResLimitInfo.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/common/ResLimitInfo.kt new file mode 100644 index 0000000000..f7dc5d33ce --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/common/ResLimitInfo.kt @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.common + +/** + * 请求资源以及对应的限流配置 + */ +data class ResLimitInfo( + // 请求资源 + val resource: String, + // 请求资源对应的限流配置 + val resourceLimit: ResourceLimit +) diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/common/ResourceLimit.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/common/ResourceLimit.kt new file mode 100644 index 0000000000..68ebc0526e --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/common/ResourceLimit.kt @@ -0,0 +1,55 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.common + +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import java.time.Duration + +/** + * 限流规则 + */ +data class ResourceLimit( + // 算法选择 + var algo: String = Algorithms.FIXED_WINDOW.name, + // 资源标识 + var resource: String = "/", + // 限流维度 + var limitDimension: String = LimitDimension.URL.name, + // 限流值 + var limit: Long = -1, + // 限流周期 + var duration: Duration = Duration.ofSeconds(1), + // 桶容量(令牌桶和漏桶使用) + var capacity: Long? = null, + // 生效范围 + var scope: String = WorkScope.LOCAL.name, + // 指定机器上运行 + var targets: List = emptyList() +) diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlRateLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlRateLimitRule.kt new file mode 100644 index 0000000000..c5eef29ac6 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlRateLimitRule.kt @@ -0,0 +1,97 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.url + +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResLimitInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +/** + * URL限流配置规则实现 + */ +class UrlRateLimitRule : RateLimitRule { + + // 指定URL规则 + private val urlLimitRules: UrlResourceLimitRule = UrlResourceLimitRule() + + override fun isEmpty(): Boolean { + return urlLimitRules.isEmpty() + } + + override fun getRateLimitRule(resInfo: ResInfo): ResLimitInfo? { + var realResource = resInfo.resource + if (realResource.isBlank()) { + return null + } + var ruleLimit = urlLimitRules.getPathResourceLimit(realResource) + if (ruleLimit == null && resInfo.extraResource.isNotEmpty()) { + for (res in resInfo.extraResource) { + ruleLimit = urlLimitRules.getPathResourceLimit(res) + if (ruleLimit != null) { + realResource = res + break + } + } + } + if (ruleLimit == null) return null + return ResLimitInfo(realResource, ruleLimit) + } + + override fun addRateLimitRule(resourceLimit: ResourceLimit) { + filterResourceLimit(resourceLimit) + urlLimitRules.addUrlResourceLimit(resourceLimit) + } + + override fun addRateLimitRules(resourceLimit: List) { + resourceLimit.forEach { + try { + addRateLimitRule(it) + } catch (e: Exception) { + logger.error("add config $it for ${this.javaClass.simpleName} failed, error is ${e.message}") + } + } + } + + override fun filterResourceLimit(resourceLimit: ResourceLimit) { + if (resourceLimit.limitDimension != LimitDimension.URL.name) { + throw InvalidResourceException(resourceLimit.limitDimension) + } + if (resourceLimit.resource.isBlank()) { + throw InvalidResourceException(resourceLimit.resource) + } + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(UrlRateLimitRule::class.java) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlRepoRateLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlRepoRateLimitRule.kt new file mode 100644 index 0000000000..5a698ded10 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlRepoRateLimitRule.kt @@ -0,0 +1,98 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.url + +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.PathResourceLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResLimitInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +/** + * 基于项目/仓库的URL限流配置规则实现 + */ +class UrlRepoRateLimitRule : RateLimitRule { + + val urlRepoLimitRules: PathResourceLimitRule = PathResourceLimitRule(pathLengthCheck = true) + + override fun isEmpty(): Boolean { + return urlRepoLimitRules.isEmpty() + } + + override fun getRateLimitRule(resInfo: ResInfo): ResLimitInfo? { + var realResource = resInfo.resource + if (realResource.isBlank()) { + return null + } + var ruleLimit = urlRepoLimitRules.getPathResourceLimit(realResource) + if (ruleLimit == null && resInfo.extraResource.isNotEmpty()) { + for (res in resInfo.extraResource) { + ruleLimit = urlRepoLimitRules.getPathResourceLimit(res) + if (ruleLimit != null) { + realResource = res + break + } + } + } + if (ruleLimit == null) return null + return ResLimitInfo(realResource, ruleLimit) + } + + override fun addRateLimitRule(resourceLimit: ResourceLimit) { + filterResourceLimit(resourceLimit) + urlRepoLimitRules.addPathResourceLimit(resourceLimit, urlRepoDimensionList) + } + + override fun addRateLimitRules(resourceLimit: List) { + resourceLimit.forEach { + try { + addRateLimitRule(it) + } catch (e: Exception) { + logger.error("add config $it for ${this.javaClass.simpleName} failed, error is ${e.message}") + } + } + } + + override fun filterResourceLimit(resourceLimit: ResourceLimit) { + if (resourceLimit.limitDimension != LimitDimension.URL_REPO.name) { + throw InvalidResourceException(resourceLimit.limitDimension) + } + if (resourceLimit.resource.isBlank()) { + throw InvalidResourceException(resourceLimit.resource) + } + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(UrlRepoRateLimitRule::class.java) + private val urlRepoDimensionList = listOf(LimitDimension.URL_REPO.name) + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlResourceLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlResourceLimitRule.kt new file mode 100644 index 0000000000..b921a4c24c --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlResourceLimitRule.kt @@ -0,0 +1,60 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.url + +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.rule.PathResourceLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.PathNode +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit + +/** + * url资源规则类: + * 如果配置了父路径，则子路径也会遵循父路径对应的配置规则，即每个子路径单独拥有对应父路径所配置的限流规则； + * 如果配置指定路径，则只有该指定路径遵循对应的配置规则，即该指定路径独享所配置的限流规则。 + */ +class UrlResourceLimitRule( + root: PathNode = PathNode("/") +) : PathResourceLimitRule(root) { + + /** + * 添加URL对应的规则 + */ + fun addUrlResourceLimit(resourceLimit: ResourceLimit) { + addPathResourceLimit(resourceLimit, urlDimensionList) + } + + fun addUrlResourceLimits(resourceLimits: List) { + resourceLimits.forEach { + addUrlResourceLimit(it) + } + } + + companion object { + private val urlDimensionList = listOf(LimitDimension.URL.name) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRateLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRateLimitRule.kt new file mode 100644 index 0000000000..9a00605c8a --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRateLimitRule.kt @@ -0,0 +1,128 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.url.user + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResLimitInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.utils.ResourcePathUtils.buildUserPath +import com.tencent.bkrepo.common.ratelimiter.utils.ResourcePathUtils.getUserAndPath +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import java.util.concurrent.ConcurrentHashMap + +/** + * 用户级别的URL限流配置规则实现 + */ +class UserUrlRateLimitRule : RateLimitRule { + + // 用户+指定url对应规则(限制每个用户对应URL的请求次数) + private val userUrlLimitRules: ConcurrentHashMap = ConcurrentHashMap() + + // 用户对应规则(限制每个用户的总请求次数) + private val userLimitRules: ConcurrentHashMap = ConcurrentHashMap() + + override fun isEmpty(): Boolean { + return userUrlLimitRules.isEmpty() && userLimitRules.isEmpty() + } + + override fun getRateLimitRule(resInfo: ResInfo): ResLimitInfo? { + var resLimitInfo = findConfigRule(resInfo.resource) + if (resLimitInfo == null) { + resLimitInfo = findConfigRule(resInfo.resource, true) + } + if (resLimitInfo == null && resInfo.extraResource.isNotEmpty()) { + val res = resInfo.extraResource.first() + val (user, _) = getUserAndPath(res) + val userRule = userLimitRules[user] + if (userRule != null) { + resLimitInfo = ResLimitInfo(res, userRule) + } + if (resLimitInfo == null && userLimitRules.containsKey(StringPool.POUND)) { + resLimitInfo = ResLimitInfo(res, userLimitRules[StringPool.POUND]!!) + } + } + return resLimitInfo + } + + override fun addRateLimitRule(resourceLimit: ResourceLimit) { + filterResourceLimit(resourceLimit) + val (userId, path) = getUserAndPath(resourceLimit.resource) + if (path.isEmpty()) { + userLimitRules[userId] = resourceLimit + } else { + val userUrlResourceLimitRule = userUrlLimitRules.getOrDefault(userId, UserUrlResourceLimitRule()) + userUrlResourceLimitRule.addUserUrlResourceLimit(resourceLimit) + userUrlLimitRules.putIfAbsent(userId, userUrlResourceLimitRule) + } + } + + override fun addRateLimitRules(resourceLimit: List) { + resourceLimit.forEach { + try { + addRateLimitRule(it) + } catch (e: Exception) { + logger.error("add config $it for ${this.javaClass.simpleName} failed, error is ${e.message}") + } + } + } + + override fun filterResourceLimit(resourceLimit: ResourceLimit) { + if (resourceLimit.limitDimension != LimitDimension.USER_URL.name) { + throw InvalidResourceException(resourceLimit.limitDimension) + } + if (resourceLimit.resource.isBlank()) { + throw InvalidResourceException(resourceLimit.resource) + } + } + + /** + * 根据资源获取配置 + */ + private fun findConfigRule(resource: String, userPattern: Boolean = false): ResLimitInfo? { + if (resource.isBlank()) { + return null + } + var (user, resWithoutUser) = getUserAndPath(resource) + if (userPattern) { + user = StringPool.POUND + } + val userUrlRule = userUrlLimitRules[user] + val ruleLimit = userUrlRule?.getPathResourceLimit(resWithoutUser) ?: return null + val resourceLimitCopy = ruleLimit.copy(resource = buildUserPath(user, ruleLimit.resource)) + return ResLimitInfo(resource, resourceLimitCopy) + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(UserUrlRateLimitRule::class.java) + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRepoRateLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRepoRateLimitRule.kt new file mode 100644 index 0000000000..6c2bf11cc9 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRepoRateLimitRule.kt @@ -0,0 +1,138 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.url.user + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResLimitInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.utils.ResourcePathUtils +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import java.util.concurrent.ConcurrentHashMap + +class UserUrlRepoRateLimitRule( + // 用户+urlRepo对应规则 + private val userUrlRepoLimitRules: ConcurrentHashMap = ConcurrentHashMap(), + // 用户对应规则 + private val userLimitRules: ConcurrentHashMap = ConcurrentHashMap() +) : RateLimitRule { + + override fun isEmpty(): Boolean { + return userUrlRepoLimitRules.isEmpty() && userLimitRules.isEmpty() + } + + override fun getRateLimitRule(resInfo: ResInfo): ResLimitInfo? { + var resLimitInfo = findConfigRule(resInfo.resource, resInfo.extraResource) + if (resLimitInfo == null) { + resLimitInfo = findConfigRule(resInfo.resource, resInfo.extraResource, true) + } + if (resLimitInfo == null && resInfo.extraResource.isNotEmpty()) { + val res = resInfo.extraResource.last() + val (user, _) = ResourcePathUtils.getUserAndPath(res) + val userRule = userLimitRules[user] + if (userRule != null) { + resLimitInfo = ResLimitInfo(res, userRule) + } + if (resLimitInfo == null && userLimitRules.containsKey(StringPool.POUND)) { + resLimitInfo = ResLimitInfo(res, userLimitRules[StringPool.POUND]!!) + } + } + return resLimitInfo + } + + override fun addRateLimitRule(resourceLimit: ResourceLimit) { + filterResourceLimit(resourceLimit) + val (userId, path) = ResourcePathUtils.getUserAndPath(resourceLimit.resource) + if (path.isEmpty()) { + userLimitRules[userId] = resourceLimit + } else { + val userUrlRepoResourceLimitRule = + userUrlRepoLimitRules.getOrDefault(userId, UserUrlRepoResourceLimitRule()) + userUrlRepoResourceLimitRule.addUserUrlRepoResourceLimit(resourceLimit) + userUrlRepoLimitRules.putIfAbsent(userId, userUrlRepoResourceLimitRule) + } + } + + override fun addRateLimitRules(resourceLimit: List) { + resourceLimit.forEach { + try { + addRateLimitRule(it) + } catch (e: Exception) { + logger.error("add config $it for ${this.javaClass.simpleName} failed, error is ${e.message}") + } + } + } + + override fun filterResourceLimit(resourceLimit: ResourceLimit) { + if (resourceLimit.limitDimension != LimitDimension.USER_URL_REPO.name) { + throw InvalidResourceException(resourceLimit.limitDimension) + } + if (resourceLimit.resource.isBlank()) { + throw InvalidResourceException(resourceLimit.resource) + } + } + + /** + * 根据资源获取配置 + */ + private fun findConfigRule( + resource: String, extraResource: List, userPattern: Boolean = false + ): ResLimitInfo? { + var realResource = resource + if (realResource.isBlank()) { + return null + } + var (user, resWithoutUser) = ResourcePathUtils.getUserAndPath(realResource) + if (userPattern) { + user = StringPool.POUND + } + val userUsageRule = userUrlRepoLimitRules[user] + var ruleLimit = userUsageRule?.getPathResourceLimit(resWithoutUser) + if (ruleLimit == null && extraResource.isNotEmpty()) { + for (res in extraResource) { + var (_, resWithoutUser) = ResourcePathUtils.getUserAndPath(res) + ruleLimit = userUsageRule?.getPathResourceLimit(resWithoutUser) + if (ruleLimit != null) { + realResource = res + break + } + } + } + if (ruleLimit == null) return null + val resourceLimitCopy = ruleLimit.copy(resource = ResourcePathUtils.buildUserPath(user, ruleLimit.resource)) + return ResLimitInfo(realResource, resourceLimitCopy) + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(UserUrlRepoRateLimitRule::class.java) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRepoResourceLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRepoResourceLimitRule.kt new file mode 100644 index 0000000000..4f2f04937b --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRepoResourceLimitRule.kt @@ -0,0 +1,72 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.url.user + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.PathResourceLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.PathNode +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.utils.ResourcePathUtils.getUserAndPath +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +class UserUrlRepoResourceLimitRule( + root: PathNode = PathNode("/"), + pathLengthCheck: Boolean = true, + private var user: String = StringPool.POUND, +) : PathResourceLimitRule(root, pathLengthCheck) { + + fun addUserUrlRepoResourceLimit(resourceLimit: ResourceLimit) { + if (resourceLimit.limitDimension !in userUrlRepoDimensionList) { + return + } + val (userId, urlPath) = getUserAndPath(resourceLimit.resource) + if (!urlPath.startsWith("/") || user.isBlank()) { + logger.warn("$resourceLimit is invalid") + throw InvalidResourceException(urlPath) + } + user = userId + val resourceLimitCopy = resourceLimit.copy(resource = urlPath) + addPathResourceLimit(resourceLimitCopy, userUrlRepoDimensionList) + } + + fun addUserUrlRepoResourceLimits(resourceLimits: List) { + resourceLimits.forEach { + addUserUrlRepoResourceLimit(it) + } + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(UserUrlRepoResourceLimitRule::class.java) + private val userUrlRepoDimensionList = listOf( + LimitDimension.USER_URL_REPO.name + ) + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlResourceLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlResourceLimitRule.kt new file mode 100644 index 0000000000..892d837d1c --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlResourceLimitRule.kt @@ -0,0 +1,74 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.url.user + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.PathResourceLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.PathNode +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.utils.ResourcePathUtils.getUserAndPath +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +/** + * 用户+url对应规则 + * 如果配置了父路径，则该用户对应访问的子路径也会遵循父路径对应的配置规则，即每个子路径单独拥有对应父路径所配置的限流规则； + * 如果配置指定路径，则只有该用户访问的指定路径遵循对应的配置规则，即该指定路径独享所配置的限流规则。 + */ +class UserUrlResourceLimitRule( + root: PathNode = PathNode("/"), + private var user: String = StringPool.POUND +) : PathResourceLimitRule(root) { + + fun addUserUrlResourceLimit(resourceLimit: ResourceLimit) { + if (resourceLimit.limitDimension !in userLimitDimensionList) { + return + } + val (userId, urlPath) = getUserAndPath(resourceLimit.resource) + if (!urlPath.startsWith("/") || user.isBlank()) { + logger.warn("$resourceLimit is invalid") + throw InvalidResourceException(urlPath) + } + user = userId + val resourceLimitCopy = resourceLimit.copy(resource = urlPath) + addPathResourceLimit(resourceLimitCopy, userLimitDimensionList) + } + + fun addUserUrlResourceLimits(resourceLimits: List) { + resourceLimits.forEach { + addUserUrlResourceLimit(it) + } + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(UserUrlResourceLimitRule::class.java) + private val userLimitDimensionList = listOf(LimitDimension.USER_URL.name) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/DownloadUsageRateLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/DownloadUsageRateLimitRule.kt new file mode 100644 index 0000000000..c79f93af95 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/DownloadUsageRateLimitRule.kt @@ -0,0 +1,55 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.usage + +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +/** + * 下载用量限流配置规则实现 + * 如果配置了仓库级别限流配置，则该对应仓库下所有请求遵循这个配置，即同一仓库下所有请求共享这个容量限额； + * 如果配置了项目级别限流配置，则对应项目下所有请求遵循这个配置，即同一项目下所有请求共享这个容量限额。 + */ +class DownloadUsageRateLimitRule : UploadUsageRateLimitRule() { + + override fun filterResourceLimit(resourceLimit: ResourceLimit) { + if (resourceLimit.limitDimension != LimitDimension.DOWNLOAD_USAGE.name) { + throw InvalidResourceException(resourceLimit.limitDimension) + } + if (resourceLimit.resource.isBlank()) { + throw InvalidResourceException(resourceLimit.resource) + } + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(DownloadUsageRateLimitRule::class.java) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/UploadUsageRateLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/UploadUsageRateLimitRule.kt new file mode 100644 index 0000000000..55a5759094 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/UploadUsageRateLimitRule.kt @@ -0,0 +1,98 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.usage + +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.PathResourceLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResLimitInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +/** + * 上传用量限流配置规则实现 + */ +open class UploadUsageRateLimitRule : RateLimitRule { + + val usageLimitRules: PathResourceLimitRule = PathResourceLimitRule(pathLengthCheck = true) + + override fun isEmpty(): Boolean { + return usageLimitRules.isEmpty() + } + + override fun getRateLimitRule(resInfo: ResInfo): ResLimitInfo? { + var realResource = resInfo.resource + if (realResource.isBlank()) { + return null + } + var ruleLimit = usageLimitRules.getPathResourceLimit(realResource) + if (ruleLimit == null && resInfo.extraResource.isNotEmpty()) { + for (res in resInfo.extraResource) { + ruleLimit = usageLimitRules.getPathResourceLimit(res) + if (ruleLimit != null) { + realResource = res + break + } + } + } + if (ruleLimit == null) return null + return ResLimitInfo(realResource, ruleLimit) + } + + override fun addRateLimitRule(resourceLimit: ResourceLimit) { + filterResourceLimit(resourceLimit) + usageLimitRules.addPathResourceLimit(resourceLimit, usageDimensionList) + } + + override fun addRateLimitRules(resourceLimit: List) { + resourceLimit.forEach { + try { + addRateLimitRule(it) + } catch (e: Exception) { + logger.error("add config $it for ${this.javaClass.simpleName} failed, error is ${e.message}") + } + } + } + + override fun filterResourceLimit(resourceLimit: ResourceLimit) { + if (resourceLimit.limitDimension != LimitDimension.UPLOAD_USAGE.name) { + throw InvalidResourceException(resourceLimit.limitDimension) + } + if (resourceLimit.resource.isBlank()) { + throw InvalidResourceException(resourceLimit.resource) + } + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(UploadUsageRateLimitRule::class.java) + private val usageDimensionList = listOf(LimitDimension.UPLOAD_USAGE.name, LimitDimension.DOWNLOAD_USAGE.name) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserDownloadUsageRateLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserDownloadUsageRateLimitRule.kt new file mode 100644 index 0000000000..49d24c8548 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserDownloadUsageRateLimitRule.kt @@ -0,0 +1,57 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.usage.user + +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import java.util.concurrent.ConcurrentHashMap + +/** + * 用户下载用量限流规则实现 + */ +class UserDownloadUsageRateLimitRule( + userUsageLimitRules: ConcurrentHashMap = ConcurrentHashMap(), + userLimitRules: ConcurrentHashMap = ConcurrentHashMap() +) : UserUploadUsageRateLimitRule(userUsageLimitRules, userLimitRules) { + + override fun filterResourceLimit(resourceLimit: ResourceLimit) { + if (resourceLimit.limitDimension != LimitDimension.USER_DOWNLOAD_USAGE.name) { + throw InvalidResourceException(resourceLimit.limitDimension) + } + if (resourceLimit.resource.isBlank()) { + throw InvalidResourceException(resourceLimit.resource) + } + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(UserDownloadUsageRateLimitRule::class.java) + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserUploadUsageRateLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserUploadUsageRateLimitRule.kt new file mode 100644 index 0000000000..3f79d1b61a --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserUploadUsageRateLimitRule.kt @@ -0,0 +1,140 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.usage.user + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResLimitInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.utils.ResourcePathUtils +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import java.util.concurrent.ConcurrentHashMap + +/** + * 用户上传用量限流配置规则实现 + */ +open class UserUploadUsageRateLimitRule( + // 用户+用量对应规则 + private val userUsageLimitRules: ConcurrentHashMap = ConcurrentHashMap(), + // 用户对应规则 + private val userLimitRules: ConcurrentHashMap = ConcurrentHashMap() +) : RateLimitRule { + + override fun isEmpty(): Boolean { + return userUsageLimitRules.isEmpty() && userLimitRules.isEmpty() + } + + override fun getRateLimitRule(resInfo: ResInfo): ResLimitInfo? { + var resLimitInfo = findConfigRule(resInfo.resource, resInfo.extraResource) + if (resLimitInfo == null) { + resLimitInfo = findConfigRule(resInfo.resource, resInfo.extraResource, true) + } + if (resLimitInfo == null && resInfo.extraResource.isNotEmpty()) { + val res = resInfo.extraResource.last() + val (user, _) = ResourcePathUtils.getUserAndPath(res) + val userRule = userLimitRules[user] + if (userRule != null) { + resLimitInfo = ResLimitInfo(res, userRule) + } + if (resLimitInfo == null && userLimitRules.containsKey(StringPool.POUND)) { + resLimitInfo = ResLimitInfo(res, userLimitRules[StringPool.POUND]!!) + } + } + return resLimitInfo + } + + override fun addRateLimitRule(resourceLimit: ResourceLimit) { + filterResourceLimit(resourceLimit) + val (userId, path) = ResourcePathUtils.getUserAndPath(resourceLimit.resource) + if (path.isEmpty()) { + userLimitRules[userId] = resourceLimit + } else { + val userUsageResourceLimitRule = userUsageLimitRules.getOrDefault(userId, UserUsageResourceLimitRule()) + userUsageResourceLimitRule.addUserUsageResourceLimit(resourceLimit) + userUsageLimitRules.putIfAbsent(userId, userUsageResourceLimitRule) + } + } + + override fun addRateLimitRules(resourceLimit: List) { + resourceLimit.forEach { + try { + addRateLimitRule(it) + } catch (e: Exception) { + logger.error("add config $it for ${this.javaClass.simpleName} failed, error is ${e.message}") + } + } + } + + override fun filterResourceLimit(resourceLimit: ResourceLimit) { + if (resourceLimit.limitDimension != LimitDimension.USER_UPLOAD_USAGE.name) { + throw InvalidResourceException(resourceLimit.limitDimension) + } + if (resourceLimit.resource.isBlank()) { + throw InvalidResourceException(resourceLimit.resource) + } + } + + /** + * 根据资源获取配置 + */ + private fun findConfigRule( + resource: String, extraResource: List, userPattern: Boolean = false + ): ResLimitInfo? { + var realResource = resource + if (realResource.isBlank()) { + return null + } + var (user, resWithoutUser) = ResourcePathUtils.getUserAndPath(realResource) + if (userPattern) { + user = StringPool.POUND + } + val userUsageRule = userUsageLimitRules[user] + var ruleLimit = userUsageRule?.getPathResourceLimit(resWithoutUser) + if (ruleLimit == null && extraResource.isNotEmpty()) { + for (res in extraResource) { + var (_, resWithoutUser) = ResourcePathUtils.getUserAndPath(res) + ruleLimit = userUsageRule?.getPathResourceLimit(resWithoutUser) + if (ruleLimit != null) { + realResource = res + break + } + } + } + if (ruleLimit == null) return null + val resourceLimitCopy = ruleLimit.copy(resource = ResourcePathUtils.buildUserPath(user, ruleLimit.resource)) + return ResLimitInfo(realResource, resourceLimitCopy) + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(UserUploadUsageRateLimitRule::class.java) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserUsageResourceLimitRule.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserUsageResourceLimitRule.kt new file mode 100644 index 0000000000..0ae86f9fef --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserUsageResourceLimitRule.kt @@ -0,0 +1,78 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.usage.user + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.PathResourceLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.PathNode +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.utils.ResourcePathUtils.getUserAndPath +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +/** + * 用户+用量对应规则 + * 如果配置了仓库级别限流配置，则该用户下对应仓库下所有请求遵循这个配置，即同一仓库下所有请求共享这个容量限额； + * 如果配置了项目级别限流配置，则用户下对应项目下所有请求遵循这个配置，即同一项目下所有请求共享这个容量限额。 + */ +class UserUsageResourceLimitRule( + root: PathNode = PathNode("/"), + pathLengthCheck: Boolean = true, + private var user: String = StringPool.POUND, +) : PathResourceLimitRule(root, pathLengthCheck) { + + fun addUserUsageResourceLimit(resourceLimit: ResourceLimit) { + if (resourceLimit.limitDimension !in userUsageDimensionList) { + return + } + val (userId, urlPath) = getUserAndPath(resourceLimit.resource) + if (!urlPath.startsWith("/") || user.isBlank()) { + logger.warn("$resourceLimit is invalid") + throw InvalidResourceException(urlPath) + } + user = userId + val resourceLimitCopy = resourceLimit.copy(resource = urlPath) + addPathResourceLimit(resourceLimitCopy, userUsageDimensionList) + } + + fun addUserUsageResourceLimits(resourceLimits: List) { + resourceLimits.forEach { + addUserUsageResourceLimit(it) + } + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(UserUsageResourceLimitRule::class.java) + private val userUsageDimensionList = listOf( + LimitDimension.USER_DOWNLOAD_USAGE.name, + LimitDimension.USER_UPLOAD_USAGE.name + ) + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/AbstractBandwidthRateLimiterService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/AbstractBandwidthRateLimiterService.kt new file mode 100644 index 0000000000..cdcea5f9e0 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/AbstractBandwidthRateLimiterService.kt @@ -0,0 +1,158 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service + +import com.tencent.bkrepo.common.ratelimiter.algorithm.RateLimiter +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import com.tencent.bkrepo.common.ratelimiter.stream.CommonRateLimitInputStream +import com.tencent.bkrepo.common.ratelimiter.stream.RateCheckContext +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.util.unit.DataSize +import java.io.InputStream +import javax.servlet.http.HttpServletRequest + +/** + * 带宽限流器抽象实现 + */ +abstract class AbstractBandwidthRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterProperties: RateLimiterProperties, + rateLimiterConfigService: RateLimiterConfigService, +) : AbstractRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService +) { + + override fun limit(request: HttpServletRequest, applyPermits: Long?) { + throw UnsupportedOperationException() + } + + /** + * 根据资源返回对应带限流实现的InputStream + */ + fun bandwidthRateStart( + request: HttpServletRequest, + inputStream: InputStream, + circuitBreakerPerSecond: DataSize, + rangeLength: Long? = null, + ): CommonRateLimitInputStream? { + val resLimitInfo = getResLimitInfo(request) ?: return null + logger.info("will check the bandwidth with length $rangeLength of ${resLimitInfo.resource}") + return try { + interceptorChain.doBeforeLimitCheck(resLimitInfo.resource, resLimitInfo.resourceLimit) + circuitBreakerCheck(resLimitInfo.resourceLimit, circuitBreakerPerSecond.toBytes()) + val rateLimiter = getAlgorithmOfRateLimiter(resLimitInfo.resource, resLimitInfo.resourceLimit) + val context = RateCheckContext( + rateLimiter = rateLimiter, latency = rateLimiterProperties.latency, + waitRound = rateLimiterProperties.waitRound, rangeLength = rangeLength, + dryRun = rateLimiterProperties.dryRun, permitsOnce = rateLimiterProperties.permitsOnce, + limitPerSecond = getPermitsPerSecond(resLimitInfo.resourceLimit) + ) + CommonRateLimitInputStream( + delegate = inputStream, + rateCheckContext = context + ) + } catch (e: AcquireLockFailedException) { + logger.warn( + "acquire lock failed for ${resLimitInfo.resource} " + + "with ${resLimitInfo.resourceLimit}, e: ${e.message}" + ) + null + } catch (e: InvalidResourceException) { + logger.warn("${resLimitInfo.resourceLimit} is invalid for ${resLimitInfo.resource} , e: ${e.message}") + null + } + } + + fun bandwidthRateLimitFinish( + request: HttpServletRequest, + exception: Exception? = null, + ) { + val resLimitInfo = getResLimitInfo(request) ?: return + afterRateLimitCheck(resLimitInfo, exception == null, exception) + } + + fun bandwidthRateLimit( + request: HttpServletRequest, + permits: Long, + circuitBreakerPerSecond: DataSize, + ) { + val resLimitInfo = getResLimitInfo(request) ?: return + rateLimitCatch( + request = request, + resLimitInfo = resLimitInfo, + applyPermits = permits, + circuitBreakerPerSecond = circuitBreakerPerSecond.toBytes() + ) { rateLimiter, realPermits -> + bandwidthLimitHandler(rateLimiter, realPermits) + } + } + + fun bandwidthLimitHandler( + rateLimiter: RateLimiter, + permits: Long + ): Boolean { + var flag = false + var failedNum = 0 + while (!flag) { + flag = rateLimiter.tryAcquire(permits) + if (!flag && failedNum < rateLimiterProperties.waitRound) { + failedNum++ + try { + Thread.sleep(rateLimiterProperties.latency) + } catch (ignore: InterruptedException) { + } + } + if (!flag && failedNum >= rateLimiterProperties.waitRound) { + return false + } + } + return true + } + + private fun getPermitsPerSecond(resourceLimit: ResourceLimit): Long { + return resourceLimit.limit / resourceLimit.duration.seconds + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger(AbstractBandwidthRateLimiterService::class.java) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/AbstractRateLimiterService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/AbstractRateLimiterService.kt new file mode 100644 index 0000000000..a9c3c60457 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/AbstractRateLimiterService.kt @@ -0,0 +1,560 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service + + +import com.fasterxml.jackson.module.kotlin.readValue +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.api.util.JsonUtils.objectMapper +import com.tencent.bkrepo.common.api.util.readJsonString +import com.tencent.bkrepo.common.artifact.constant.PROJECT_ID +import com.tencent.bkrepo.common.artifact.constant.REPO_NAME +import com.tencent.bkrepo.common.query.enums.OperationType +import com.tencent.bkrepo.common.query.model.QueryModel +import com.tencent.bkrepo.common.query.model.Rule +import com.tencent.bkrepo.common.ratelimiter.algorithm.DistributedFixedWindowRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.DistributedLeakyRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.DistributedSlidingWindowRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.DistributedTokenBucketRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.FixedWindowRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.LeakyRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.RateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.SlidingWindowRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.TokenBucketRateLimiter +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.interceptor.MonitorRateLimiterInterceptorAdaptor +import com.tencent.bkrepo.common.ratelimiter.interceptor.RateLimiterInterceptor +import com.tencent.bkrepo.common.ratelimiter.interceptor.RateLimiterInterceptorChain +import com.tencent.bkrepo.common.ratelimiter.interceptor.TargetRateLimiterInterceptorAdaptor +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.bandwidth.DownloadBandwidthRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.bandwidth.UploadBandwidthRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResLimitInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.url.UrlRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.url.UrlRepoRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.url.user.UserUrlRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.url.user.UserUrlRepoRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.usage.DownloadUsageRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.usage.UploadUsageRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.usage.user.UserDownloadUsageRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.usage.user.UserUploadUsageRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import com.tencent.bkrepo.common.service.servlet.MultipleReadHttpRequest +import java.util.concurrent.ConcurrentHashMap +import javax.servlet.http.HttpServletRequest +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Value +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.http.HttpMethod +import org.springframework.http.MediaType +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.util.unit.DataSize +import org.springframework.web.servlet.HandlerMapping + +/** + * 限流器抽象实现 + */ +abstract class AbstractRateLimiterService( + private val taskScheduler: ThreadPoolTaskScheduler, + val rateLimiterProperties: RateLimiterProperties, + private val rateLimiterMetrics: RateLimiterMetrics, + private val redisTemplate: RedisTemplate? = null, + private val rateLimiterConfigService: RateLimiterConfigService, +) : RateLimiterService { + + @Value("\${spring.application.name}") + var moduleName: String = StringPool.EMPTY + + + // 资源对应限限流算法缓存 + private var rateLimiterCache: ConcurrentHashMap = ConcurrentHashMap(256) + + val interceptorChain: RateLimiterInterceptorChain = + RateLimiterInterceptorChain( + mutableListOf( + MonitorRateLimiterInterceptorAdaptor(rateLimiterMetrics), + TargetRateLimiterInterceptorAdaptor(rateLimiterConfigService) + ) + ) + + // 限流规则配置 + var rateLimitRule: RateLimitRule? = null + + // 当前限流规则配置hashcode + var currentRuleHashCode: Int? = null + + init { + taskScheduler.scheduleWithFixedDelay(this::refreshRateLimitRule, rateLimiterProperties.refreshDuration * 1000) + } + + /** + * 获取对应资源限流规则配置 + */ + fun getResLimitInfo(request: HttpServletRequest): ResLimitInfo? { + if (!rateLimiterProperties.enabled) { + return null + } + val resLimitInfo = try { + val resInfo = ResInfo( + resource = buildResource(request), + extraResource = buildExtraResource(request) + ) + rateLimitRule?.getRateLimitRule(resInfo) + } catch (e: InvalidResourceException) { + logger.warn("Config is invalid for request ${request.requestURI}, e: ${e.message}") + null + } + + if (resLimitInfo == null) { + return null + } + return resLimitInfo + } + + override fun limit(request: HttpServletRequest, applyPermits: Long?) { + if (!rateLimiterProperties.enabled) { + return + } + if (ignoreRequest(request)) return + if (rateLimitRule == null || rateLimitRule!!.isEmpty()) return + val resLimitInfo = getResLimitInfo(request) ?: return + rateLimitCatch( + request = request, + resLimitInfo = resLimitInfo, + applyPermits = applyPermits, + ) { rateLimiter, permits -> + rateLimiter.tryAcquire(permits) + } + } + + override fun addInterceptor(interceptor: RateLimiterInterceptor) { + this.interceptorChain.addInterceptor(interceptor) + } + + override fun addInterceptors(interceptors: List) { + if (interceptors.isNotEmpty()) { + this.interceptorChain.addInterceptors(interceptors) + } + } + + /** + * 生成资源对应的唯一key + */ + abstract fun generateKey(resource: String, resourceLimit: ResourceLimit): String + + /** + * 根据请求获取对应的资源，用于查找对应限流规则 + */ + abstract fun buildResource(request: HttpServletRequest): String + + /** + * 根据请求获取对其他资源信息，用于查找对应限流规则 + */ + abstract fun buildExtraResource(request: HttpServletRequest): List + + /** + * 根据请求获取需要申请的许可数 + */ + abstract fun getApplyPermits(request: HttpServletRequest, applyPermits: Long?): Long + + /** + * 限流器实现对应的维度 + */ + abstract fun getLimitDimensions(): List + + /** + * 获取对应限流规则配置实现 + */ + abstract fun getRateLimitRuleClass(): Class + + /** + * 对请求进行过滤，不进行限流 + */ + open fun ignoreRequest(request: HttpServletRequest): Boolean { + return false + } + + /** + * 根据资源和限流规则生成对应限流算法 + */ + open fun createAlgorithmOfRateLimiter(resource: String, resourceLimit: ResourceLimit): RateLimiter { + if (resourceLimit.limit < 0) { + throw InvalidResourceException("config limit is ${resourceLimit.limit}") + } + return when (resourceLimit.algo) { + Algorithms.FIXED_WINDOW.name -> { + buildFixedWindowRateLimiter(resource, resourceLimit) + } + + Algorithms.TOKEN_BUCKET.name -> { + buildTokenBucketRateLimiter(resource, resourceLimit) + } + + Algorithms.SLIDING_WINDOW.name -> { + buildSlidingWindowRateLimiter(resource, resourceLimit) + } + + Algorithms.LEAKY_BUCKET.name -> { + buildLeakyRateLimiter(resource, resourceLimit) + } + + else -> { + throw InvalidResourceException("config algo is ${resourceLimit.algo}") + } + } + } + + fun getRepoInfoFromAttribute(request: HttpServletRequest): Pair { + var projectId: String? = null + var repoName: String? = null + try { + projectId = ((request.getAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE)) + as Map<*, *>)["projectId"] as String? + repoName = ((request.getAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE)) + as Map<*, *>)["repoName"] as String? + } catch (ignore: Exception) { + } + if (projectId.isNullOrEmpty()) { + throw InvalidResourceException("Could not find projectId from request ${request.requestURI}") + } + return Pair(projectId, repoName) + } + + fun getRepoInfoFromBody(request: HttpServletRequest): Pair { + val limit = DataSize.ofMegabytes(1).toBytes() + val lengthCondition = request.contentLength in 1..limit + val typeCondition = request.contentType?.startsWith(MediaType.APPLICATION_JSON_VALUE) == true + // 限制缓存大小 + if (lengthCondition && typeCondition) { + val multiReadRequest = MultipleReadHttpRequest(request, limit) + val (projectId, repoName) = getRepoInfoFromQueryModel(multiReadRequest) + if (!projectId.isNullOrEmpty()) return Pair(projectId, repoName) + val (newProjectId, newRepoName) = getRepoInfoFromOtherRequest(multiReadRequest) + if (newProjectId.isNullOrEmpty()) { + throw InvalidResourceException("Could not find projectId from request ${request.requestURI}") + } + return Pair(newProjectId, newRepoName) + } + throw InvalidResourceException("Could not find projectId from body of request ${request.requestURI}") + } + + private fun getRepoInfoFromQueryModel(multiReadRequest: MultipleReadHttpRequest): Pair { + try { + val queryModel = multiReadRequest.inputStream.readJsonString() + val rule = queryModel.rule + if (rule is Rule.NestedRule && rule.relation == Rule.NestedRule.RelationType.AND) { + return findRepoInfoFromRule(rule) + } + } catch (ignore: Exception) { + } + return Pair(null, null) + } + + private fun getRepoInfoFromOtherRequest(multiReadRequest: MultipleReadHttpRequest): Pair { + try { + val mappedValue = objectMapper.readValue>(multiReadRequest.inputStream) + return Pair((mappedValue[PROJECT_ID] as? String), (mappedValue[REPO_NAME] as? String)) + } catch (ignore: Exception) { + return Pair(null, null) + } + } + + private fun findRepoInfoFromRule(rule: Rule.NestedRule): Pair { + var projectId: String? = null + var repoName: String? = null + findKeyRule(PROJECT_ID, rule.rules)?.let { + it.value.toString().apply { projectId = this } + } + findKeyRule(REPO_NAME, rule.rules)?.let { + if (it.operation == OperationType.EQ) { + it.value.toString().apply { repoName = this } + } + } + return Pair(projectId, repoName) + } + + private fun findKeyRule(key: String, rules: List): Rule.QueryRule? { + for (rule in rules) { + if (rule is Rule.QueryRule && rule.field == key) { + return rule + } + } + return null + } + + /** + * 配置规则刷新 + */ + fun refreshRateLimitRule() { + if (!rateLimiterProperties.enabled) return + val usageRuleConfigs = rateLimiterProperties.rules.filter { + it.limitDimension in getLimitDimensions() + } + val databaseConfig = try { + rateLimiterConfigService.findByModuleNameAndLimitDimension( + moduleName, getLimitDimensions().first() + ) + } catch (e: Exception) { + logger.error("system error: $e") + listOf() + } + val configs = usageRuleConfigs.plus(databaseConfig.map { tRateLimit -> + ResourceLimit( + algo = tRateLimit.algo, + resource = tRateLimit.resource, + limit = tRateLimit.limit, + limitDimension = tRateLimit.limitDimension, + duration = tRateLimit.duration, + capacity = tRateLimit.capacity, + scope = tRateLimit.scope, + targets = tRateLimit.targets + ) + }) + // 配置规则变更后需要清理缓存的限流算法实现 + val newRuleHashCode = configs.hashCode() + if (currentRuleHashCode == newRuleHashCode) { + if (rateLimiterCache.size > rateLimiterProperties.cacheCapacity) { + clearLimiterCache() + } + return + } + val usageRules = getRuleClass() ?: return + usageRules.addRateLimitRules(configs) + rateLimitRule = usageRules + clearLimiterCache() + currentRuleHashCode = newRuleHashCode + logger.info("rules in ${this.javaClass.simpleName} for request has been refreshed!") + } + + private fun getRuleClass(): RateLimitRule? { + return when (getRateLimitRuleClass()) { + UrlRateLimitRule::class.java -> UrlRateLimitRule() + UploadUsageRateLimitRule::class.java -> UploadUsageRateLimitRule() + DownloadUsageRateLimitRule::class.java -> DownloadUsageRateLimitRule() + UserDownloadUsageRateLimitRule::class.java -> UserDownloadUsageRateLimitRule() + UserUploadUsageRateLimitRule::class.java -> UserUploadUsageRateLimitRule() + UserUrlRateLimitRule::class.java -> UserUrlRateLimitRule() + UploadBandwidthRateLimitRule::class.java -> UploadBandwidthRateLimitRule() + DownloadBandwidthRateLimitRule::class.java -> DownloadBandwidthRateLimitRule() + UrlRepoRateLimitRule::class.java -> UrlRepoRateLimitRule() + UserUrlRepoRateLimitRule::class.java -> UserUrlRepoRateLimitRule() + else -> null + } + } + + private fun clearLimiterCache() { + rateLimiterCache.forEach { + try { + it.value.removeCacheLimit(it.key) + } catch (e: Exception) { + logger.warn("clear limiter cache error: ${e.cause}, ${e.message}") + } + } + rateLimiterCache.clear() + } + + private fun beforeRateLimitCheck( + request: HttpServletRequest, + applyPermits: Long? = null, + resLimitInfo: ResLimitInfo, + circuitBreakerPerSecond: Long? = null, + ): Pair { + with(resLimitInfo) { + val realPermits = getApplyPermits(request, applyPermits) + interceptorChain.doBeforeLimitCheck(resource, resourceLimit) + circuitBreakerCheck(resourceLimit, circuitBreakerPerSecond) + val rateLimiter = getAlgorithmOfRateLimiter(resource, resourceLimit) + return Pair(rateLimiter, realPermits) + } + } + + fun afterRateLimitCheck( + resLimitInfo: ResLimitInfo, + pass: Boolean, + exception: Exception? = null, + ) { + with(resLimitInfo) { + interceptorChain.doAfterLimitCheck(resource, resourceLimit, pass, exception) + } + } + + fun rateLimitCatch( + request: HttpServletRequest, + resLimitInfo: ResLimitInfo, + applyPermits: Long? = null, + circuitBreakerPerSecond: Long? = null, + action: (RateLimiter, Long) -> Boolean + ) { + var exception: Exception? = null + var pass = false + try { + val (rateLimiter, permits) = beforeRateLimitCheck( + request = request, + applyPermits = applyPermits, + resLimitInfo = resLimitInfo, + circuitBreakerPerSecond = circuitBreakerPerSecond + ) + val realPermits = permits + pass = action(rateLimiter, realPermits) + if (!pass) { + val msg = "${resLimitInfo.resource} has exceeded max rate limit: " + + "${resLimitInfo.resourceLimit.limit} /${resLimitInfo.resourceLimit.duration}" + if (rateLimiterProperties.dryRun) { + logger.warn(msg) + } else { + throw OverloadException(msg) + } + } + } catch (e: OverloadException) { + throw e + } catch (e: AcquireLockFailedException) { + logger.warn( + "acquire lock failed for ${resLimitInfo.resource}" + + " with ${resLimitInfo.resourceLimit}, e: ${e.message}" + ) + exception = e + } catch (e: InvalidResourceException) { + logger.warn("${resLimitInfo.resourceLimit} is invalid ${resLimitInfo.resource} , e: ${e.message}") + exception = e + } catch (e: Exception) { + logger.error("internal error: $e") + exception = e + } finally { + afterRateLimitCheck(resLimitInfo, pass, exception) + } + } + + /** + * 获取对应限流算法实现 + */ + fun getAlgorithmOfRateLimiter( + resource: String, resourceLimit: ResourceLimit + ): RateLimiter { + val limitKey = generateKey(resource, resourceLimit) + var rateLimiter = rateLimiterCache[limitKey] + if (rateLimiter == null) { + val newRateLimiter = createAlgorithmOfRateLimiter(limitKey, resourceLimit) + rateLimiter = rateLimiterCache.putIfAbsent(limitKey, newRateLimiter) + if (rateLimiter == null) { + rateLimiter = newRateLimiter + } + } + return rateLimiter + } + + /** + * （特殊操作）如果配置的限流规则比熔断配置小，则直接限流 + */ + fun circuitBreakerCheck( + resourceLimit: ResourceLimit, + circuitBreakerPerSecond: Long? = null, + ) { + if (circuitBreakerPerSecond == null) return + val permitsPerSecond = resourceLimit.limit / resourceLimit.duration.seconds + if (circuitBreakerPerSecond >= permitsPerSecond) { + throw OverloadException( + "The circuit breaker is activated when too many download requests are made to the service!" + ) + } + } + + private fun buildFixedWindowRateLimiter( + resource: String, + resourceLimit: ResourceLimit, + ): RateLimiter { + return if (resourceLimit.scope == WorkScope.LOCAL.name) { + FixedWindowRateLimiter(resourceLimit.limit, resourceLimit.duration) + } else { + DistributedFixedWindowRateLimiter( + resource, resourceLimit.limit, resourceLimit.duration, redisTemplate!! + ) + } + } + + private fun buildTokenBucketRateLimiter( + resource: String, + resourceLimit: ResourceLimit, + ): RateLimiter { + val permitsPerSecond = (resourceLimit.limit / resourceLimit.duration.seconds.toDouble()) + return if (resourceLimit.scope == WorkScope.LOCAL.name) { + TokenBucketRateLimiter(permitsPerSecond) + } else { + if (resourceLimit.capacity == null || resourceLimit.capacity!! <= 0) { + throw InvalidResourceException("Resource limit config $resourceLimit is illegal") + } + DistributedTokenBucketRateLimiter( + resource, permitsPerSecond, resourceLimit.capacity!!, redisTemplate!! + ) + } + } + + private fun buildSlidingWindowRateLimiter( + resource: String, + resourceLimit: ResourceLimit, + ): RateLimiter { + return if (resourceLimit.scope == WorkScope.LOCAL.name) { + SlidingWindowRateLimiter(resourceLimit.limit, resourceLimit.duration) + } else { + DistributedSlidingWindowRateLimiter( + resource, resourceLimit.limit, resourceLimit.duration, redisTemplate!! + ) + } + } + + private fun buildLeakyRateLimiter( + resource: String, + resourceLimit: ResourceLimit, + ): RateLimiter { + if (resourceLimit.capacity == null || resourceLimit.capacity!! <= 0) { + throw InvalidResourceException("Resource limit config $resourceLimit is illegal") + } + val rate = resourceLimit.limit / resourceLimit.duration.seconds.toDouble() + return if (resourceLimit.scope == WorkScope.LOCAL.name) { + LeakyRateLimiter(rate, resourceLimit.capacity!!) + } else { + DistributedLeakyRateLimiter( + resource, rate, resourceLimit.capacity!!, redisTemplate!! + ) + } + } + + companion object { + val logger: Logger = LoggerFactory.getLogger(AbstractRateLimiterService::class.java) + val UPLOAD_REQUEST_METHOD = listOf(HttpMethod.POST.name, HttpMethod.PUT.name, HttpMethod.PATCH.name) + val DOWNLOAD_REQUEST_METHOD = listOf(HttpMethod.GET.name) + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/RateLimiterService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/RateLimiterService.kt new file mode 100644 index 0000000000..5d1e0761d0 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/RateLimiterService.kt @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service + +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.ratelimiter.interceptor.RateLimiterInterceptor +import javax.servlet.http.HttpServletRequest + +interface RateLimiterService { + + /** + * 限流判断 + */ + @Throws(AcquireLockFailedException::class, InvalidResourceException::class, OverloadException::class) + fun limit(request: HttpServletRequest, applyPermits: Long? = null) + + fun addInterceptor(interceptor: RateLimiterInterceptor) + + fun addInterceptors(interceptors: List) +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/RequestLimitCheckService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/RequestLimitCheckService.kt new file mode 100644 index 0000000000..315ebe8011 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/RequestLimitCheckService.kt @@ -0,0 +1,176 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service + +import com.tencent.bkrepo.common.ratelimiter.RateLimiterAutoConfiguration +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.service.bandwidth.DownloadBandwidthRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.bandwidth.UploadBandwidthRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.url.UrlRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.url.UrlRepoRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.url.user.UserUrlRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.url.user.UserUrlRepoRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.usage.DownloadUsageRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.usage.UploadUsageRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.usage.user.UserDownloadUsageRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.usage.user.UserUploadUsageRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.stream.CommonRateLimitInputStream +import com.tencent.bkrepo.common.service.util.HttpContextHolder +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.beans.factory.annotation.Qualifier +import org.springframework.util.unit.DataSize +import java.io.InputStream +import javax.servlet.http.HttpServletRequest + +class RequestLimitCheckService( + private val rateLimiterProperties: RateLimiterProperties, +) { + + @Autowired + @Qualifier(RateLimiterAutoConfiguration.URL_REPO_RATELIMITER_SERVICE) + private lateinit var urlRepoRateLimiterService: UrlRepoRateLimiterService + + @Autowired + @Qualifier(RateLimiterAutoConfiguration.URL_RATELIMITER_SERVICE) + private lateinit var urlRateLimiterService: UrlRateLimiterService + + @Autowired + @Qualifier(RateLimiterAutoConfiguration.USER_URL_REPO_RATELIMITER_SERVICE) + private lateinit var userUrlRepoRateLimiterService: UserUrlRepoRateLimiterService + + @Autowired + @Qualifier(RateLimiterAutoConfiguration.UPLOAD_USAGE_RATELIMITER_SERVICE) + private lateinit var uploadUsageRateLimiterService: UploadUsageRateLimiterService + + @Autowired + @Qualifier(RateLimiterAutoConfiguration.USER_URL_RATELIMITER_SERVICE) + private lateinit var userUrlRateLimiterService: UserUrlRateLimiterService + + @Autowired + @Qualifier(RateLimiterAutoConfiguration.USER_UPLOAD_USAGE_RATELIMITER_SERVICE) + private lateinit var userUploadUsageRateLimiterService: UserUploadUsageRateLimiterService + + @Autowired + @Qualifier(RateLimiterAutoConfiguration.DOWNLOAD_USAGE_RATELIMITER_SERVICE) + private lateinit var downloadUsageRateLimiterService: DownloadUsageRateLimiterService + + @Autowired + @Qualifier(RateLimiterAutoConfiguration.USER_DOWNLOAD_USAGE_RATELIMITER_SERVICE) + private lateinit var userDownloadUsageRateLimiterService: UserDownloadUsageRateLimiterService + + @Autowired + @Qualifier(RateLimiterAutoConfiguration.DOWNLOAD_BANDWIDTH_RATELIMITER_SERVICE) + private lateinit var downloadBandwidthRateLimiterService: DownloadBandwidthRateLimiterService + + @Autowired + @Qualifier(RateLimiterAutoConfiguration.UPLOAD_BANDWIDTH_RATELIMITER_ERVICE) + private lateinit var uploadBandwidthRateLimiterService: UploadBandwidthRateLimiterService + + fun preLimitCheck(request: HttpServletRequest) { + if (!rateLimiterProperties.enabled) { + return + } + // TODO 可以优化 + urlRepoRateLimiterService.limit(request) + userUrlRepoRateLimiterService.limit(request) + userUrlRateLimiterService.limit(request) + userUploadUsageRateLimiterService.limit(request) + urlRateLimiterService.limit(request) + uploadUsageRateLimiterService.limit(request) + } + + fun postLimitCheck(applyPermits: Long) { + if (!rateLimiterProperties.enabled) { + return + } + val request = getRequest() ?: return + downloadUsageRateLimiterService.limit(request, applyPermits) + userDownloadUsageRateLimiterService.limit(request, applyPermits) + + } + + fun bandwidthCheck( + inputStream: InputStream, + circuitBreakerPerSecond: DataSize, + rangeLength: Long? = null, + ): CommonRateLimitInputStream? { + if (!rateLimiterProperties.enabled) { + return null + } + val request = getRequest() ?: return null + if (!downloadBandwidthRateLimiterService.ignoreRequest(request)) { + return downloadBandwidthRateLimiterService.bandwidthRateStart( + request, inputStream, circuitBreakerPerSecond, rangeLength + ) + } + if (!uploadBandwidthRateLimiterService.ignoreRequest(request)) { + return uploadBandwidthRateLimiterService.bandwidthRateStart( + request, inputStream, circuitBreakerPerSecond, rangeLength + ) + } + return null + } + + fun bandwidthFinish(exception: Exception? = null) { + if (!rateLimiterProperties.enabled) { + return + } + val request = getRequest() ?: return + if (!downloadBandwidthRateLimiterService.ignoreRequest(request)) { + return downloadBandwidthRateLimiterService.bandwidthRateLimitFinish( + request, exception + ) + } + if (!uploadBandwidthRateLimiterService.ignoreRequest(request)) { + return uploadBandwidthRateLimiterService.bandwidthRateLimitFinish( + request, exception + ) + } + } + + fun uploadBandwidthCheck( + applyPermits: Long, + circuitBreakerPerSecond: DataSize, + ) { + if (!rateLimiterProperties.enabled) { + return + } + val request = getRequest() ?: return + uploadBandwidthRateLimiterService.bandwidthRateLimit( + request, applyPermits, circuitBreakerPerSecond + ) + } + + private fun getRequest(): HttpServletRequest? { + return try { + HttpContextHolder.getRequest() + } catch (e: IllegalArgumentException) { + return null + } + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/bandwidth/DownloadBandwidthRateLimiterService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/bandwidth/DownloadBandwidthRateLimiterService.kt new file mode 100644 index 0000000000..ddfd54edc5 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/bandwidth/DownloadBandwidthRateLimiterService.kt @@ -0,0 +1,76 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.bandwidth + +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.bandwidth.DownloadBandwidthRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import javax.servlet.http.HttpServletRequest + +/** + * 下载带宽限流器实现，针对project/repo进行限流 + */ +class DownloadBandwidthRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService +) : UploadBandwidthRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService +) { + + override fun getLimitDimensions(): List { + return listOf( + LimitDimension.DOWNLOAD_BANDWIDTH.name + ) + } + + override fun ignoreRequest(request: HttpServletRequest): Boolean { + return request.method !in DOWNLOAD_REQUEST_METHOD + } + + override fun getRateLimitRuleClass(): Class { + return DownloadBandwidthRateLimitRule::class.java + } + + override fun generateKey(resource: String, resourceLimit: ResourceLimit): String { + return KEY_PREFIX + "DownloadBandwidth:$resource" + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/bandwidth/UploadBandwidthRateLimiterService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/bandwidth/UploadBandwidthRateLimiterService.kt new file mode 100644 index 0000000000..1c99e58b9c --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/bandwidth/UploadBandwidthRateLimiterService.kt @@ -0,0 +1,101 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.bandwidth + + +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.bandwidth.UploadBandwidthRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.service.AbstractBandwidthRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import javax.servlet.http.HttpServletRequest + +/** + * 上传带宽限流器实现, 针对project/repo进行限流 + */ +open class UploadBandwidthRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService +) : AbstractBandwidthRateLimiterService( + taskScheduler, + rateLimiterMetrics, + redisTemplate, + rateLimiterProperties, + rateLimiterConfigService +) { + + override fun buildResource(request: HttpServletRequest): String { + val (projectId, repoName) = getRepoInfoFromAttribute(request) + return if (repoName.isNullOrEmpty()) { + "/$projectId/" + } else { + "/$projectId/$repoName/" + } + } + + override fun buildExtraResource(request: HttpServletRequest): List { + val (projectId, repoName) = getRepoInfoFromAttribute(request) + if (repoName.isNullOrEmpty()) return emptyList() + return listOf("/$projectId/") + } + + override fun getApplyPermits(request: HttpServletRequest, applyPermits: Long?): Long { + if (applyPermits == null) { + throw AcquireLockFailedException("apply permits is null") + } + return applyPermits + } + + override fun getLimitDimensions(): List { + return listOf( + LimitDimension.UPLOAD_BANDWIDTH.name + ) + } + + override fun getRateLimitRuleClass(): Class { + return UploadBandwidthRateLimitRule::class.java + } + + override fun ignoreRequest(request: HttpServletRequest): Boolean { + return request.method !in UPLOAD_REQUEST_METHOD + } + + override fun generateKey(resource: String, resourceLimit: ResourceLimit): String { + return KEY_PREFIX + "UploadBandwidth:$resource" + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/UrlRateLimiterService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/UrlRateLimiterService.kt new file mode 100644 index 0000000000..8df65479fc --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/UrlRateLimiterService.kt @@ -0,0 +1,85 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.url + + +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.url.UrlRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import javax.servlet.http.HttpServletRequest + +/** + * url限流器实现 + */ +class UrlRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService +) : AbstractRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService +) { + + + override fun buildResource(request: HttpServletRequest): String { + return request.requestURI + } + + override fun buildExtraResource(request: HttpServletRequest): List { + return emptyList() + } + + override fun getApplyPermits(request: HttpServletRequest, applyPermits: Long?): Long { + return 1 + } + + override fun getLimitDimensions(): List { + return listOf(LimitDimension.URL.name) + } + + override fun getRateLimitRuleClass(): Class { + return UrlRateLimitRule::class.java + } + + override fun generateKey(resource: String, resourceLimit: ResourceLimit): String { + return KEY_PREFIX + "Url:$resource" + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/UrlRepoRateLimiterService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/UrlRepoRateLimiterService.kt new file mode 100644 index 0000000000..3189f7f0b1 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/UrlRepoRateLimiterService.kt @@ -0,0 +1,112 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.url + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.url.UrlRepoRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.web.servlet.HandlerMapping.BEST_MATCHING_PATTERN_ATTRIBUTE +import javax.servlet.http.HttpServletRequest + +/** + * urlRepo限流器实现 + */ +class UrlRepoRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService +) : AbstractRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService +) { + + override fun ignoreRequest(request: HttpServletRequest): Boolean { + if (rateLimiterProperties.specialUrls.contains(StringPool.POUND)) { + return false + } + return !rateLimiterProperties.specialUrls.contains(request.getAttribute(BEST_MATCHING_PATTERN_ATTRIBUTE)) + } + + override fun buildResource(request: HttpServletRequest): String { + val (projectId, repoName) = try { + getRepoInfoFromAttribute(request) + } catch (e: InvalidResourceException) { + getRepoInfoFromBody(request) + } + return if (repoName.isNullOrEmpty()) { + "/$projectId/" + } else { + "/$projectId/$repoName/" + } + } + + override fun buildExtraResource(request: HttpServletRequest): List { + val (projectId, repoName) = try { + getRepoInfoFromAttribute(request) + } catch (e: InvalidResourceException) { + getRepoInfoFromBody(request) + } + val result = mutableListOf() + if (!repoName.isNullOrEmpty()) { + result.add("/$projectId/") + } + return result + } + + override fun getApplyPermits(request: HttpServletRequest, applyPermits: Long?): Long { + return 1 + } + + override fun getLimitDimensions(): List { + return listOf(LimitDimension.URL_REPO.name) + } + + override fun getRateLimitRuleClass(): Class { + return UrlRepoRateLimitRule::class.java + } + + override fun generateKey(resource: String, resourceLimit: ResourceLimit): String { + return KEY_PREFIX + "UrlRepo:$resource" + } + +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/user/UserUrlRateLimiterService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/user/UserUrlRateLimiterService.kt new file mode 100644 index 0000000000..a98975b789 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/user/UserUrlRateLimiterService.kt @@ -0,0 +1,90 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.url.user + + +import com.tencent.bkrepo.common.api.constant.ANONYMOUS_USER +import com.tencent.bkrepo.common.api.constant.USER_KEY +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.url.user.UserUrlRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import com.tencent.bkrepo.common.service.util.HttpContextHolder +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import javax.servlet.http.HttpServletRequest + +/** + * 用户+url限流器实现 + */ +class UserUrlRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService, +) : AbstractRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService +) { + + override fun buildResource(request: HttpServletRequest): String { + val userId = HttpContextHolder.getRequestOrNull()?.getAttribute(USER_KEY) as? String ?: ANONYMOUS_USER + val realUrl = request.requestURI + return "$userId:$realUrl" + } + + override fun buildExtraResource(request: HttpServletRequest): List { + val userId = HttpContextHolder.getRequestOrNull()?.getAttribute(USER_KEY) as? String ?: ANONYMOUS_USER + return listOf("$userId:") + } + + override fun getApplyPermits(request: HttpServletRequest, applyPermits: Long?): Long { + return 1 + } + + override fun getLimitDimensions(): List { + return listOf(LimitDimension.USER_URL.name) + } + + override fun getRateLimitRuleClass(): Class { + return UserUrlRateLimitRule::class.java + } + + override fun generateKey(resource: String, resourceLimit: ResourceLimit): String { + return KEY_PREFIX + "UserUrl:$resource" + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/user/UserUrlRepoRateLimiterService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/user/UserUrlRepoRateLimiterService.kt new file mode 100644 index 0000000000..b6e29a06a7 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/user/UserUrlRepoRateLimiterService.kt @@ -0,0 +1,118 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.url.user + +import com.tencent.bkrepo.common.api.constant.ANONYMOUS_USER +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.api.constant.USER_KEY +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.url.user.UserUrlRepoRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import com.tencent.bkrepo.common.service.util.HttpContextHolder +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.web.servlet.HandlerMapping.BEST_MATCHING_PATTERN_ATTRIBUTE +import javax.servlet.http.HttpServletRequest + +/** + * user+urlRepo限流器实现 + */ +class UserUrlRepoRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + private val rateLimiterConfigService: RateLimiterConfigService +) : AbstractRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService +) { + + override fun ignoreRequest(request: HttpServletRequest): Boolean { + if (rateLimiterProperties.specialUrls.contains(StringPool.POUND)) { + return false + } + return !rateLimiterProperties.specialUrls.contains(request.getAttribute(BEST_MATCHING_PATTERN_ATTRIBUTE)) + } + + override fun buildResource(request: HttpServletRequest): String { + val userId = HttpContextHolder.getRequestOrNull()?.getAttribute(USER_KEY) as? String ?: ANONYMOUS_USER + val (projectId, repoName) = try { + getRepoInfoFromAttribute(request) + } catch (e: InvalidResourceException) { + getRepoInfoFromBody(request) + } + return if (repoName.isNullOrEmpty()) { + "$userId:/$projectId/" + } else { + "$userId:/$projectId/$repoName/" + } + } + + override fun buildExtraResource(request: HttpServletRequest): List { + val userId = HttpContextHolder.getRequestOrNull()?.getAttribute(USER_KEY) as? String ?: ANONYMOUS_USER + val (projectId, repoName) = try { + getRepoInfoFromAttribute(request) + } catch (e: InvalidResourceException) { + getRepoInfoFromBody(request) + } + val result = mutableListOf() + if (!repoName.isNullOrEmpty()) { + result.add("$userId:/$projectId/") + } + result.add("$userId:") + return result + } + + override fun getApplyPermits(request: HttpServletRequest, applyPermits: Long?): Long { + return 1 + } + + override fun getLimitDimensions(): List { + return listOf(LimitDimension.USER_URL_REPO.name) + } + + override fun getRateLimitRuleClass(): Class { + return UserUrlRepoRateLimitRule::class.java + } + + override fun generateKey(resource: String, resourceLimit: ResourceLimit): String { + return KEY_PREFIX + "UserUrlRepo:$resource" + } + +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/DownloadUsageRateLimiterService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/DownloadUsageRateLimiterService.kt new file mode 100644 index 0000000000..8d8a0cbc64 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/DownloadUsageRateLimiterService.kt @@ -0,0 +1,84 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.usage + +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.usage.DownloadUsageRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import javax.servlet.http.HttpServletRequest + +/** + * 下载容量限流器实现, 针对project和repo + */ +class DownloadUsageRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService +) : UploadUsageRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService +) { + + override fun getApplyPermits(request: HttpServletRequest, applyPermits: Long?): Long { + if (applyPermits == null) { + throw AcquireLockFailedException("apply permits is null") + } + return applyPermits + } + + override fun getLimitDimensions(): List { + return listOf( + LimitDimension.DOWNLOAD_USAGE.name + ) + } + + override fun ignoreRequest(request: HttpServletRequest): Boolean { + return request.method !in DOWNLOAD_REQUEST_METHOD + } + + override fun getRateLimitRuleClass(): Class { + return DownloadUsageRateLimitRule::class.java + } + + override fun generateKey(resource: String, resourceLimit: ResourceLimit): String { + return KEY_PREFIX + "DownloadUsage:$resource" + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/UploadUsageRateLimiterService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/UploadUsageRateLimiterService.kt new file mode 100644 index 0000000000..2a38210253 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/UploadUsageRateLimiterService.kt @@ -0,0 +1,110 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.usage + + +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.usage.UploadUsageRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import javax.servlet.http.HttpServletRequest + +/** + * 上传容量限流器实现，针对project和repo + */ +open class UploadUsageRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService, +) : AbstractRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService +) { + + override fun buildResource(request: HttpServletRequest): String { + val (projectId, repoName) = getRepoInfoFromAttribute(request) + return if (repoName.isNullOrEmpty()) { + "/$projectId/" + } else { + "/$projectId/$repoName/" + } + } + + override fun buildExtraResource(request: HttpServletRequest): List { + val (projectId, repoName) = getRepoInfoFromAttribute(request) + val result = mutableListOf() + if (!repoName.isNullOrEmpty()) { + result.add("/$projectId/") + } + return result + } + + override fun getApplyPermits(request: HttpServletRequest, applyPermits: Long?): Long { + return when (request.method) { + in UPLOAD_REQUEST_METHOD -> { + var length = request.contentLengthLong + if (length == -1L) { + logger.warn("content length of ${request.requestURI} is -1") + length = 0 + } + length + } + else -> 0 + } + } + + override fun getLimitDimensions(): List { + return listOf( + LimitDimension.UPLOAD_USAGE.name + ) + } + + override fun getRateLimitRuleClass(): Class { + return UploadUsageRateLimitRule::class.java + } + + override fun ignoreRequest(request: HttpServletRequest): Boolean { + return request.method !in UPLOAD_REQUEST_METHOD + } + + override fun generateKey(resource: String, resourceLimit: ResourceLimit): String { + return KEY_PREFIX + "UploadUsage:$resource" + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/user/UserDownloadUsageRateLimiterService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/user/UserDownloadUsageRateLimiterService.kt new file mode 100644 index 0000000000..3952414e63 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/user/UserDownloadUsageRateLimiterService.kt @@ -0,0 +1,84 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.usage.user + +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.usage.user.UserDownloadUsageRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import javax.servlet.http.HttpServletRequest + +/** + * 用户下载容量限流器实现，针对user、project和repo + */ +class UserDownloadUsageRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService +) : UserUploadUsageRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService +) { + + override fun getApplyPermits(request: HttpServletRequest, applyPermits: Long?): Long { + if (applyPermits == null) { + throw AcquireLockFailedException("response content is null") + } + return applyPermits + } + + override fun getLimitDimensions(): List { + return listOf( + LimitDimension.USER_DOWNLOAD_USAGE.name + ) + } + + override fun ignoreRequest(request: HttpServletRequest): Boolean { + return request.method !in DOWNLOAD_REQUEST_METHOD + } + + override fun getRateLimitRuleClass(): Class { + return UserDownloadUsageRateLimitRule::class.java + } + + override fun generateKey(resource: String, resourceLimit: ResourceLimit): String { + return KEY_PREFIX + "UserDownloadUsage:$resource" + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/user/UserUploadUsageRateLimiterService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/user/UserUploadUsageRateLimiterService.kt new file mode 100644 index 0000000000..42c714d3ad --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/user/UserUploadUsageRateLimiterService.kt @@ -0,0 +1,115 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.usage.user + +import com.tencent.bkrepo.common.api.constant.ANONYMOUS_USER +import com.tencent.bkrepo.common.api.constant.USER_KEY +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.rule.RateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.usage.user.UserUploadUsageRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterService +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import com.tencent.bkrepo.common.service.util.HttpContextHolder +import org.springframework.data.redis.core.RedisTemplate +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import javax.servlet.http.HttpServletRequest + +/** + * 用户上传容量限流器实现, 针对user、project和repo + */ +open class UserUploadUsageRateLimiterService( + taskScheduler: ThreadPoolTaskScheduler, + rateLimiterProperties: RateLimiterProperties, + rateLimiterMetrics: RateLimiterMetrics, + redisTemplate: RedisTemplate? = null, + rateLimiterConfigService: RateLimiterConfigService +) : AbstractRateLimiterService( + taskScheduler, + rateLimiterProperties, + rateLimiterMetrics, + redisTemplate, + rateLimiterConfigService +) { + + override fun buildResource(request: HttpServletRequest): String { + val (projectId, repoName) = getRepoInfoFromAttribute(request) + val userId = HttpContextHolder.getRequestOrNull()?.getAttribute(USER_KEY) as? String ?: ANONYMOUS_USER + return if (repoName.isNullOrEmpty()) { + "$userId:/$projectId/" + } else { + "$userId:/$projectId/$repoName/" + } + } + + override fun buildExtraResource(request: HttpServletRequest): List { + val (projectId, repoName) = getRepoInfoFromAttribute(request) + val userId = HttpContextHolder.getRequestOrNull()?.getAttribute(USER_KEY) as? String ?: ANONYMOUS_USER + val result = mutableListOf() + if (!repoName.isNullOrEmpty()) { + result.add("$userId:/$projectId/") + } + result.add("$userId:") + return result + } + + override fun getApplyPermits(request: HttpServletRequest, applyPermits: Long?): Long { + return when (request.method) { + in UPLOAD_REQUEST_METHOD -> { + var length = request.contentLengthLong + if (length == -1L) { + logger.warn("content length of ${request.requestURI} is -1") + length = 0 + } + length + } + else -> 0 + } + } + + override fun getLimitDimensions(): List { + return listOf( + LimitDimension.USER_UPLOAD_USAGE.name + ) + } + + override fun getRateLimitRuleClass(): Class { + return UserUploadUsageRateLimitRule::class.java + } + + override fun ignoreRequest(request: HttpServletRequest): Boolean { + return request.method !in UPLOAD_REQUEST_METHOD + } + + override fun generateKey(resource: String, resourceLimit: ResourceLimit): String { + return KEY_PREFIX + "UserUploadUsage:$resource" + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/user/RateLimiterConfigService.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/user/RateLimiterConfigService.kt new file mode 100644 index 0000000000..fa8c6c0431 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/service/user/RateLimiterConfigService.kt @@ -0,0 +1,139 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.user + +import com.tencent.bkrepo.common.ratelimiter.model.RateLimitCreatOrUpdateRequest +import com.tencent.bkrepo.common.ratelimiter.model.TRateLimit +import com.tencent.bkrepo.common.ratelimiter.repository.RateLimitRepository +import java.time.Duration +import org.springframework.beans.factory.annotation.Value +import org.springframework.stereotype.Service + +@Service +class RateLimiterConfigService( + private val rateLimitRepository: RateLimitRepository +) { + + @Value("\${spring.cloud.client.ip-address}") + var host: String = "127.0.0.1" + + fun list(): List { + return rateLimitRepository.findAll() + } + + fun create(fileCacheRequest: RateLimitCreatOrUpdateRequest) { + with(fileCacheRequest) { + rateLimitRepository.insert( + TRateLimit( + id = null, + resource = resource, + limitDimension = limitDimension, + algo = algo, + limit = limit, + duration = Duration.ofSeconds(duration), + capacity = capacity, + scope = scope, + moduleName = moduleName + ) + ) + } + } + + fun checkExist(id: String): Boolean { + return rateLimitRepository.existsById(id) + } + + fun checkExist(fileCacheRequest: RateLimitCreatOrUpdateRequest): Boolean { + with(fileCacheRequest) { + return rateLimitRepository.existsByResourceAndLimitDimension(resource, limitDimension) + } + } + + fun delete(id: String) { + rateLimitRepository.removeById(id) + } + + fun getById(id: String): TRateLimit? { + return rateLimitRepository.findById(id) + } + + fun update(fileCacheRequest: RateLimitCreatOrUpdateRequest) { + with(fileCacheRequest) { + targets?.let { + rateLimitRepository.save( + TRateLimit( + id = id, + resource = resource, + limitDimension = limitDimension, + algo = algo, + limit = limit, + duration = Duration.ofSeconds(duration), + capacity = capacity, + scope = scope, + moduleName = moduleName, + targets = it + ) + ) + } ?: run { + rateLimitRepository.save( + TRateLimit( + id = id, + resource = resource, + limitDimension = limitDimension, + algo = algo, + limit = limit, + duration = Duration.ofSeconds(duration), + capacity = capacity, + scope = scope, + moduleName = moduleName + ) + ) + } + } + } + + fun findByModuleNameAndLimitDimension(moduleName: String, limitDimension: String): List { + return rateLimitRepository.findByModuleNameAndLimitDimension(moduleName, limitDimension) + } + + fun findByResourceAndLimitDimension(resource: String, limitDimension: String): List { + return rateLimitRepository.findByResourceAndLimitDimension(resource, limitDimension) + } + + fun findByModuleNameAndLimitDimensionAndResource( + resource: String, + moduleName: List, + limitDimension: String + ): TRateLimit? { + return rateLimitRepository.findByModuleNameAndLimitDimensionAndResource(resource, moduleName, limitDimension) + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/stream/CommonRateLimitInputStream.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/stream/CommonRateLimitInputStream.kt new file mode 100644 index 0000000000..2ce9d3cc9b --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/stream/CommonRateLimitInputStream.kt @@ -0,0 +1,127 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.stream + +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.artifact.stream.DelegateInputStream +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import java.io.InputStream + +class CommonRateLimitInputStream( + delegate: InputStream, + private val rateCheckContext: RateCheckContext +) : DelegateInputStream(delegate) { + + private var bytesRead: Long = 0 + private var applyNum: Long = 0 + + override fun read(): Int { + tryAcquire(1) + val data = super.read() + if (data != -1) { + bytesRead++ + } + return data + } + + override fun read(byteArray: ByteArray): Int { + tryAcquire(byteArray.size) + val readLen = super.read(byteArray) + if (readLen != -1) { + bytesRead += readLen + } + return readLen + } + + override fun read(byteArray: ByteArray, off: Int, len: Int): Int { + tryAcquire(len) + val readLen = super.read(byteArray, off, len) + if (readLen != -1) { + bytesRead += readLen + } + return readLen + } + + private fun tryAcquire(bytes: Int) { + with(rateCheckContext) { + if (rangeLength == null || rangeLength!! <= 0) { + // 当不知道文件大小时，没办法进行大小预估，无法降低申请频率， 只能每次读取都进行判断 + acquire(bytes.toLong()) + } else { + // 避免频繁申请，增加耗时，降低申请频率， 每次申请一定数量 + // 当申请的bytes比limitPerSecond还大时,直接限流 + if (limitPerSecond < bytes) { + if (rateCheckContext.dryRun) { + return + } + throw OverloadException("request reached bandwidth limit") + } + // 此处避免限流带宽大小比每次申请的还少的情况下，每次都被限流 + val realPermitOnce = limitPerSecond.coerceAtMost(permitsOnce) + if (bytesRead == 0L || (bytesRead + bytes) > applyNum) { + val leftLength = rangeLength!! - bytesRead + val permits = if (leftLength >= 0) { + leftLength.coerceAtMost(realPermitOnce) + } else { + // 当剩余文件大小小于0时，说明文件大小不正确，无法确认剩余多少 + realPermitOnce + } + acquire(permits) + applyNum += permits + } + } + } + } + + private fun acquire(permits: Long) { + var flag = false + var failedNum = 0 + while (!flag) { + // 当限制小于读取大小时，会进入死循环，增加等待轮次，如果达到等待轮次上限后还是无法获取，则抛异常结束 + try { + flag = rateCheckContext.rateLimiter.tryAcquire(permits) + } catch (ignore: AcquireLockFailedException) { + return + } + if (!flag && failedNum < rateCheckContext.waitRound) { + failedNum++ + try { + Thread.sleep(rateCheckContext.latency * failedNum) + } catch (ignore: InterruptedException) { + } + continue + } + if (!flag && failedNum >= rateCheckContext.waitRound) { + if (rateCheckContext.dryRun) { + return + } + throw OverloadException("request reached bandwidth limit") + } + } + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/stream/RateCheckContext.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/stream/RateCheckContext.kt new file mode 100644 index 0000000000..588266debe --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/stream/RateCheckContext.kt @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.stream + +import com.tencent.bkrepo.common.ratelimiter.algorithm.RateLimiter + +data class RateCheckContext( + var rateLimiter: RateLimiter, + var latency: Long, + var waitRound: Int, + var limitPerSecond: Long, + var rangeLength: Long? = null, + var dryRun: Boolean = false, + var permitsOnce: Long = 1024 * 1024, +) diff --git a/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/utils/ResourcePathUtils.kt b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/utils/ResourcePathUtils.kt new file mode 100644 index 0000000000..3e777bdedf --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/kotlin/com/tencent/bkrepo/common/ratelimiter/utils/ResourcePathUtils.kt @@ -0,0 +1,99 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.utils + +import com.tencent.bkrepo.common.api.constant.CharPool +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import java.net.URI +import java.net.URISyntaxException + +object ResourcePathUtils { + + /** + * 分割路径， 支持带template变量路径 + */ + @Throws(InvalidResourceException::class) + fun tokenizeResourcePath(resourcePath: String): List { + if (resourcePath.isBlank()) { + return emptyList() + } + if (!resourcePath.startsWith("/")) { + throw InvalidResourceException("invalid resource path: $resourcePath") + } + val dirs = resourcePath.split("/").toTypedArray() + val dirList: MutableList = ArrayList() + for (i in dirs.indices) { + if (dirs[i].contains("?") + && (!dirs[i].startsWith("{") || !dirs[i].endsWith("}"))) { + throw InvalidResourceException("invalid resource path: $resourcePath") + } + if (dirs[i].isNotEmpty()) { + dirList.add(dirs[i]) + } + } + return dirList + } + + /** + * 获取URL路径，不带host等信息 + */ + fun getPathOfUrl(url: String): String? { + if (url.isBlank()) { + return null + } + val uri: URI + try { + uri = URI(url) + } catch (e: URISyntaxException) { + throw InvalidResourceException(url) + } + val path = uri.path + if (path.isNullOrEmpty()) { + return "/" + } + return path + } + + /** + * 截取user和path + */ + fun getUserAndPath(resource: String): Pair { + val index = resource.indexOfFirst { it == CharPool.COLON } + if (index == -1) { + throw InvalidResourceException(resource) + } + return Pair(resource.substring(0, index), resource.substring(index + 1)) + } + + /** + * 根据userId和path生成对应配置格式 + */ + fun buildUserPath(userId: String, resource: String): String { + return "$userId:$resource" + } +} diff --git a/src/backend/common/common-ratelimiter/src/main/resources/META-INF/spring.factories b/src/backend/common/common-ratelimiter/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..4de49fafff --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.bkrepo.common.ratelimiter.RateLimiterAutoConfiguration diff --git a/src/backend/common/common-ratelimiter/src/main/resources/fix-window-rate-limiter.lua b/src/backend/common/common-ratelimiter/src/main/resources/fix-window-rate-limiter.lua new file mode 100644 index 0000000000..9d3c918b1b --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/resources/fix-window-rate-limiter.lua @@ -0,0 +1,14 @@ +local key = KEYS[1] +local limit = tonumber(ARGV[1]) +local adder = tonumber(ARGV[2]) +local ttl = tonumber(ARGV[3]) +local current = tonumber(redis.call('get', key) or "0") +if current + adder > limit then + return 0 +else + redis.call('incrby', key, adder) + if (current == 0) then + redis.call('EXPIRE', key, ttl) + end + return 1 +end \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/main/resources/leaky-rate-limiter.lua b/src/backend/common/common-ratelimiter/src/main/resources/leaky-rate-limiter.lua new file mode 100644 index 0000000000..7819c1cb6a --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/resources/leaky-rate-limiter.lua @@ -0,0 +1,58 @@ +local leaky_bucket_key = KEYS[1] +-- last update key +local last_bucket_key = KEYS[2] +-- the rate of leak water +local rate = tonumber(ARGV[1]) +-- capacity +local capacity = tonumber(ARGV[2]) +-- request count +local requested = tonumber(ARGV[3]) +-- current timestamp seconds +local now = tonumber(ARGV[4]) + +-- the key life time +local key_lifetime = math.ceil((capacity / rate) + 1) + +-- the yield of water in the bucket default 0 +local key_bucket_count = tonumber(redis.call("GET", leaky_bucket_key)) or 0 + +-- the last update time default now +local last_time = tonumber(redis.call("GET", last_bucket_key)) or now + +-- the time difference +local millis_since_last_leak = now - last_time + +-- the yield of water had lasted +local leaks = millis_since_last_leak * rate + +if leaks > 0 then + -- clean up the bucket + if leaks >= key_bucket_count then + key_bucket_count = 0 + else + -- compute the yield of water in the bucket + key_bucket_count = key_bucket_count - leaks + end + last_time = now +end + +-- is allowed pass default not allow +local is_allow = 0 + +local new_bucket_count = key_bucket_count + requested +-- allow +if new_bucket_count <= capacity then + is_allow = 1 +else + -- not allow + return {is_allow, new_bucket_count} +end + +-- update the key bucket water yield +redis.call("SETEX", leaky_bucket_key, key_lifetime, new_bucket_count) + +-- update last update time +redis.call("SETEX", last_bucket_key, key_lifetime, now) + +-- return +return {is_allow, new_bucket_count} diff --git a/src/backend/common/common-ratelimiter/src/main/resources/sliding-window-rate-limiter.lua b/src/backend/common/common-ratelimiter/src/main/resources/sliding-window-rate-limiter.lua new file mode 100644 index 0000000000..7c8b5d5cd5 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/resources/sliding-window-rate-limiter.lua @@ -0,0 +1,36 @@ +-- key: 限流器的键名 +-- limit: 限流器的容量 +-- interval: 时间窗口的长度（单位为秒） +-- count: 一次获取的令牌数量 +local key = KEYS[1] +local limit = tonumber(ARGV[1]) +local interval = tonumber(ARGV[2]) +local count = tonumber(ARGV[3]) +local now_sec = tonumber(ARGV[4]) +local random = tonumber(ARGV[5]) + +-- 删除时间窗口之外的令牌 +redis.call('zremrangebyscore', key, 0, now_sec - interval) + +-- 获取当前时间窗口内的令牌数量 +local current = tonumber(redis.call('zcard', key)) + +-- 如果当前令牌数量已经达到限流器的容量，则不再生成新的令牌 +if current >= limit then + return 0 +end + +-- 计算需要生成的令牌数量 +local remaining = limit - current +local allowed_num = 0 +if (remaining < count) then + return { allowed_num, remaining } +end +allowed_num = 1 +-- 生成令牌，并返回生成的令牌数量 +local tokens = {} +for i = 1, count do + redis.call('zadd', key, now_sec, random..i) +end +redis.call('expire', key, interval) +return { allowed_num, remaining } diff --git a/src/backend/common/common-ratelimiter/src/main/resources/token-bucket-rate-limiter.lua b/src/backend/common/common-ratelimiter/src/main/resources/token-bucket-rate-limiter.lua new file mode 100644 index 0000000000..b0c8f44645 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/main/resources/token-bucket-rate-limiter.lua @@ -0,0 +1,51 @@ +local tokens_key = KEYS[1] +local timestamp_key = KEYS[2] +-- 每秒填充速率 +local rate = tonumber(ARGV[1]) +-- 令牌桶最大数量 +local capacity = tonumber(ARGV[2]) +-- 消耗令牌数量 +local requested = tonumber(ARGV[3]) +-- now 当前时间秒 +local now = tonumber(ARGV[4]) + +-- 计算令牌桶填充满需要多久 +local fill_time = capacity/rate +-- *2保证时间充足 +local ttl = math.floor(fill_time*2) + +-- 防止小于0 +if ttl < 1 then + ttl = 10 +end +-- 获取令牌桶内剩余数量 +local last_tokens = tonumber(redis.call("get", tokens_key)) +-- 第一次没有数值，设置桶为满的 +if last_tokens == nil then + last_tokens = capacity +end +-- 获取上次更新时间 +local last_refreshed = tonumber(redis.call("get", timestamp_key)) +if last_refreshed == nil then + last_refreshed = 0 +end +-- 本次校验和上次更新时间的间隔 +local delta = math.max(0, now-last_refreshed) +-- 填充令牌，计算新的令牌桶剩余令牌数，填充不超过令牌桶上限 +local filled_tokens = math.min(capacity, last_tokens+(delta*rate)) + +-- 判断令牌数量是否足够 +local allowed = filled_tokens >= requested +local new_tokens = filled_tokens +local allowed_num = 0 +if allowed then + -- 如成功，令牌桶剩余令牌数减消耗令牌数 + new_tokens = filled_tokens - requested + allowed_num = 1 +end + +-- 设置令牌桶剩余令牌数，令牌桶最后填充时间now, ttl超时时间 +redis.call("setex", tokens_key, ttl, new_tokens) +redis.call("setex", timestamp_key, ttl, now) + +return { allowed_num, new_tokens } diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/RateLimiterAutoConfigurationTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/RateLimiterAutoConfigurationTest.kt new file mode 100644 index 0000000000..3d08fcdb28 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/RateLimiterAutoConfigurationTest.kt @@ -0,0 +1,35 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter + +import org.springframework.boot.SpringBootConfiguration +import org.springframework.boot.autoconfigure.EnableAutoConfiguration + +@SpringBootConfiguration +@EnableAutoConfiguration +class RateLimiterAutoConfigurationTest \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedFixedWindowRateLimiterTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedFixedWindowRateLimiterTest.kt new file mode 100644 index 0000000000..33d1a40b0d --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedFixedWindowRateLimiterTest.kt @@ -0,0 +1,105 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.api.util.HumanReadable +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import org.springframework.test.annotation.DirtiesContext +import java.time.Duration +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import kotlin.concurrent.thread +import kotlin.system.measureTimeMillis + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class DistributedFixedWindowRateLimiterTest : DistributedTest() { + + + @Test + fun testTryAcquire() { + val key = KEY + "testTryAcquire" + val ratelimiter = DistributedFixedWindowRateLimiter(key, 5, Duration.ofSeconds(1), redisTemplate) + println(System.currentTimeMillis()) + val passed1 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed1) + Thread.sleep(1000) + val passed2 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed2) + val passed3 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed3) + val passed4 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed4) + val passed5 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed5) + println(System.currentTimeMillis()) + val passed6 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed6) + println(System.currentTimeMillis()) + val passed7 = ratelimiter.tryAcquire(1) + Assertions.assertFalse(passed7) + clean(key) + } + + @Test + fun testTryAcquireOnMultiThreads() { + val key = KEY + "testTryAcquireOnMultiThreads" + val ratelimiter = DistributedFixedWindowRateLimiter(key, 5, Duration.ofSeconds(1), redisTemplate) + var successNum = 0 + var failedNum = 0 + var errorNum = 0 + val readers = Runtime.getRuntime().availableProcessors() + val countDownLatch = CountDownLatch(readers) + val elapsedTime = measureTimeMillis { + repeat(readers) { + thread { + try { + val passed = ratelimiter.tryAcquire(1) + if (passed) { + successNum++ + } else { + failedNum++ + } + } catch (e: Exception) { + errorNum++ + } + countDownLatch.countDown() + } + } + } + countDownLatch.await() + println("elapse: ${HumanReadable.time(elapsedTime, TimeUnit.MILLISECONDS)}") + println("successNum $successNum, failedNum $failedNum. errorNum $errorNum") + clean(key) + } + + + companion object { + private const val KEY = "DistributedFixedKey" + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedLeakyRateLimiterTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedLeakyRateLimiterTest.kt new file mode 100644 index 0000000000..4815debe65 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedLeakyRateLimiterTest.kt @@ -0,0 +1,105 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.api.util.HumanReadable +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import org.springframework.test.annotation.DirtiesContext +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import kotlin.concurrent.thread +import kotlin.system.measureTimeMillis + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class DistributedLeakyRateLimiterTest : DistributedTest() { + + @Test + fun testTryAcquire() { + val key = KEY + "testTryAcquire" + val ratelimiter = DistributedLeakyRateLimiter(key, 5.0, 5, redisTemplate) + val passed1 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed1) + println(System.currentTimeMillis()) + Thread.sleep(1000) + val passed2 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed2) + val passed3 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed3) + val passed4 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed4) + val passed5 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed5) + println(System.currentTimeMillis()) + val passed6 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed6) + println(System.currentTimeMillis()) + val passed7 = ratelimiter.tryAcquire(1) + Assertions.assertFalse(passed7) + Thread.sleep(1000) + val passed8 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed8) + clean(key) + } + + @Test + fun testTryAcquireOnMultiThreads() { + val key = KEY + "testTryAcquireOnMultiThreads" + val ratelimiter = DistributedLeakyRateLimiter(key, 5.0, 5, redisTemplate) + var successNum = 0 + var failedNum = 0 + var errorNum = 0 + val readers = Runtime.getRuntime().availableProcessors() + val countDownLatch = CountDownLatch(readers) + val elapsedTime = measureTimeMillis { + repeat(readers) { + thread { + try { + val passed = ratelimiter.tryAcquire(1) + if (passed) { + successNum++ + } else { + failedNum++ + } + } catch (e: Exception) { + errorNum++ + } + countDownLatch.countDown() + } + } + } + countDownLatch.await() + println("elapse: ${HumanReadable.time(elapsedTime, TimeUnit.MILLISECONDS)}") + println("successNum $successNum, failedNum $failedNum. errorNum $errorNum") + clean(key) + } + + companion object { + private const val KEY = "DistributedLeakyKey" + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedSlidingWindowRateLimiterTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedSlidingWindowRateLimiterTest.kt new file mode 100644 index 0000000000..3bd57c3844 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedSlidingWindowRateLimiterTest.kt @@ -0,0 +1,101 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.api.util.HumanReadable +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import org.springframework.test.annotation.DirtiesContext +import java.time.Duration +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import kotlin.concurrent.thread +import kotlin.system.measureTimeMillis + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class DistributedSlidingWindowRateLimiterTest : DistributedTest() { + @Test + fun testTryAcquire() { + val key = KEY + "testTryAcquire" + val ratelimiter = DistributedSlidingWindowRateLimiter(key, 5, Duration.ofSeconds(1), redisTemplate) + val passed1 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed1) + val passed2 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed2) + val passed3 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed3) + val passed4 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed4) + val passed5 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed5) + var passed6 = ratelimiter.tryAcquire(1) + Assertions.assertFalse(passed6) + Thread.sleep(1000) + var passed7 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed7) + val passed8 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed8) + clean(key) + } + + @Test + fun testTryAcquireOnMultiThreads() { + val key = KEY + "testTryAcquireOnMultiThreads" + val ratelimiter = DistributedSlidingWindowRateLimiter(key, 5, Duration.ofSeconds(1), redisTemplate) + var successNum = 0 + var failedNum = 0 + var errorNum = 0 + val readers = Runtime.getRuntime().availableProcessors() + val countDownLatch = CountDownLatch(readers) + val elapsedTime = measureTimeMillis { + repeat(readers) { + thread { + try { + val passed = ratelimiter.tryAcquire(1) + if (passed) { + successNum++ + } else { + failedNum++ + } + } catch (e: Exception) { + errorNum++ + } + countDownLatch.countDown() + } + } + } + countDownLatch.await() + println("elapse: ${HumanReadable.time(elapsedTime, TimeUnit.MILLISECONDS)}") + println("successNum $successNum, failedNum $failedNum. errorNum $errorNum") + clean(key) + } + + companion object { + private const val KEY = "DistributedSlidingKey" + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedTest.kt new file mode 100644 index 0000000000..ef38441481 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedTest.kt @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.ratelimiter.rule.BaseRuleTest +import com.tencent.bkrepo.common.redis.RedisAutoConfiguration +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.autoconfigure.ImportAutoConfiguration +import org.springframework.boot.test.autoconfigure.data.redis.DataRedisTest +import org.springframework.data.redis.core.RedisTemplate + +@DataRedisTest +@ImportAutoConfiguration(RedisTestConfiguration::class, RedisAutoConfiguration::class) +open class DistributedTest : BaseRuleTest() { + + @Autowired + lateinit var redisTemplate: RedisTemplate + + fun clean(key: String) { + redisTemplate.delete(key) + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedTokenBucketRateLimiterTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedTokenBucketRateLimiterTest.kt new file mode 100644 index 0000000000..e8efdd8ee0 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/DistributedTokenBucketRateLimiterTest.kt @@ -0,0 +1,103 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.api.util.HumanReadable +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import org.springframework.test.annotation.DirtiesContext +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import kotlin.concurrent.thread +import kotlin.system.measureTimeMillis + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class DistributedTokenBucketRateLimiterTest : DistributedTest() { + @Test + fun testTryAcquire1() { + val key = KEY + "testTryAcquire1" + var ratelimiter = DistributedTokenBucketRateLimiter(key, 5.0, 5, redisTemplate) + val passed1 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed1) + Thread.sleep(1000) + val passed2 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed2) + println(System.currentTimeMillis()) + val passed3 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed3) + println(System.currentTimeMillis()) + val passed4 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed4) + println(System.currentTimeMillis()) + val passed5 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed5) + println(System.currentTimeMillis()) + val passed6 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed6) + val passed7 = ratelimiter.tryAcquire(1) + Assertions.assertFalse(passed7) + clean(key) + } + + + @Test + fun testTryAcquireOnMultiThreads() { + val key = KEY + "testTryAcquireOnMultiThreads" + var ratelimiter = DistributedTokenBucketRateLimiter(key, 5.0, 5, redisTemplate) + var successNum = 0 + var failedNum = 0 + var errorNum = 0 + val readers = Runtime.getRuntime().availableProcessors() + val countDownLatch = CountDownLatch(readers) + val elapsedTime = measureTimeMillis { + repeat(readers) { + thread { + try { + val passed = ratelimiter.tryAcquire(1) + if (passed) { + successNum++ + } else { + failedNum++ + } + } catch (e: Exception) { + errorNum++ + } + countDownLatch.countDown() + } + } + } + countDownLatch.await() + println("elapse: ${HumanReadable.time(elapsedTime, TimeUnit.MILLISECONDS)}") + println("successNum $successNum, failedNum $failedNum. errorNum $errorNum") + clean(key) + } + + companion object { + private const val KEY = "DistributedTokenKey" + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/FixedWindowRateLimiterTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/FixedWindowRateLimiterTest.kt new file mode 100644 index 0000000000..49487c7165 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/FixedWindowRateLimiterTest.kt @@ -0,0 +1,106 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.google.common.base.Stopwatch +import com.google.common.base.Ticker +import com.tencent.bkrepo.common.api.util.HumanReadable +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.mockito.Mockito +import java.time.Duration +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import kotlin.concurrent.thread +import kotlin.random.Random +import kotlin.system.measureTimeMillis + +class FixedWindowRateLimiterTest { + + @Test + fun testTryAcquire() { + val ticker = Mockito.mock(Ticker::class.java) + Mockito.`when`(ticker.read()).thenReturn(0 * 1000 * 1000L) + val ratelimiter: RateLimiter = FixedWindowRateLimiter(5, Duration.ofSeconds(1), Stopwatch.createStarted(ticker)) + Mockito.`when`(ticker.read()).thenReturn(100 * 1000 * 1000L) + val passed1 = ratelimiter.tryAcquire(1) + assertTrue(passed1) + Mockito.`when`(ticker.read()).thenReturn(200 * 1000 * 1000L) + val passed2 = ratelimiter.tryAcquire(1) + assertTrue(passed2) + Mockito.`when`(ticker.read()).thenReturn(300 * 1000 * 1000L) + val passed3 = ratelimiter.tryAcquire(1) + assertTrue(passed3) + Mockito.`when`(ticker.read()).thenReturn(400 * 1000 * 1000L) + val passed4 = ratelimiter.tryAcquire(1) + assertTrue(passed4) + Mockito.`when`(ticker.read()).thenReturn(500 * 1000 * 1000L) + val passed5 = ratelimiter.tryAcquire(1) + assertTrue(passed5) + Mockito.`when`(ticker.read()).thenReturn(600 * 1000 * 1000L) + val passed6 = ratelimiter.tryAcquire(1) + assertFalse(passed6) + Mockito.`when`(ticker.read()).thenReturn(1001 * 1000 * 1000L) + val passed7 = ratelimiter.tryAcquire(1) + assertTrue(passed7) + } + + @Test + fun testTryAcquireOnMultiThreads() { + val ticker = Mockito.mock(Ticker::class.java) + Mockito.`when`(ticker.read()).thenReturn(0 * 1000 * 1000L) + val ratelimiter: RateLimiter = FixedWindowRateLimiter(5, Duration.ofSeconds(1), Stopwatch.createStarted(ticker)) + var successNum = 0 + var failedNum = 0 + var errorNum = 0 + val readers = Runtime.getRuntime().availableProcessors() + val countDownLatch = CountDownLatch(readers) + val elapsedTime = measureTimeMillis { + repeat(readers) { + thread { + try { + Thread.sleep((Random.nextInt(5) * 2).toLong()) + val passed = ratelimiter.tryAcquire(1) + if (passed) { + successNum++ + } else { + failedNum++ + } + } catch (e: Exception) { + errorNum++ + } + countDownLatch.countDown() + } + } + } + countDownLatch.await() + println("elapse: ${HumanReadable.time(elapsedTime, TimeUnit.MILLISECONDS)}") + println("successNum $successNum, failedNum $failedNum. errorNum $errorNum") + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/LeakyRateLimiterTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/LeakyRateLimiterTest.kt new file mode 100644 index 0000000000..77924e66d6 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/LeakyRateLimiterTest.kt @@ -0,0 +1,91 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.api.util.HumanReadable +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import kotlin.concurrent.thread +import kotlin.system.measureTimeMillis + +class LeakyRateLimiterTest { + + @Test + fun testTryAcquire() { + val ratelimiter = LeakyRateLimiter(5.0, 5) + val passed1 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed1) + val passed2 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed2) + val passed3 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed3) + val passed4 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed4) + val passed5 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed5) + val passed6 = ratelimiter.tryAcquire(1) + Assertions.assertFalse(passed6) + try { + Thread.sleep(1000) + } catch (ignore: InterruptedException) { + } + val passed7 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed7) + } + + @Test + fun testTryAcquireOnMultiThreads() { + val ratelimiter = LeakyRateLimiter(5.0, 5) + var successNum = 0 + var failedNum = 0 + var errorNun = 0 + val readers = Runtime.getRuntime().availableProcessors() + val countDownLatch = CountDownLatch(readers) + val elapsedTime = measureTimeMillis { + repeat(readers) { + thread { + try { + val passed = ratelimiter.tryAcquire(1) + if (passed) { + successNum++ + } else { + failedNum++ + } + } catch (e: Exception) { + errorNun++ + } + countDownLatch.countDown() + } + } + } + countDownLatch.await() + println("elapse: ${HumanReadable.time(elapsedTime, TimeUnit.MILLISECONDS)}") + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/RedisTestConfiguration.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/RedisTestConfiguration.kt new file mode 100644 index 0000000000..710c226a80 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/RedisTestConfiguration.kt @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import org.springframework.boot.test.context.TestConfiguration +import redis.embedded.RedisServer +import javax.annotation.PostConstruct +import javax.annotation.PreDestroy + + +@TestConfiguration +class RedisTestConfiguration { + private val redisServer = RedisServer.builder().build() + + @PostConstruct + fun postConstruct() { + redisServer.start() + } + + @PreDestroy + fun preDestroy() { + redisServer.stop() + } +} diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/SlidingWindowRateLimiterTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/SlidingWindowRateLimiterTest.kt new file mode 100644 index 0000000000..6d742545bb --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/SlidingWindowRateLimiterTest.kt @@ -0,0 +1,92 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.api.util.HumanReadable +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import java.time.Duration +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import kotlin.concurrent.thread +import kotlin.system.measureTimeMillis + +class SlidingWindowRateLimiterTest { + + @Test + fun testTryAcquire() { + val ratelimiter = SlidingWindowRateLimiter(5, Duration.ofMillis(100)) + val passed1 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed1) + val passed2 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed2) + val passed3 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed3) + val passed4 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed4) + val passed5 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed5) + Thread.sleep(1200) + var passed6 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed6) + var passed7 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed7) + val passed8 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed8) + } + + @Test + fun testTryAcquireOnMultiThreads() { + val ratelimiter = SlidingWindowRateLimiter(5, Duration.ofMillis(100)) + var successNum = 0 + var failedNum = 0 + var errorNum = 0 + val readers = Runtime.getRuntime().availableProcessors() + val countDownLatch = CountDownLatch(readers) + val elapsedTime = measureTimeMillis { + repeat(readers) { + thread { + try { + val passed = ratelimiter.tryAcquire(1) + if (passed) { + successNum++ + } else { + failedNum++ + } + } catch (e: Exception) { + errorNum++ + } + countDownLatch.countDown() + } + } + } + countDownLatch.await() + println("elapse: ${HumanReadable.time(elapsedTime, TimeUnit.MILLISECONDS)}") + println("successNum $successNum, failedNum $failedNum. errorNum $errorNum") + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/TokenBucketRateLimiterTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/TokenBucketRateLimiterTest.kt new file mode 100644 index 0000000000..96cea7389a --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/algorithm/TokenBucketRateLimiterTest.kt @@ -0,0 +1,97 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.algorithm + +import com.tencent.bkrepo.common.api.util.HumanReadable +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import kotlin.concurrent.thread +import kotlin.system.measureTimeMillis + +class TokenBucketRateLimiterTest { + + @Test + fun testTryAcquire() { + var ratelimiter = TokenBucketRateLimiter(5.0) + val passed1 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed1) + Thread.sleep(1000) + val passed2 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed2) + println(System.currentTimeMillis()) + val passed3 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed3) + println(System.currentTimeMillis()) + val passed4 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed4) + println(System.currentTimeMillis()) + val passed5 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed5) + println(System.currentTimeMillis()) + val passed6 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed6) + val passed7 = ratelimiter.tryAcquire(1) + Assertions.assertFalse(passed7) + } + + + @Test + fun testTryAcquireOnMultiThreads() { + val ratelimiter = TokenBucketRateLimiter(5.0) + val passed1 = ratelimiter.tryAcquire(1) + Assertions.assertTrue(passed1) + Thread.sleep(1000) + var successNum = 0 + var failedNum = 0 + var errorNum = 0 + val readers = Runtime.getRuntime().availableProcessors() + val countDownLatch = CountDownLatch(readers) + val elapsedTime = measureTimeMillis { + repeat(readers) { + thread { + try { + val passed = ratelimiter.tryAcquire(1) + if (passed) { + successNum++ + } else { + failedNum++ + } + } catch (e: Exception) { + errorNum++ + } + countDownLatch.countDown() + } + } + } + countDownLatch.await() + println("elapse: ${HumanReadable.time(elapsedTime, TimeUnit.MILLISECONDS)}") + println("successNum $successNum, failedNum $failedNum. errorNum $errorNum") + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimiterInterceptorChainTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimiterInterceptorChainTest.kt new file mode 100644 index 0000000000..f850ea4252 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/interceptor/RateLimiterInterceptorChainTest.kt @@ -0,0 +1,151 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.interceptor + +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.repository.RateLimitRepository +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import java.time.Duration +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows + + +class RateLimiterInterceptorChainTest { + + open class InterceptorA : RateLimiterInterceptor { + + override fun beforeLimitCheck(resource: String, resourceLimit: ResourceLimit) { + list.add(identity() + ":before") + } + + + override fun afterLimitCheck( + resource: String, resourceLimit: ResourceLimit?, + result: Boolean, e: Exception? + ) { + list.add(identity() + ":after") + } + + protected open fun identity(): String { + return InterceptorA::class.java.simpleName + } + } + + class InterceptorB : InterceptorA() { + override fun identity(): String { + return InterceptorB::class.java.simpleName + } + } + + class InterceptorC : InterceptorA() { + override fun identity(): String { + return InterceptorC::class.java.simpleName + } + } + + @BeforeEach + fun reInit() { + list.clear() + } + + @Test + fun testDoBeforeLimit() { + val chain = RateLimiterInterceptorChain() + chain.addInterceptor(InterceptorA()) + chain.addInterceptor(InterceptorB()) + chain.addInterceptor(InterceptorC()) + val resourceLimit = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project1/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + chain.doBeforeLimitCheck("test1", resourceLimit) + assertEquals(Companion.list.size, 3) + assertEquals(Companion.list[0], "InterceptorA:before") + assertEquals(Companion.list[1], "InterceptorB:before") + assertEquals(Companion.list[2], "InterceptorC:before") + } + + @Test + fun testTargetDoBeforeLimit() { + val chain = RateLimiterInterceptorChain() + val rateLimitRepository: RateLimitRepository = RateLimitRepository() + chain.addInterceptor(TargetRateLimiterInterceptorAdaptor(RateLimiterConfigService(rateLimitRepository))) + val resourceLimit = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project1/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name, + targets = listOf("127.0.0.2") + ) + assertThrows { + chain.doBeforeLimitCheck("test1", resourceLimit) + } + } + + @Test + fun testDoAfterLimit() { + val chain = RateLimiterInterceptorChain() + chain.addInterceptor(InterceptorC()) + chain.addInterceptor(InterceptorB()) + chain.addInterceptor(InterceptorA()) + chain.doAfterLimitCheck("test1", null, true, null) + assertEquals(Companion.list.size, 3) + assertEquals(Companion.list[0], "InterceptorC:after") + assertEquals(Companion.list[1], "InterceptorB:after") + assertEquals(Companion.list[2], "InterceptorA:after") + } + + @Test + fun testIsEmpty() { + val chain = RateLimiterInterceptorChain() + val isEmpty = chain.isEmpty() + Assertions.assertTrue(isEmpty) + } + + @Test + fun testClear() { + val chain = RateLimiterInterceptorChain() + chain.addInterceptor(InterceptorB()) + chain.addInterceptor(InterceptorA()) + chain.addInterceptor(InterceptorC()) + assertEquals(chain.size(), 3) + chain.clear() + assertEquals(chain.size(), 0) + } + + companion object { + val list: MutableList = ArrayList() + } +} diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/BaseRuleTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/BaseRuleTest.kt new file mode 100644 index 0000000000..b309208bbf --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/BaseRuleTest.kt @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule + +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.junit.jupiter.api.Assertions + +open class BaseRuleTest { + + fun assertEqualsLimitInfo(actualInfo: ResourceLimit?, expectedInfo: ResourceLimit?) { + Assertions.assertEquals(actualInfo?.limit, expectedInfo?.limit) + Assertions.assertEquals(actualInfo?.algo, expectedInfo?.algo) + Assertions.assertEquals(actualInfo?.limitDimension, expectedInfo?.limitDimension) + Assertions.assertEquals(actualInfo?.capacity, expectedInfo?.capacity) + Assertions.assertEquals(actualInfo?.resource, expectedInfo?.resource) + Assertions.assertEquals(actualInfo?.scope, expectedInfo?.scope) + Assertions.assertEquals(actualInfo?.duration, expectedInfo?.duration) + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/DownloadBandwidthRateLimitRuleTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/DownloadBandwidthRateLimitRuleTest.kt new file mode 100644 index 0000000000..26817c58e9 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/DownloadBandwidthRateLimitRuleTest.kt @@ -0,0 +1,236 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.bandwidth + +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.BaseRuleTest +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import java.time.Duration + +class DownloadBandwidthRateLimitRuleTest : BaseRuleTest() { + + private val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/", + limitDimension = LimitDimension.DOWNLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l2 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project1/repo1/", + limitDimension = LimitDimension.DOWNLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l3 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/repo1/", + limitDimension = LimitDimension.DOWNLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l4 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project1/*/", + limitDimension = LimitDimension.DOWNLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l5 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/*/", + limitDimension = LimitDimension.DOWNLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l6 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/", + limitDimension = LimitDimension.DOWNLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l7 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/repo3/", + limitDimension = LimitDimension.DOWNLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @Test + fun testIsEmpty() { + val downloadBandwidthRateLimitRule = DownloadBandwidthRateLimitRule() + assertEquals(downloadBandwidthRateLimitRule.isEmpty(), true) + downloadBandwidthRateLimitRule.addRateLimitRule(l1) + assertEquals(downloadBandwidthRateLimitRule.isEmpty(), false) + } + + @Test + fun testBandwidthRateLimitRuleAndGetRateLimitRule() { + val downloadBandwidthRateLimitRule = DownloadBandwidthRateLimitRule() + downloadBandwidthRateLimitRule.addRateLimitRule(l1) + downloadBandwidthRateLimitRule.addRateLimitRule(l2) + downloadBandwidthRateLimitRule.addRateLimitRule(l3) + downloadBandwidthRateLimitRule.addRateLimitRule(l4) + + var resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + var actualInfo = downloadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + assertEquals(actualInfo?.resource, "/project1/repo1/") + + resInfo = ResInfo("/project1/repo2/", listOf("/project1/")) + actualInfo = downloadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + assertEquals(actualInfo?.resource, "/project1/repo2/") + + resInfo = ResInfo("/project2/repo2/", listOf("/project2/")) + actualInfo = downloadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, null) + assertEquals(actualInfo?.resource, null) + + resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + actualInfo = downloadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEquals(actualInfo?.resource, "/project2/repo1/") + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = downloadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + assertEquals(actualInfo?.resource, "/project3/") + + downloadBandwidthRateLimitRule.addRateLimitRule(l6) + + resInfo = ResInfo("/project4/repo4/", listOf("/project4/")) + actualInfo = downloadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + assertEquals(actualInfo?.resource, "/project4/") + + downloadBandwidthRateLimitRule.addRateLimitRule(l5) + + resInfo = ResInfo("/project4/repo4/", listOf("/project4/")) + actualInfo = downloadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + assertEquals(actualInfo?.resource, "/project4/repo4/") + + resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + actualInfo = downloadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + assertEquals(actualInfo?.resource, "/project1/repo1/") + + resInfo = ResInfo("/project1/repo2/", listOf("/project1/")) + actualInfo = downloadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + assertEquals(actualInfo?.resource, "/project1/repo2/") + + resInfo = ResInfo("/project2/repo2/", listOf("/project2/")) + actualInfo = downloadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + assertEquals(actualInfo?.resource, "/project2/repo2/") + + resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + actualInfo = downloadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEquals(actualInfo?.resource, "/project2/repo1/") + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = downloadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + assertEquals(actualInfo?.resource, "/project3/") + + downloadBandwidthRateLimitRule.addRateLimitRule(l7) + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = downloadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l7) + assertEquals(actualInfo?.resource, "/project3/repo3/") + } + + @Test + fun testBandwidthRateLimitRuleAndGetRateLimitRuleWithEmptyRule() { + val rule = DownloadBandwidthRateLimitRule() + var resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + var info = rule.getRateLimitRule(resInfo) + assertNull(info) + resInfo = ResInfo("/", listOf()) + info = rule.getRateLimitRule(resInfo) + assertNull(info) + } + + @Test + fun testBandwidthRateLimitRuleAndGetRateLimitRuleWithResEmpty() { + val rule = DownloadBandwidthRateLimitRule() + rule.addRateLimitRule(l1) + var resInfo = ResInfo("", listOf()) + var actualInfo = rule.getRateLimitRule(resInfo) + assertNull(actualInfo?.resourceLimit) + resInfo = ResInfo("//", listOf()) + actualInfo = rule.getRateLimitRule(resInfo) + assertNull(actualInfo?.resourceLimit) + } + + @Test + fun testBandwidthRateLimitRuleAndGetRateLimitRuleWithDifferentOrder() { + val rule = DownloadBandwidthRateLimitRule() + rule.addRateLimitRule(l3) + rule.addRateLimitRule(l2) + + var resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + var actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + + resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testBandwidthRateLimitRuleAndGetRateLimitRuleWithDuplicatedLimitInfos() { + val rule = DownloadBandwidthRateLimitRule() + rule.addRateLimitRule(l3) + rule.addRateLimitRule(l3) + val resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + val actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testBandwidthRateLimitRuleWithInvalidLimitInfo() { + val rule = DownloadBandwidthRateLimitRule() + val rl = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/2/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + assertThrows { rule.addRateLimitRule(rl) } + } + + @Test + fun testBandwidthRateLimitRuleWithInvalidUrl() { + val rule = DownloadBandwidthRateLimitRule() + rule.addRateLimitRule(l1) + rule.addRateLimitRule(l2) + val resInfo = ResInfo("invalid url", listOf("invalid url")) + assertThrows { rule.getRateLimitRule(resInfo) } + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/UploadBandwidthRateLimitRuleTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/UploadBandwidthRateLimitRuleTest.kt new file mode 100644 index 0000000000..33ac193447 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/bandwidth/UploadBandwidthRateLimitRuleTest.kt @@ -0,0 +1,236 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.bandwidth + +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.BaseRuleTest +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import java.time.Duration + +class UploadBandwidthRateLimitRuleTest : BaseRuleTest() { + + private val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/", + limitDimension = LimitDimension.UPLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l2 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project1/repo1/", + limitDimension = LimitDimension.UPLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l3 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/repo1/", + limitDimension = LimitDimension.UPLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l4 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project1/*/", + limitDimension = LimitDimension.UPLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l5 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/*/", + limitDimension = LimitDimension.UPLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l6 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/", + limitDimension = LimitDimension.UPLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l7 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/repo3/", + limitDimension = LimitDimension.UPLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @Test + fun testIsEmpty() { + val uploadBandwidthRateLimitRule = UploadBandwidthRateLimitRule() + assertEquals(uploadBandwidthRateLimitRule.isEmpty(), true) + uploadBandwidthRateLimitRule.addRateLimitRule(l1) + assertEquals(uploadBandwidthRateLimitRule.isEmpty(), false) + } + + @Test + fun testBandwidthRateLimitRuleAndGetRateLimitRule() { + val uploadBandwidthRateLimitRule = UploadBandwidthRateLimitRule() + uploadBandwidthRateLimitRule.addRateLimitRule(l1) + uploadBandwidthRateLimitRule.addRateLimitRule(l2) + uploadBandwidthRateLimitRule.addRateLimitRule(l3) + uploadBandwidthRateLimitRule.addRateLimitRule(l4) + + var resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + var actualInfo = uploadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + assertEquals(actualInfo?.resource, "/project1/repo1/") + + resInfo = ResInfo("/project1/repo2/", listOf("/project1/")) + actualInfo = uploadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + assertEquals(actualInfo?.resource, "/project1/repo2/") + + resInfo = ResInfo("/project2/repo2/", listOf("/project2/")) + actualInfo = uploadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, null) + assertEquals(actualInfo?.resource, null) + + resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + actualInfo = uploadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEquals(actualInfo?.resource, "/project2/repo1/") + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = uploadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + assertEquals(actualInfo?.resource, "/project3/") + + uploadBandwidthRateLimitRule.addRateLimitRule(l6) + + resInfo = ResInfo("/project4/repo4/", listOf("/project4/")) + actualInfo = uploadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + assertEquals(actualInfo?.resource, "/project4/") + + uploadBandwidthRateLimitRule.addRateLimitRule(l5) + + resInfo = ResInfo("/project4/repo4/", listOf("/project4/")) + actualInfo = uploadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + assertEquals(actualInfo?.resource, "/project4/repo4/") + + resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + actualInfo = uploadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + assertEquals(actualInfo?.resource, "/project1/repo1/") + + resInfo = ResInfo("/project1/repo2/", listOf("/project1/")) + actualInfo = uploadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + assertEquals(actualInfo?.resource, "/project1/repo2/") + + resInfo = ResInfo("/project2/repo2/", listOf("/project2/")) + actualInfo = uploadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + assertEquals(actualInfo?.resource, "/project2/repo2/") + + resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + actualInfo = uploadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEquals(actualInfo?.resource, "/project2/repo1/") + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = uploadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + assertEquals(actualInfo?.resource, "/project3/") + + uploadBandwidthRateLimitRule.addRateLimitRule(l7) + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = uploadBandwidthRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l7) + assertEquals(actualInfo?.resource, "/project3/repo3/") + } + + @Test + fun testBandwidthRateLimitRuleAndGetRateLimitRuleWithEmptyRule() { + val rule = UploadBandwidthRateLimitRule() + var resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + var info = rule.getRateLimitRule(resInfo) + assertNull(info) + resInfo = ResInfo("/", listOf()) + info = rule.getRateLimitRule(resInfo) + assertNull(info) + } + + @Test + fun testBandwidthRateLimitRuleAndGetRateLimitRuleWithResEmpty() { + val rule = UploadBandwidthRateLimitRule() + rule.addRateLimitRule(l1) + var resInfo = ResInfo("", listOf()) + var actualInfo = rule.getRateLimitRule(resInfo) + assertNull(actualInfo?.resourceLimit) + resInfo = ResInfo("//", listOf()) + actualInfo = rule.getRateLimitRule(resInfo) + assertNull(actualInfo?.resourceLimit) + } + + @Test + fun testBandwidthRateLimitRuleAndGetRateLimitRuleWithDifferentOrder() { + val rule = UploadBandwidthRateLimitRule() + rule.addRateLimitRule(l3) + rule.addRateLimitRule(l2) + + var resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + var actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + + resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testBandwidthRateLimitRuleAndGetRateLimitRuleWithDuplicatedLimitInfos() { + val rule = UploadBandwidthRateLimitRule() + rule.addRateLimitRule(l3) + rule.addRateLimitRule(l3) + val resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + val actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testBandwidthRateLimitRuleWithInvalidLimitInfo() { + val rule = UploadBandwidthRateLimitRule() + val rl = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/2/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + assertThrows { rule.addRateLimitRule(rl) } + } + + @Test + fun testBandwidthRateLimitRuleWithInvalidUrl() { + val rule = UploadBandwidthRateLimitRule() + rule.addRateLimitRule(l1) + rule.addRateLimitRule(l2) + val resInfo = ResInfo("invalid url", listOf("invalid url")) + assertThrows { rule.getRateLimitRule(resInfo) } + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlRateLimitRuleTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlRateLimitRuleTest.kt new file mode 100644 index 0000000000..a043023d71 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlRateLimitRuleTest.kt @@ -0,0 +1,296 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.url + +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.BaseRuleTest +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import java.time.Duration + +class UrlRateLimitRuleTest : BaseRuleTest() { + + private val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l2 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project1/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l3 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project1/repo1/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l4 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project2/*/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l5 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/*/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l6 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l7 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/repo3/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l8 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/{(^[a-zA-Z]*\$)}/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l9 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/{(^[0-9]*\$)}/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l10 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/{repo}}/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l11 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @Test + fun testIsEmpty() { + val urlRateLimitRule = UrlRateLimitRule() + assertEquals(urlRateLimitRule.isEmpty(), true) + urlRateLimitRule.addRateLimitRule(l1) + assertEquals(urlRateLimitRule.isEmpty(), false) + } + + @Test + fun testUrlRateLimitRuleAndGetRateLimitRule() { + val urlRateLimitRule = UrlRateLimitRule() + urlRateLimitRule.addRateLimitRule(l1) + urlRateLimitRule.addRateLimitRule(l2) + urlRateLimitRule.addRateLimitRule(l3) + urlRateLimitRule.addRateLimitRule(l4) + + var resInfo = ResInfo("///") + var actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertNull(actualInfo) + + resInfo = ResInfo("/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + assertEquals(actualInfo?.resource, "/") + + resInfo = ResInfo("/project1/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + assertEquals(actualInfo?.resource, "/project1/") + + resInfo = ResInfo("/project2/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + assertEquals(actualInfo?.resource, "/project2/") + + urlRateLimitRule.addRateLimitRule(l11) + resInfo = ResInfo("/project3/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + assertEquals(actualInfo?.resource, "/project3/") + + resInfo = ResInfo("/project1/repo1/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEquals(actualInfo?.resource, "/project1/repo1/") + + resInfo = ResInfo("/project3/repo3/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + assertEquals(actualInfo?.resource, "/project3/repo3/") + + urlRateLimitRule.addRateLimitRule(l6) + + resInfo = ResInfo("/project4/repo4/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + assertEquals(actualInfo?.resource, "/project4/repo4/") + + urlRateLimitRule.addRateLimitRule(l5) + + resInfo = ResInfo("/project4/repo4/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + assertEquals(actualInfo?.resource, "/project4/repo4/") + + resInfo = ResInfo("/project1/repo1/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEquals(actualInfo?.resource, "/project1/repo1/") + + resInfo = ResInfo("/project1/repo2/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + assertEquals(actualInfo?.resource, "/project1/repo2/") + + resInfo = ResInfo("/project2/repo2/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + assertEquals(actualInfo?.resource, "/project2/repo2/") + + resInfo = ResInfo("/project3/repo1/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + assertEquals(actualInfo?.resource, "/project3/repo1/") + + resInfo = ResInfo("/project4/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + assertEquals(actualInfo?.resource, "/project4/") + + resInfo = ResInfo("/project4/repo4/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + assertEquals(actualInfo?.resource, "/project4/repo4/") + + urlRateLimitRule.addRateLimitRule(l7) + urlRateLimitRule.addRateLimitRule(l8) + urlRateLimitRule.addRateLimitRule(l9) + + resInfo = ResInfo("/project3/repo3/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l7) + assertEquals(actualInfo?.resource, "/project3/repo3/") + + resInfo = ResInfo("/project3/repo4/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + assertEquals(actualInfo?.resource, "/project3/repo4/") + + resInfo = ResInfo("/project3/xxxx/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l8) + assertEquals(actualInfo?.resource, "/project3/xxxx/") + + resInfo = ResInfo("/project3/1234/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l9) + assertEquals(actualInfo?.resource, "/project3/1234/") + + urlRateLimitRule.addRateLimitRule(l10) + resInfo = ResInfo("/project3/xxxx/") + actualInfo = urlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l10) + assertEquals(actualInfo?.resource, "/project3/xxxx/") + + + } + + @Test + fun testUrlRateLimitRuleAndGetRateLimitRuleWithEmptyRule() { + val rule = UrlRateLimitRule() + var resInfo = ResInfo("/project1/repo1/") + var info = rule.getRateLimitRule(resInfo) + assertNull(info) + resInfo = ResInfo("/", listOf()) + info = rule.getRateLimitRule(resInfo) + assertNull(info) + } + + @Test + fun testUrlRateLimitRuleAndGetRateLimitRuleWithResEmpty() { + val rule = UrlRateLimitRule() + rule.addRateLimitRule(l1) + var resInfo = ResInfo("", listOf()) + var actualInfo = rule.getRateLimitRule(resInfo) + assertNull(actualInfo?.resourceLimit) + resInfo = ResInfo("//", listOf()) + actualInfo = rule.getRateLimitRule(resInfo) + assertNull(actualInfo?.resourceLimit) + } + + @Test + fun testUrlRateLimitRuleAndGetRateLimitRuleWithDifferentOrder() { + val rule = UrlRateLimitRule() + rule.addRateLimitRule(l5) + + val resInfo = ResInfo("/project1/repo1/") + var actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + + rule.addRateLimitRule(l3) + + actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testUrlRateLimitRuleAndGetRateLimitRuleWithDuplicatedLimitInfos() { + val rule = UrlRateLimitRule() + rule.addRateLimitRule(l3) + rule.addRateLimitRule(l3) + val resInfo = ResInfo("/project1/repo1/") + val actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testUrlRateLimitRuleWithInvalidLimitInfo() { + val rule = UrlRateLimitRule() + val rl = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/2/", + limitDimension = LimitDimension.UPLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + assertThrows { rule.addRateLimitRule(rl) } + } + + @Test + fun testUrlRateLimitRuleWithInvalidUrl() { + val rule = UrlRateLimitRule() + rule.addRateLimitRule(l1) + rule.addRateLimitRule(l2) + val resInfo = ResInfo("invalid url", listOf("invalid url")) + assertThrows { rule.getRateLimitRule(resInfo) } + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlRepoRateLimitRuleTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlRepoRateLimitRuleTest.kt new file mode 100644 index 0000000000..f74009cf46 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/UrlRepoRateLimitRuleTest.kt @@ -0,0 +1,236 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.url + +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.BaseRuleTest +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import java.time.Duration +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows + +class UrlRepoRateLimitRuleTest : BaseRuleTest() { + + private val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/", + limitDimension = LimitDimension.URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l2 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project1/repo1/", + limitDimension = LimitDimension.URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l3 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/repo1/", + limitDimension = LimitDimension.URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l4 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project1/*/", + limitDimension = LimitDimension.URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l5 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/*/", + limitDimension = LimitDimension.URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l6 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/", + limitDimension = LimitDimension.URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l7 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/repo3/", + limitDimension = LimitDimension.URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @Test + fun testIsEmpty() { + val urlRepoRateLimitRule = UrlRepoRateLimitRule() + assertEquals(urlRepoRateLimitRule.isEmpty(), true) + urlRepoRateLimitRule.addRateLimitRule(l1) + assertEquals(urlRepoRateLimitRule.isEmpty(), false) + } + + @Test + fun testUserUrlRateLimitRuleAndGetRateLimitRule() { + val urlRepoRateLimitRule = UrlRepoRateLimitRule() + urlRepoRateLimitRule.addRateLimitRule(l1) + urlRepoRateLimitRule.addRateLimitRule(l2) + urlRepoRateLimitRule.addRateLimitRule(l3) + urlRepoRateLimitRule.addRateLimitRule(l4) + + var resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + var actualInfo = urlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + assertEquals(actualInfo?.resource, "/project1/repo1/") + + resInfo = ResInfo("/project1/repo2/", listOf("/project1/")) + actualInfo = urlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + assertEquals(actualInfo?.resource, "/project1/repo2/") + + resInfo = ResInfo("/project2/repo2/", listOf("/project2/")) + actualInfo = urlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, null) + assertEquals(actualInfo?.resource, null) + + resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + actualInfo = urlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEquals(actualInfo?.resource, "/project2/repo1/") + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = urlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + assertEquals(actualInfo?.resource, "/project3/") + + urlRepoRateLimitRule.addRateLimitRule(l6) + + resInfo = ResInfo("/project4/repo4/", listOf("/project4/")) + actualInfo = urlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + assertEquals(actualInfo?.resource, "/project4/") + + urlRepoRateLimitRule.addRateLimitRule(l5) + + resInfo = ResInfo("/project4/repo4/", listOf("/project4/")) + actualInfo = urlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + assertEquals(actualInfo?.resource, "/project4/repo4/") + + resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + actualInfo = urlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + assertEquals(actualInfo?.resource, "/project1/repo1/") + + resInfo = ResInfo("/project1/repo2/", listOf("/project1/")) + actualInfo = urlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + assertEquals(actualInfo?.resource, "/project1/repo2/") + + resInfo = ResInfo("/project2/repo2/", listOf("/project2/")) + actualInfo = urlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + assertEquals(actualInfo?.resource, "/project2/repo2/") + + resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + actualInfo = urlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEquals(actualInfo?.resource, "/project2/repo1/") + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = urlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + assertEquals(actualInfo?.resource, "/project3/") + + urlRepoRateLimitRule.addRateLimitRule(l7) + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = urlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l7) + assertEquals(actualInfo?.resource, "/project3/repo3/") + } + + @Test + fun testUserUrlRateLimitRuleAndGetRateLimitRuleWithEmptyRule() { + val rule = UrlRepoRateLimitRule() + var resInfo = ResInfo("/project1/repo1/") + var info = rule.getRateLimitRule(resInfo) + assertNull(info) + resInfo = ResInfo("/", listOf()) + info = rule.getRateLimitRule(resInfo) + assertNull(info) + } + + @Test + fun testUserUrlRateLimitRuleAndGetRateLimitRuleWithResEmpty() { + val rule = UrlRepoRateLimitRule() + rule.addRateLimitRule(l1) + var resInfo = ResInfo("", listOf()) + var actualInfo = rule.getRateLimitRule(resInfo) + assertNull(actualInfo?.resourceLimit) + resInfo = ResInfo("//", listOf()) + actualInfo = rule.getRateLimitRule(resInfo) + assertNull(actualInfo?.resourceLimit) + } + + @Test + fun testUserUrlRateLimitRuleAndGetRateLimitRuleWithDifferentOrder() { + val rule = UrlRepoRateLimitRule() + rule.addRateLimitRule(l5) + + val resInfo = ResInfo("/project1/repo1/") + var actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + + rule.addRateLimitRule(l3) + + actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testUserUrlRateLimitRuleAndGetRateLimitRuleWithDuplicatedLimitInfos() { + val rule = UrlRepoRateLimitRule() + rule.addRateLimitRule(l3) + rule.addRateLimitRule(l3) + val resInfo = ResInfo("/project1/repo1/") + val actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testUserUrlRateLimitRuleWithInvalidLimitInfo() { + val rule = UrlRepoRateLimitRule() + val rl = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/2/", + limitDimension = LimitDimension.UPLOAD_BANDWIDTH.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + assertThrows { rule.addRateLimitRule(rl) } + } + + @Test + fun testUserUrlRateLimitRuleWithInvalidUrl() { + val rule = UrlRepoRateLimitRule() + rule.addRateLimitRule(l1) + rule.addRateLimitRule(l2) + val resInfo = ResInfo("invalid url", listOf("invalid url")) + assertThrows { rule.getRateLimitRule(resInfo) } + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRateLimitRuleTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRateLimitRuleTest.kt new file mode 100644 index 0000000000..ae75f399a4 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRateLimitRuleTest.kt @@ -0,0 +1,339 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.url.user + +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.BaseRuleTest +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import java.time.Duration + +class UserUrlRateLimitRuleTest : BaseRuleTest() { + + private val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "*:/", + limitDimension = LimitDimension.USER_URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l2 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project1/", + limitDimension = LimitDimension.USER_URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l3 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project1/repo1/", + limitDimension = LimitDimension.USER_URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l4 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project2/*/", + limitDimension = LimitDimension.USER_URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l5 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/*/*/", + limitDimension = LimitDimension.USER_URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l6 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/*/", + limitDimension = LimitDimension.USER_URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l7 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/repo3/", + limitDimension = LimitDimension.USER_URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l8 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/{(^[a-zA-Z]*\$)}/", + limitDimension = LimitDimension.USER_URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l9 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/{(^[0-9]*\$)}/", + limitDimension = LimitDimension.USER_URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l10 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/{repo}}/", + limitDimension = LimitDimension.USER_URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l11 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/", + limitDimension = LimitDimension.USER_URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l12 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project1/repo1/", + limitDimension = LimitDimension.USER_URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + private val l13 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "*:", + limitDimension = LimitDimension.USER_URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + private val l14 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:", + limitDimension = LimitDimension.USER_URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @Test + fun testIsEmpty() { + val userUrlRateLimitRule = UserUrlRateLimitRule() + assertEquals(userUrlRateLimitRule.isEmpty(), true) + userUrlRateLimitRule.addRateLimitRule(l1) + assertEquals(userUrlRateLimitRule.isEmpty(), false) + } + + @Test + fun testUserUrlRateLimitRuleAndGetRateLimitRule() { + val userUrlRateLimitRule = UserUrlRateLimitRule() + userUrlRateLimitRule.addRateLimitRule(l13) + var resInfo = ResInfo("user1:/project3/repo2/", listOf("user1:")) + var actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l13) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + userUrlRateLimitRule.addRateLimitRule(l14) + resInfo = ResInfo("user1:/project3/repo2/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l14) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + + userUrlRateLimitRule.addRateLimitRule(l1) + + resInfo = ResInfo("user1:///", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l14) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + + resInfo = ResInfo("user1:/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + Assertions.assertEquals(actualInfo?.resource, "user1:/") + + + + userUrlRateLimitRule.addRateLimitRule(l2) + userUrlRateLimitRule.addRateLimitRule(l3) + userUrlRateLimitRule.addRateLimitRule(l4) + + resInfo = ResInfo("user1:/project1/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/") + + resInfo = ResInfo("user1:/project2/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + Assertions.assertEquals(actualInfo?.resource, "user1:/project2/") + + userUrlRateLimitRule.addRateLimitRule(l11) + resInfo = ResInfo("user1:/project3/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/") + + resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/repo1/") + + resInfo = ResInfo("user1:/project3/repo3/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/repo3/") + + userUrlRateLimitRule.addRateLimitRule(l6) + + resInfo = ResInfo("user1:/project4/repo4/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/repo4/") + + userUrlRateLimitRule.addRateLimitRule(l5) + + resInfo = ResInfo("user1:/project4/repo4/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/repo4/") + + resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/repo1/") + + resInfo = ResInfo("user1:/project1/repo2/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/repo2/") + + resInfo = ResInfo("user1:/project2/repo2/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + Assertions.assertEquals(actualInfo?.resource, "user1:/project2/repo2/") + + resInfo = ResInfo("user1:/project3/repo1/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/repo1/") + + resInfo = ResInfo("user1:/project4/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/") + + resInfo = ResInfo("user1:/project4/repo4/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/repo4/") + + userUrlRateLimitRule.addRateLimitRule(l7) + userUrlRateLimitRule.addRateLimitRule(l8) + userUrlRateLimitRule.addRateLimitRule(l9) + + resInfo = ResInfo("user1:/project3/repo3/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l7) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/repo3/") + + resInfo = ResInfo("user1:/project3/repo4/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/repo4/") + + resInfo = ResInfo("user1:/project3/xxxx/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l8) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/xxxx/") + + resInfo = ResInfo("user1:/project3/1234/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l9) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/1234/") + + userUrlRateLimitRule.addRateLimitRule(l10) + resInfo = ResInfo("user1:/project3/xxxx/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l10) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/xxxx/") + + + resInfo = ResInfo("user2:/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + Assertions.assertEquals(actualInfo?.resource, "user2:/") + + resInfo = ResInfo("user2:/project3/xxxx/", listOf("user1:")) + actualInfo = userUrlRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + Assertions.assertEquals(actualInfo?.resource, "user2:/project3/xxxx/") + + } + + @Test + fun testUserUrlRateLimitRuleAndGetRateLimitRuleWithEmptyRule() { + val rule = UserUrlRateLimitRule() + var resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:")) + var info = rule.getRateLimitRule(resInfo) + Assertions.assertNull(info) + resInfo = ResInfo("user1:/", listOf("user1:")) + info = rule.getRateLimitRule(resInfo) + Assertions.assertNull(info) + } + + @Test + fun testUserUrlRateLimitRuleAndGetRateLimitRuleWithResEmpty() { + val rule = UserUrlRateLimitRule() + rule.addRateLimitRule(l1) + var resInfo = ResInfo("", listOf("user1:")) + var actualInfo = rule.getRateLimitRule(resInfo) + Assertions.assertNull(actualInfo?.resourceLimit) + } + + @Test + fun testUserUrlRateLimitRuleAndGetRateLimitRuleWithDifferentOrder() { + val rule = UserUrlRateLimitRule() + rule.addRateLimitRule(l5) + + val resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:")) + var actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + + rule.addRateLimitRule(l3) + + actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testUserUrlRateLimitRuleAndGetRateLimitRuleWithDuplicatedLimitInfos() { + val rule = UserUrlRateLimitRule() + rule.addRateLimitRule(l3) + rule.addRateLimitRule(l12) + val resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:")) + val actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l12) + } + + @Test + fun testUserUrlRateLimitRuleWithInvalidLimitInfo() { + val rule = UserUrlRateLimitRule() + val rl = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/2/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + assertThrows { rule.addRateLimitRule(rl) } + } + + @Test + fun testUserUrlRateLimitRuleWithInvalidUrl() { + val rule = UserUrlRateLimitRule() + rule.addRateLimitRule(l1) + rule.addRateLimitRule(l2) + val resInfo = ResInfo("invalid url") + assertThrows { rule.getRateLimitRule(resInfo) } + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRepoRateLimitRuleTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRepoRateLimitRuleTest.kt new file mode 100644 index 0000000000..efdd708692 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/url/user/UserUrlRepoRateLimitRuleTest.kt @@ -0,0 +1,340 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.url.user + +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.BaseRuleTest +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import java.time.Duration +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows + +class UserUrlRepoRateLimitRuleTest : BaseRuleTest() { + + private val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "*:/", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l2 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project1/", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l3 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project1/repo1/", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l4 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project2/*/", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l5 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/*/*/", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l6 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/*/", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l7 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/repo3/", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l8 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/{(^[a-zA-Z]*\$)}/", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l9 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/{(^[0-9]*\$)}/", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l10 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/{repo}}/", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l11 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l12 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project1/repo1/", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + private val l13 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "*:", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + private val l14 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @Test + fun testIsEmpty() { + val userUrlRepoRateLimitRule = UserUrlRepoRateLimitRule() + assertEquals(userUrlRepoRateLimitRule.isEmpty(), true) + userUrlRepoRateLimitRule.addRateLimitRule(l1) + assertEquals(userUrlRepoRateLimitRule.isEmpty(), false) + } + + @Test + fun testUserUrlRepoRateLimitRuleAndGetRateLimitRule() { + val userUrlRepoRateLimitRule = UserUrlRepoRateLimitRule() + userUrlRepoRateLimitRule.addRateLimitRule(l13) + var resInfo = ResInfo("user1:/project3/repo2/", listOf("user1:/project3/", "user1:")) + var actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l13) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + userUrlRepoRateLimitRule.addRateLimitRule(l14) + + resInfo = ResInfo("user1:/project3/repo2/", listOf("user1:/project3/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l14) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + + userUrlRepoRateLimitRule.addRateLimitRule(l1) + + resInfo = ResInfo("user1:///", listOf("user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l14) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + + resInfo = ResInfo("user1:/", listOf("user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + Assertions.assertEquals(actualInfo?.resource, "user1:/") + + + + userUrlRepoRateLimitRule.addRateLimitRule(l2) + userUrlRepoRateLimitRule.addRateLimitRule(l3) + userUrlRepoRateLimitRule.addRateLimitRule(l4) + + resInfo = ResInfo("user1:/project1/", listOf("user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/") + + resInfo = ResInfo("user1:/project2/", listOf("user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l14) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + userUrlRepoRateLimitRule.addRateLimitRule(l11) + + resInfo = ResInfo("user1:/project3/", listOf("user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/") + + resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:/project1/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/repo1/") + + resInfo = ResInfo("user1:/project3/repo3/", listOf("user1:/project3/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/") + + userUrlRepoRateLimitRule.addRateLimitRule(l6) + + resInfo = ResInfo("user1:/project4/repo4/", listOf("user1:/project4/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/") + + userUrlRepoRateLimitRule.addRateLimitRule(l5) + + resInfo = ResInfo("user1:/project4/repo4/", listOf("user1:/project4/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/repo4/") + + resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:/project1/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/repo1/") + + resInfo = ResInfo("user1:/project1/repo2/", listOf("user1:/project1/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/") + + resInfo = ResInfo("user1:/project2/repo2/", listOf("user1:/project2/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + Assertions.assertEquals(actualInfo?.resource, "user1:/project2/repo2/") + + resInfo = ResInfo("user1:/project3/repo1/", listOf("user1:/project3/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/") + + resInfo = ResInfo("user1:/project4/", listOf("user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/") + + resInfo = ResInfo("user1:/project4/repo4/", listOf("user1:/project4/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/repo4/") + + userUrlRepoRateLimitRule.addRateLimitRule(l7) + userUrlRepoRateLimitRule.addRateLimitRule(l8) + userUrlRepoRateLimitRule.addRateLimitRule(l9) + + resInfo = ResInfo("user1:/project3/repo3/", listOf("user1:/project3/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l7) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/repo3/") + + resInfo = ResInfo("user1:/project3/repo4/", listOf("user1:/project3/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/") + + resInfo = ResInfo("user1:/project3/xxxx/", listOf("user1:/project3/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l8) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/xxxx/") + + resInfo = ResInfo("user1:/project3/1234/", listOf("user1:/project3/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l9) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/1234/") + + userUrlRepoRateLimitRule.addRateLimitRule(l10) + resInfo = ResInfo("user1:/project3/xxxx/", listOf("user1:/project3/", "user1:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l10) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/xxxx/") + + + resInfo = ResInfo("user2:/", listOf("user2:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + Assertions.assertEquals(actualInfo?.resource, "user2:/") + + resInfo = ResInfo("user2:/project3/xxxx/", listOf("user2:/project3/", "user2:")) + actualInfo = userUrlRepoRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l13) + Assertions.assertEquals(actualInfo?.resource, "user2:") + } + + @Test + fun testUserUrlRepoRateLimitRuleAndGetRateLimitRuleWithEmptyRule() { + val rule = UserUrlRepoRateLimitRule() + var resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:")) + var info = rule.getRateLimitRule(resInfo) + Assertions.assertNull(info) + resInfo = ResInfo("user1:/", listOf("user1:")) + info = rule.getRateLimitRule(resInfo) + Assertions.assertNull(info) + } + + @Test + fun testUserUrlRepoRateLimitRuleAndGetRateLimitRuleWithResEmpty() { + val rule = UserUrlRepoRateLimitRule() + rule.addRateLimitRule(l1) + var resInfo = ResInfo("", listOf("user1:")) + var actualInfo = rule.getRateLimitRule(resInfo) + Assertions.assertNull(actualInfo?.resourceLimit) + } + + @Test + fun testUserUrlRepoRateLimitRuleAndGetRateLimitRuleWithDifferentOrder() { + val rule = UserUrlRepoRateLimitRule() + rule.addRateLimitRule(l5) + + val resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:")) + var actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + + rule.addRateLimitRule(l3) + + actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testUserUrlRepoRateLimitRuleAndGetRateLimitRuleWithDuplicatedLimitInfos() { + val rule = UserUrlRepoRateLimitRule() + rule.addRateLimitRule(l3) + rule.addRateLimitRule(l12) + val resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:")) + val actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l12) + } + + @Test + fun testUserUrlRepoRateLimitRuleWithInvalidLimitInfo() { + val rule = UserUrlRepoRateLimitRule() + val rl = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/2/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + assertThrows { rule.addRateLimitRule(rl) } + } + + @Test + fun testUserUrlRepoRateLimitRuleWithInvalidUrl() { + val rule = UserUrlRepoRateLimitRule() + rule.addRateLimitRule(l1) + rule.addRateLimitRule(l2) + val resInfo = ResInfo("invalid url") + assertThrows { rule.getRateLimitRule(resInfo) } + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/DownloadUsageRateLimitRuleTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/DownloadUsageRateLimitRuleTest.kt new file mode 100644 index 0000000000..04aac62941 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/DownloadUsageRateLimitRuleTest.kt @@ -0,0 +1,236 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.usage + +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.BaseRuleTest +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import java.time.Duration + +class DownloadUsageRateLimitRuleTest : BaseRuleTest() { + + private val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/", + limitDimension = LimitDimension.DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l2 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project1/repo1/", + limitDimension = LimitDimension.DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l3 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/repo1/", + limitDimension = LimitDimension.DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l4 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project1/*/", + limitDimension = LimitDimension.DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l5 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/*/", + limitDimension = LimitDimension.DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l6 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/", + limitDimension = LimitDimension.DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l7 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/repo3/", + limitDimension = LimitDimension.DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @Test + fun testIsEmpty() { + val downloadUsageRateLimitRule = DownloadUsageRateLimitRule() + assertEquals(downloadUsageRateLimitRule.isEmpty(), true) + downloadUsageRateLimitRule.addRateLimitRule(l1) + assertEquals(downloadUsageRateLimitRule.isEmpty(), false) + } + + @Test + fun testUsageRateLimitRuleAndGetRateLimitRule() { + val downloadUsageRateLimitRule = DownloadUsageRateLimitRule() + downloadUsageRateLimitRule.addRateLimitRule(l1) + downloadUsageRateLimitRule.addRateLimitRule(l2) + downloadUsageRateLimitRule.addRateLimitRule(l3) + downloadUsageRateLimitRule.addRateLimitRule(l4) + + var resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + var actualInfo = downloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + assertEquals(actualInfo?.resource, "/project1/repo1/") + + resInfo = ResInfo("/project1/repo2/", listOf("/project1/")) + actualInfo = downloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + assertEquals(actualInfo?.resource, "/project1/repo2/") + + resInfo = ResInfo("/project2/repo2/", listOf("/project2/")) + actualInfo = downloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, null) + assertEquals(actualInfo?.resource, null) + + resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + actualInfo = downloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEquals(actualInfo?.resource, "/project2/repo1/") + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = downloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + assertEquals(actualInfo?.resource, "/project3/") + + downloadUsageRateLimitRule.addRateLimitRule(l6) + + resInfo = ResInfo("/project4/repo4/", listOf("/project4/")) + actualInfo = downloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + assertEquals(actualInfo?.resource, "/project4/") + + downloadUsageRateLimitRule.addRateLimitRule(l5) + + resInfo = ResInfo("/project4/repo4/", listOf("/project4/")) + actualInfo = downloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + assertEquals(actualInfo?.resource, "/project4/repo4/") + + resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + actualInfo = downloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + assertEquals(actualInfo?.resource, "/project1/repo1/") + + resInfo = ResInfo("/project1/repo2/", listOf("/project1/")) + actualInfo = downloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + assertEquals(actualInfo?.resource, "/project1/repo2/") + + resInfo = ResInfo("/project2/repo2/", listOf("/project2/")) + actualInfo = downloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + assertEquals(actualInfo?.resource, "/project2/repo2/") + + resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + actualInfo = downloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEquals(actualInfo?.resource, "/project2/repo1/") + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = downloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + assertEquals(actualInfo?.resource, "/project3/") + + downloadUsageRateLimitRule.addRateLimitRule(l7) + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = downloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l7) + assertEquals(actualInfo?.resource, "/project3/repo3/") + } + + @Test + fun testUsageRateLimitRuleAndGetRateLimitRuleWithEmptyRule() { + val rule = DownloadUsageRateLimitRule() + var resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + var info = rule.getRateLimitRule(resInfo) + assertNull(info) + resInfo = ResInfo("/", listOf()) + info = rule.getRateLimitRule(resInfo) + assertNull(info) + } + + @Test + fun testUsageRateLimitRuleAndGetRateLimitRuleWithResEmpty() { + val rule = DownloadUsageRateLimitRule() + rule.addRateLimitRule(l1) + var resInfo = ResInfo("", listOf()) + var actualInfo = rule.getRateLimitRule(resInfo) + assertNull(actualInfo?.resourceLimit) + resInfo = ResInfo("//", listOf()) + actualInfo = rule.getRateLimitRule(resInfo) + assertNull(actualInfo?.resourceLimit) + } + + @Test + fun testUsageRateLimitRuleAndGetRateLimitRuleWithDifferentOrder() { + val rule = DownloadUsageRateLimitRule() + rule.addRateLimitRule(l3) + rule.addRateLimitRule(l2) + + var resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + var actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + + resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testUsageRateLimitRuleAndGetRateLimitRuleWithDuplicatedLimitInfos() { + val rule = DownloadUsageRateLimitRule() + rule.addRateLimitRule(l3) + rule.addRateLimitRule(l3) + val resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + val actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testUsageRateLimitRuleWithInvalidLimitInfo() { + val rule = DownloadUsageRateLimitRule() + val rl = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/2/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + assertThrows { rule.addRateLimitRule(rl) } + } + + @Test + fun testUsageRateLimitRuleWithInvalidUrl() { + val rule = DownloadUsageRateLimitRule() + rule.addRateLimitRule(l1) + rule.addRateLimitRule(l2) + val resInfo = ResInfo("invalid url", listOf("invalid url")) + assertThrows { rule.getRateLimitRule(resInfo) } + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/UploadUsageRateLimitRuleTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/UploadUsageRateLimitRuleTest.kt new file mode 100644 index 0000000000..8d634f519b --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/UploadUsageRateLimitRuleTest.kt @@ -0,0 +1,236 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.usage + +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.BaseRuleTest +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import java.time.Duration + +class UploadUsageRateLimitRuleTest : BaseRuleTest() { + + private val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/", + limitDimension = LimitDimension.UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l2 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project1/repo1/", + limitDimension = LimitDimension.UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l3 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/repo1/", + limitDimension = LimitDimension.UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l4 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project1/*/", + limitDimension = LimitDimension.UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l5 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/*/", + limitDimension = LimitDimension.UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l6 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/", + limitDimension = LimitDimension.UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l7 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/project3/repo3/", + limitDimension = LimitDimension.UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @Test + fun testIsEmpty() { + val uploadUsageRateLimitRule = UploadUsageRateLimitRule() + assertEquals(uploadUsageRateLimitRule.isEmpty(), true) + uploadUsageRateLimitRule.addRateLimitRule(l1) + assertEquals(uploadUsageRateLimitRule.isEmpty(), false) + } + + @Test + fun testUsageRateLimitRuleAndGetRateLimitRule() { + val uploadUsageRateLimitRule = UploadUsageRateLimitRule() + uploadUsageRateLimitRule.addRateLimitRule(l1) + uploadUsageRateLimitRule.addRateLimitRule(l2) + uploadUsageRateLimitRule.addRateLimitRule(l3) + uploadUsageRateLimitRule.addRateLimitRule(l4) + + var resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + var actualInfo = uploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + assertEquals(actualInfo?.resource, "/project1/repo1/") + + resInfo = ResInfo("/project1/repo2/", listOf("/project1/")) + actualInfo = uploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + assertEquals(actualInfo?.resource, "/project1/repo2/") + + resInfo = ResInfo("/project2/repo2/", listOf("/project2/")) + actualInfo = uploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, null) + assertEquals(actualInfo?.resource, null) + + resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + actualInfo = uploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEquals(actualInfo?.resource, "/project2/repo1/") + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = uploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + assertEquals(actualInfo?.resource, "/project3/") + + uploadUsageRateLimitRule.addRateLimitRule(l6) + + resInfo = ResInfo("/project4/repo4/", listOf("/project4/")) + actualInfo = uploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + assertEquals(actualInfo?.resource, "/project4/") + + uploadUsageRateLimitRule.addRateLimitRule(l5) + + resInfo = ResInfo("/project4/repo4/", listOf("/project4/")) + actualInfo = uploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + assertEquals(actualInfo?.resource, "/project4/repo4/") + + resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + actualInfo = uploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + assertEquals(actualInfo?.resource, "/project1/repo1/") + + resInfo = ResInfo("/project1/repo2/", listOf("/project1/")) + actualInfo = uploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + assertEquals(actualInfo?.resource, "/project1/repo2/") + + resInfo = ResInfo("/project2/repo2/", listOf("/project2/")) + actualInfo = uploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + assertEquals(actualInfo?.resource, "/project2/repo2/") + + resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + actualInfo = uploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEquals(actualInfo?.resource, "/project2/repo1/") + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = uploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + assertEquals(actualInfo?.resource, "/project3/") + + uploadUsageRateLimitRule.addRateLimitRule(l7) + + resInfo = ResInfo("/project3/repo3/", listOf("/project3/")) + actualInfo = uploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l7) + assertEquals(actualInfo?.resource, "/project3/repo3/") + } + + @Test + fun testUsageRateLimitRuleAndGetRateLimitRuleWithEmptyRule() { + val rule = UploadUsageRateLimitRule() + var resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + var info = rule.getRateLimitRule(resInfo) + assertNull(info) + resInfo = ResInfo("/", listOf()) + info = rule.getRateLimitRule(resInfo) + assertNull(info) + } + + @Test + fun testUsageRateLimitRuleAndGetRateLimitRuleWithResEmpty() { + val rule = UploadUsageRateLimitRule() + rule.addRateLimitRule(l1) + var resInfo = ResInfo("", listOf()) + var actualInfo = rule.getRateLimitRule(resInfo) + assertNull(actualInfo?.resourceLimit) + resInfo = ResInfo("//", listOf()) + actualInfo = rule.getRateLimitRule(resInfo) + assertNull(actualInfo?.resourceLimit) + } + + @Test + fun testUsageRateLimitRuleAndGetRateLimitRuleWithDifferentOrder() { + val rule = UploadUsageRateLimitRule() + rule.addRateLimitRule(l3) + rule.addRateLimitRule(l2) + + var resInfo = ResInfo("/project1/repo1/", listOf("/project1/")) + var actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + + resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testUsageRateLimitRuleAndGetRateLimitRuleWithDuplicatedLimitInfos() { + val rule = UploadUsageRateLimitRule() + rule.addRateLimitRule(l3) + rule.addRateLimitRule(l3) + val resInfo = ResInfo("/project2/repo1/", listOf("/project2/")) + val actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testUsageRateLimitRuleWithInvalidLimitInfo() { + val rule = UploadUsageRateLimitRule() + val rl = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/2/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + assertThrows { rule.addRateLimitRule(rl) } + } + + @Test + fun testUsageRateLimitRuleWithInvalidUrl() { + val rule = UploadUsageRateLimitRule() + rule.addRateLimitRule(l1) + rule.addRateLimitRule(l2) + val resInfo = ResInfo("invalid url", listOf("invalid url")) + assertThrows { rule.getRateLimitRule(resInfo) } + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserDownloadUsageRateLimitRuleTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserDownloadUsageRateLimitRuleTest.kt new file mode 100644 index 0000000000..5ec3ae822b --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserDownloadUsageRateLimitRuleTest.kt @@ -0,0 +1,340 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.usage.user + +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.BaseRuleTest +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import java.time.Duration + +class UserDownloadUsageRateLimitRuleTest : BaseRuleTest() { + + private val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "*:/", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l2 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project1/", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l3 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project1/repo1/", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l4 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project2/*/", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l5 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/*/*/", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l6 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/*/", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l7 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/repo3/", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l8 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/{(^[a-zA-Z]*\$)}/", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l9 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/{(^[0-9]*\$)}/", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l10 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/{repo}}/", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l11 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l12 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project1/repo1/", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + private val l13 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "*:", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + private val l14 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @Test + fun testIsEmpty() { + val userDownloadUsageRateLimitRule = UserDownloadUsageRateLimitRule() + assertEquals(userDownloadUsageRateLimitRule.isEmpty(), true) + userDownloadUsageRateLimitRule.addRateLimitRule(l1) + assertEquals(userDownloadUsageRateLimitRule.isEmpty(), false) + } + + @Test + fun testUserDownloadUsageRateLimitRuleAndGetRateLimitRule() { + val userDownloadUsageRateLimitRule = UserDownloadUsageRateLimitRule() + userDownloadUsageRateLimitRule.addRateLimitRule(l13) + var resInfo = ResInfo("user1:/project3/repo2/", listOf("user1:/project3/", "user1:")) + var actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l13) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + userDownloadUsageRateLimitRule.addRateLimitRule(l14) + + resInfo = ResInfo("user1:/project3/repo2/", listOf("user1:/project3/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l14) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + + userDownloadUsageRateLimitRule.addRateLimitRule(l1) + + resInfo = ResInfo("user1:///", listOf("user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l14) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + + resInfo = ResInfo("user1:/", listOf("user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + Assertions.assertEquals(actualInfo?.resource, "user1:/") + + + + userDownloadUsageRateLimitRule.addRateLimitRule(l2) + userDownloadUsageRateLimitRule.addRateLimitRule(l3) + userDownloadUsageRateLimitRule.addRateLimitRule(l4) + + resInfo = ResInfo("user1:/project1/", listOf("user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/") + + resInfo = ResInfo("user1:/project2/", listOf("user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l14) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + userDownloadUsageRateLimitRule.addRateLimitRule(l11) + + resInfo = ResInfo("user1:/project3/", listOf("user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/") + + resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:/project1/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/repo1/") + + resInfo = ResInfo("user1:/project3/repo3/", listOf("user1:/project3/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/") + + userDownloadUsageRateLimitRule.addRateLimitRule(l6) + + resInfo = ResInfo("user1:/project4/repo4/", listOf("user1:/project4/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/") + + userDownloadUsageRateLimitRule.addRateLimitRule(l5) + + resInfo = ResInfo("user1:/project4/repo4/", listOf("user1:/project4/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/repo4/") + + resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:/project1/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/repo1/") + + resInfo = ResInfo("user1:/project1/repo2/", listOf("user1:/project1/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/") + + resInfo = ResInfo("user1:/project2/repo2/", listOf("user1:/project2/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + Assertions.assertEquals(actualInfo?.resource, "user1:/project2/repo2/") + + resInfo = ResInfo("user1:/project3/repo1/", listOf("user1:/project3/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/") + + resInfo = ResInfo("user1:/project4/", listOf("user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/") + + resInfo = ResInfo("user1:/project4/repo4/", listOf("user1:/project4/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/repo4/") + + userDownloadUsageRateLimitRule.addRateLimitRule(l7) + userDownloadUsageRateLimitRule.addRateLimitRule(l8) + userDownloadUsageRateLimitRule.addRateLimitRule(l9) + + resInfo = ResInfo("user1:/project3/repo3/", listOf("user1:/project3/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l7) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/repo3/") + + resInfo = ResInfo("user1:/project3/repo4/", listOf("user1:/project3/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/") + + resInfo = ResInfo("user1:/project3/xxxx/", listOf("user1:/project3/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l8) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/xxxx/") + + resInfo = ResInfo("user1:/project3/1234/", listOf("user1:/project3/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l9) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/1234/") + + userDownloadUsageRateLimitRule.addRateLimitRule(l10) + resInfo = ResInfo("user1:/project3/xxxx/", listOf("user1:/project3/", "user1:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l10) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/xxxx/") + + + resInfo = ResInfo("user2:/", listOf("user2:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + Assertions.assertEquals(actualInfo?.resource, "user2:/") + + resInfo = ResInfo("user2:/project3/xxxx/", listOf("user2:/project3/", "user2:")) + actualInfo = userDownloadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l13) + Assertions.assertEquals(actualInfo?.resource, "user2:") + + } + + @Test + fun testUserDownloadUsageRateLimitRuleAndGetRateLimitRuleWithEmptyRule() { + val rule = UserDownloadUsageRateLimitRule() + var resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:/project1/", "user1:")) + var info = rule.getRateLimitRule(resInfo) + Assertions.assertNull(info) + resInfo = ResInfo("user1:/", listOf("user1:")) + info = rule.getRateLimitRule(resInfo) + Assertions.assertNull(info) + } + + @Test + fun testUserDownloadUsageRateLimitRuleAndGetRateLimitRuleWithResEmpty() { + val rule = UserDownloadUsageRateLimitRule() + rule.addRateLimitRule(l1) + var resInfo = ResInfo("", listOf("")) + assertThrows { rule.getRateLimitRule(resInfo) } + } + + @Test + fun testUserDownloadUsageRateLimitRuleAndGetRateLimitRuleWithDifferentOrder() { + val rule = UserDownloadUsageRateLimitRule() + rule.addRateLimitRule(l5) + + var resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:/project1/", "user1:")) + var actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + + rule.addRateLimitRule(l3) + + actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testUserDownloadUsageRateLimitRuleAndGetRateLimitRuleWithDuplicatedLimitInfos() { + val rule = UserDownloadUsageRateLimitRule() + rule.addRateLimitRule(l3) + rule.addRateLimitRule(l12) + var resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:/project1/", "user1:")) + val actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l12) + } + + @Test + fun testUserDownloadUsageRateLimitRuleWithInvalidLimitInfo() { + val rule = UserDownloadUsageRateLimitRule() + val rl = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/2/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + assertThrows { rule.addRateLimitRule(rl) } + } + + @Test + fun testUserDownloadUsageRateLimitRuleWithInvalidUrl() { + val rule = UserDownloadUsageRateLimitRule() + rule.addRateLimitRule(l1) + rule.addRateLimitRule(l2) + val resInfo = ResInfo("invalid url") + assertThrows { rule.getRateLimitRule(resInfo) } + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserUploadUsageRateLimitRuleTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserUploadUsageRateLimitRuleTest.kt new file mode 100644 index 0000000000..65fa3d0723 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/rule/usage/user/UserUploadUsageRateLimitRuleTest.kt @@ -0,0 +1,340 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.rule.usage.user + +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.rule.BaseRuleTest +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import java.time.Duration + +class UserUploadUsageRateLimitRuleTest : BaseRuleTest() { + + private val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "*:/", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l2 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project1/", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l3 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project1/repo1/", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l4 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project2/*/", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l5 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/*/*/", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l6 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/*/", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l7 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/repo3/", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l8 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/{(^[a-zA-Z]*\$)}/", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l9 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/{(^[0-9]*\$)}/", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l10 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/{repo}}/", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l11 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project3/", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + private val l12 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:/project1/repo1/", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + private val l13 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "*:", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + private val l14 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "user1:", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @Test + fun testIsEmpty() { + val userUploadUsageRateLimitRule = UserUploadUsageRateLimitRule() + assertEquals(userUploadUsageRateLimitRule.isEmpty(), true) + userUploadUsageRateLimitRule.addRateLimitRule(l1) + assertEquals(userUploadUsageRateLimitRule.isEmpty(), false) + } + + @Test + fun testUserUploadUsageRateLimitRuleAndGetRateLimitRule() { + val userUploadUsageRateLimitRule = UserUploadUsageRateLimitRule() + userUploadUsageRateLimitRule.addRateLimitRule(l13) + var resInfo = ResInfo("user1:/project3/repo2/", listOf("user1:/project3/", "user1:")) + var actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l13) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + userUploadUsageRateLimitRule.addRateLimitRule(l14) + + resInfo = ResInfo("user1:/project3/repo2/", listOf("user1:/project3/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l14) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + + userUploadUsageRateLimitRule.addRateLimitRule(l1) + + resInfo = ResInfo("user1:///", listOf("user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l14) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + + resInfo = ResInfo("user1:/", listOf("user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + Assertions.assertEquals(actualInfo?.resource, "user1:/") + + + + userUploadUsageRateLimitRule.addRateLimitRule(l2) + userUploadUsageRateLimitRule.addRateLimitRule(l3) + userUploadUsageRateLimitRule.addRateLimitRule(l4) + + resInfo = ResInfo("user1:/project1/", listOf("user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/") + + resInfo = ResInfo("user1:/project2/", listOf("user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l14) + Assertions.assertEquals(actualInfo?.resource, "user1:") + + userUploadUsageRateLimitRule.addRateLimitRule(l11) + + resInfo = ResInfo("user1:/project3/", listOf("user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/") + + resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:/project1/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/repo1/") + + resInfo = ResInfo("user1:/project3/repo3/", listOf("user1:/project3/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/") + + userUploadUsageRateLimitRule.addRateLimitRule(l6) + + resInfo = ResInfo("user1:/project4/repo4/", listOf("user1:/project4/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/") + + userUploadUsageRateLimitRule.addRateLimitRule(l5) + + resInfo = ResInfo("user1:/project4/repo4/", listOf("user1:/project4/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/repo4/") + + resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:/project1/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/repo1/") + + resInfo = ResInfo("user1:/project1/repo2/", listOf("user1:/project1/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l2) + Assertions.assertEquals(actualInfo?.resource, "user1:/project1/") + + resInfo = ResInfo("user1:/project2/repo2/", listOf("user1:/project2/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l4) + Assertions.assertEquals(actualInfo?.resource, "user1:/project2/repo2/") + + resInfo = ResInfo("user1:/project3/repo1/", listOf("user1:/project3/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/") + + resInfo = ResInfo("user1:/project4/", listOf("user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l6) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/") + + resInfo = ResInfo("user1:/project4/repo4/", listOf("user1:/project4/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + Assertions.assertEquals(actualInfo?.resource, "user1:/project4/repo4/") + + userUploadUsageRateLimitRule.addRateLimitRule(l7) + userUploadUsageRateLimitRule.addRateLimitRule(l8) + userUploadUsageRateLimitRule.addRateLimitRule(l9) + + resInfo = ResInfo("user1:/project3/repo3/", listOf("user1:/project3/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l7) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/repo3/") + + resInfo = ResInfo("user1:/project3/repo4/", listOf("user1:/project3/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l11) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/") + + resInfo = ResInfo("user1:/project3/xxxx/", listOf("user1:/project3/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l8) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/xxxx/") + + resInfo = ResInfo("user1:/project3/1234/", listOf("user1:/project3/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l9) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/1234/") + + userUploadUsageRateLimitRule.addRateLimitRule(l10) + resInfo = ResInfo("user1:/project3/xxxx/", listOf("user1:/project3/", "user1:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l10) + Assertions.assertEquals(actualInfo?.resource, "user1:/project3/xxxx/") + + + resInfo = ResInfo("user2:/", listOf("user2:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l1) + Assertions.assertEquals(actualInfo?.resource, "user2:/") + + resInfo = ResInfo("user2:/project3/xxxx/", listOf("user2:/project3/", "user2:")) + actualInfo = userUploadUsageRateLimitRule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l13) + Assertions.assertEquals(actualInfo?.resource, "user2:") + + } + + @Test + fun testUserUploadUsageRateLimitRuleAndGetRateLimitRuleWithEmptyRule() { + val rule = UserUploadUsageRateLimitRule() + var resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:/project1/", "user1:")) + var info = rule.getRateLimitRule(resInfo) + Assertions.assertNull(info) + resInfo = ResInfo("user1:/", listOf("user1:")) + info = rule.getRateLimitRule(resInfo) + Assertions.assertNull(info) + } + + @Test + fun testUserUploadUsageRateLimitRuleAndGetRateLimitRuleWithResEmpty() { + val rule = UserUploadUsageRateLimitRule() + rule.addRateLimitRule(l1) + var resInfo = ResInfo("", listOf("")) + assertThrows { rule.getRateLimitRule(resInfo) } + } + + @Test + fun testUserUploadUsageRateLimitRuleAndGetRateLimitRuleWithDifferentOrder() { + val rule = UserUploadUsageRateLimitRule() + rule.addRateLimitRule(l5) + + var resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:/project1/", "user1:")) + var actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l5) + + rule.addRateLimitRule(l3) + + actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + } + + @Test + fun testUserUploadUsageRateLimitRuleAndGetRateLimitRuleWithDuplicatedLimitInfos() { + val rule = UserUploadUsageRateLimitRule() + rule.addRateLimitRule(l3) + rule.addRateLimitRule(l12) + var resInfo = ResInfo("user1:/project1/repo1/", listOf("user1:/project1/", "user1:")) + val actualInfo = rule.getRateLimitRule(resInfo) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l3) + assertEqualsLimitInfo(actualInfo?.resourceLimit, l12) + } + + @Test + fun testUserUploadUsageRateLimitRuleWithInvalidLimitInfo() { + val rule = UserUploadUsageRateLimitRule() + val rl = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/2/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + assertThrows { rule.addRateLimitRule(rl) } + } + + @Test + fun testUserUploadUsageRateLimitRuleWithInvalidUrl() { + val rule = UserUploadUsageRateLimitRule() + rule.addRateLimitRule(l1) + rule.addRateLimitRule(l2) + val resInfo = ResInfo("invalid url") + assertThrows { rule.getRateLimitRule(resInfo) } + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/AbstractRateLimiterServiceTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/AbstractRateLimiterServiceTest.kt new file mode 100644 index 0000000000..090da664cf --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/AbstractRateLimiterServiceTest.kt @@ -0,0 +1,275 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service + +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.ratelimiter.algorithm.DistributedFixedWindowRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.DistributedLeakyRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.DistributedSlidingWindowRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.DistributedTest +import com.tencent.bkrepo.common.ratelimiter.algorithm.DistributedTokenBucketRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.FixedWindowRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.LeakyRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.SlidingWindowRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.TokenBucketRateLimiter +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import com.tencent.bkrepo.common.ratelimiter.metrics.RateLimiterMetrics +import com.tencent.bkrepo.common.ratelimiter.repository.RateLimitRepository +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import io.micrometer.core.instrument.MeterRegistry +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.TestInstance +import org.springframework.boot.test.mock.mockito.MockBean +import org.springframework.mock.web.MockHttpServletRequest +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import java.time.Duration + + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +open class AbstractRateLimiterServiceTest : DistributedTest() { + + lateinit var request: MockHttpServletRequest + lateinit var rateLimiterService: RateLimiterService + lateinit var rateLimiterProperties: RateLimiterProperties + lateinit var taskScheduler: ThreadPoolTaskScheduler + lateinit var rateLimiterMetrics: RateLimiterMetrics + lateinit var rateLimiterConfigService: RateLimiterConfigService + + @MockBean + private lateinit var meterRegistry: MeterRegistry + + @MockBean + private lateinit var rateLimitRepository: RateLimitRepository + + fun init() { + request = MockHttpServletRequest() + val scheduler = ThreadPoolTaskScheduler() + scheduler.initialize() + taskScheduler = scheduler + rateLimiterProperties = RateLimiterProperties() + rateLimiterMetrics = RateLimiterMetrics(meterRegistry) + rateLimiterConfigService = RateLimiterConfigService(rateLimitRepository) + } + + + open fun createAlgorithmOfRateLimiterTest() { + val resource = (rateLimiterService as AbstractRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as AbstractRateLimiterService).buildExtraResource(request) + ) + val resourceLimit = (rateLimiterService as AbstractRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + // 测试固定窗口本地算法生成 + var rateLimiter = (rateLimiterService as AbstractRateLimiterService) + .createAlgorithmOfRateLimiter(resource, resourceLimit!!.resourceLimit) + Assertions.assertInstanceOf( + FixedWindowRateLimiter::class.java, + rateLimiter + ) + // 测试固定窗口分布式算法生成 + val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/", + limitDimension = LimitDimension.URL.name, limit = 1, + duration = Duration.ofSeconds(1), scope = WorkScope.GLOBAL.name + ) + rateLimiter = (rateLimiterService as AbstractRateLimiterService).createAlgorithmOfRateLimiter(resource, l1) + Assertions.assertInstanceOf( + DistributedFixedWindowRateLimiter::class.java, + rateLimiter + ) + // 测试limit < 0的场景 + val l2 = ResourceLimit( + algo = Algorithms.SLIDING_WINDOW.name, resource = "/project3/{(^[a-zA-Z]*\$)}/", + limitDimension = LimitDimension.URL.name, limit = -1, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + Assertions.assertThrows(InvalidResourceException::class.java) { + (rateLimiterService as AbstractRateLimiterService).createAlgorithmOfRateLimiter(resource, l2) + } + // 测试滑动窗口本地算法生成 + l2.limit = 1 + rateLimiter = (rateLimiterService as AbstractRateLimiterService).createAlgorithmOfRateLimiter(resource, l2) + Assertions.assertInstanceOf( + SlidingWindowRateLimiter::class.java, + rateLimiter + ) + // 测试滑动窗口分布式算法生成 + l2.scope = WorkScope.GLOBAL.name + rateLimiter = (rateLimiterService as AbstractRateLimiterService).createAlgorithmOfRateLimiter(resource, l2) + Assertions.assertInstanceOf( + DistributedSlidingWindowRateLimiter::class.java, + rateLimiter + ) + // 测试不支持的算法类型 + val l3 = ResourceLimit( + algo = "", resource = "/project3/{(^[0-9]*\$)}/", + limitDimension = LimitDimension.URL.name, limit = 52428800, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + Assertions.assertThrows(InvalidResourceException::class.java) { + (rateLimiterService as AbstractRateLimiterService).createAlgorithmOfRateLimiter(resource, l3) + } + //测试 capacity 不合规的场景 + val l4 = ResourceLimit( + algo = Algorithms.LEAKY_BUCKET.name, resource = "/project3/{repo}}/", + limitDimension = LimitDimension.URL.name, limit = 1, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + Assertions.assertThrows(InvalidResourceException::class.java) { + (rateLimiterService as AbstractRateLimiterService).createAlgorithmOfRateLimiter(resource, l4) + } + l4.capacity = -2 + Assertions.assertThrows(InvalidResourceException::class.java) { + (rateLimiterService as AbstractRateLimiterService).createAlgorithmOfRateLimiter(resource, l4) + } + // 测试漏桶本地算法生成 + l4.capacity = 5 + rateLimiter = (rateLimiterService as AbstractRateLimiterService).createAlgorithmOfRateLimiter(resource, l4) + Assertions.assertInstanceOf( + LeakyRateLimiter::class.java, + rateLimiter + ) + // 测试漏桶分布式算法生成 + l4.scope = WorkScope.GLOBAL.name + rateLimiter = (rateLimiterService as AbstractRateLimiterService).createAlgorithmOfRateLimiter(resource, l4) + Assertions.assertInstanceOf( + DistributedLeakyRateLimiter::class.java, + rateLimiter + ) + // 测试令牌桶本地算法生成 + val l5 = ResourceLimit( + algo = Algorithms.TOKEN_BUCKET.name, resource = "/project3/", + limitDimension = LimitDimension.URL.name, limit = 1, capacity = 5, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + rateLimiter = (rateLimiterService as AbstractRateLimiterService).createAlgorithmOfRateLimiter(resource, l5) + Assertions.assertInstanceOf( + TokenBucketRateLimiter::class.java, + rateLimiter + ) + // 测试令牌桶分布式算法生成 + l5.scope = WorkScope.GLOBAL.name + rateLimiter = (rateLimiterService as AbstractRateLimiterService).createAlgorithmOfRateLimiter(resource, l5) + Assertions.assertInstanceOf( + DistributedTokenBucketRateLimiter::class.java, + rateLimiter + ) + } + + + open fun refreshRateLimitRuleTest() { + val hashCode = rateLimiterProperties.rules.hashCode() + (rateLimiterService as AbstractRateLimiterService).refreshRateLimitRule() + Assertions.assertEquals(hashCode, (rateLimiterService as AbstractRateLimiterService).currentRuleHashCode) + } + + + open fun getAlgorithmOfRateLimiterTest() { + val resource = (rateLimiterService as AbstractRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as AbstractRateLimiterService).buildExtraResource(request) + ) + val resourceLimit = (rateLimiterService as AbstractRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + assertEqualsLimitInfo(resourceLimit!!.resourceLimit, rateLimiterProperties.rules.first()) + val rateLimiter = (rateLimiterService as AbstractRateLimiterService) + .getAlgorithmOfRateLimiter(resource, resourceLimit.resourceLimit) + Assertions.assertInstanceOf( + FixedWindowRateLimiter::class.java, + rateLimiter + ) + Assertions.assertEquals( + rateLimiter, + (rateLimiterService as AbstractRateLimiterService) + .getAlgorithmOfRateLimiter(resource, resourceLimit.resourceLimit) + ) + } + + + open fun getResLimitInfoTest() { + val resourceLimit = (rateLimiterService as AbstractRateLimiterService).getResLimitInfo(request) + Assertions.assertNotNull(resourceLimit) + assertEqualsLimitInfo(resourceLimit!!.resourceLimit, rateLimiterProperties.rules.first()) + } + + + open fun circuitBreakerCheckTest() { + + var circuitBreakerPerSecond: Long? = null + (rateLimiterService as AbstractRateLimiterService).circuitBreakerCheck( + rateLimiterProperties.rules.first(), circuitBreakerPerSecond + ) + + circuitBreakerPerSecond = Long.MAX_VALUE + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as AbstractRateLimiterService).circuitBreakerCheck( + rateLimiterProperties.rules.first(), circuitBreakerPerSecond + ) + } + circuitBreakerPerSecond = Long.MIN_VALUE + (rateLimiterService as AbstractRateLimiterService).circuitBreakerCheck( + rateLimiterProperties.rules.first(), circuitBreakerPerSecond + ) + } + + + open fun rateLimitCatchTest() { + val resourceLimit = (rateLimiterService as AbstractRateLimiterService).getResLimitInfo(request) + Assertions.assertNotNull(resourceLimit) + val rateLimiter = (rateLimiterService as AbstractRateLimiterService) + .getAlgorithmOfRateLimiter(resourceLimit!!.resource, resourceLimit.resourceLimit) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as AbstractRateLimiterService).rateLimitCatch( + request = request, + resLimitInfo = resourceLimit, + applyPermits = Long.MAX_VALUE, + circuitBreakerPerSecond = null, + ) { rateLimiter, permits -> + false + } + } + rateLimiterProperties.dryRun = true + (rateLimiterService as AbstractRateLimiterService).rateLimitCatch( + request = request, + resLimitInfo = resourceLimit, + applyPermits = Long.MAX_VALUE, + circuitBreakerPerSecond = null, + ) { rateLimiter, permits -> + false + } + rateLimiterProperties.dryRun = false + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/bandwidth/DownloadBandwidthRateLimiterServiceTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/bandwidth/DownloadBandwidthRateLimiterServiceTest.kt new file mode 100644 index 0000000000..c3126425fc --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/bandwidth/DownloadBandwidthRateLimiterServiceTest.kt @@ -0,0 +1,239 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.bandwidth + +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import com.tencent.bkrepo.common.ratelimiter.rule.bandwidth.DownloadBandwidthRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterServiceTest +import java.time.Duration +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.test.annotation.DirtiesContext +import org.springframework.web.servlet.HandlerMapping + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class DownloadBandwidthRateLimiterServiceTest : AbstractRateLimiterServiceTest() { + + val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/", + limitDimension = LimitDimension.DOWNLOAD_BANDWIDTH.name, limit = 10, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @BeforeAll + fun before() { + init() + rateLimiterProperties.enabled = true + rateLimiterProperties.rules = listOf(l1) + request.requestURI = "/blueking/generic-local/test.txt" + val uriVariables: MutableMap = HashMap() + uriVariables["projectId"] = "blueking" + uriVariables["repoName"] = "generic-local" + request.setAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE, uriVariables) + request.method = "GET" + val scheduler = ThreadPoolTaskScheduler() + scheduler.initialize() + rateLimiterService = DownloadBandwidthRateLimiterService( + taskScheduler = scheduler, + rateLimiterProperties = rateLimiterProperties, + redisTemplate = redisTemplate, + rateLimiterMetrics = rateLimiterMetrics, + rateLimiterConfigService = rateLimiterConfigService + ) + (rateLimiterService as DownloadBandwidthRateLimiterService).refreshRateLimitRule() + } + + @Test + override fun createAlgorithmOfRateLimiterTest() { + super.createAlgorithmOfRateLimiterTest() + } + + @Test + override fun refreshRateLimitRuleTest() { + super.refreshRateLimitRuleTest() + } + + @Test + override fun getAlgorithmOfRateLimiterTest() { + super.getAlgorithmOfRateLimiterTest() + } + + @Test + override fun getResLimitInfoTest() { + super.getResLimitInfoTest() + } + + @Test + override fun circuitBreakerCheckTest() { + super.circuitBreakerCheckTest() + } + + @Test + override fun rateLimitCatchTest() { + super.rateLimitCatchTest() + } + + @Test + fun ignoreRequestTest() { + Assertions.assertEquals( + false, + (rateLimiterService as DownloadBandwidthRateLimiterService).ignoreRequest(request) + ) + request.method = "POST" + Assertions.assertEquals( + true, + (rateLimiterService as DownloadBandwidthRateLimiterService).ignoreRequest(request) + ) + request.method = "PATCH" + Assertions.assertEquals( + true, + (rateLimiterService as DownloadBandwidthRateLimiterService).ignoreRequest(request) + ) + request.method = "PUT" + Assertions.assertEquals( + true, + (rateLimiterService as DownloadBandwidthRateLimiterService).ignoreRequest(request) + ) + request.method = "GET" + } + + @Test + fun getRepoInfoTest() { + val (projectId, repoName) = + (rateLimiterService as DownloadBandwidthRateLimiterService).getRepoInfoFromAttribute(request) + Assertions.assertEquals("blueking", projectId) + Assertions.assertEquals("generic-local", repoName) + + } + + @Test + fun buildResourceTest() { + Assertions.assertEquals( + "/blueking/generic-local/", + (rateLimiterService as DownloadBandwidthRateLimiterService).buildResource(request) + ) + } + + @Test + fun buildExtraResourceTest() { + Assertions.assertEquals( + listOf("/blueking/"), + (rateLimiterService as DownloadBandwidthRateLimiterService).buildExtraResource(request) + ) + } + + @Test + fun getApplyPermitsTest() { + Assertions.assertThrows(AcquireLockFailedException::class.java) { + (rateLimiterService as DownloadBandwidthRateLimiterService).getApplyPermits(request, null) + } + Assertions.assertEquals( + 10, + (rateLimiterService as DownloadBandwidthRateLimiterService).getApplyPermits(request, 10) + ) + } + + @Test + fun getRateLimitRuleClassTest() { + Assertions.assertEquals( + DownloadBandwidthRateLimitRule::class.java, + (rateLimiterService as DownloadBandwidthRateLimiterService).getRateLimitRuleClass() + ) + } + + @Test + fun getLimitDimensionsTest() { + Assertions.assertEquals( + listOf(LimitDimension.DOWNLOAD_BANDWIDTH.name), + (rateLimiterService as DownloadBandwidthRateLimiterService).getLimitDimensions() + ) + } + + @Test + fun generateKeyTest() { + val resource = (rateLimiterService as DownloadBandwidthRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as DownloadBandwidthRateLimiterService).buildExtraResource(request) + ) + var resourceLimit = (rateLimiterService as DownloadBandwidthRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "DownloadBandwidth:" + "/blueking/", + (rateLimiterService as DownloadBandwidthRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + l1.resource = "/blueking/generic-local/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as DownloadBandwidthRateLimiterService).refreshRateLimitRule() + resourceLimit = (rateLimiterService as DownloadBandwidthRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "DownloadBandwidth:" + "/blueking/generic-local/", + (rateLimiterService as DownloadBandwidthRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + l1.resource = "/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as DownloadBandwidthRateLimiterService).refreshRateLimitRule() + } + + @Test + fun generateKeyTestNull() { + val resource = (rateLimiterService as DownloadBandwidthRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as DownloadBandwidthRateLimiterService).buildExtraResource(request) + ) + l1.resource = "/test/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as DownloadBandwidthRateLimiterService).refreshRateLimitRule() + val resourceLimit = (rateLimiterService as DownloadBandwidthRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNull(resourceLimit) + + l1.resource = "/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as DownloadBandwidthRateLimiterService).refreshRateLimitRule() + } + + @Test + fun limitTest() { + Assertions.assertThrows(UnsupportedOperationException::class.java) { rateLimiterService.limit(request) } + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/bandwidth/UploadBandwidthRateLimiterServiceTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/bandwidth/UploadBandwidthRateLimiterServiceTest.kt new file mode 100644 index 0000000000..108c7eb237 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/bandwidth/UploadBandwidthRateLimiterServiceTest.kt @@ -0,0 +1,295 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.bandwidth + +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import com.tencent.bkrepo.common.ratelimiter.rule.bandwidth.UploadBandwidthRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterServiceTest +import java.time.Duration +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.test.annotation.DirtiesContext +import org.springframework.util.unit.DataSize +import org.springframework.web.servlet.HandlerMapping + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class UploadBandwidthRateLimiterServiceTest : AbstractRateLimiterServiceTest() { + + val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/", + limitDimension = LimitDimension.UPLOAD_BANDWIDTH.name, limit = 10, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @BeforeAll + fun before() { + init() + rateLimiterProperties.enabled = true + rateLimiterProperties.rules = listOf(l1) + request.requestURI = "/blueking/generic-local/test.txt" + val uriVariables: MutableMap = HashMap() + uriVariables["projectId"] = "blueking" + uriVariables["repoName"] = "generic-local" + request.setAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE, uriVariables) + val content = byteArrayOf(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) + request.setContent(content) + request.method = "PUT" + val scheduler = ThreadPoolTaskScheduler() + scheduler.initialize() + rateLimiterService = UploadBandwidthRateLimiterService( + taskScheduler = scheduler, + rateLimiterProperties = rateLimiterProperties, + redisTemplate = redisTemplate, + rateLimiterMetrics = rateLimiterMetrics, + rateLimiterConfigService = rateLimiterConfigService + ) + (rateLimiterService as UploadBandwidthRateLimiterService).refreshRateLimitRule() + } + + @Test + override fun createAlgorithmOfRateLimiterTest() { + super.createAlgorithmOfRateLimiterTest() + } + + @Test + override fun refreshRateLimitRuleTest() { + super.refreshRateLimitRuleTest() + } + + @Test + override fun getAlgorithmOfRateLimiterTest() { + super.getAlgorithmOfRateLimiterTest() + } + + @Test + override fun getResLimitInfoTest() { + super.getResLimitInfoTest() + } + + @Test + override fun circuitBreakerCheckTest() { + super.circuitBreakerCheckTest() + } + + @Test + override fun rateLimitCatchTest() { + super.rateLimitCatchTest() + } + + @Test + fun ignoreRequestTest() { + Assertions.assertEquals(false, (rateLimiterService as UploadBandwidthRateLimiterService).ignoreRequest(request)) + request.method = "POST" + Assertions.assertEquals(false, (rateLimiterService as UploadBandwidthRateLimiterService).ignoreRequest(request)) + request.method = "PATCH" + Assertions.assertEquals(false, (rateLimiterService as UploadBandwidthRateLimiterService).ignoreRequest(request)) + request.method = "GET" + Assertions.assertEquals(true, (rateLimiterService as UploadBandwidthRateLimiterService).ignoreRequest(request)) + request.method = "PUT" + } + + @Test + fun getRepoInfoTest() { + val (projectId, repoName) = (rateLimiterService as UploadBandwidthRateLimiterService).getRepoInfoFromAttribute( + request + ) + Assertions.assertEquals("blueking", projectId) + Assertions.assertEquals("generic-local", repoName) + + } + + @Test + fun buildResourceTest() { + Assertions.assertEquals( + "/blueking/generic-local/", + (rateLimiterService as UploadBandwidthRateLimiterService).buildResource(request) + ) + } + + @Test + fun buildExtraResourceTest() { + Assertions.assertEquals( + listOf("/blueking/"), + (rateLimiterService as UploadBandwidthRateLimiterService).buildExtraResource(request) + ) + } + + @Test + fun getApplyPermitsTest() { + Assertions.assertThrows(AcquireLockFailedException::class.java) { + (rateLimiterService as UploadBandwidthRateLimiterService).getApplyPermits(request, null) + } + Assertions.assertEquals( + 10, + (rateLimiterService as UploadBandwidthRateLimiterService).getApplyPermits(request, 10) + ) + } + + @Test + fun getRateLimitRuleClassTest() { + Assertions.assertEquals( + UploadBandwidthRateLimitRule::class.java, + (rateLimiterService as UploadBandwidthRateLimiterService).getRateLimitRuleClass() + ) + } + + @Test + fun getLimitDimensionsTest() { + Assertions.assertEquals( + listOf(LimitDimension.UPLOAD_BANDWIDTH.name), + (rateLimiterService as UploadBandwidthRateLimiterService).getLimitDimensions() + ) + } + + @Test + fun generateKeyTest() { + val resource = (rateLimiterService as UploadBandwidthRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UploadBandwidthRateLimiterService).buildExtraResource(request) + ) + var resourceLimit = (rateLimiterService as UploadBandwidthRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UploadBandwidth:" + "/blueking/", + (rateLimiterService as UploadBandwidthRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + l1.resource = "/blueking/generic-local/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UploadBandwidthRateLimiterService).refreshRateLimitRule() + resourceLimit = (rateLimiterService as UploadBandwidthRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UploadBandwidth:" + "/blueking/generic-local/", + (rateLimiterService as UploadBandwidthRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + l1.resource = "/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UploadBandwidthRateLimiterService).refreshRateLimitRule() + } + + @Test + fun generateKeyTestNull() { + val resource = (rateLimiterService as UploadBandwidthRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UploadBandwidthRateLimiterService).buildExtraResource(request) + ) + l1.resource = "/test/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UploadBandwidthRateLimiterService).refreshRateLimitRule() + val resourceLimit = (rateLimiterService as UploadBandwidthRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNull(resourceLimit) + + l1.resource = "/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UploadBandwidthRateLimiterService).refreshRateLimitRule() + } + + @Test + fun limitTest() { + Assertions.assertThrows(UnsupportedOperationException::class.java) { + (rateLimiterService as UploadBandwidthRateLimiterService).limit(request) + } + } + + @Test + fun bandwidthRateLimitTest() { + (rateLimiterService as UploadBandwidthRateLimiterService).bandwidthRateLimit( + request = request, + permits = 1, + circuitBreakerPerSecond = DataSize.ofBytes(0) + ) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UploadBandwidthRateLimiterService).bandwidthRateLimit( + request = request, + permits = 10000, + circuitBreakerPerSecond = DataSize.ofBytes(0) + ) + } + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UploadBandwidthRateLimiterService).bandwidthRateLimit( + request = request, + permits = 1, + circuitBreakerPerSecond = DataSize.ofTerabytes(1) + ) + } + } + + @Test + fun bandwidthRateLimit1Test() { + val content = "1234567891" + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UploadBandwidthRateLimiterService).bandwidthRateStart( + request = request, + inputStream = content.byteInputStream(), + circuitBreakerPerSecond = DataSize.ofTerabytes(1), + rangeLength = null + ) + } + (rateLimiterService as UploadBandwidthRateLimiterService).bandwidthRateStart( + request = request, + inputStream = content.byteInputStream(), + circuitBreakerPerSecond = DataSize.ofBytes(1), + rangeLength = null + ) + } + + @Test + fun bandwidthLimitHandlerTest() { + val resourceLimit = (rateLimiterService as UploadBandwidthRateLimiterService).getResLimitInfo(request) + Assertions.assertNotNull(resourceLimit) + assertEqualsLimitInfo(resourceLimit!!.resourceLimit, rateLimiterProperties.rules.first()) + val rateLimiter = (rateLimiterService as UploadBandwidthRateLimiterService) + .getAlgorithmOfRateLimiter(resourceLimit.resource, resourceLimit.resourceLimit) + Assertions.assertEquals( + true, + (rateLimiterService as UploadBandwidthRateLimiterService).bandwidthLimitHandler(rateLimiter, 1) + ) + Assertions.assertEquals( + false, + (rateLimiterService as UploadBandwidthRateLimiterService).bandwidthLimitHandler(rateLimiter, 100) + ) + } + + +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/UrlRateLimiterServiceTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/UrlRateLimiterServiceTest.kt new file mode 100644 index 0000000000..ec3baac926 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/UrlRateLimiterServiceTest.kt @@ -0,0 +1,215 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.url + +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.url.UrlRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterServiceTest +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.test.annotation.DirtiesContext +import java.time.Duration + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class UrlRateLimiterServiceTest : AbstractRateLimiterServiceTest() { + + val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/", + limitDimension = LimitDimension.URL.name, limit = 1, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @BeforeAll + fun before() { + init() + rateLimiterProperties.enabled = true + rateLimiterProperties.rules = listOf(l1) + request.requestURI = "/blueking/generic-local/test.txt" + val scheduler = ThreadPoolTaskScheduler() + scheduler.initialize() + rateLimiterService = UrlRateLimiterService( + taskScheduler = scheduler, + rateLimiterProperties = rateLimiterProperties, + redisTemplate = redisTemplate, + rateLimiterMetrics = rateLimiterMetrics, + rateLimiterConfigService = rateLimiterConfigService + ) + (rateLimiterService as UrlRateLimiterService).refreshRateLimitRule() + } + + @Test + override fun createAlgorithmOfRateLimiterTest() { + super.createAlgorithmOfRateLimiterTest() + } + + @Test + override fun refreshRateLimitRuleTest() { + super.refreshRateLimitRuleTest() + } + + @Test + override fun getAlgorithmOfRateLimiterTest() { + super.getAlgorithmOfRateLimiterTest() + } + + @Test + override fun getResLimitInfoTest() { + super.getResLimitInfoTest() + } + + @Test + override fun circuitBreakerCheckTest() { + super.circuitBreakerCheckTest() + } + + @Test + override fun rateLimitCatchTest() { + super.rateLimitCatchTest() + } + + @Test + fun ignoreRequestTest() { + Assertions.assertEquals(false, (rateLimiterService as UrlRateLimiterService).ignoreRequest(request)) + } + + @Test + fun buildResourceTest() { + Assertions.assertEquals( + "/blueking/generic-local/test.txt", + (rateLimiterService as UrlRateLimiterService).buildResource(request) + ) + request.requestURI = "/api/node/batch/blueking/generic-local" + Assertions.assertEquals( + "/api/node/batch/blueking/generic-local", + (rateLimiterService as UrlRateLimiterService).buildResource(request) + ) + request.requestURI = "/blueking/generic-local/test.txt" + } + + @Test + fun buildExtraResourceTest() { + Assertions.assertEquals( + emptyList(), + (rateLimiterService as UrlRateLimiterService).buildExtraResource(request) + ) + } + + @Test + fun getApplyPermitsTest() { + Assertions.assertEquals(1, (rateLimiterService as UrlRateLimiterService).getApplyPermits(request, null)) + } + + @Test + fun getRateLimitRuleClassTest() { + Assertions.assertEquals( + UrlRateLimitRule::class.java, + (rateLimiterService as UrlRateLimiterService).getRateLimitRuleClass() + ) + } + + @Test + fun getLimitDimensionsTest() { + Assertions.assertEquals( + listOf(LimitDimension.URL.name), + (rateLimiterService as UrlRateLimiterService).getLimitDimensions() + ) + } + + @Test + fun generateKeyTest() { + val resource = (rateLimiterService as UrlRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UrlRateLimiterService).buildExtraResource(request) + ) + val resourceLimit = (rateLimiterService as UrlRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "Url:" + "/blueking/generic-local/test.txt", + (rateLimiterService as UrlRateLimiterService).generateKey( + resourceLimit!!.resource, resourceLimit.resourceLimit + ) + ) + } + + @Test + fun generateKeyTestNull() { + l1.resource = "/test/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UrlRateLimiterService).refreshRateLimitRule() + + val resource = (rateLimiterService as UrlRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UrlRateLimiterService).buildExtraResource(request) + ) + val resourceLimit = (rateLimiterService as UrlRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNull(resourceLimit) + + l1.resource = "/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UrlRateLimiterService).refreshRateLimitRule() + } + + @Test + fun limitTest() { + // 本地限流验证 + (rateLimiterService as UrlRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UrlRateLimiterService).limit(request) + } + + // 分布式算法验证 + l1.scope = WorkScope.GLOBAL.name + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UrlRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UrlRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UrlRateLimiterService).limit(request) + } + + l1.resource = "/blueking/" + l1.limit = 2 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UrlRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UrlRateLimiterService).limit(request) + (rateLimiterService as UrlRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UrlRateLimiterService).limit(request) + } + } + +} diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/UrlRepoRateLimiterServiceTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/UrlRepoRateLimiterServiceTest.kt new file mode 100644 index 0000000000..80af9fbb42 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/UrlRepoRateLimiterServiceTest.kt @@ -0,0 +1,287 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.url + +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.api.util.toJsonString +import com.tencent.bkrepo.common.artifact.constant.PROJECT_ID +import com.tencent.bkrepo.common.artifact.constant.REPO_NAME +import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory +import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.url.UrlRepoRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterServiceTest +import com.tencent.bkrepo.repository.pojo.repo.UserRepoCreateRequest +import com.tencent.bkrepo.repository.pojo.search.NodeQueryBuilder +import java.time.Duration +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.test.annotation.DirtiesContext +import org.springframework.web.servlet.HandlerMapping +import org.springframework.web.servlet.HandlerMapping.BEST_MATCHING_PATTERN_ATTRIBUTE + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class UrlRepoRateLimiterServiceTest : AbstractRateLimiterServiceTest() { + + val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/", + limitDimension = LimitDimension.URL_REPO.name, limit = 1, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @BeforeAll + fun before() { + init() + rateLimiterProperties.enabled = true + rateLimiterProperties.rules = listOf(l1) + request.requestURI = "/blueking/generic-local/test.txt" + val uriVariables: MutableMap = HashMap() + uriVariables["projectId"] = "blueking" + uriVariables["repoName"] = "generic-local" + request.setAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE, uriVariables) + request.setAttribute(BEST_MATCHING_PATTERN_ATTRIBUTE, "/{projectId}/{repoName}/**") + request.contentType = "application/json" + val scheduler = ThreadPoolTaskScheduler() + scheduler.initialize() + rateLimiterService = UrlRepoRateLimiterService( + taskScheduler = scheduler, + rateLimiterProperties = rateLimiterProperties, + redisTemplate = redisTemplate, + rateLimiterMetrics = rateLimiterMetrics, + rateLimiterConfigService = rateLimiterConfigService + ) + (rateLimiterService as UrlRepoRateLimiterService).refreshRateLimitRule() + } + + @Test + override fun createAlgorithmOfRateLimiterTest() { + super.createAlgorithmOfRateLimiterTest() + } + + @Test + override fun refreshRateLimitRuleTest() { + super.refreshRateLimitRuleTest() + } + + @Test + override fun getAlgorithmOfRateLimiterTest() { + super.getAlgorithmOfRateLimiterTest() + } + + @Test + override fun getResLimitInfoTest() { + super.getResLimitInfoTest() + } + + @Test + override fun circuitBreakerCheckTest() { + super.circuitBreakerCheckTest() + } + + @Test + override fun rateLimitCatchTest() { + super.rateLimitCatchTest() + } + + @Test + fun ignoreRequestTest() { + rateLimiterProperties.specialUrls = listOf("*") + Assertions.assertEquals(false, (rateLimiterService as UrlRepoRateLimiterService).ignoreRequest(request)) + rateLimiterProperties.specialUrls = listOf("/{projectId}/{repoName}/**") + Assertions.assertEquals(false, (rateLimiterService as UrlRepoRateLimiterService).ignoreRequest(request)) + rateLimiterProperties.specialUrls = listOf() + Assertions.assertEquals(true, (rateLimiterService as UrlRepoRateLimiterService).ignoreRequest(request)) + } + + @Test + fun buildResourceTest() { + Assertions.assertEquals( + "/blueking/generic-local/", + (rateLimiterService as UrlRepoRateLimiterService).buildResource(request) + ) + request.requestURI = "/api/node/batch/blueking/generic-local" + Assertions.assertEquals( + "/blueking/generic-local/", + (rateLimiterService as UrlRepoRateLimiterService).buildResource(request) + ) + request.requestURI = "/blueking/generic-local/test.txt" + } + + @Test + fun buildExtraResourceTest() { + Assertions.assertEquals( + listOf("/blueking/"), + (rateLimiterService as UrlRepoRateLimiterService).buildExtraResource(request) + ) + } + + @Test + fun getApplyPermitsTest() { + Assertions.assertEquals(1, (rateLimiterService as UrlRepoRateLimiterService).getApplyPermits(request, null)) + } + + @Test + fun getRateLimitRuleClassTest() { + Assertions.assertEquals( + UrlRepoRateLimitRule::class.java, + (rateLimiterService as UrlRepoRateLimiterService).getRateLimitRuleClass() + ) + } + + @Test + fun getLimitDimensionsTest() { + Assertions.assertEquals( + listOf(LimitDimension.URL_REPO.name), + (rateLimiterService as UrlRepoRateLimiterService).getLimitDimensions() + ) + } + + @Test + fun generateKeyTest() { + val resource = (rateLimiterService as UrlRepoRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UrlRepoRateLimiterService).buildExtraResource(request) + ) + val resourceLimit = (rateLimiterService as UrlRepoRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UrlRepo:" + "/blueking/", + (rateLimiterService as UrlRepoRateLimiterService).generateKey( + resourceLimit!!.resource, resourceLimit.resourceLimit + ) + ) + } + + @Test + fun getRepoInfoTest() { + val (projectId, repoName) = (rateLimiterService as UrlRepoRateLimiterService).getRepoInfoFromAttribute( + request + ) + Assertions.assertEquals("blueking", projectId) + Assertions.assertEquals("generic-local", repoName) + } + + @Test + fun getRepoInfoFromBodyTest() { + request.requestURI = "/api/node/search" + request.contentType = "application/json" + val queryModelBuilder = NodeQueryBuilder() + .select(PROJECT_ID, REPO_NAME) + .sortByAsc("fullPath") + .page(1, 10) + .projectId("test-projectId") + .repoName("test-repoName") + + val queryModel = queryModelBuilder.build() + request.setContent(queryModel.toJsonString().toByteArray()) + val (projectId, repoName) = (rateLimiterService as UrlRepoRateLimiterService).getRepoInfoFromBody( + request + ) + Assertions.assertEquals("test-projectId", projectId) + Assertions.assertEquals("test-repoName", repoName) + + request.setContent( + UserRepoCreateRequest( + projectId = "test-projectId1", + name = "test-repoName1", + type = RepositoryType.GENERIC, + category = RepositoryCategory.COMPOSITE, + display = false + ).toJsonString().toByteArray() + ) + val (projectId1, repoName1) = (rateLimiterService as UrlRepoRateLimiterService).getRepoInfoFromBody( + request + ) + Assertions.assertEquals("test-projectId1", projectId1) + Assertions.assertEquals(null, repoName1) + + request.requestURI = "/blueking/generic-local/test.txt" + } + + @Test + fun generateKeyTestNull() { + l1.resource = "/test/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UrlRepoRateLimiterService).refreshRateLimitRule() + + val resource = (rateLimiterService as UrlRepoRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UrlRepoRateLimiterService).buildExtraResource(request) + ) + val resourceLimit = (rateLimiterService as UrlRepoRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNull(resourceLimit) + + l1.resource = "/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UrlRepoRateLimiterService).refreshRateLimitRule() + } + + @Test + fun limitTest() { + rateLimiterProperties.specialUrls = listOf("*") + // 本地限流验证 + (rateLimiterService as UrlRepoRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UrlRepoRateLimiterService).limit(request) + } + + // 分布式算法验证 + l1.scope = WorkScope.GLOBAL.name + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UrlRepoRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UrlRepoRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UrlRepoRateLimiterService).limit(request) + } + + l1.resource = "/blueking/" + l1.limit = 2 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UrlRepoRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UrlRepoRateLimiterService).limit(request) + (rateLimiterService as UrlRepoRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UrlRepoRateLimiterService).limit(request) + } + + l1.resource = "/*/" + l1.limit = 1 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UrlRepoRateLimiterService).refreshRateLimitRule() + } + +} diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/user/UserUrlRateLimiterServiceTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/user/UserUrlRateLimiterServiceTest.kt new file mode 100644 index 0000000000..fe89d23cbb --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/user/UserUrlRateLimiterServiceTest.kt @@ -0,0 +1,263 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.url.user + +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.url.user.UserUrlRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterServiceTest +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.test.annotation.DirtiesContext +import org.springframework.web.context.request.RequestContextHolder +import org.springframework.web.context.request.ServletRequestAttributes +import java.time.Duration + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class UserUrlRateLimiterServiceTest : AbstractRateLimiterServiceTest() { + + val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "*:/", + limitDimension = LimitDimension.USER_URL.name, limit = 1, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @BeforeAll + fun before() { + init() + rateLimiterProperties.enabled = true + rateLimiterProperties.rules = listOf(l1) + request.requestURI = "/blueking/generic-local/test.txt" + request.setAttribute("userId", "admin") + val attributes = ServletRequestAttributes(request) + RequestContextHolder.setRequestAttributes(attributes) + val scheduler = ThreadPoolTaskScheduler() + scheduler.initialize() + rateLimiterService = UserUrlRateLimiterService( + taskScheduler = scheduler, + rateLimiterProperties = rateLimiterProperties, + redisTemplate = redisTemplate, + rateLimiterMetrics = rateLimiterMetrics, + rateLimiterConfigService = rateLimiterConfigService + ) + (rateLimiterService as UserUrlRateLimiterService).refreshRateLimitRule() + } + + @Test + override fun createAlgorithmOfRateLimiterTest() { + super.createAlgorithmOfRateLimiterTest() + } + + @Test + override fun refreshRateLimitRuleTest() { + super.refreshRateLimitRuleTest() + } + + @Test + override fun getAlgorithmOfRateLimiterTest() { + super.getAlgorithmOfRateLimiterTest() + } + + @Test + override fun getResLimitInfoTest() { + super.getResLimitInfoTest() + } + + @Test + override fun circuitBreakerCheckTest() { + super.circuitBreakerCheckTest() + } + + @Test + override fun rateLimitCatchTest() { + super.rateLimitCatchTest() + } + + @Test + fun ignoreRequestTest() { + Assertions.assertEquals(false, (rateLimiterService as UserUrlRateLimiterService).ignoreRequest(request)) + } + + @Test + fun buildResourceTest() { + Assertions.assertEquals( + "admin:/blueking/generic-local/test.txt", + (rateLimiterService as UserUrlRateLimiterService).buildResource(request) + ) + request.requestURI = "/api/node/batch/blueking/generic-local" + request.setAttribute("userId", "test") + Assertions.assertEquals( + "test:/api/node/batch/blueking/generic-local", + (rateLimiterService as UserUrlRateLimiterService).buildResource(request) + ) + request.requestURI = "/blueking/generic-local/test.txt" + request.setAttribute("userId", "admin") + } + + @Test + fun buildExtraResourceTest() { + Assertions.assertEquals( + listOf("admin:"), + (rateLimiterService as UserUrlRateLimiterService).buildExtraResource(request) + ) + } + + @Test + fun getApplyPermitsTest() { + Assertions.assertEquals( + 1, + (rateLimiterService as UserUrlRateLimiterService).getApplyPermits(request, null) + ) + } + + @Test + fun getRateLimitRuleClassTest() { + Assertions.assertEquals( + UserUrlRateLimitRule::class.java, + (rateLimiterService as UserUrlRateLimiterService).getRateLimitRuleClass() + ) + } + + @Test + fun getLimitDimensionsTest() { + Assertions.assertEquals( + listOf(LimitDimension.USER_URL.name), + (rateLimiterService as UserUrlRateLimiterService).getLimitDimensions() + ) + } + + @Test + fun generateKeyTest() { + val resource = (rateLimiterService as UserUrlRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UserUrlRateLimiterService).buildExtraResource(request) + ) + var resourceLimit = (rateLimiterService as UserUrlRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UserUrl:" + "admin:/blueking/generic-local/test.txt", + (rateLimiterService as UserUrlRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + l1.resource = "admin:" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRateLimiterService).refreshRateLimitRule() + + resourceLimit = (rateLimiterService as UserUrlRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UserUrl:" + "admin:", + (rateLimiterService as UserUrlRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + + l1.resource = "*:/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRateLimiterService).refreshRateLimitRule() + } + + @Test + fun generateKeyTestNull() { + l1.resource = "test:" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRateLimiterService).refreshRateLimitRule() + + val resource = (rateLimiterService as UserUrlRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UserUrlRateLimiterService).buildExtraResource(request) + ) + var resourceLimit = (rateLimiterService as UserUrlRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNull(resourceLimit) + + l1.resource = "test:/blueking/generic-local/test.txt" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRateLimiterService).refreshRateLimitRule() + resourceLimit = (rateLimiterService as UserUrlRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNull(resourceLimit) + + + l1.resource = "*:/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRateLimiterService).refreshRateLimitRule() + } + + @Test + fun limitTest() { + // 本地限流验证 + (rateLimiterService as UserUrlRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserUrlRateLimiterService).limit(request) + } + + // 分布式算法验证 + l1.scope = WorkScope.GLOBAL.name + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UserUrlRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserUrlRateLimiterService).limit(request) + } + + l1.resource = "admin:/blueking/" + l1.limit = 2 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UserUrlRateLimiterService).limit(request) + (rateLimiterService as UserUrlRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserUrlRateLimiterService).limit(request) + } + + l1.resource = "admin:" + l1.limit = 1 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UserUrlRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserUrlRateLimiterService).limit(request) + } + + l1.resource = "test:/blueking" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UserUrlRateLimiterService).limit(request) + (rateLimiterService as UserUrlRateLimiterService).limit(request) + Assertions.assertDoesNotThrow { (rateLimiterService as UserUrlRateLimiterService).limit(request) } + } + + +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/user/UserUrlRepoRateLimiterServiceTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/user/UserUrlRepoRateLimiterServiceTest.kt new file mode 100644 index 0000000000..74fa8ccfd0 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/url/user/UserUrlRepoRateLimiterServiceTest.kt @@ -0,0 +1,337 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.url.user + +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.api.util.toJsonString +import com.tencent.bkrepo.common.artifact.constant.PROJECT_ID +import com.tencent.bkrepo.common.artifact.constant.REPO_NAME +import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory +import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.url.user.UserUrlRepoRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterServiceTest +import com.tencent.bkrepo.repository.pojo.repo.UserRepoCreateRequest +import com.tencent.bkrepo.repository.pojo.search.NodeQueryBuilder +import java.time.Duration +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.test.annotation.DirtiesContext +import org.springframework.web.context.request.RequestContextHolder +import org.springframework.web.context.request.ServletRequestAttributes +import org.springframework.web.servlet.HandlerMapping +import org.springframework.web.servlet.HandlerMapping.BEST_MATCHING_PATTERN_ATTRIBUTE + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class UserUrlRepoRateLimiterServiceTest : AbstractRateLimiterServiceTest() { + + val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "*:/*/", + limitDimension = LimitDimension.USER_URL_REPO.name, limit = 1, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @BeforeAll + fun before() { + init() + rateLimiterProperties.enabled = true + rateLimiterProperties.rules = listOf(l1) + request.requestURI = "/blueking/generic-local/test.txt" + request.setAttribute("userId", "admin") + request.requestURI = "/blueking/generic-local/test.txt" + val uriVariables: MutableMap = HashMap() + uriVariables["projectId"] = "blueking" + uriVariables["repoName"] = "generic-local" + request.setAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE, uriVariables) + request.setAttribute(BEST_MATCHING_PATTERN_ATTRIBUTE, "/{projectId}/{repoName}/**") + val attributes = ServletRequestAttributes(request) + RequestContextHolder.setRequestAttributes(attributes) + val scheduler = ThreadPoolTaskScheduler() + scheduler.initialize() + rateLimiterService = UserUrlRepoRateLimiterService( + taskScheduler = scheduler, + rateLimiterProperties = rateLimiterProperties, + redisTemplate = redisTemplate, + rateLimiterMetrics = rateLimiterMetrics, + rateLimiterConfigService = rateLimiterConfigService + ) + (rateLimiterService as UserUrlRepoRateLimiterService).refreshRateLimitRule() + } + + @Test + override fun createAlgorithmOfRateLimiterTest() { + super.createAlgorithmOfRateLimiterTest() + } + + @Test + override fun refreshRateLimitRuleTest() { + super.refreshRateLimitRuleTest() + } + + @Test + override fun getAlgorithmOfRateLimiterTest() { + super.getAlgorithmOfRateLimiterTest() + } + + @Test + override fun getResLimitInfoTest() { + super.getResLimitInfoTest() + } + + @Test + override fun circuitBreakerCheckTest() { + super.circuitBreakerCheckTest() + } + + @Test + override fun rateLimitCatchTest() { + super.rateLimitCatchTest() + } + + @Test + fun ignoreRequestTest() { + rateLimiterProperties.specialUrls = listOf("*") + Assertions.assertEquals(false, (rateLimiterService as UserUrlRepoRateLimiterService).ignoreRequest(request)) + rateLimiterProperties.specialUrls = listOf("/{projectId}/{repoName}/**") + Assertions.assertEquals(false, (rateLimiterService as UserUrlRepoRateLimiterService).ignoreRequest(request)) + rateLimiterProperties.specialUrls = listOf() + Assertions.assertEquals(true, (rateLimiterService as UserUrlRepoRateLimiterService).ignoreRequest(request)) + } + + @Test + fun buildResourceTest() { + Assertions.assertEquals( + "admin:/blueking/generic-local/", + (rateLimiterService as UserUrlRepoRateLimiterService).buildResource(request) + ) + request.requestURI = "/api/node/batch/blueking/generic-local" + request.setAttribute("userId", "test") + Assertions.assertEquals( + "test:/blueking/generic-local/", + (rateLimiterService as UserUrlRepoRateLimiterService).buildResource(request) + ) + request.requestURI = "/blueking/generic-local/test.txt" + request.setAttribute("userId", "admin") + } + + @Test + fun buildExtraResourceTest() { + Assertions.assertEquals( + listOf("admin:/blueking/", "admin:"), + (rateLimiterService as UserUrlRepoRateLimiterService).buildExtraResource(request) + ) + } + + @Test + fun getApplyPermitsTest() { + Assertions.assertEquals( + 1, + (rateLimiterService as UserUrlRepoRateLimiterService).getApplyPermits(request, null) + ) + } + + @Test + fun getRateLimitRuleClassTest() { + Assertions.assertEquals( + UserUrlRepoRateLimitRule::class.java, + (rateLimiterService as UserUrlRepoRateLimiterService).getRateLimitRuleClass() + ) + } + + @Test + fun getLimitDimensionsTest() { + Assertions.assertEquals( + listOf(LimitDimension.USER_URL_REPO.name), + (rateLimiterService as UserUrlRepoRateLimiterService).getLimitDimensions() + ) + } + + @Test + fun generateKeyTest() { + val resource = (rateLimiterService as UserUrlRepoRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UserUrlRepoRateLimiterService).buildExtraResource(request) + ) + var resourceLimit = + (rateLimiterService as UserUrlRepoRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UserUrlRepo:" + "admin:/blueking/", + (rateLimiterService as UserUrlRepoRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + l1.resource = "admin:" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRepoRateLimiterService).refreshRateLimitRule() + + resourceLimit = (rateLimiterService as UserUrlRepoRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UserUrlRepo:" + "admin:", + (rateLimiterService as UserUrlRepoRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + + l1.resource = "*:/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRepoRateLimiterService).refreshRateLimitRule() + } + + + @Test + fun getRepoInfoTest() { + val (projectId, repoName) = (rateLimiterService as UserUrlRepoRateLimiterService).getRepoInfoFromAttribute( + request + ) + Assertions.assertEquals("blueking", projectId) + Assertions.assertEquals("generic-local", repoName) + } + + @Test + fun getRepoInfoFromBodyTest() { + request.requestURI = "/api/node/search" + request.contentType = "application/json" + val queryModelBuilder = NodeQueryBuilder() + .select(PROJECT_ID, REPO_NAME) + .sortByAsc("fullPath") + .page(1, 10) + .projectId("test-projectId") + .repoName("test-repoName") + + val queryModel = queryModelBuilder.build() + request.setContent(queryModel.toJsonString().toByteArray()) + val (projectId, repoName) = (rateLimiterService as UserUrlRepoRateLimiterService).getRepoInfoFromBody( + request + ) + Assertions.assertEquals("test-projectId", projectId) + Assertions.assertEquals("test-repoName", repoName) + + request.setContent( + UserRepoCreateRequest( + projectId = "test-projectId1", + name = "test-repoName1", + type = RepositoryType.GENERIC, + category = RepositoryCategory.COMPOSITE, + display = false + ).toJsonString().toByteArray() + ) + val (projectId1, repoName1) = (rateLimiterService as UserUrlRepoRateLimiterService).getRepoInfoFromBody( + request + ) + Assertions.assertEquals("test-projectId1", projectId1) + Assertions.assertEquals(null, repoName1) + + request.requestURI = "/blueking/generic-local/test.txt" + } + + @Test + fun generateKeyTestNull() { + l1.resource = "test:" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRepoRateLimiterService).refreshRateLimitRule() + + val resource = (rateLimiterService as UserUrlRepoRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UserUrlRepoRateLimiterService).buildExtraResource(request) + ) + var resourceLimit = + (rateLimiterService as UserUrlRepoRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNull(resourceLimit) + + l1.resource = "test:/blueking/generic-local/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRepoRateLimiterService).refreshRateLimitRule() + resourceLimit = (rateLimiterService as UserUrlRepoRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNull(resourceLimit) + + + l1.resource = "*:/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRepoRateLimiterService).refreshRateLimitRule() + } + + @Test + fun limitTest() { + rateLimiterProperties.specialUrls = listOf("*") + // 本地限流验证 + (rateLimiterService as UserUrlRepoRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserUrlRepoRateLimiterService).limit(request) + } + + // 分布式算法验证 + l1.scope = WorkScope.GLOBAL.name + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRepoRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UserUrlRepoRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserUrlRepoRateLimiterService).limit(request) + } + + l1.resource = "admin:/blueking/" + l1.limit = 2 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRepoRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UserUrlRepoRateLimiterService).limit(request) + (rateLimiterService as UserUrlRepoRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserUrlRepoRateLimiterService).limit(request) + } + + l1.resource = "admin:" + l1.limit = 1 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRepoRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UserUrlRepoRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserUrlRepoRateLimiterService).limit(request) + } + + l1.resource = "test:/blueking" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRepoRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UserUrlRepoRateLimiterService).limit(request) + (rateLimiterService as UserUrlRepoRateLimiterService).limit(request) + Assertions.assertDoesNotThrow { (rateLimiterService as UserUrlRepoRateLimiterService).limit(request) } + + l1.resource = "*:/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUrlRepoRateLimiterService).refreshRateLimitRule() + } + + +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/DownloadUsageRateLimiterServiceTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/DownloadUsageRateLimiterServiceTest.kt new file mode 100644 index 0000000000..16cdf93eb1 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/DownloadUsageRateLimiterServiceTest.kt @@ -0,0 +1,262 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.usage + +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.usage.DownloadUsageRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterServiceTest +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.test.annotation.DirtiesContext +import org.springframework.web.servlet.HandlerMapping +import java.time.Duration + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class DownloadUsageRateLimiterServiceTest : AbstractRateLimiterServiceTest() { + + val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/", + limitDimension = LimitDimension.DOWNLOAD_USAGE.name, limit = 10, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @BeforeAll + fun before() { + init() + rateLimiterProperties.enabled = true + rateLimiterProperties.rules = listOf(l1) + request.requestURI = "/blueking/generic-local/test.txt" + val uriVariables: MutableMap = HashMap() + uriVariables["projectId"] = "blueking" + uriVariables["repoName"] = "generic-local" + request.setAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE, uriVariables) + request.method = "GET" + val scheduler = ThreadPoolTaskScheduler() + scheduler.initialize() + rateLimiterService = DownloadUsageRateLimiterService( + taskScheduler = scheduler, + rateLimiterProperties = rateLimiterProperties, + redisTemplate = redisTemplate, + rateLimiterMetrics = rateLimiterMetrics, + rateLimiterConfigService = rateLimiterConfigService + ) + (rateLimiterService as DownloadUsageRateLimiterService).refreshRateLimitRule() + } + + @Test + override fun createAlgorithmOfRateLimiterTest() { + super.createAlgorithmOfRateLimiterTest() + } + + @Test + override fun refreshRateLimitRuleTest() { + super.refreshRateLimitRuleTest() + } + + @Test + override fun getAlgorithmOfRateLimiterTest() { + super.getAlgorithmOfRateLimiterTest() + } + + @Test + override fun getResLimitInfoTest() { + super.getResLimitInfoTest() + } + + @Test + override fun circuitBreakerCheckTest() { + super.circuitBreakerCheckTest() + } + + @Test + override fun rateLimitCatchTest() { + super.rateLimitCatchTest() + } + + @Test + fun ignoreRequestTest() { + Assertions.assertEquals(false, (rateLimiterService as DownloadUsageRateLimiterService).ignoreRequest(request)) + request.method = "POST" + Assertions.assertEquals(true, (rateLimiterService as DownloadUsageRateLimiterService).ignoreRequest(request)) + request.method = "PATCH" + Assertions.assertEquals(true, (rateLimiterService as DownloadUsageRateLimiterService).ignoreRequest(request)) + request.method = "PUT" + Assertions.assertEquals(true, (rateLimiterService as DownloadUsageRateLimiterService).ignoreRequest(request)) + request.method = "HEAD" + Assertions.assertEquals(true, (rateLimiterService as DownloadUsageRateLimiterService).ignoreRequest(request)) + request.method = "GET" + } + + @Test + fun getRepoInfoTest() { + val (projectId, repoName) = + (rateLimiterService as DownloadUsageRateLimiterService).getRepoInfoFromAttribute(request) + Assertions.assertEquals("blueking", projectId) + Assertions.assertEquals("generic-local", repoName) + + } + + @Test + fun buildResourceTest() { + Assertions.assertEquals( + "/blueking/generic-local/", + (rateLimiterService as DownloadUsageRateLimiterService).buildResource(request) + ) + } + + @Test + fun buildExtraResourceTest() { + Assertions.assertEquals( + listOf("/blueking/"), + (rateLimiterService as DownloadUsageRateLimiterService).buildExtraResource(request) + ) + } + + @Test + fun getApplyPermitsTest() { + Assertions.assertThrows(AcquireLockFailedException::class.java) { + (rateLimiterService as DownloadUsageRateLimiterService).getApplyPermits(request, null) + } + + Assertions.assertEquals( + 10, + (rateLimiterService as DownloadUsageRateLimiterService).getApplyPermits(request, 10) + ) + } + + @Test + fun getRateLimitRuleClassTest() { + Assertions.assertEquals( + DownloadUsageRateLimitRule::class.java, + (rateLimiterService as DownloadUsageRateLimiterService).getRateLimitRuleClass() + ) + } + + @Test + fun getLimitDimensionsTest() { + Assertions.assertEquals( + listOf(LimitDimension.DOWNLOAD_USAGE.name), + (rateLimiterService as DownloadUsageRateLimiterService).getLimitDimensions() + ) + } + + @Test + fun generateKeyTest() { + val resource = (rateLimiterService as DownloadUsageRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as DownloadUsageRateLimiterService).buildExtraResource(request) + ) + var resourceLimit = (rateLimiterService as DownloadUsageRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "DownloadUsage:" + "/blueking/", + (rateLimiterService as DownloadUsageRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + l1.resource = "/blueking/generic-local/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as DownloadUsageRateLimiterService).refreshRateLimitRule() + resourceLimit = (rateLimiterService as DownloadUsageRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "DownloadUsage:" + "/blueking/generic-local/", + (rateLimiterService as DownloadUsageRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + l1.resource = "/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as DownloadUsageRateLimiterService).refreshRateLimitRule() + } + + @Test + fun generateKeyTestNull() { + val resource = (rateLimiterService as DownloadUsageRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as DownloadUsageRateLimiterService).buildExtraResource(request) + ) + l1.resource = "/test/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as DownloadUsageRateLimiterService).refreshRateLimitRule() + val resourceLimit = (rateLimiterService as DownloadUsageRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNull(resourceLimit) + + l1.resource = "/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as DownloadUsageRateLimiterService).refreshRateLimitRule() + } + + @Test + fun limitTest() { + // 本地限流验证 + (rateLimiterService as DownloadUsageRateLimiterService).limit(request, 10) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as DownloadUsageRateLimiterService).limit(request, 10) + } + + // 分布式算法验证 + l1.scope = WorkScope.GLOBAL.name + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as DownloadUsageRateLimiterService).refreshRateLimitRule() + (rateLimiterService as DownloadUsageRateLimiterService).limit(request, 10) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as DownloadUsageRateLimiterService).limit(request, 10) + } + + l1.resource = "/blueking/" + l1.limit = 2 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as DownloadUsageRateLimiterService).refreshRateLimitRule() + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as DownloadUsageRateLimiterService).limit(request, 10) + } + + l1.resource = "/blueking/" + l1.limit = 20 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as DownloadUsageRateLimiterService).refreshRateLimitRule() + (rateLimiterService as DownloadUsageRateLimiterService).limit(request, 10) + (rateLimiterService as DownloadUsageRateLimiterService).limit(request, 10) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as DownloadUsageRateLimiterService).limit(request, 10) + } + } + +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/UploadUsageRateLimiterServiceTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/UploadUsageRateLimiterServiceTest.kt new file mode 100644 index 0000000000..96e2e9a3bb --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/UploadUsageRateLimiterServiceTest.kt @@ -0,0 +1,263 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.usage + +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.usage.UploadUsageRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterServiceTest +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.test.annotation.DirtiesContext +import org.springframework.web.servlet.HandlerMapping +import java.time.Duration + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class UploadUsageRateLimiterServiceTest : AbstractRateLimiterServiceTest() { + + val l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "/*/", + limitDimension = LimitDimension.UPLOAD_USAGE.name, limit = 10, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @BeforeAll + fun before() { + init() + rateLimiterProperties.enabled = true + rateLimiterProperties.rules = listOf(l1) + request.requestURI = "/blueking/generic-local/test.txt" + val uriVariables: MutableMap = HashMap() + uriVariables["projectId"] = "blueking" + uriVariables["repoName"] = "generic-local" + request.setAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE, uriVariables) + val content = byteArrayOf(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) + request.setContent(content) + request.method = "PUT" + val scheduler = ThreadPoolTaskScheduler() + scheduler.initialize() + rateLimiterService = UploadUsageRateLimiterService( + taskScheduler = scheduler, + rateLimiterProperties = rateLimiterProperties, + redisTemplate = redisTemplate, + rateLimiterMetrics = rateLimiterMetrics, + rateLimiterConfigService = rateLimiterConfigService + ) + (rateLimiterService as UploadUsageRateLimiterService).refreshRateLimitRule() + } + + @Test + override fun createAlgorithmOfRateLimiterTest() { + super.createAlgorithmOfRateLimiterTest() + } + + @Test + override fun refreshRateLimitRuleTest() { + super.refreshRateLimitRuleTest() + } + + @Test + override fun getAlgorithmOfRateLimiterTest() { + super.getAlgorithmOfRateLimiterTest() + } + + @Test + override fun getResLimitInfoTest() { + super.getResLimitInfoTest() + } + + @Test + override fun circuitBreakerCheckTest() { + super.circuitBreakerCheckTest() + } + + @Test + override fun rateLimitCatchTest() { + super.rateLimitCatchTest() + } + + @Test + fun ignoreRequestTest() { + Assertions.assertEquals(false, (rateLimiterService as UploadUsageRateLimiterService).ignoreRequest(request)) + request.method = "POST" + Assertions.assertEquals(false, (rateLimiterService as UploadUsageRateLimiterService).ignoreRequest(request)) + request.method = "PATCH" + Assertions.assertEquals(false, (rateLimiterService as UploadUsageRateLimiterService).ignoreRequest(request)) + request.method = "GET" + Assertions.assertEquals(true, (rateLimiterService as UploadUsageRateLimiterService).ignoreRequest(request)) + request.method = "PUT" + } + + @Test + fun getRepoInfoTest() { + val (projectId, repoName) = + (rateLimiterService as UploadUsageRateLimiterService).getRepoInfoFromAttribute(request) + Assertions.assertEquals("blueking", projectId) + Assertions.assertEquals("generic-local", repoName) + + } + + @Test + fun buildResourceTest() { + Assertions.assertEquals( + "/blueking/generic-local/", + (rateLimiterService as UploadUsageRateLimiterService).buildResource(request) + ) + } + + @Test + fun buildExtraResourceTest() { + Assertions.assertEquals( + listOf("/blueking/"), + (rateLimiterService as UploadUsageRateLimiterService).buildExtraResource(request) + ) + } + + @Test + fun getApplyPermitsTest() { + Assertions.assertEquals( + 10, + (rateLimiterService as UploadUsageRateLimiterService).getApplyPermits(request, null) + ) + request.method = "GET" + Assertions.assertEquals( + 0, + (rateLimiterService as UploadUsageRateLimiterService).getApplyPermits(request, null) + ) + request.method = "PUT" + } + + @Test + fun getRateLimitRuleClassTest() { + Assertions.assertEquals( + UploadUsageRateLimitRule::class.java, + (rateLimiterService as UploadUsageRateLimiterService).getRateLimitRuleClass() + ) + } + + @Test + fun getLimitDimensionsTest() { + Assertions.assertEquals( + listOf(LimitDimension.UPLOAD_USAGE.name), + (rateLimiterService as UploadUsageRateLimiterService).getLimitDimensions() + ) + } + + @Test + fun generateKeyTest() { + val resource = (rateLimiterService as UploadUsageRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UploadUsageRateLimiterService).buildExtraResource(request) + ) + var resourceLimit = (rateLimiterService as UploadUsageRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UploadUsage:" + "/blueking/", + (rateLimiterService as UploadUsageRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + l1.resource = "/blueking/generic-local/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UploadUsageRateLimiterService).refreshRateLimitRule() + resourceLimit = (rateLimiterService as UploadUsageRateLimiterService).rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UploadUsage:" + "/blueking/generic-local/", + (rateLimiterService as UploadUsageRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + l1.resource = "/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UploadUsageRateLimiterService).refreshRateLimitRule() + } + + @Test + fun generateKeyTestNull() { + val resource = (rateLimiterService as UploadUsageRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UploadUsageRateLimiterService).buildExtraResource(request) + ) + l1.resource = "/test/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UploadUsageRateLimiterService).refreshRateLimitRule() + val resourceLimit = (rateLimiterService as UploadUsageRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNull(resourceLimit) + + l1.resource = "/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UploadUsageRateLimiterService).refreshRateLimitRule() + } + + @Test + fun limitTest() { + // 本地限流验证 + (rateLimiterService as UploadUsageRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UploadUsageRateLimiterService).limit(request) + } + + // 分布式算法验证 + l1.scope = WorkScope.GLOBAL.name + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UploadUsageRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UploadUsageRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UploadUsageRateLimiterService).limit(request) + } + + l1.resource = "/blueking/" + l1.limit = 2 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UploadUsageRateLimiterService).refreshRateLimitRule() + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UploadUsageRateLimiterService).limit(request) + } + + l1.resource = "/blueking/" + l1.limit = 20 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UploadUsageRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UploadUsageRateLimiterService).limit(request) + (rateLimiterService as UploadUsageRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UploadUsageRateLimiterService).limit(request) + } + } + +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/user/UserDownloadUsageRateLimiterServiceTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/user/UserDownloadUsageRateLimiterServiceTest.kt new file mode 100644 index 0000000000..89829d5b98 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/user/UserDownloadUsageRateLimiterServiceTest.kt @@ -0,0 +1,309 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.usage.user + +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.exception.AcquireLockFailedException +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.usage.user.UserDownloadUsageRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterServiceTest +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.test.annotation.DirtiesContext +import org.springframework.web.context.request.RequestContextHolder +import org.springframework.web.context.request.ServletRequestAttributes +import org.springframework.web.servlet.HandlerMapping +import java.time.Duration + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class UserDownloadUsageRateLimiterServiceTest : AbstractRateLimiterServiceTest() { + + var l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "*:/*/", + limitDimension = LimitDimension.USER_DOWNLOAD_USAGE.name, limit = 10, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @BeforeAll + fun before() { + init() + rateLimiterProperties.enabled = true + rateLimiterProperties.rules = listOf(l1) + request.requestURI = "/blueking/generic-local/test.txt" + request.setAttribute("userId", "admin") + val uriVariables: MutableMap = HashMap() + uriVariables["projectId"] = "blueking" + uriVariables["repoName"] = "generic-local" + request.setAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE, uriVariables) + request.method = "GET" + val attributes = ServletRequestAttributes(request) + RequestContextHolder.setRequestAttributes(attributes) + val scheduler = ThreadPoolTaskScheduler() + scheduler.initialize() + rateLimiterService = UserDownloadUsageRateLimiterService( + taskScheduler = scheduler, + rateLimiterProperties = rateLimiterProperties, + redisTemplate = redisTemplate, + rateLimiterMetrics = rateLimiterMetrics, + rateLimiterConfigService = rateLimiterConfigService + ) + (rateLimiterService as UserDownloadUsageRateLimiterService).refreshRateLimitRule() + } + + @Test + override fun createAlgorithmOfRateLimiterTest() { + super.createAlgorithmOfRateLimiterTest() + } + + @Test + override fun refreshRateLimitRuleTest() { + super.refreshRateLimitRuleTest() + } + + @Test + override fun getAlgorithmOfRateLimiterTest() { + super.getAlgorithmOfRateLimiterTest() + } + + @Test + override fun getResLimitInfoTest() { + super.getResLimitInfoTest() + } + + @Test + override fun circuitBreakerCheckTest() { + super.circuitBreakerCheckTest() + } + + @Test + override fun rateLimitCatchTest() { + super.rateLimitCatchTest() + } + + @Test + fun ignoreRequestTest() { + Assertions.assertEquals( + false, + (rateLimiterService as UserDownloadUsageRateLimiterService).ignoreRequest(request) + ) + request.method = "POST" + Assertions.assertEquals( + true, + (rateLimiterService as UserDownloadUsageRateLimiterService).ignoreRequest(request) + ) + request.method = "PATCH" + + Assertions.assertEquals( + true, + (rateLimiterService as UserDownloadUsageRateLimiterService).ignoreRequest(request) + ) + request.method = "PUT" + + Assertions.assertEquals( + true, + (rateLimiterService as UserDownloadUsageRateLimiterService).ignoreRequest(request) + ) + request.method = "GET" + } + + @Test + fun getRepoInfoTest() { + val (projectId, repoName) = + (rateLimiterService as UserDownloadUsageRateLimiterService).getRepoInfoFromAttribute(request) + Assertions.assertEquals("blueking", projectId) + Assertions.assertEquals("generic-local", repoName) + + } + + @Test + fun buildResourceTest() { + Assertions.assertEquals( + "admin:/blueking/generic-local/", + (rateLimiterService as UserDownloadUsageRateLimiterService).buildResource(request) + ) + } + + @Test + fun buildExtraResourceTest() { + Assertions.assertEquals( + listOf("admin:/blueking/", "admin:"), + (rateLimiterService as UserDownloadUsageRateLimiterService).buildExtraResource(request) + ) + } + + @Test + fun getApplyPermitsTest() { + Assertions.assertThrows(AcquireLockFailedException::class.java) { + (rateLimiterService as UserDownloadUsageRateLimiterService).getApplyPermits(request, null) + } + + Assertions.assertEquals( + 10, + (rateLimiterService as UserDownloadUsageRateLimiterService).getApplyPermits(request, 10) + ) + + } + + @Test + fun getRateLimitRuleClassTest() { + Assertions.assertEquals( + UserDownloadUsageRateLimitRule::class.java, + (rateLimiterService as UserDownloadUsageRateLimiterService).getRateLimitRuleClass() + ) + } + + @Test + fun getLimitDimensionsTest() { + Assertions.assertEquals( + listOf(LimitDimension.USER_DOWNLOAD_USAGE.name), + (rateLimiterService as UserDownloadUsageRateLimiterService).getLimitDimensions() + ) + } + + @Test + fun generateKeyTestNull() { + val resource = (rateLimiterService as UserDownloadUsageRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UserDownloadUsageRateLimiterService).buildExtraResource(request) + ) + + l1.resource = "test:/blueking/generic-local/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserDownloadUsageRateLimiterService).refreshRateLimitRule() + var resourceLimit = (rateLimiterService as UserDownloadUsageRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNull(resourceLimit) + + l1.resource = "test:" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserDownloadUsageRateLimiterService).refreshRateLimitRule() + resourceLimit = (rateLimiterService as UserDownloadUsageRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNull(resourceLimit) + + l1.resource = "*:/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserDownloadUsageRateLimiterService).refreshRateLimitRule() + } + + @Test + fun generateKeyTest() { + val resource = (rateLimiterService as UserDownloadUsageRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UserDownloadUsageRateLimiterService).buildExtraResource(request) + ) + var resourceLimit = (rateLimiterService as UserDownloadUsageRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UserDownloadUsage:" + "admin:/blueking/", + (rateLimiterService as UserDownloadUsageRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + + l1.resource = "admin:/blueking/generic-local/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserDownloadUsageRateLimiterService).refreshRateLimitRule() + resourceLimit = (rateLimiterService as UserDownloadUsageRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UserDownloadUsage:" + "admin:/blueking/generic-local/", + (rateLimiterService as UserDownloadUsageRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + + l1.resource = "admin:" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserDownloadUsageRateLimiterService).refreshRateLimitRule() + resourceLimit = (rateLimiterService as UserDownloadUsageRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UserDownloadUsage:" + "admin:", + (rateLimiterService as UserDownloadUsageRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + + l1.resource = "*:/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserDownloadUsageRateLimiterService).refreshRateLimitRule() + } + + @Test + fun limitTest() { + // 本地限流验证 + (rateLimiterService as UserDownloadUsageRateLimiterService).limit(request, 10) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserDownloadUsageRateLimiterService).limit(request, 10) + } + + // 分布式算法验证 + l1.scope = WorkScope.GLOBAL.name + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserDownloadUsageRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UserDownloadUsageRateLimiterService).limit(request, 10) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserDownloadUsageRateLimiterService).limit(request, 10) + } + + l1.resource = "admin:/blueking/" + l1.limit = 2 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserDownloadUsageRateLimiterService).refreshRateLimitRule() + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserDownloadUsageRateLimiterService).limit(request, 10) + } + + l1.resource = "admin:" + l1.limit = 2 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserDownloadUsageRateLimiterService).refreshRateLimitRule() + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserDownloadUsageRateLimiterService).limit(request, 10) + } + + l1.resource = "test:" + l1.limit = 20 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserDownloadUsageRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UserDownloadUsageRateLimiterService).limit(request, 10) + (rateLimiterService as UserDownloadUsageRateLimiterService).limit(request, 10) + Assertions.assertDoesNotThrow { (rateLimiterService as UserDownloadUsageRateLimiterService).limit(request, 10) } + } + +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/user/UserUploadUsageRateLimiterServiceTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/user/UserUploadUsageRateLimiterServiceTest.kt new file mode 100644 index 0000000000..9b0cc43f92 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/service/usage/user/UserUploadUsageRateLimiterServiceTest.kt @@ -0,0 +1,298 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.service.usage.user + +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.ratelimiter.constant.KEY_PREFIX +import com.tencent.bkrepo.common.ratelimiter.enums.Algorithms +import com.tencent.bkrepo.common.ratelimiter.enums.LimitDimension +import com.tencent.bkrepo.common.ratelimiter.enums.WorkScope +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResInfo +import com.tencent.bkrepo.common.ratelimiter.rule.common.ResourceLimit +import com.tencent.bkrepo.common.ratelimiter.rule.usage.user.UserUploadUsageRateLimitRule +import com.tencent.bkrepo.common.ratelimiter.service.AbstractRateLimiterServiceTest +import java.time.Duration +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler +import org.springframework.test.annotation.DirtiesContext +import org.springframework.web.context.request.RequestContextHolder +import org.springframework.web.context.request.ServletRequestAttributes +import org.springframework.web.servlet.HandlerMapping + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class UserUploadUsageRateLimiterServiceTest : AbstractRateLimiterServiceTest() { + + var l1 = ResourceLimit( + algo = Algorithms.FIXED_WINDOW.name, resource = "*:/*/", + limitDimension = LimitDimension.USER_UPLOAD_USAGE.name, limit = 10, + duration = Duration.ofSeconds(1), scope = WorkScope.LOCAL.name + ) + + @BeforeAll + fun before() { + init() + rateLimiterProperties.enabled = true + rateLimiterProperties.rules = listOf(l1) + request.requestURI = "/blueking/generic-local/test.txt" + request.setAttribute("userId", "admin") + val uriVariables: MutableMap = HashMap() + uriVariables["projectId"] = "blueking" + uriVariables["repoName"] = "generic-local" + request.setAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE, uriVariables) + val content = byteArrayOf(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) + request.setContent(content) + request.method = "PUT" + val attributes = ServletRequestAttributes(request) + RequestContextHolder.setRequestAttributes(attributes) + val scheduler = ThreadPoolTaskScheduler() + scheduler.initialize() + rateLimiterService = UserUploadUsageRateLimiterService( + taskScheduler = scheduler, + rateLimiterProperties = rateLimiterProperties, + redisTemplate = redisTemplate, + rateLimiterMetrics = rateLimiterMetrics, + rateLimiterConfigService = rateLimiterConfigService + ) + (rateLimiterService as UserUploadUsageRateLimiterService).refreshRateLimitRule() + } + + @Test + override fun createAlgorithmOfRateLimiterTest() { + super.createAlgorithmOfRateLimiterTest() + } + + @Test + override fun refreshRateLimitRuleTest() { + super.refreshRateLimitRuleTest() + } + + @Test + override fun getAlgorithmOfRateLimiterTest() { + super.getAlgorithmOfRateLimiterTest() + } + + @Test + override fun getResLimitInfoTest() { + super.getResLimitInfoTest() + } + + @Test + override fun circuitBreakerCheckTest() { + super.circuitBreakerCheckTest() + } + + @Test + override fun rateLimitCatchTest() { + super.rateLimitCatchTest() + } + + @Test + fun ignoreRequestTest() { + Assertions.assertEquals(false, (rateLimiterService as UserUploadUsageRateLimiterService).ignoreRequest(request)) + request.method = "POST" + Assertions.assertEquals(false, (rateLimiterService as UserUploadUsageRateLimiterService).ignoreRequest(request)) + request.method = "PATCH" + Assertions.assertEquals(false, (rateLimiterService as UserUploadUsageRateLimiterService).ignoreRequest(request)) + request.method = "GET" + Assertions.assertEquals(true, (rateLimiterService as UserUploadUsageRateLimiterService).ignoreRequest(request)) + request.method = "PUT" + } + + @Test + fun getRepoInfoTest() { + val (projectId, repoName) = (rateLimiterService as UserUploadUsageRateLimiterService).getRepoInfoFromAttribute( + request + ) + Assertions.assertEquals("blueking", projectId) + Assertions.assertEquals("generic-local", repoName) + + } + + @Test + fun buildResourceTest() { + Assertions.assertEquals( + "admin:/blueking/generic-local/", + (rateLimiterService as UserUploadUsageRateLimiterService).buildResource(request) + ) + } + + @Test + fun buildExtraResourceTest() { + Assertions.assertEquals( + listOf("admin:/blueking/", "admin:"), + (rateLimiterService as UserUploadUsageRateLimiterService).buildExtraResource(request) + ) + } + + @Test + fun getApplyPermitsTest() { + Assertions.assertEquals( + 10, + (rateLimiterService as UserUploadUsageRateLimiterService).getApplyPermits(request, null) + ) + request.method = "GET" + Assertions.assertEquals( + 0, + (rateLimiterService as UserUploadUsageRateLimiterService).getApplyPermits(request, null) + ) + request.method = "PUT" + } + + @Test + fun getRateLimitRuleClassTest() { + Assertions.assertEquals( + UserUploadUsageRateLimitRule::class.java, + (rateLimiterService as UserUploadUsageRateLimiterService).getRateLimitRuleClass() + ) + } + + @Test + fun getLimitDimensionsTest() { + Assertions.assertEquals( + listOf(LimitDimension.USER_UPLOAD_USAGE.name), + (rateLimiterService as UserUploadUsageRateLimiterService).getLimitDimensions() + ) + } + + @Test + fun generateKeyTestNull() { + val resource = (rateLimiterService as UserUploadUsageRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UserUploadUsageRateLimiterService).buildExtraResource(request) + ) + + l1.resource = "test:/blueking/generic-local/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUploadUsageRateLimiterService).refreshRateLimitRule() + var resourceLimit = (rateLimiterService as UserUploadUsageRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNull(resourceLimit) + + l1.resource = "test:" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUploadUsageRateLimiterService).refreshRateLimitRule() + resourceLimit = (rateLimiterService as UserUploadUsageRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNull(resourceLimit) + + l1.resource = "*:/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUploadUsageRateLimiterService).refreshRateLimitRule() + } + + @Test + fun generateKeyTest() { + val resource = (rateLimiterService as UserUploadUsageRateLimiterService).buildResource(request) + val resInfo = ResInfo( + resource = resource, + extraResource = (rateLimiterService as UserUploadUsageRateLimiterService).buildExtraResource(request) + ) + var resourceLimit = (rateLimiterService as UserUploadUsageRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UserUploadUsage:" + "admin:/blueking/", + (rateLimiterService as UserUploadUsageRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + + l1.resource = "admin:/blueking/generic-local/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUploadUsageRateLimiterService).refreshRateLimitRule() + resourceLimit = (rateLimiterService as UserUploadUsageRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UserUploadUsage:" + "admin:/blueking/generic-local/", + (rateLimiterService as UserUploadUsageRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + + l1.resource = "admin:" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUploadUsageRateLimiterService).refreshRateLimitRule() + resourceLimit = (rateLimiterService as UserUploadUsageRateLimiterService) + .rateLimitRule?.getRateLimitRule(resInfo) + Assertions.assertNotNull(resourceLimit) + Assertions.assertEquals( + KEY_PREFIX + "UserUploadUsage:" + "admin:", + (rateLimiterService as UserUploadUsageRateLimiterService) + .generateKey(resourceLimit!!.resource, resourceLimit.resourceLimit) + ) + + l1.resource = "*:/*/" + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUploadUsageRateLimiterService).refreshRateLimitRule() + } + + @Test + fun limitTest() { + // 本地限流验证 + (rateLimiterService as UserUploadUsageRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserUploadUsageRateLimiterService).limit(request) + } + + // 分布式算法验证 + l1.scope = WorkScope.GLOBAL.name + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUploadUsageRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UserUploadUsageRateLimiterService).limit(request) + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserUploadUsageRateLimiterService).limit(request) + } + + l1.resource = "admin:/blueking/" + l1.limit = 2 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUploadUsageRateLimiterService).refreshRateLimitRule() + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserUploadUsageRateLimiterService).limit(request) + } + + l1.resource = "admin:" + l1.limit = 2 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUploadUsageRateLimiterService).refreshRateLimitRule() + Assertions.assertThrows(OverloadException::class.java) { + (rateLimiterService as UserUploadUsageRateLimiterService).limit(request) + } + + l1.resource = "test:" + l1.limit = 20 + rateLimiterProperties.rules = listOf(l1) + (rateLimiterService as UserUploadUsageRateLimiterService).refreshRateLimitRule() + (rateLimiterService as UserUploadUsageRateLimiterService).limit(request) + (rateLimiterService as UserUploadUsageRateLimiterService).limit(request) + Assertions.assertDoesNotThrow { (rateLimiterService as UserUploadUsageRateLimiterService).limit(request) } + } + +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/stream/CommonRateLimitInputStreamTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/stream/CommonRateLimitInputStreamTest.kt new file mode 100644 index 0000000000..d554ac25a1 --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/stream/CommonRateLimitInputStreamTest.kt @@ -0,0 +1,160 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.stream + +import com.google.common.base.Stopwatch +import com.google.common.base.Ticker +import com.tencent.bkrepo.common.api.util.HumanReadable +import com.tencent.bkrepo.common.ratelimiter.algorithm.DistributedFixedWindowRateLimiter +import com.tencent.bkrepo.common.ratelimiter.algorithm.DistributedTest +import com.tencent.bkrepo.common.ratelimiter.algorithm.FixedWindowRateLimiter +import com.tencent.bkrepo.common.api.exception.OverloadException +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance +import org.mockito.Mockito +import org.springframework.test.annotation.DirtiesContext +import java.time.Duration +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import kotlin.concurrent.thread +import kotlin.system.measureTimeMillis + +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class CommonRateLimitInputStreamTest : DistributedTest() { + + lateinit var ticker: Ticker + private val content = "1234567891" + val keyStr = "CommonRateLimitInputStreamTest" + + @BeforeAll + fun before() { + ticker = Mockito.mock(Ticker::class.java) + } + + @Test + fun readTestOncePermitsGreaterThanLength() { + val (context, _) = createContext(1024 * 1024) + inputStreamReadTest(context) + } + + @Test + fun readTestOncePermitsLessThanLength() { + val (context, _) = createContext(5) + inputStreamReadTest(context) + } + + @Test + fun readTestOncePermitsEqualLength() { + val (context, _) = createContext(10) + inputStreamReadTest(context) + } + + @Test + fun readTestOncePermitsGreaterThanLimit() { + val (context, _) = createContext(10, 5) + CommonRateLimitInputStream( + delegate = content.byteInputStream(), + rateCheckContext = context + ).use { `is` -> + val buf = ByteArray(3) + `is`.read(buf) + Assertions.assertThrows(OverloadException::class.java) { `is`.read(buf) } + } + } + + + + @Test + fun readTestOnMultiThreads() { + val (context, key) = createContext(10, 100, true, keyStr) + var successNum = 0 + var failedNum = 0 + var errorNum = 0 + val readers = Runtime.getRuntime().availableProcessors() + val countDownLatch = CountDownLatch(readers) + val elapsedTime = measureTimeMillis { + repeat(readers) { + thread { + try { + inputStreamReadTest(context) + successNum++ + } catch (e: Exception) { + errorNum++ + } + countDownLatch.countDown() + } + } + } + countDownLatch.await() + println("elapse: ${HumanReadable.time(elapsedTime, TimeUnit.MILLISECONDS)}") + println("successNum $successNum, failedNum $failedNum. errorNum $errorNum") + key?.let { clean(key) } + } + + private fun inputStreamReadTest(context: RateCheckContext) { + CommonRateLimitInputStream( + delegate = content.byteInputStream(), + rateCheckContext = context + ).use { `is` -> + val buf = ByteArray(3) + assertEquals(`is`.read(buf, 0, 3), 3) + assertEquals(String(buf), "123") + assertEquals(`is`.read().toChar(), '4') + assertEquals(`is`.read(buf), 3) + assertEquals(String(buf), "567") + assertEquals(`is`.read().toChar(), '8') + assertEquals(`is`.read().toChar(), '9') + assertEquals(`is`.read().toChar(), '1') + assertEquals(`is`.read(), -1) + println("finished") + } + + } + + private fun createContext( + permitsOnce: Long, limit: Long = 1024 * 1024 * 100, + distributed: Boolean = false, keyStr: String? = null, + ): Pair { + val (rateLimiter, key) = if (distributed) { + Pair(DistributedFixedWindowRateLimiter(keyStr!!, limit, Duration.ofSeconds(1), redisTemplate), keyStr) + } else { + Pair(FixedWindowRateLimiter(limit, Duration.ofSeconds(1), Stopwatch.createStarted(ticker)), keyStr) + } + return Pair( + RateCheckContext( + rateLimiter = rateLimiter, latency = 10, + waitRound = 3, rangeLength = content.length.toLong(), + dryRun = false, permitsOnce = permitsOnce, limitPerSecond = limit + ), key + ) + } +} \ No newline at end of file diff --git a/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/utils/ResourcePathUtilsTest.kt b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/utils/ResourcePathUtilsTest.kt new file mode 100644 index 0000000000..893b43f6da --- /dev/null +++ b/src/backend/common/common-ratelimiter/src/test/kotlin/com/tencent/bkrepo/common/ratelimiter/utils/ResourcePathUtilsTest.kt @@ -0,0 +1,155 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.ratelimiter.utils + +import com.tencent.bkrepo.common.ratelimiter.exception.InvalidResourceException +import org.hamcrest.CoreMatchers +import org.hamcrest.MatcherAssert +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.fail +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows + +class ResourcePathUtilsTest { + + @Test + fun testTokenizeUrlPath() { + val url = "/test1/test2" + try { + val actualSegments = ResourcePathUtils.tokenizeResourcePath(url) + assertEquals(actualSegments.size, 2) + MatcherAssert.assertThat(actualSegments, CoreMatchers.hasItems("test1", "test2")) + } catch (e: InvalidResourceException) { + fail("tokenizeUrlPath should not throw InvalidResourceException.") + } + } + + @Test + fun testTokenizeUrlPathWithUrlPatten() { + val url = "/test1/test2/{projectId}/{repoName}/{*:(.*)}" + try { + val actualSegments = ResourcePathUtils.tokenizeResourcePath(url) + assertEquals(actualSegments.size, 5) + MatcherAssert.assertThat( + actualSegments, + CoreMatchers.hasItems("test1", "test2", "{projectId}", "{repoName}", "{*:(.*)}") + ) + } catch (e: InvalidResourceException) { + fail("tokenizeUrlPath should not throw InvalidResourceException.") + } + } + + @Test + fun testTokenizeUrlPathWithEmptyUrl() { + try { + val actualSegments = ResourcePathUtils.tokenizeResourcePath("") + assertNotNull(actualSegments) + assertEquals(actualSegments.size, 0) + val actualSegments2 = ResourcePathUtils.tokenizeResourcePath("/") + assertNotNull(actualSegments2) + assertEquals(actualSegments2.size, 0) + } catch (e: InvalidResourceException) { + fail("tokenizeUrlPath should not throw InvalidResourceException.") + } + } + + @Test + fun testGetUrlPath() { + try { + var actualPath: String? = ResourcePathUtils.getPathOfUrl("http://www.bkrepo.com/") + assertEquals(actualPath, "/") + actualPath = ResourcePathUtils.getPathOfUrl("http://www.bkrepo.com") + assertEquals(actualPath, "/") + actualPath = ResourcePathUtils.getPathOfUrl("http://www.bkrepo.com/test1/test2") + assertEquals(actualPath, "/test1/test2") + actualPath = ResourcePathUtils.getPathOfUrl("http://www.bkrepo.com/test1/test2?user=xxx") + assertEquals(actualPath, "/test1/test2") + actualPath = ResourcePathUtils.getPathOfUrl("/test1/test2") + assertEquals(actualPath, "/test1/test2") + actualPath = ResourcePathUtils.getPathOfUrl("/test1/test2?user=xxx") + assertEquals(actualPath, "/test1/test2") + actualPath = ResourcePathUtils.getPathOfUrl("/test1/test2/") + assertEquals(actualPath, "/test1/test2/") + } catch (e: InvalidResourceException) { + fail("getPathOfUrl() should not throw exception here.") + } + } + + @Test + fun testGetUrlPathWithEmptyUrl() { + try { + val actualPath: String? = ResourcePathUtils.getPathOfUrl("") + assertNull(actualPath) + } catch (e: InvalidResourceException) { + fail("getPathOfUrl() should not throw exception here.") + } + } + + @Test + fun testGetUserAndPathWithEmptyUrl() { + try { + assertThrows { ResourcePathUtils.getUserAndPath("") } + } catch (e: InvalidResourceException) { + fail("getUserAndPath() should not throw exception here.") + } + } + + @Test + fun testGetUserAndPath() { + try { + assertThrows { ResourcePathUtils.getUserAndPath("a") } + var (userId, path) = ResourcePathUtils.getUserAndPath("a:") + assertEquals(userId, "a") + assertEquals(path, "") + val (userId1, path1) = ResourcePathUtils.getUserAndPath("a:1") + assertEquals(userId1, "a") + assertEquals(path1, "1") + val (userId2, path2) = ResourcePathUtils.getUserAndPath("a:1:12") + assertEquals(userId2, "a") + assertEquals(path2, "1:12") + } catch (e: InvalidResourceException) { + fail("getPathOfUrl() should not throw exception here.") + } + } + + @Test + fun testBuildUserPath() { + var actualPath: String? = ResourcePathUtils.buildUserPath("a", "b") + assertEquals(actualPath, "a:b") + actualPath = ResourcePathUtils.buildUserPath("1", "2") + assertEquals(actualPath, "1:2") + } + + @Test + fun testBuildUserPathWithEmptyUrl() { + var actualPath: String? = ResourcePathUtils.buildUserPath("a", "") + assertEquals(actualPath, "a:") + } +} \ No newline at end of file diff --git a/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/SecurityAutoConfiguration.kt b/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/SecurityAutoConfiguration.kt index c666134bca..2f5bd0c80b 100644 --- a/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/SecurityAutoConfiguration.kt +++ b/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/SecurityAutoConfiguration.kt @@ -31,31 +31,19 @@ package com.tencent.bkrepo.common.security -import com.tencent.bkrepo.auth.api.ServiceExternalPermissionClient -import com.tencent.bkrepo.auth.api.ServicePermissionClient -import com.tencent.bkrepo.auth.api.ServiceUserClient -import com.tencent.bkrepo.common.api.pojo.ClusterArchitecture -import com.tencent.bkrepo.common.api.pojo.ClusterNodeType import com.tencent.bkrepo.common.security.actuator.ActuatorAuthConfiguration import com.tencent.bkrepo.common.security.crypto.CryptoConfiguration import com.tencent.bkrepo.common.security.exception.SecurityExceptionHandler import com.tencent.bkrepo.common.security.http.HttpAuthConfiguration -import com.tencent.bkrepo.common.security.http.core.HttpAuthProperties import com.tencent.bkrepo.common.security.interceptor.devx.DevXAccessInterceptor import com.tencent.bkrepo.common.security.interceptor.devx.DevXProperties import com.tencent.bkrepo.common.security.manager.AuthenticationManager -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.security.manager.PrincipalManager import com.tencent.bkrepo.common.security.manager.ci.CIPermissionManager import com.tencent.bkrepo.common.security.manager.ci.CIPermissionProperties -import com.tencent.bkrepo.common.security.manager.edge.EdgePermissionManager -import com.tencent.bkrepo.common.security.manager.proxy.ProxyPermissionManager import com.tencent.bkrepo.common.security.permission.PermissionConfiguration import com.tencent.bkrepo.common.security.proxy.ProxyAuthConfiguration import com.tencent.bkrepo.common.security.service.ServiceAuthConfiguration -import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.api.ProjectClient -import com.tencent.bkrepo.repository.api.RepositoryClient import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication @@ -66,7 +54,6 @@ import org.springframework.context.annotation.Import import org.springframework.web.servlet.config.annotation.InterceptorRegistry import org.springframework.web.servlet.config.annotation.WebMvcConfigurer -@Suppress("SpringJavaInjectionPointsAutowiringInspection") @Configuration @ConditionalOnWebApplication @Import( @@ -78,50 +65,12 @@ import org.springframework.web.servlet.config.annotation.WebMvcConfigurer ActuatorAuthConfiguration::class, CryptoConfiguration::class, AuthenticationManager::class, - CIPermissionManager::class + CIPermissionManager::class, + PrincipalManager::class ) @EnableConfigurationProperties(DevXProperties::class, CIPermissionProperties::class) class SecurityAutoConfiguration { - @Bean - @Suppress("LongParameterList") - fun permissionManager( - projectClient: ProjectClient, - repositoryClient: RepositoryClient, - permissionResource: ServicePermissionClient, - externalPermissionResource: ServiceExternalPermissionClient, - userResource: ServiceUserClient, - nodeClient: NodeClient, - clusterProperties: ClusterProperties, - httpAuthProperties: HttpAuthProperties - ): PermissionManager { - return if (clusterProperties.role == ClusterNodeType.EDGE - && clusterProperties.architecture == ClusterArchitecture.COMMIT_EDGE - && clusterProperties.commitEdge.auth.center - ) { - EdgePermissionManager( - projectClient = projectClient, - repositoryClient = repositoryClient, - permissionResource = permissionResource, - externalPermissionResource = externalPermissionResource, - userResource = userResource, - nodeClient = nodeClient, - clusterProperties = clusterProperties, - httpAuthProperties = httpAuthProperties - ) - } else { - PermissionManager( - projectClient = projectClient, - repositoryClient = repositoryClient, - permissionResource = permissionResource, - externalPermissionResource = externalPermissionResource, - userResource = userResource, - nodeClient = nodeClient, - httpAuthProperties = httpAuthProperties - ) - } - } - @Bean @ConditionalOnProperty(value = ["devx.enabled"]) fun devXAccessInterceptorConfigure( @@ -151,26 +100,4 @@ class SecurityAutoConfiguration { fun devXAccessInterceptor(properties: DevXProperties): DevXAccessInterceptor { return DevXAccessInterceptor(properties) } - - @Bean - @ConditionalOnMissingBean - fun proxyPermissionManager( - projectClient: ProjectClient, - repositoryClient: RepositoryClient, - permissionResource: ServicePermissionClient, - externalPermissionResource: ServiceExternalPermissionClient, - userResource: ServiceUserClient, - nodeClient: NodeClient, - httpAuthProperties: HttpAuthProperties - ): ProxyPermissionManager { - return ProxyPermissionManager( - projectClient = projectClient, - repositoryClient = repositoryClient, - permissionResource = permissionResource, - externalPermissionResource = externalPermissionResource, - userResource = userResource, - nodeClient = nodeClient, - httpAuthProperties = httpAuthProperties - ) - } } diff --git a/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/actuator/ActuatorAuthConfiguration.kt b/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/actuator/ActuatorAuthConfiguration.kt index 1d98929cac..6c3e205c20 100644 --- a/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/actuator/ActuatorAuthConfiguration.kt +++ b/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/actuator/ActuatorAuthConfiguration.kt @@ -32,7 +32,7 @@ package com.tencent.bkrepo.common.security.actuator import com.tencent.bkrepo.common.security.manager.AuthenticationManager -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.security.manager.PrincipalManager import org.springframework.boot.actuate.autoconfigure.endpoint.web.CorsEndpointProperties import org.springframework.boot.actuate.autoconfigure.endpoint.web.WebEndpointProperties import org.springframework.boot.actuate.autoconfigure.web.server.ManagementPortType @@ -47,7 +47,6 @@ import org.springframework.boot.actuate.endpoint.web.servlet.WebMvcEndpointHandl import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import org.springframework.core.env.Environment -import java.util.ArrayList @Configuration class ActuatorAuthConfiguration { @@ -83,8 +82,8 @@ class ActuatorAuthConfiguration { @Bean fun actuatorAuthInterceptor( authenticationManager: AuthenticationManager, - permissionManager: PermissionManager + principalManager: PrincipalManager ): ActuatorAuthInterceptor { - return ActuatorAuthInterceptor(authenticationManager, permissionManager) + return ActuatorAuthInterceptor(authenticationManager, principalManager) } } diff --git a/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/actuator/ActuatorAuthInterceptor.kt b/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/actuator/ActuatorAuthInterceptor.kt index 21eb1f65c6..2f2b333646 100644 --- a/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/actuator/ActuatorAuthInterceptor.kt +++ b/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/actuator/ActuatorAuthInterceptor.kt @@ -37,7 +37,7 @@ import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.security.exception.AuthenticationException import com.tencent.bkrepo.common.security.exception.PermissionException import com.tencent.bkrepo.common.security.manager.AuthenticationManager -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.security.manager.PrincipalManager import com.tencent.bkrepo.common.security.permission.PrincipalType import org.springframework.util.AntPathMatcher import org.springframework.web.servlet.handler.HandlerInterceptorAdapter @@ -47,7 +47,7 @@ import javax.servlet.http.HttpServletResponse class ActuatorAuthInterceptor( private val authenticationManager: AuthenticationManager, - private val permissionManager: PermissionManager + private val principalManager: PrincipalManager ) : HandlerInterceptorAdapter() { private val antPathMatcher = AntPathMatcher() @@ -65,7 +65,7 @@ class ActuatorAuthInterceptor( val parts = decodedHeader.split(StringPool.COLON) require(parts.size >= 2) val userId = authenticationManager.checkUserAccount(parts[0], parts[1]) - permissionManager.checkPrincipal(userId, PrincipalType.ADMIN) + principalManager.checkPrincipal(userId, PrincipalType.ADMIN) return true } catch (exception: AuthenticationException) { throw exception diff --git a/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/interceptor/devx/DevXProperties.kt b/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/interceptor/devx/DevXProperties.kt index 03ed0819f3..d54195a26f 100644 --- a/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/interceptor/devx/DevXProperties.kt +++ b/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/interceptor/devx/DevXProperties.kt @@ -131,4 +131,16 @@ data class DevXProperties( * 应用devX拦截器的接口 */ var includePatterns: List = emptyList(), + + /** + * 校验devx token接口url + */ + var validateTokenUrl: String = "", + + /** + * 校验devx token接口的认证token + */ + var authToken: String = "", + + ) diff --git a/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/manager/PrincipalManager.kt b/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/manager/PrincipalManager.kt new file mode 100644 index 0000000000..db33a6a926 --- /dev/null +++ b/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/manager/PrincipalManager.kt @@ -0,0 +1,54 @@ +package com.tencent.bkrepo.common.security.manager + +import com.tencent.bkrepo.auth.api.ServiceUserClient +import com.tencent.bkrepo.common.api.constant.ANONYMOUS_USER +import com.tencent.bkrepo.common.security.exception.AuthenticationException +import com.tencent.bkrepo.common.security.exception.PermissionException +import com.tencent.bkrepo.common.security.permission.PrincipalType +import com.tencent.bkrepo.common.security.util.SecurityUtils +import org.slf4j.LoggerFactory + +class PrincipalManager( + private val serviceUserClient: ServiceUserClient +) { + + fun checkPrincipal(userId: String, principalType: PrincipalType) { + val platformId = SecurityUtils.getPlatformId() + checkAnonymous(userId, platformId) + + if (principalType == PrincipalType.ADMIN) { + if (!isAdminUser(userId)) { + throw PermissionException() + } + } else if (principalType == PrincipalType.PLATFORM) { + if (userId.isEmpty()) { + logger.warn("platform auth with empty userId[$platformId,$userId]") + } + if (platformId == null && !isAdminUser(userId)) { + throw PermissionException() + } + } else if (principalType == PrincipalType.GENERAL) { + if (userId.isEmpty() || userId == ANONYMOUS_USER) { + throw PermissionException() + } + } + } + + /** + * 检查是否为匿名用户，如果是匿名用户则返回401并提示登录 + */ + private fun checkAnonymous(userId: String, platformId: String?) { + if (userId == ANONYMOUS_USER && platformId == null) { + throw AuthenticationException() + } + } + + private fun isAdminUser(userId: String): Boolean { + return serviceUserClient.userInfoById(userId).data?.admin == true + } + + companion object { + private val logger = LoggerFactory.getLogger(PrincipalManager::class.java) + } + +} diff --git a/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/permission/DefaultPermissionCheckHandler.kt b/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/permission/DefaultPermissionCheckHandler.kt index 18d45b3802..2d389a89be 100644 --- a/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/permission/DefaultPermissionCheckHandler.kt +++ b/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/permission/DefaultPermissionCheckHandler.kt @@ -31,13 +31,13 @@ package com.tencent.bkrepo.common.security.permission -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.security.manager.PrincipalManager open class DefaultPermissionCheckHandler( - private val permissionManager: PermissionManager + private val principalManager: PrincipalManager ) : PermissionCheckHandler { override fun onPrincipalCheck(userId: String, principal: Principal) { - permissionManager.checkPrincipal(userId, principal.type) + principalManager.checkPrincipal(userId, principal.type) } } diff --git a/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/permission/PermissionConfiguration.kt b/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/permission/PermissionConfiguration.kt index 7fce064139..7960acc49b 100644 --- a/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/permission/PermissionConfiguration.kt +++ b/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/permission/PermissionConfiguration.kt @@ -31,7 +31,7 @@ package com.tencent.bkrepo.common.security.permission -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.security.manager.PrincipalManager import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration @@ -46,7 +46,7 @@ class PermissionConfiguration { @Bean @ConditionalOnMissingBean - fun permissionCheckHandler(permissionManager: PermissionManager): PermissionCheckHandler { - return DefaultPermissionCheckHandler(permissionManager) + fun permissionCheckHandler(principalManager: PrincipalManager): PermissionCheckHandler { + return DefaultPermissionCheckHandler(principalManager) } } diff --git a/src/backend/common/common-service/service-servlet/src/main/kotlin/com/tencent/bkrepo/common/service/exception/GlobalExceptionHandler.kt b/src/backend/common/common-service/service-servlet/src/main/kotlin/com/tencent/bkrepo/common/service/exception/GlobalExceptionHandler.kt index 1df6f9540c..c80bd8746c 100644 --- a/src/backend/common/common-service/service-servlet/src/main/kotlin/com/tencent/bkrepo/common/service/exception/GlobalExceptionHandler.kt +++ b/src/backend/common/common-service/service-servlet/src/main/kotlin/com/tencent/bkrepo/common/service/exception/GlobalExceptionHandler.kt @@ -33,10 +33,15 @@ package com.tencent.bkrepo.common.service.exception import com.fasterxml.jackson.module.kotlin.MissingKotlinParameterException import com.tencent.bkrepo.common.api.constant.HttpStatus +import com.tencent.bkrepo.common.api.constant.MediaTypes import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.api.exception.OverloadException import com.tencent.bkrepo.common.api.exception.TooManyRequestsException import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.service.log.LoggerHolder +import com.tencent.bkrepo.common.service.util.HttpContextHolder +import com.tencent.bkrepo.common.service.util.LocaleMessageUtils import org.springframework.core.Ordered import org.springframework.core.annotation.Order import org.springframework.http.converter.HttpMessageNotReadableException @@ -143,6 +148,21 @@ class GlobalExceptionHandler : AbstractExceptionHandler() { return response(exception) } + @ExceptionHandler(OverloadException::class) + fun handleException(exception: OverloadException): Response? { + if (HttpContextHolder.getResponse().isCommitted) { + // 当返回已写入部分数据后，无法正常返回429 + val errorMessage = LocaleMessageUtils.getLocalizedMessage(exception.messageCode, exception.params) + LoggerHolder.logErrorCodeException(exception, "[${exception.messageCode.getCode()}]$errorMessage") + HttpContextHolder.getResponse().outputStream.close() + return null + } else { + HttpContextHolder.getResponse().reset() + HttpContextHolder.getResponse().contentType = MediaTypes.APPLICATION_JSON_WITHOUT_CHARSET + return response(exception) + } + } + @ExceptionHandler(Exception::class) fun handleException(exception: Exception): Response { return response(exception) diff --git a/src/backend/common/common-service/service-servlet/src/main/resources/logback-config.xml b/src/backend/common/common-service/service-servlet/src/main/resources/logback-config.xml index 1a0057f1f5..1255bb6b23 100644 --- a/src/backend/common/common-service/service-servlet/src/main/resources/logback-config.xml +++ b/src/backend/common/common-service/service-servlet/src/main/resources/logback-config.xml @@ -10,12 +10,35 @@ + + + + + + ${AUDIT_EVENT_LOG_FILE} + + ${AUDIT_EVENT_LOG_FILE}-%d{yyyy-MM-dd}.log.%i + 1GB + 3 + 5GB + + + ${AUDIT_EVENT_LOG_PATTERN} + UTF-8 + + + + + + + + ${logging.app.file} diff --git a/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/config/ReceiveProperties.kt b/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/config/ReceiveProperties.kt index 506efdbddb..67248ff1c3 100644 --- a/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/config/ReceiveProperties.kt +++ b/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/config/ReceiveProperties.kt @@ -72,5 +72,9 @@ data class ReceiveProperties( /** * 每秒接收数据量 */ - var rateLimit: DataSize = DataSize.ofBytes(-1) + var rateLimit: DataSize = DataSize.ofBytes(-1), + /** + * 限速熔断阈值，当仓库配置的rateLimit小于等于限速熔断阈值时则直接将请求断开 + */ + var circuitBreakerThreshold: DataSize = DataSize.ofKilobytes(1), ) diff --git a/src/backend/composer/api-composer/build.gradle.kts b/src/backend/composer/api-composer/build.gradle.kts index 709034334e..9361a7502f 100644 --- a/src/backend/composer/api-composer/build.gradle.kts +++ b/src/backend/composer/api-composer/build.gradle.kts @@ -30,6 +30,7 @@ */ dependencies{ + api(project(":repository:api-repository")) implementation(project(":common:common-api")) implementation(project(":common:common-artifact:artifact-api")) compileOnly("org.springframework:spring-web") diff --git a/src/backend/composer/biz-composer/src/main/kotlin/com/tencent/bkrepo/composer/controller/ComposerResourceController.kt b/src/backend/composer/biz-composer/src/main/kotlin/com/tencent/bkrepo/composer/controller/ComposerResourceController.kt index 57bbf363bb..8bf95bd2d0 100644 --- a/src/backend/composer/biz-composer/src/main/kotlin/com/tencent/bkrepo/composer/controller/ComposerResourceController.kt +++ b/src/backend/composer/biz-composer/src/main/kotlin/com/tencent/bkrepo/composer/controller/ComposerResourceController.kt @@ -31,7 +31,15 @@ package com.tencent.bkrepo.composer.controller +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_CREATE_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_DOWNLOAD_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.composer.api.ComposerResource import com.tencent.bkrepo.composer.artifact.ComposerArtifactInfo @@ -45,6 +53,24 @@ class ComposerResourceController( @Autowired private val composerService: ComposerService ) : ComposerResource { + + @AuditEntry( + actionId = NODE_DOWNLOAD_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DOWNLOAD_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#composerArtifactInfo?.getArtifactFullPath()", + instanceNames = "#composerArtifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#composerArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#composerArtifactInfo?.repoName") + ], + scopeId = "#composerArtifactInfo?.projectId", + content = ActionAuditContent.NODE_DOWNLOAD_CONTENT + ) override fun installRequire(composerArtifactInfo: ComposerArtifactInfo) { composerService.installRequire(composerArtifactInfo) } @@ -65,6 +91,23 @@ class ComposerResourceController( } } + @AuditEntry( + actionId = NODE_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#composerArtifactInfo?.getArtifactFullPath()", + instanceNames = "#composerArtifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#composerArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#composerArtifactInfo?.repoName") + ], + scopeId = "#composerArtifactInfo?.projectId", + content = ActionAuditContent.NODE_UPLOAD_CONTENT + ) override fun deploy(composerArtifactInfo: ComposerArtifactInfo, file: ArtifactFile) { composerService.deploy(composerArtifactInfo, file) } diff --git a/src/backend/composer/biz-composer/src/main/kotlin/com/tencent/bkrepo/composer/controller/ComposerWebResourceController.kt b/src/backend/composer/biz-composer/src/main/kotlin/com/tencent/bkrepo/composer/controller/ComposerWebResourceController.kt index 9c939bad09..0f19094737 100644 --- a/src/backend/composer/biz-composer/src/main/kotlin/com/tencent/bkrepo/composer/controller/ComposerWebResourceController.kt +++ b/src/backend/composer/biz-composer/src/main/kotlin/com/tencent/bkrepo/composer/controller/ComposerWebResourceController.kt @@ -27,7 +27,14 @@ package com.tencent.bkrepo.composer.controller +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.REPO_EDIT_ACTION +import com.tencent.bkrepo.common.artifact.audit.REPO_RESOURCE import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.composer.api.ComposerWebResource import com.tencent.bkrepo.composer.artifact.ComposerArtifactInfo @@ -38,11 +45,48 @@ import org.springframework.web.bind.annotation.RestController class ComposerWebResourceController( private val composerWebService: ComposerWebService ) : ComposerWebResource { + + @AuditEntry( + actionId = REPO_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = REPO_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#composerArtifactInfo?.repoName", + instanceNames = "#composerArtifactInfo?.repoName" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#composerArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.NAME_TEMPLATE, value = "#packageKey") + ], + scopeId = "#composerArtifactInfo?.projectId", + content = ActionAuditContent.REPO_PACKAGE_DELETE_CONTENT + ) override fun deletePackage(composerArtifactInfo: ComposerArtifactInfo, packageKey: String): Response { composerWebService.deletePackage(composerArtifactInfo, packageKey) return ResponseBuilder.success() } + @AuditEntry( + actionId = REPO_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = REPO_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#composerArtifactInfo?.repoName", + instanceNames = "#composerArtifactInfo?.repoName" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#composerArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.NAME_TEMPLATE, value = "#packageKey"), + AuditAttribute(name = ActionAuditContent.VERSION_TEMPLATE, value = "#version") + + ], + scopeId = "#composerArtifactInfo?.projectId", + content = ActionAuditContent.REPO_PACKAGE_VERSION_DELETE_CONTENT + ) override fun deleteVersion( composerArtifactInfo: ComposerArtifactInfo, packageKey: String, diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/ConanController.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/ConanController.kt index 227e2b04b6..2ad6d20855 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/ConanController.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/ConanController.kt @@ -27,10 +27,18 @@ package com.tencent.bkrepo.conan.controller +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_CREATE_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_DOWNLOAD_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.conan.constant.CONANFILE import com.tencent.bkrepo.conan.constant.CONANINFO @@ -181,6 +189,23 @@ class ConanController( /** * 获取package下文件 */ + @AuditEntry( + actionId = NODE_DOWNLOAD_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DOWNLOAD_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#conanArtifactInfo?.getArtifactFullPath()", + instanceNames = "#conanArtifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#conanArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#conanArtifactInfo?.repoName") + ], + scopeId = "#conanArtifactInfo?.projectId", + content = ActionAuditContent.NODE_DOWNLOAD_CONTENT + ) @GetMapping(PACKAGE_REVISION_FILE_V2) @Permission(type = ResourceType.REPO, action = PermissionAction.READ) fun getPackageRevisionFile( @@ -192,6 +217,23 @@ class ConanController( /** * 上传文件 */ + @AuditEntry( + actionId = NODE_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#conanArtifactInfo?.getArtifactFullPath()", + instanceNames = "#conanArtifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#conanArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#conanArtifactInfo?.repoName") + ], + scopeId = "#conanArtifactInfo?.projectId", + content = ActionAuditContent.NODE_UPLOAD_CONTENT + ) @PutMapping(PACKAGE_REVISION_FILE_V2) @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) fun uploadPackageRevisionFile( @@ -215,6 +257,23 @@ class ConanController( /** * 获取recipe下文件 */ + @AuditEntry( + actionId = NODE_DOWNLOAD_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DOWNLOAD_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#conanArtifactInfo?.getArtifactFullPath()", + instanceNames = "#conanArtifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#conanArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#conanArtifactInfo?.repoName") + ], + scopeId = "#conanArtifactInfo?.projectId", + content = ActionAuditContent.NODE_DOWNLOAD_CONTENT + ) @GetMapping(RECIPE_REVISION_FILE_V2) @Permission(type = ResourceType.REPO, action = PermissionAction.READ) fun getRecipeRevisionFile( @@ -226,6 +285,23 @@ class ConanController( /** * 上传文件 */ + @AuditEntry( + actionId = NODE_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#conanArtifactInfo?.getArtifactFullPath()", + instanceNames = "#conanArtifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#conanArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#conanArtifactInfo?.repoName") + ], + scopeId = "#conanArtifactInfo?.projectId", + content = ActionAuditContent.NODE_UPLOAD_CONTENT + ) @PutMapping(RECIPE_REVISION_FILE_V2) @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) fun uploadRecipeRevisionFile( diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/ConanDeleteController.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/ConanDeleteController.kt index a9723684ba..eb5c668d9d 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/ConanDeleteController.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/ConanDeleteController.kt @@ -27,10 +27,18 @@ package com.tencent.bkrepo.conan.controller +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord +import com.tencent.bk.audit.context.ActionAuditContext import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_DELETE_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.conan.constant.DEFAULT_REVISION_V1 import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo @@ -63,6 +71,23 @@ class ConanDeleteController( * Remove any existing recipes or its packages created. * Will remove all revisions, packages and package revisions (parent folder) */ + @AuditEntry( + actionId = NODE_DELETE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DELETE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#conanArtifactInfo?.getArtifactFullPath()", + instanceNames = "#conanArtifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#conanArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#conanArtifactInfo?.repoName") + ], + scopeId = "#conanArtifactInfo?.projectId", + content = ActionAuditContent.NODE_DELETE_CONTENT + ) @DeleteMapping(REMOVE_RECIPE_V1) @Permission(type = ResourceType.REPO, action = PermissionAction.DELETE) fun removeRecipe( @@ -75,12 +100,30 @@ class ConanDeleteController( /** * if packageIds is empty, then will remove all packages */ + @AuditEntry( + actionId = NODE_DELETE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DELETE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#conanArtifactInfo?.getArtifactFullPath()", + instanceNames = "#conanArtifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#conanArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#conanArtifactInfo?.repoName") + ], + scopeId = "#conanArtifactInfo?.projectId", + content = ActionAuditContent.NODE_DELETE_CONTENT + ) @PostMapping(REMOVE_PACKAGES_V1) @Permission(type = ResourceType.REPO, action = PermissionAction.DELETE) fun removePackages( @ArtifactPathVariable conanArtifactInfo: ConanArtifactInfo, @RequestBody removeRequest: PackageIdRemoveRequest ): ResponseEntity { + ActionAuditContext.current().setInstance(removeRequest) conanDeleteService.removePackages(conanArtifactInfo, DEFAULT_REVISION_V1, removeRequest.packageIds) return ConanCommonController.buildResponse(StringPool.EMPTY) } @@ -89,12 +132,30 @@ class ConanDeleteController( * The remove files is a part of the upload process, * where the revision in v1 will always be DEFAULT_REVISION_V1 */ + @AuditEntry( + actionId = NODE_DELETE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DELETE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#conanArtifactInfo?.getArtifactFullPath()", + instanceNames = "#conanArtifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#conanArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#conanArtifactInfo?.repoName") + ], + scopeId = "#conanArtifactInfo?.projectId", + content = ActionAuditContent.NODE_DELETE_CONTENT + ) @PostMapping(REMOVE_FILES_V1) @Permission(type = ResourceType.REPO, action = PermissionAction.DELETE) fun removeRecipeFiles( @ArtifactPathVariable conanArtifactInfo: ConanArtifactInfo, @RequestBody fileRemoveRequest: FileRemoveRequest ): ResponseEntity { + ActionAuditContext.current().setInstance(fileRemoveRequest) conanDeleteService.removeRecipeFiles(conanArtifactInfo, fileRemoveRequest.files) return ConanCommonController.buildResponse(StringPool.EMPTY) } @@ -103,6 +164,23 @@ class ConanDeleteController( * Remove any existing recipes or its packages created. * Will remove all revisions, packages and package revisions (parent folder) */ + @AuditEntry( + actionId = NODE_DELETE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DELETE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#conanArtifactInfo?.getArtifactFullPath()", + instanceNames = "#conanArtifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#conanArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#conanArtifactInfo?.repoName") + ], + scopeId = "#conanArtifactInfo?.projectId", + content = ActionAuditContent.NODE_DELETE_CONTENT + ) @DeleteMapping(REMOVE_RECIPE_V2, REMOVE_RECIPE_REVISIONS_V2) @Permission(type = ResourceType.REPO, action = PermissionAction.DELETE) fun removeRecipeV2( @@ -118,6 +196,23 @@ class ConanDeleteController( * - If PRev is NOT specified but RRev is specified (package_recipe_revision_url) * it will remove all the package revisions */ + @AuditEntry( + actionId = NODE_DELETE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DELETE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#conanArtifactInfo?.getArtifactFullPath()", + instanceNames = "#conanArtifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#conanArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#conanArtifactInfo?.repoName") + ], + scopeId = "#conanArtifactInfo?.projectId", + content = ActionAuditContent.NODE_DELETE_CONTENT + ) @DeleteMapping(REMOVE_PACKAGE_RECIPE_REVISION_V2, REMOVE_PACKAGE_REVISION_V2) @Permission(type = ResourceType.REPO, action = PermissionAction.DELETE) fun removePackagesV2( @@ -130,6 +225,23 @@ class ConanDeleteController( /** * Remove all packages from a RREV */ + @AuditEntry( + actionId = NODE_DELETE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DELETE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#conanArtifactInfo?.getArtifactFullPath()", + instanceNames = "#conanArtifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#conanArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#conanArtifactInfo?.repoName") + ], + scopeId = "#conanArtifactInfo?.projectId", + content = ActionAuditContent.NODE_DELETE_CONTENT + ) @DeleteMapping(REMOVE_ALL_PACKAGE_UNDER_REVISION_V2) @Permission(type = ResourceType.REPO, action = PermissionAction.DELETE) fun removeAllPackagesV2( diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/ConanUploadDownloadController.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/ConanUploadDownloadController.kt index f400fcc61d..9dd23e9b20 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/ConanUploadDownloadController.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/ConanUploadDownloadController.kt @@ -27,10 +27,18 @@ package com.tencent.bkrepo.conan.controller +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_CREATE_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_DOWNLOAD_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo import com.tencent.bkrepo.conan.pojo.artifact.ConanArtifactInfo.Companion.UPLOAD_FILE_V1 @@ -50,6 +58,23 @@ class ConanUploadDownloadController( /** * 获取文件 */ + @AuditEntry( + actionId = NODE_DOWNLOAD_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DOWNLOAD_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#conanArtifactInfo?.getArtifactFullPath()", + instanceNames = "#conanArtifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#conanArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#conanArtifactInfo?.repoName") + ], + scopeId = "#conanArtifactInfo?.projectId", + content = ActionAuditContent.NODE_DOWNLOAD_CONTENT + ) @GetMapping(UPLOAD_FILE_V1, UPLOAD_PACKAGE_FILE_V1) @Permission(type = ResourceType.REPO, action = PermissionAction.READ) fun getFile( @@ -61,6 +86,23 @@ class ConanUploadDownloadController( /** * 上传文件 */ + @AuditEntry( + actionId = NODE_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#conanArtifactInfo?.getArtifactFullPath()", + instanceNames = "#conanArtifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#conanArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#conanArtifactInfo?.repoName") + ], + scopeId = "#conanArtifactInfo?.projectId", + content = ActionAuditContent.NODE_UPLOAD_CONTENT + ) @PutMapping(UPLOAD_FILE_V1, UPLOAD_PACKAGE_FILE_V1) @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) fun uploadFile( diff --git a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/UserConanController.kt b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/UserConanController.kt index 2f958a6050..aac41e8dc8 100644 --- a/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/UserConanController.kt +++ b/src/backend/conan/biz-conan/src/main/kotlin/com/tencent/bkrepo/conan/controller/UserConanController.kt @@ -27,10 +27,17 @@ package com.tencent.bkrepo.conan.controller +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.REPO_EDIT_ACTION +import com.tencent.bkrepo.common.artifact.audit.REPO_RESOURCE import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.conan.pojo.ConanDomainInfo @@ -73,6 +80,23 @@ class UserConanController( return ResponseBuilder.success(conanDeleteService.detailVersion(artifactInfo, packageKey, version)) } + @AuditEntry( + actionId = REPO_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = REPO_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#artifactInfo?.repoName", + instanceNames = "#artifactInfo?.repoName" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.NAME_TEMPLATE, value = "#packageKey") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.REPO_PACKAGE_DELETE_CONTENT + ) @ApiOperation("删除仓库下的包") @DeleteMapping(CONAN_PACKAGE_DELETE_URL) @Permission(type = ResourceType.REPO, action = PermissionAction.DELETE) @@ -86,6 +110,25 @@ class UserConanController( return ResponseBuilder.success() } + @AuditEntry( + actionId = REPO_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = REPO_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#artifactInfo?.repoName", + instanceNames = "#artifactInfo?.repoName" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.NAME_TEMPLATE, value = "#packageKey"), + AuditAttribute(name = ActionAuditContent.VERSION_TEMPLATE, value = "#version") + + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.REPO_PACKAGE_VERSION_DELETE_CONTENT + ) @ApiOperation("删除仓库下的包版本") @DeleteMapping(CONAN_VERSION_DELETE_URL) @Permission(type = ResourceType.REPO, action = PermissionAction.DELETE) diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/component/PermissionHelper.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/component/PermissionHelper.kt index d8fc27d7f8..27921e9e25 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/component/PermissionHelper.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/component/PermissionHelper.kt @@ -29,7 +29,7 @@ package com.tencent.bkrepo.ddc.component import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.ddc.utils.DdcUtils.DIR_BLOBS import org.springframework.stereotype.Component diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/CompressedBlobController.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/CompressedBlobController.kt index cab45c63f9..0f32f45b70 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/CompressedBlobController.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/CompressedBlobController.kt @@ -27,9 +27,17 @@ package com.tencent.bkrepo.ddc.controller +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_CREATE_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_DOWNLOAD_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE import com.tencent.bkrepo.ddc.artifact.CompressedBlobArtifactInfo import com.tencent.bkrepo.ddc.component.PermissionHelper import com.tencent.bkrepo.ddc.service.CompressedBlobService @@ -50,6 +58,23 @@ class CompressedBlobController( private val permissionHelper: PermissionHelper, ) { + @AuditEntry( + actionId = NODE_DOWNLOAD_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DOWNLOAD_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_DOWNLOAD_CONTENT + ) @ApiOperation("获取压缩后的缓存") @GetMapping( "/{repoName}/{$PATH_VARIABLE_CONTENT_ID}", @@ -64,6 +89,23 @@ class CompressedBlobController( compressedBlobService.get(artifactInfo) } + @AuditEntry( + actionId = NODE_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_UPLOAD_CONTENT + ) @ApiOperation("上传压缩后的缓存") @PutMapping("/{repoName}/{$PATH_VARIABLE_CONTENT_ID}", consumes = [MEDIA_TYPE_UNREAL_UNREAL_COMPRESSED_BUFFER]) fun put( diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/LegacyReferencesController.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/LegacyReferencesController.kt index d3ced46ef3..1f460a4b48 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/LegacyReferencesController.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/LegacyReferencesController.kt @@ -27,9 +27,18 @@ package com.tencent.bkrepo.ddc.controller +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_DELETE_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_DOWNLOAD_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE +import com.tencent.bkrepo.common.artifact.audit.NODE_CREATE_ACTION import com.tencent.bkrepo.ddc.artifact.ReferenceArtifactInfo import com.tencent.bkrepo.ddc.artifact.ReferenceArtifactInfo.Companion.PATH_VARIABLE_BUCKET import com.tencent.bkrepo.ddc.artifact.ReferenceArtifactInfo.Companion.PATH_VARIABLE_REF_ID @@ -52,6 +61,24 @@ class LegacyReferencesController( private val referenceArtifactService: ReferenceArtifactService, private val permissionHelper: PermissionHelper, ) { + + @AuditEntry( + actionId = NODE_DOWNLOAD_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DOWNLOAD_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_DOWNLOAD_CONTENT + ) @ApiOperation("获取ref") @GetMapping( "/ddc/{repoName}/{${PATH_VARIABLE_BUCKET}}/{${PATH_VARIABLE_REF_ID}}.raw", @@ -71,6 +98,23 @@ class LegacyReferencesController( referenceArtifactService.downloadRef(artifactInfo) } + @AuditEntry( + actionId = NODE_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_UPLOAD_CONTENT + ) @PutMapping( "/ddc/{repoName}/{${PATH_VARIABLE_BUCKET}}/{${PATH_VARIABLE_REF_ID}}", consumes = [MediaType.APPLICATION_OCTET_STREAM_VALUE] @@ -85,6 +129,23 @@ class LegacyReferencesController( referenceArtifactService.createRef(artifactInfo, file) } + @AuditEntry( + actionId = NODE_DELETE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DELETE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_DELETE_CONTENT + ) @DeleteMapping("/ddc/{repoName}/{${PATH_VARIABLE_BUCKET}}/{${PATH_VARIABLE_REF_ID}}") fun delete( @ApiParam(value = "ddc ref", required = true) diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/ReferencesController.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/ReferencesController.kt index c5ae5a29d9..ce7e37cb75 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/ReferencesController.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/ReferencesController.kt @@ -27,11 +27,19 @@ package com.tencent.bkrepo.ddc.controller +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.common.api.exception.BadRequestException import com.tencent.bkrepo.common.api.message.CommonMessageCode.PARAMETER_INVALID import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_DOWNLOAD_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE +import com.tencent.bkrepo.common.artifact.audit.NODE_CREATE_ACTION import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.ddc.artifact.ReferenceArtifactInfo import com.tencent.bkrepo.ddc.artifact.repository.DdcLocalRepository.Companion.HEADER_NAME_HASH @@ -58,6 +66,23 @@ class ReferencesController( private val permissionHelper: PermissionHelper, ) { + @AuditEntry( + actionId = NODE_DOWNLOAD_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DOWNLOAD_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_DOWNLOAD_CONTENT + ) @ApiOperation("获取ref") @GetMapping( "/{repoName}/{$PATH_VARIABLE_BUCKET}/{$PATH_VARIABLE_REF_ID}", @@ -79,6 +104,23 @@ class ReferencesController( referenceArtifactService.downloadRef(artifactInfo) } + @AuditEntry( + actionId = NODE_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_UPLOAD_CONTENT + ) @ApiOperation("开始创建ref") @PutMapping( "/{repoName}/{$PATH_VARIABLE_BUCKET}/{$PATH_VARIABLE_REF_ID}", diff --git a/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/pojo/DevxLoginResponse.kt b/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/pojo/DevxLoginResponse.kt index b15c000d0e..cc796258be 100644 --- a/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/pojo/DevxLoginResponse.kt +++ b/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/pojo/DevxLoginResponse.kt @@ -29,5 +29,6 @@ package com.tencent.bkrepo.fs.server.pojo data class DevxLoginResponse( val projectId: String, - val token: String + val token: String, + val workspaceName: String ) diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/LoginHandler.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/LoginHandler.kt index 6f52a15ab7..3cd2eaab72 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/LoginHandler.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/LoginHandler.kt @@ -32,6 +32,7 @@ import com.tencent.bkrepo.auth.pojo.user.CreateUserRequest import com.tencent.bkrepo.auth.pojo.user.CreateUserToProjectRequest import com.tencent.bkrepo.common.api.constant.BASIC_AUTH_PREFIX import com.tencent.bkrepo.common.api.constant.HttpHeaders +import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.api.util.BasicAuthUtils import com.tencent.bkrepo.common.artifact.constant.PROJECT_ID import com.tencent.bkrepo.common.artifact.constant.REPO_NAME @@ -42,6 +43,7 @@ import com.tencent.bkrepo.fs.server.constant.JWT_CLAIMS_PERMIT import com.tencent.bkrepo.fs.server.constant.JWT_CLAIMS_REPOSITORY import com.tencent.bkrepo.fs.server.context.ReactiveArtifactContextHolder import com.tencent.bkrepo.fs.server.pojo.DevxLoginResponse +import com.tencent.bkrepo.fs.server.request.DevxLoginRequest import com.tencent.bkrepo.fs.server.request.IoaLoginRequest import com.tencent.bkrepo.fs.server.service.PermissionService import com.tencent.bkrepo.fs.server.utils.DevxWorkspaceUtils @@ -86,11 +88,20 @@ class LoginHandler( } suspend fun devxLogin(request: ServerRequest): ServerResponse { - val workspace = DevxWorkspaceUtils.getWorkspace().awaitSingleOrNull() ?: throw AuthenticationException() + val devxToken = request.bodyToMono(DevxLoginRequest::class.java).awaitSingleOrNull()?.token val repoName = request.pathVariable(REPO_NAME) - val userId = createUser(workspace) - val token = createToken(workspace.projectId, repoName, userId) - val response = DevxLoginResponse(workspace.projectId, token) + val response = if (devxToken.isNullOrEmpty()) { + val workspace = DevxWorkspaceUtils.getWorkspace().awaitSingleOrNull() ?: throw AuthenticationException() + val userId = createUser(workspace) + val token = createToken(workspace.projectId, repoName, userId) + DevxLoginResponse(workspace.projectId, token, StringPool.EMPTY) + } else { + val devxTokenInfo = DevxWorkspaceUtils.validateToken(devxToken).awaitSingle() + createUser(devxTokenInfo.userId) + val token = createToken(devxTokenInfo.projectId, repoName, devxTokenInfo.userId) + DevxLoginResponse(devxTokenInfo.projectId, token, devxTokenInfo.workspaceName) + } + return ReactiveResponseBuilder.success(response) } diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/request/DevxLoginRequest.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/request/DevxLoginRequest.kt new file mode 100644 index 0000000000..1069930d65 --- /dev/null +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/request/DevxLoginRequest.kt @@ -0,0 +1,32 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.fs.server.request + +data class DevxLoginRequest( + val token: String? +) diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/response/DevxTokenInfo.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/response/DevxTokenInfo.kt new file mode 100644 index 0000000000..8904396f99 --- /dev/null +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/response/DevxTokenInfo.kt @@ -0,0 +1,34 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.fs.server.response + +data class DevxTokenInfo( + val userId: String, + val projectId: String, + val workspaceName: String +) diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/storage/CoArtifactDataReceiver.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/storage/CoArtifactDataReceiver.kt index 38f8546a90..6d1b681f1b 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/storage/CoArtifactDataReceiver.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/storage/CoArtifactDataReceiver.kt @@ -30,11 +30,15 @@ package com.tencent.bkrepo.fs.server.storage import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.artifact.stream.DigestCalculateListener import com.tencent.bkrepo.common.artifact.stream.closeQuietly +import com.tencent.bkrepo.common.storage.config.MonitorProperties import com.tencent.bkrepo.common.storage.config.ReceiveProperties import com.tencent.bkrepo.common.storage.monitor.StorageHealthMonitor import com.tencent.bkrepo.common.storage.monitor.Throughput +import com.tencent.bkrepo.common.storage.util.createFile +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.reactive.awaitSingle import kotlinx.coroutines.reactor.awaitSingleOrNull +import kotlinx.coroutines.withContext import org.slf4j.LoggerFactory import org.springframework.core.io.buffer.DataBuffer import org.springframework.core.io.buffer.DataBufferUtils @@ -45,6 +49,7 @@ import reactor.core.publisher.Sinks import reactor.core.scheduler.Schedulers import java.io.ByteArrayInputStream import java.io.File +import java.io.IOException import java.io.InputStream import java.nio.channels.AsynchronousFileChannel import java.nio.file.Files @@ -56,10 +61,22 @@ import java.nio.file.StandardOpenOption * */ class CoArtifactDataReceiver( receiveProperties: ReceiveProperties, + monitorProperties: MonitorProperties, private var path: Path, private val fileName: String = generateRandomName(), ) : StorageHealthMonitor.Observer { + /** + * 传输过程中发生存储降级时，是否将数据转移到本地磁盘 + */ + private val enableTransfer = monitorProperties.enableTransfer + + /** + * 数据传输buffer大小 + */ + private val bufferSize = receiveProperties.bufferSize.toBytes().toInt() + + /** * 文件内存接收阈值 * */ @@ -93,13 +110,7 @@ class CoArtifactDataReceiver( /** * 文件异步接收通道 * */ - private val channel: AsynchronousFileChannel by lazy { - AsynchronousFileChannel.open( - filePath, - StandardOpenOption.WRITE, - StandardOpenOption.CREATE_NEW, - ) - } + private var channel: AsynchronousFileChannel? = null /** * 是否降级 @@ -111,6 +122,11 @@ class CoArtifactDataReceiver( * */ var finished: Boolean = false + /** + * 数据是否转移到本地磁盘 + */ + private var hasTransferred: Boolean = false + /** * 降级路径 * */ @@ -157,12 +173,14 @@ class CoArtifactDataReceiver( checkFallback() val len = buffer.readableByteCount() if (pos + len > fileSizeThreshold && inMemory) { + initChannel() flushToFile() - DataBufferUtils.write(Mono.just(buffer), channel, pos).awaitSingle() + DataBufferUtils.write(Mono.just(buffer), channel!!, pos).awaitSingle() } else if (inMemory) { buffer.read(cacheData!!, pos.toInt(), len) } else { - DataBufferUtils.write(Mono.just(buffer), channel, pos).awaitSingle() + initChannel() + DataBufferUtils.write(Mono.just(buffer), channel!!, pos).awaitSingle() } buffer.readPosition(0) digest(buffer) @@ -173,13 +191,30 @@ class CoArtifactDataReceiver( if (inMemory) { val cacheData = cacheData!!.copyOfRange(0, pos.toInt()) val buf = DefaultDataBufferFactory.sharedInstance.wrap(cacheData) - DataBufferUtils.write(Mono.just(buf), channel).awaitSingle() + val filePath = this.filePath.apply { this.createFile() } + channel = withContext(Dispatchers.IO) { + AsynchronousFileChannel.open(filePath, StandardOpenOption.WRITE, StandardOpenOption.CREATE) + } + DataBufferUtils.write(Mono.just(buf), channel!!).awaitSingle() inMemory = false // help gc this.cacheData = null } } + private suspend fun initChannel() { + if (channel == null) { + channel = withContext(Dispatchers.IO) { + Files.createDirectories(filePath.parent) + AsynchronousFileChannel.open( + filePath, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE_NEW, + ) + } + } + } + private fun digest(buffer: DataBuffer) { val len = buffer.readableByteCount() val digestArray = ByteArray(len) @@ -188,23 +223,71 @@ class CoArtifactDataReceiver( } override fun unhealthy(fallbackPath: Path?, reason: String?) { - if (!finished && !fallback) { + if (!finished && !fallback && !hasTransferred) { fallBackPath = fallbackPath fallback = true logger.warn("Path[$path] is unhealthy, fallback to use [$fallBackPath], reason: $reason") } } - private fun checkFallback() { - if (!fallback) { + /** + * 检查是否需要fall back操作 + */ + private suspend fun checkFallback() { + if (!fallback || hasTransferred) { return } if (fallBackPath == null || fallBackPath == path) { - logger.info("Fallback path is null or equals to primary path,skip") + logger.info("Fallback path is null or equals to primary path, skip transfer data") + hasTransferred = true return } - if (inMemory) { - path = fallBackPath!! + // originalPath表示NFS位置， fallBackPath表示本地磁盘位置 + val originalPath = path + // 更新当前path为本地磁盘 + path = fallBackPath!! + // transfer date + if (!inMemory) { + // 当文件已经落到NFS + if (enableTransfer) { + // 开Transfer功能时，从NFS转移到本地盘 + cleanOriginalChannel() + val originalFile = originalPath.resolve(fileName) + val filePath = this.filePath.apply { this.createFile() } + val dataBuffer = DataBufferUtils.read(originalPath, DefaultDataBufferFactory(), bufferSize) + channel = withContext(Dispatchers.IO) { + AsynchronousFileChannel.open( + filePath, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE_NEW, + ) + } + DataBufferUtils.write(dataBuffer, channel!!, 0).awaitSingle() + withContext(Dispatchers.IO) { + Files.deleteIfExists(originalFile) + } + logger.info("Success to transfer data from [$originalPath] to [$path]") + } else { + // 禁用Transfer功能时，忽略操作，继续使用NFS + path = originalPath + fallback = false + } + } + hasTransferred = true + } + + /** + * 关闭原始输出流 + */ + private fun cleanOriginalChannel() { + try { + channel?.force(true) + } catch (ignored: IOException) { + } + + try { + channel?.close() + } catch (ignored: IOException) { } } @@ -217,14 +300,14 @@ class CoArtifactDataReceiver( return Throughput(pos, endTime - startTime) } finally { if (!inMemory) { - channel.closeQuietly() + channel?.closeQuietly() } } } fun close() { if (!inMemory) { - channel.closeQuietly() + channel?.closeQuietly() Files.deleteIfExists(filePath) logger.info("Delete path $filePath") } diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/storage/CoArtifactFile.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/storage/CoArtifactFile.kt index 9c3c346cc2..48d64980e9 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/storage/CoArtifactFile.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/storage/CoArtifactFile.kt @@ -66,6 +66,7 @@ class CoArtifactFile( val path = storageCredentials.upload.location.toPath() receiver = CoArtifactDataReceiver( storageProperties.receive, + storageProperties.monitor, path ) monitor.add(receiver) @@ -108,7 +109,8 @@ class CoArtifactFile( } override fun isFallback(): Boolean { - return false + runBlocking { finish() } + return receiver.fallback } override fun isInLocalDisk(): Boolean { diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/utils/DevxWorkspaceUtils.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/utils/DevxWorkspaceUtils.kt index 8889e1b739..f8ea63ae4b 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/utils/DevxWorkspaceUtils.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/utils/DevxWorkspaceUtils.kt @@ -30,6 +30,8 @@ package com.tencent.bkrepo.fs.server.utils import com.github.benmanes.caffeine.cache.AsyncLoadingCache import com.github.benmanes.caffeine.cache.Caffeine import com.google.common.util.concurrent.ThreadFactoryBuilder +import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.api.util.toJsonString import com.tencent.bkrepo.common.security.interceptor.devx.ApiAuth import com.tencent.bkrepo.common.security.interceptor.devx.DevXCvmWorkspace @@ -38,8 +40,12 @@ import com.tencent.bkrepo.common.security.interceptor.devx.DevXWorkSpace import com.tencent.bkrepo.common.security.interceptor.devx.PageResponse import com.tencent.bkrepo.common.security.interceptor.devx.QueryResponse import com.tencent.bkrepo.fs.server.context.ReactiveRequestContextHolder +import com.tencent.bkrepo.fs.server.response.DevxTokenInfo +import com.tencent.devops.api.pojo.Response +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.reactor.awaitSingle import kotlinx.coroutines.reactor.mono +import kotlinx.coroutines.withContext import org.slf4j.LoggerFactory import org.springframework.core.ParameterizedTypeReference import org.springframework.http.HttpStatus @@ -53,6 +59,7 @@ import reactor.netty.http.client.HttpClient import reactor.netty.http.client.PrematureCloseException import reactor.netty.resources.ConnectionProvider import reactor.util.retry.RetryBackoffSpec +import java.net.URLDecoder import java.time.Duration import java.util.concurrent.Executors @@ -170,6 +177,29 @@ class DevxWorkspaceUtils( } } + suspend fun validateToken(devxToken: String): Mono { + val token = withContext(Dispatchers.IO) { + URLDecoder.decode(devxToken, Charsets.UTF_8.name()) + } + return httpClient + .get() + .uri("${devXProperties.validateTokenUrl}?dToken=$token") + .header("X-DEVOPS-BK-TOKEN", devXProperties.authToken) + .exchangeToMono { + mono { parseDevxTokenInfo(it) } + } + } + + private suspend fun parseDevxTokenInfo(response: ClientResponse): DevxTokenInfo { + return if (response.statusCode() != HttpStatus.OK) { + val errorMsg = response.awaitBody() + logger.error("${response.statusCode()} $errorMsg") + throw ErrorCodeException(CommonMessageCode.RESOURCE_EXPIRED, "token") + } else { + response.awaitBody>().data!! + } + } + private fun WebClient.RequestHeadersSpec<*>.doRequest( type: ParameterizedTypeReference>, handler: (res: QueryResponse?) -> R diff --git a/src/backend/fs/boot-fs-server/src/test/kotlin/com/tencent/com/bkrepo/fs/storage/CoArtifactDataReceiverTest.kt b/src/backend/fs/boot-fs-server/src/test/kotlin/com/tencent/com/bkrepo/fs/storage/CoArtifactDataReceiverTest.kt index 5087db8537..38bc0a8683 100644 --- a/src/backend/fs/boot-fs-server/src/test/kotlin/com/tencent/com/bkrepo/fs/storage/CoArtifactDataReceiverTest.kt +++ b/src/backend/fs/boot-fs-server/src/test/kotlin/com/tencent/com/bkrepo/fs/storage/CoArtifactDataReceiverTest.kt @@ -28,6 +28,7 @@ package com.tencent.com.bkrepo.fs.storage import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.storage.config.MonitorProperties import com.tencent.bkrepo.common.storage.config.ReceiveProperties import com.tencent.bkrepo.common.storage.util.toPath import com.tencent.bkrepo.fs.server.storage.CoArtifactDataReceiver @@ -140,6 +141,7 @@ class CoArtifactDataReceiverTest { fileSizeThreshold = DataSize.ofBytes(fileSizeThreshold), rateLimit = DataSize.ofBytes(-1) ) - return CoArtifactDataReceiver(receive, primaryPath, filename) + val monitorProperties = MonitorProperties(enabled = true, enableTransfer = true) + return CoArtifactDataReceiver(receive, monitorProperties, primaryPath, filename) } } diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt index f0a7d3c4f0..5c49f8be96 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt @@ -29,6 +29,7 @@ package com.tencent.bkrepo.generic.artifact import com.google.common.cache.CacheBuilder import com.google.common.cache.CacheLoader +import com.tencent.bk.audit.context.ActionAuditContext import com.tencent.bkrepo.auth.constant.CUSTOM import com.tencent.bkrepo.auth.constant.PIPELINE import com.tencent.bkrepo.common.api.constant.CharPool @@ -65,6 +66,7 @@ import com.tencent.bkrepo.common.artifact.stream.EmptyInputStream import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.artifact.util.chunked.ChunkedUploadUtils import com.tencent.bkrepo.common.artifact.util.http.HttpRangeUtils +import com.tencent.bkrepo.common.metadata.service.node.PipelineNodeService import com.tencent.bkrepo.common.query.model.Rule import com.tencent.bkrepo.common.security.manager.ci.CIPermissionManager import com.tencent.bkrepo.common.security.manager.ci.CIPermissionManager.Companion.METADATA_BUILD_ID @@ -110,7 +112,6 @@ import com.tencent.bkrepo.replication.pojo.task.objects.ReplicaObjectInfo import com.tencent.bkrepo.replication.pojo.task.request.ReplicaTaskCreateRequest import com.tencent.bkrepo.replication.pojo.task.setting.ConflictStrategy import com.tencent.bkrepo.replication.pojo.task.setting.ReplicaSetting -import com.tencent.bkrepo.repository.api.PipelineNodeClient import com.tencent.bkrepo.repository.constant.NODE_DETAIL_LIST_KEY import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.bkrepo.repository.pojo.node.NodeDetail @@ -135,7 +136,7 @@ import kotlin.reflect.full.memberProperties class GenericLocalRepository( private val replicaTaskClient: ReplicaTaskClient, private val clusterNodeClient: ClusterNodeClient, - private val pipelineNodeClient: PipelineNodeClient, + private val pipelineNodeService: PipelineNodeService, private val ciPermissionManager: CIPermissionManager ) : LocalRepository() { @@ -632,7 +633,7 @@ class GenericLocalRepository( return if (isSearchPipelineRoot) { // 仅在查询流水线仓库第一页时返回用户有权限的流水线目录 if (queryModel.page.pageNumber == DEFAULT_PAGE_NUMBER) { - pipelineNodeClient.listPipeline(context.projectId, context.repoName).data!!.map { node -> + pipelineNodeService.listPipeline(context.projectId, context.repoName).map { node -> val nodePropMap = LinkedHashMap() NodeInfo::class.memberProperties .filter { it.name != NodeInfo::deleted.name } @@ -734,7 +735,7 @@ class GenericLocalRepository( val headerNames = request.headerNames for (headerName in headerNames) { if (headerName.startsWith(BKREPO_META_PREFIX, true)) { - val key = headerName.substring(BKREPO_META_PREFIX.length).trim().toLowerCase() + val key = headerName.substring(BKREPO_META_PREFIX.length).trim().lowercase(Locale.getDefault()) if (key.isNotBlank()) { metadata[key] = HeaderUtils.getUrlDecodedHeader(headerName)!! } @@ -874,6 +875,7 @@ class GenericLocalRepository( md5 = fileInfo.md5, size = fileInfo.size ) + ActionAuditContext.current().setInstance(nodeRequest) nodeService.createNode(nodeRequest) return property } diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/GenericController.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/GenericController.kt index e24bc6f9f4..fa9a336e7e 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/GenericController.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/GenericController.kt @@ -31,6 +31,10 @@ package com.tencent.bkrepo.generic.controller +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.api.exception.ErrorCodeException @@ -45,9 +49,14 @@ import com.tencent.bkrepo.common.artifact.constant.ARTIFACT_INFO_KEY import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.router.Router import com.tencent.bkrepo.common.artifact.util.PipelineRepoUtils +import com.tencent.bkrepo.common.metadata.permission.PermissionManager +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_DELETE_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_DOWNLOAD_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE +import com.tencent.bkrepo.common.artifact.audit.NODE_CREATE_ACTION import com.tencent.bkrepo.common.mongo.dao.util.Pages import com.tencent.bkrepo.common.query.model.QueryModel -import com.tencent.bkrepo.common.security.manager.PermissionManager import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.common.service.util.ResponseBuilder @@ -85,12 +94,46 @@ class GenericController( private val compressedFileService: CompressedFileService, ) { + @AuditEntry( + actionId = NODE_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_UPLOAD_CONTENT + ) @PutMapping(GENERIC_MAPPING_URI) @Permission(ResourceType.NODE, PermissionAction.WRITE) fun upload(@ArtifactPathVariable artifactInfo: GenericArtifactInfo, file: ArtifactFile) { uploadService.upload(artifactInfo, file) } + @AuditEntry( + actionId = NODE_DELETE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DELETE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_DELETE_CONTENT + ) @Permission(ResourceType.NODE, PermissionAction.DELETE) @DeleteMapping(GENERIC_MAPPING_URI) fun delete( @@ -101,6 +144,23 @@ class GenericController( return ResponseBuilder.success() } + @AuditEntry( + actionId = NODE_DOWNLOAD_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DOWNLOAD_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_DOWNLOAD_CONTENT + ) @Permission(ResourceType.NODE, PermissionAction.DOWNLOAD) @GetMapping(GENERIC_MAPPING_URI) @Router @@ -128,6 +188,23 @@ class GenericController( return ResponseBuilder.success() } + @AuditEntry( + actionId = NODE_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_UPLOAD_CONTENT + ) @Permission(ResourceType.NODE, PermissionAction.WRITE) @PutMapping(BLOCK_MAPPING_URI) fun completeBlockUpload( @@ -149,6 +226,23 @@ class GenericController( return ResponseBuilder.success(uploadService.listBlock(userId, uploadId, artifactInfo)) } + @AuditEntry( + actionId = NODE_DOWNLOAD_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DOWNLOAD_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#batchDownloadPaths?.paths", + instanceNames = "#batchDownloadPaths?.paths" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#repoName") + ], + scopeId = "#projectId", + content = ActionAuditContent.NODE_DOWNLOAD_CONTENT + ) @RequestMapping(BATCH_MAPPING_URI, method = [RequestMethod.GET, RequestMethod.POST]) fun batchDownload( @PathVariable projectId: String, diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/TemporaryAccessController.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/TemporaryAccessController.kt index f74e13c4b8..661a010118 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/TemporaryAccessController.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/controller/TemporaryAccessController.kt @@ -27,6 +27,11 @@ package com.tencent.bkrepo.generic.controller +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord +import com.tencent.bk.audit.context.ActionAuditContext import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.token.TemporaryTokenCreateRequest import com.tencent.bkrepo.auth.pojo.token.TokenType @@ -38,7 +43,11 @@ import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable import com.tencent.bkrepo.common.artifact.metrics.ChunkArtifactTransferMetrics import com.tencent.bkrepo.common.artifact.router.Router -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.permission.PermissionManager +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_DOWNLOAD_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE +import com.tencent.bkrepo.common.artifact.audit.NODE_CREATE_ACTION import com.tencent.bkrepo.common.security.permission.Principal import com.tencent.bkrepo.common.security.permission.PrincipalType import com.tencent.bkrepo.common.service.util.HttpContextHolder @@ -104,6 +113,33 @@ class TemporaryAccessController( } } + @AuditEntry( + actionId = NODE_DOWNLOAD_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DOWNLOAD_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()", + ), + attributes = [ + AuditAttribute( + name = ActionAuditContent.PROJECT_CODE_TEMPLATE, + value = "#artifactInfo?.projectId" + ), + AuditAttribute( + name = ActionAuditContent.REPO_NAME_TEMPLATE, + value = "#artifactInfo?.repoName" + ), + AuditAttribute( + name = ActionAuditContent.TOKEN_TEMPLATE, + value = "#token" + ) + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_SHARE_DOWNLOAD_CONTENT + ) @ApiOperation("下载分享文件") @Router @CrossOrigin @@ -115,11 +151,31 @@ class TemporaryAccessController( artifactInfo: GenericArtifactInfo ) { val downloadUser = downloadUserId ?: userId + ActionAuditContext.current().addExtendData("downloadUser", downloadUser) val tokenInfo = temporaryAccessService.validateToken(token, artifactInfo, TokenType.DOWNLOAD) temporaryAccessService.downloadByShare(downloadUser, tokenInfo.createdBy, artifactInfo) temporaryAccessService.decrementPermits(tokenInfo) } + + @AuditEntry( + actionId = NODE_DOWNLOAD_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DOWNLOAD_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName"), + AuditAttribute(name = ActionAuditContent.TOKEN_TEMPLATE, value = "#token") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_DOWNLOAD_WITH_TOKEN_CONTENT + ) @Router @CrossOrigin @GetMapping("/download/$GENERIC_MAPPING_URI") @@ -132,6 +188,24 @@ class TemporaryAccessController( temporaryAccessService.decrementPermits(tokenInfo) } + @AuditEntry( + actionId = NODE_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName"), + AuditAttribute(name = ActionAuditContent.TOKEN_TEMPLATE, value = "#token") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_UPLOAD_WITH_TOKEN_CONTENT + ) @CrossOrigin @PutMapping("/upload/$GENERIC_MAPPING_URI") fun uploadByToken( diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/ProxyService.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/ProxyService.kt index e04c804811..8efa909f40 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/ProxyService.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/ProxyService.kt @@ -45,7 +45,7 @@ import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.artifact.stream.bound import com.tencent.bkrepo.common.artifact.util.http.HttpRangeUtils -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.security.util.AESUtils import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/TemporaryAccessService.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/TemporaryAccessService.kt index 3ec533da62..f9fea00d69 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/TemporaryAccessService.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/TemporaryAccessService.kt @@ -61,7 +61,7 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHold import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadContext import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HeaderUtils import com.tencent.bkrepo.common.service.util.HttpContextHolder @@ -136,6 +136,7 @@ class TemporaryAccessService( val repo = repositoryService.getRepoDetail(projectId, repoName) ?: throw ErrorCodeException(ArtifactMessageCode.REPOSITORY_NOT_FOUND, repoName) val context = ArtifactDownloadContext(repo = repo, userId = downloadUser) + HttpContextHolder.getRequest().setAttribute(USER_KEY, downloadUser) context.shareUserId = shareBy val repository = ArtifactContextHolder.getRepository(context.repositoryDetail.category) repository.download(context) diff --git a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt index 801869ba46..198562119e 100644 --- a/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt +++ b/src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/UploadService.kt @@ -31,6 +31,7 @@ package com.tencent.bkrepo.generic.service +import com.tencent.bk.audit.context.ActionAuditContext import com.tencent.bkrepo.common.api.exception.BadRequestException import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.message.CommonMessageCode @@ -148,21 +149,21 @@ class UploadService( } // 保存节点 val repository = ArtifactContextHolder.getRepository(RepositoryCategory.LOCAL) as GenericLocalRepository - nodeService.createNode( - NodeCreateRequest( - projectId = artifactInfo.projectId, - repoName = artifactInfo.repoName, - folder = false, - fullPath = artifactInfo.getArtifactFullPath(), - sha256 = mergedFileInfo.sha256, - md5 = mergedFileInfo.md5, - size = mergedFileInfo.size, - overwrite = getBooleanHeader(HEADER_OVERWRITE), - operator = userId, - expires = getLongHeader(HEADER_EXPIRES), - nodeMetadata = repository.resolveMetadata(HttpContextHolder.getRequest()) - ) + val request = NodeCreateRequest( + projectId = artifactInfo.projectId, + repoName = artifactInfo.repoName, + folder = false, + fullPath = artifactInfo.getArtifactFullPath(), + sha256 = mergedFileInfo.sha256, + md5 = mergedFileInfo.md5, + size = mergedFileInfo.size, + overwrite = getBooleanHeader(HEADER_OVERWRITE), + operator = userId, + expires = getLongHeader(HEADER_EXPIRES), + nodeMetadata = repository.resolveMetadata(HttpContextHolder.getRequest()) ) + ActionAuditContext.current().setInstance(request) + nodeService.createNode(request) logger.info("User[${SecurityUtils.getPrincipal()}] complete upload [$artifactInfo] success.") } diff --git a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/controller/api/ChartManipulationController.kt b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/controller/api/ChartManipulationController.kt index aa7a567113..a1ebe42b3b 100644 --- a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/controller/api/ChartManipulationController.kt +++ b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/controller/api/ChartManipulationController.kt @@ -31,8 +31,16 @@ package com.tencent.bkrepo.helm.controller.api +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.common.artifact.api.ArtifactFileMap import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_DELETE_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE +import com.tencent.bkrepo.common.artifact.audit.NODE_CREATE_ACTION import com.tencent.bkrepo.helm.pojo.artifact.HelmArtifactInfo import com.tencent.bkrepo.helm.pojo.artifact.HelmArtifactInfo.Companion.CHART_DELETE_VERSION_URL import com.tencent.bkrepo.helm.pojo.artifact.HelmArtifactInfo.Companion.HELM_PUSH_PLUGIN_URL @@ -56,6 +64,23 @@ class ChartManipulationController( /** * helm push */ + @AuditEntry( + actionId = NODE_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_UPLOAD_CONTENT + ) @PostMapping(HELM_PUSH_URL, HELM_PUSH_PLUGIN_URL) @ResponseStatus(HttpStatus.CREATED) fun upload( @@ -69,6 +94,23 @@ class ChartManipulationController( /** * helm push prov */ + @AuditEntry( + actionId = NODE_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_UPLOAD_CONTENT + ) @PostMapping(HELM_PUSH_PROV_URL) @ResponseStatus(HttpStatus.CREATED) fun uploadProv( @@ -82,6 +124,23 @@ class ChartManipulationController( /** * delete chart version */ + @AuditEntry( + actionId = NODE_DELETE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DELETE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_DELETE_CONTENT + ) @DeleteMapping(CHART_DELETE_VERSION_URL) fun deleteVersion( @RequestAttribute userId: String, diff --git a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/controller/api/ChartRepositoryController.kt b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/controller/api/ChartRepositoryController.kt index 8f6c28a118..fed37cabb0 100644 --- a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/controller/api/ChartRepositoryController.kt +++ b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/controller/api/ChartRepositoryController.kt @@ -31,8 +31,15 @@ package com.tencent.bkrepo.helm.controller.api +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_DOWNLOAD_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.helm.pojo.artifact.HelmArtifactInfo import com.tencent.bkrepo.helm.pojo.artifact.HelmArtifactInfo.Companion.HELM_INDEX_YAML_URL @@ -72,6 +79,23 @@ class ChartRepositoryController( /** * retrieved when you run helm install chartmuseum/mychart */ + @AuditEntry( + actionId = NODE_DOWNLOAD_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DOWNLOAD_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_DOWNLOAD_CONTENT + ) @GetMapping(HELM_INSTALL_URL) fun installTgz(@ArtifactPathVariable artifactInfo: HelmArtifactInfo) { chartRepositoryService.installTgz(artifactInfo) @@ -80,6 +104,23 @@ class ChartRepositoryController( /** * retrieved when you run helm install with the --verify flag */ + @AuditEntry( + actionId = NODE_DOWNLOAD_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DOWNLOAD_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_DOWNLOAD_CONTENT + ) @GetMapping(HELM_PROV_INSTALL_URL) fun installProv(@ArtifactPathVariable artifactInfo: HelmArtifactInfo) { chartRepositoryService.installProv(artifactInfo) diff --git a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/controller/api/UserHelmController.kt b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/controller/api/UserHelmController.kt index c0f1ab60b4..18f209a39c 100644 --- a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/controller/api/UserHelmController.kt +++ b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/controller/api/UserHelmController.kt @@ -31,8 +31,15 @@ package com.tencent.bkrepo.helm.controller.api +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.REPO_EDIT_ACTION +import com.tencent.bkrepo.common.artifact.audit.REPO_RESOURCE import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.helm.pojo.HelmDomainInfo import com.tencent.bkrepo.helm.pojo.artifact.HelmArtifactInfo @@ -76,6 +83,23 @@ class UserHelmController( return ResponseBuilder.success(chartRepositoryService.detailVersion(userId, artifactInfo, packageKey, version)) } + @AuditEntry( + actionId = REPO_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = REPO_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#artifactInfo?.repoName", + instanceNames = "#artifactInfo?.repoName" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.NAME_TEMPLATE, value = "#packageKey") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.REPO_PACKAGE_DELETE_CONTENT + ) @ApiOperation("删除仓库下的包") @DeleteMapping(CHART_PACKAGE_DELETE_URL) fun deletePackage( @@ -88,6 +112,25 @@ class UserHelmController( return ResponseBuilder.success() } + @AuditEntry( + actionId = REPO_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = REPO_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#artifactInfo?.repoName", + instanceNames = "#artifactInfo?.repoName" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.NAME_TEMPLATE, value = "#packageKey"), + AuditAttribute(name = ActionAuditContent.VERSION_TEMPLATE, value = "#version") + + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.REPO_PACKAGE_VERSION_DELETE_CONTENT + ) @ApiOperation("删除仓库下的包版本") @DeleteMapping(CHART_VERSION_DELETE_URL) fun deleteVersion( diff --git a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/ChartRepositoryServiceImpl.kt b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/ChartRepositoryServiceImpl.kt index 2dfd4427df..eeca8f4174 100644 --- a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/ChartRepositoryServiceImpl.kt +++ b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/ChartRepositoryServiceImpl.kt @@ -44,6 +44,7 @@ import com.tencent.bkrepo.common.query.model.PageLimit import com.tencent.bkrepo.common.query.model.QueryModel import com.tencent.bkrepo.common.query.model.Rule import com.tencent.bkrepo.common.query.model.Sort +import com.tencent.bkrepo.common.api.exception.OverloadException import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HttpContextHolder @@ -226,6 +227,8 @@ class ChartRepositoryServiceImpl( context.putAttribute(FULL_PATH, HelmUtils.getIndexCacheYamlFullPath()) try { ArtifactContextHolder.getRepository().download(context) + } catch (e: OverloadException) { + throw e } catch (e: Exception) { logger.warn("Error occurred while downloading index.yaml, error: ${e.message}") throw HelmFileNotFoundException( @@ -333,6 +336,8 @@ class ChartRepositoryServiceImpl( context.putAttribute(FILE_TYPE, CHART) try { ArtifactContextHolder.getRepository().download(context) + } catch (e: OverloadException) { + throw e } catch (e: ArtifactDownloadForbiddenException) { throw e } catch (e: Exception) { @@ -350,6 +355,8 @@ class ChartRepositoryServiceImpl( context.putAttribute(FILE_TYPE, PROV) try { ArtifactContextHolder.getRepository().download(context) + } catch (e: OverloadException) { + throw e } catch (e: Exception) { logger.warn("Error occurred while installing prov, error: ${e.message}") throw HelmFileNotFoundException( diff --git a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/FixToolServiceImpl.kt b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/FixToolServiceImpl.kt index 486c9d458d..25cae16791 100644 --- a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/FixToolServiceImpl.kt +++ b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/FixToolServiceImpl.kt @@ -369,9 +369,9 @@ class FixToolServiceImpl : FixToolService, AbstractChartService() { private fun resolveNode(record: Map): NodeInfo { return NodeInfo( createdBy = record["createdBy"] as String, - createdDate = record["createdDate"] as String, + createdDate = (record["createdDate"] as LocalDateTime).toString(), lastModifiedBy = record["lastModifiedBy"] as String, - lastModifiedDate = record["lastModifiedDate"] as String, + lastModifiedDate = (record["lastModifiedDate"] as LocalDateTime).toString(), folder = record["folder"] as Boolean, path = record["path"] as String, name = record["name"] as String, diff --git a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/HelmOperationService.kt b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/HelmOperationService.kt index 6a8b373bb8..4c8e96d666 100644 --- a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/HelmOperationService.kt +++ b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/service/impl/HelmOperationService.kt @@ -37,7 +37,7 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHold import com.tencent.bkrepo.common.artifact.repository.context.ArtifactRemoveContext import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel import com.tencent.bkrepo.common.artifact.util.PackageKeys -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.helm.config.HelmProperties import com.tencent.bkrepo.helm.exception.HelmFileNotFoundException diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ArtifactAccessLogEmbeddingJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ArtifactAccessLogEmbeddingJob.kt index cb4987ac31..9b9f5bc106 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ArtifactAccessLogEmbeddingJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/preload/ArtifactAccessLogEmbeddingJob.kt @@ -77,12 +77,14 @@ class ArtifactAccessLogEmbeddingJob( override fun getLockAtMostFor(): Duration = Duration.ofDays(7L) override fun doStart0(jobContext: JobContext) { + val deprecatedVectorStore = createVectorStore(2L) val lastMonthVectorStore = createVectorStore(1L) val curMonthVectorStore = createVectorStore(0L) + val deprecatedCollectionExists = deprecatedVectorStore.collectionExists() var lastMonthCollectionExists = lastMonthVectorStore.collectionExists() val curMonthCollectionExists = curMonthVectorStore.collectionExists() - if (lastMonthCollectionExists && !curMonthCollectionExists) { + if (lastMonthCollectionExists && !curMonthCollectionExists && !deprecatedCollectionExists) { // 可能由于数据生成过程被中断导致存在上月数据不存在当月数据，需要删除重新生成 lastMonthVectorStore.dropCollection() lastMonthCollectionExists = false @@ -112,8 +114,7 @@ class ArtifactAccessLogEmbeddingJob( logger.info("insert data into collection[${curMonthVectorStore.collectionName()}] success") // 删除过期数据 - val deprecatedVectorStore = createVectorStore(2L) - if (deprecatedVectorStore.collectionExists()) { + if (deprecatedCollectionExists) { logger.info("deprecated collection[${deprecatedVectorStore.collectionName()}] exists") deprecatedVectorStore.dropCollection() logger.info("drop collection [${deprecatedVectorStore.collectionName()}] success") diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/ArtifactCleanupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/ArtifactCleanupJob.kt index c40c9b7abc..0fbcf547d4 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/ArtifactCleanupJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/ArtifactCleanupJob.kt @@ -33,7 +33,10 @@ import com.tencent.bkrepo.common.api.util.readJsonString import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.pojo.configuration.RepositoryConfiguration +import com.tencent.bkrepo.common.metadata.model.TNode import com.tencent.bkrepo.common.metadata.service.node.NodeService +import com.tencent.bkrepo.common.metadata.service.repo.QuotaService +import com.tencent.bkrepo.common.metadata.util.NodeQueryHelper import com.tencent.bkrepo.common.security.constant.MS_AUTH_HEADER_SECURITY_TOKEN import com.tencent.bkrepo.common.security.service.ServiceAuthManager import com.tencent.bkrepo.common.service.log.LoggerHolder @@ -45,12 +48,14 @@ import com.tencent.bkrepo.job.batch.base.JobContext import com.tencent.bkrepo.job.config.properties.ArtifactCleanupJobProperties import com.tencent.bkrepo.job.exception.JobExecuteException import com.tencent.bkrepo.repository.constant.SYSTEM_USER +import com.tencent.bkrepo.repository.pojo.node.NodeListOption import org.springframework.beans.factory.annotation.Value import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.cloud.client.discovery.DiscoveryClient import org.springframework.data.mongodb.core.find import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.and import org.springframework.data.mongodb.core.query.isEqualTo import org.springframework.http.HttpEntity import org.springframework.http.HttpHeaders @@ -61,6 +66,7 @@ import org.springframework.web.client.RestTemplate import java.time.Duration import java.time.LocalDateTime import java.time.format.DateTimeFormatter +import java.util.Locale import kotlin.reflect.KClass /** @@ -72,7 +78,8 @@ class ArtifactCleanupJob( private val properties: ArtifactCleanupJobProperties, private val nodeService: NodeService, private val discoveryClient: DiscoveryClient, - private val serviceAuthManager: ServiceAuthManager + private val serviceAuthManager: ServiceAuthManager, + private val quotaService: QuotaService, ) : DefaultContextMongoDbJob(properties) { private val restTemplate = RestTemplate() @@ -104,17 +111,17 @@ class ArtifactCleanupJob( if (filterConfig(row.projectId, row.name, cleanupStrategy)) return logger.info( "Will clean the artifacts in repo ${row.projectId}|${row.name} " + - "with cleanup strategy $cleanupStrategy" + "with cleanup strategy $cleanupStrategy" ) when (row.type) { RepositoryType.GENERIC.name -> { // 清理generic制品 - deleteNodes(row.projectId, row.name, cleanupStrategy) + deleteNodes(row, cleanupStrategy) } RepositoryType.DOCKER.name, RepositoryType.OCI.name, RepositoryType.HELM.name - -> { + -> { // 清理镜像制品 deletePackages( projectId = row.projectId, @@ -128,7 +135,7 @@ class ArtifactCleanupJob( } catch (e: Exception) { throw JobExecuteException( "Failed to send cleanup repository for " + - "repo ${row.projectId}|${row.name}, error: ${e.message}", e + "repo ${row.projectId}|${row.name}, error: ${e.message}", e ) } } @@ -159,7 +166,7 @@ class ArtifactCleanupJob( } - private fun deleteNodes(projectId: String, repoName: String, cleanupStrategy: CleanupStrategy) { + private fun deleteNodes(row: RepoData, cleanupStrategy: CleanupStrategy) { val cleanupDate = when (cleanupStrategy.cleanupType) { // 只保留距离当前时间天数以内的制品 CleanupStrategyEnum.RETENTION_DAYS.value -> { @@ -171,13 +178,10 @@ class ArtifactCleanupJob( } else -> return } - doNodeCleanup(projectId, repoName, cleanupDate, cleanupStrategy.cleanTargets) + doNodeCleanup(row, cleanupDate, cleanupStrategy.cleanTargets) } - private fun doNodeCleanup( - projectId: String, repoName: String, cleanupDate: LocalDateTime, - cleanupFolders: List? = null - ) { + private fun doNodeCleanup(row: RepoData, cleanupDate: LocalDateTime, cleanupFolders: List? = null) { val folders = if (cleanupFolders.isNullOrEmpty()) { listOf(PathUtils.ROOT) } else { @@ -185,23 +189,44 @@ class ArtifactCleanupJob( } folders.forEach { try { - nodeService.deleteBeforeDate( - projectId = projectId, - repoName = repoName, - path = PathUtils.toPath(it), + val path = PathUtils.toPath(it) + val result = nodeService.deleteBeforeDate( + projectId = row.projectId, + repoName = row.name, + path = path, date = cleanupDate, - operator = SYSTEM_USER + operator = SYSTEM_USER, + decreaseVolume = false ) + decreaseVolume(row, cleanupDate, path, result.deletedTime) } catch (e: Exception) { logger.warn( - "Request of clean nodes $it in repo $projectId|$repoName failed, error is ${e.message}," + - " cause is ${e.cause?.message}!, will sleep ${properties.sleepSeconds} seconds" + "Request of clean nodes $it in repo ${row.projectId}|${row.name} failed, error is ${e.message}," + + " cause is ${e.cause?.message}!" ) - Thread.sleep(properties.sleepSeconds * 1000) } } } + /** + * 清理容量，只针对配置了容量限制的才实时更新 + */ + private fun decreaseVolume(row: RepoData, cleanupDate: LocalDateTime, path: String, deletedTime: LocalDateTime) { + if (row.quota == null) return + val option = NodeListOption(includeFolder = false, deep = true) + val timeCriteria = Criteria().orOperator( + Criteria().and(TNode::lastAccessDate).lt(cleanupDate).and(TNode::lastModifiedDate).lt(cleanupDate), + Criteria().and(TNode::lastAccessDate).`is`(null).and(TNode::lastModifiedDate).lt(cleanupDate), + ) + val criteria = NodeQueryHelper.nodeListCriteria(row.projectId, row.name, path, option) + .andOperator(timeCriteria) + + val deletedCriteria = criteria.and(TNode::deleted).isEqualTo(deletedTime) + val deletedSize = nodeService.aggregateComputeSize(deletedCriteria) + logger.info("decrease volume $deletedSize for repo ${row.projectId}|${row.name}") + quotaService.decreaseUsedVolume(row.projectId, row.name, deletedSize) + } + private fun deletePackages( projectId: String, repoName: String, cleanupStrategy: CleanupStrategy, @@ -237,9 +262,12 @@ class ArtifactCleanupJob( private fun doPackageVersionCleanup( - projectId: String, repoName: String, - cleanupStrategy: CleanupStrategy, packageName: String, - versionList: List, repoType: String + projectId: String, + repoName: String, + cleanupStrategy: CleanupStrategy, + packageName: String, + versionList: List, + repoType: String ) { when (cleanupStrategy.cleanupType) { // 只保留距离当前时间天数以内的制品 @@ -283,7 +311,7 @@ class ArtifactCleanupJob( ) { val (urlPath, serviceInstance) = when (repoType) { RepositoryType.DOCKER.name, RepositoryType.OCI.name -> { - val dockerServiceId = buildServiceName(RepositoryType.DOCKER.name.toLowerCase()) + val dockerServiceId = buildServiceName(RepositoryType.DOCKER.name.lowercase(Locale.getDefault())) val instance = discoveryClient.getInstances(dockerServiceId).firstOrNull() Pair( "/service/third/version/delete/$projectId/$repoName", @@ -291,7 +319,7 @@ class ArtifactCleanupJob( ) } RepositoryType.HELM.name -> { - val dockerServiceId = buildServiceName(RepositoryType.HELM.name.toLowerCase()) + val dockerServiceId = buildServiceName(RepositoryType.HELM.name.lowercase(Locale.getDefault())) val instance = discoveryClient.getInstances(dockerServiceId).firstOrNull() Pair( "/service/index/version/delete/$projectId/$repoName", @@ -345,6 +373,8 @@ class ArtifactCleanupJob( val projectId: String by map val type: String by map val configuration: String by map + val quota: Long? = map["quota"] as Long? + val used: Long? = map["used"] as Long? } data class CleanupStrategy( diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/FileReferenceCleanupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/FileReferenceCleanupJob.kt index b873b40d59..a451becd6e 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/FileReferenceCleanupJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/FileReferenceCleanupJob.kt @@ -34,6 +34,7 @@ import com.tencent.bkrepo.archive.api.ArchiveClient import com.tencent.bkrepo.archive.constant.DEFAULT_KEY import com.tencent.bkrepo.archive.request.ArchiveFileRequest import com.tencent.bkrepo.archive.request.DeleteCompressRequest +import com.tencent.bkrepo.common.artifact.constant.DEFAULT_STORAGE_KEY import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService import com.tencent.bkrepo.common.mongo.constant.ID import com.tencent.bkrepo.common.service.log.LoggerHolder @@ -84,8 +85,21 @@ class FileReferenceCleanupJob( * */ private lateinit var bf: BloomFilter + /** + * + * 两个credentials使用相同的存储时（例如同一个对象存储桶），可能导致数据误删， + * 例如存储迁移的场景，迁移前后的存储桶一样仅缓存路径改变的情况 + * 此时需要获取相同存储的映射关系，避免迁移结束旧存储引用减到0后将后端存储的数据删除，导致数据丢失 + * + * 设置映射关系后会检查映射的StorageCredentialsKey是否存在对应引用，存在时将不删实际存储文件仅删除存储自身的引用 + */ + @Volatile + private lateinit var storageKeyMapping: Map> + override fun createJobContext(): FileJobContext { bf = buildBloomFilter() + storageKeyMapping = storageCredentialService.getStorageKeyMapping() + logger.info("storage key mapping: [$storageKeyMapping]") return FileJobContext() } @@ -136,13 +150,17 @@ class FileReferenceCleanupJob( return } var successToDeleted = cleanupRelatedResources(sha256, credentialsKey) - if (storageService.exist(sha256, storageCredentials)) { + val existsRefOfMappingStorage = existsRefOfMappingStorage(row, collectionName) + if (!existsRefOfMappingStorage && storageService.exist(sha256, storageCredentials)) { storageService.delete(sha256, storageCredentials) successToDeleted = true } if (!successToDeleted) { context.fileMissing.incrementAndGet() - logger.warn("File[$sha256] is missing on [$storageCredentials], skip cleaning up.") + logger.warn( + "File[$sha256] is missing on [${storageCredentials?.key}] or " + + "existsRefOfMappingStorage[$existsRefOfMappingStorage], skip cleaning up." + ) } mongoTemplate.remove(Query(Criteria(ID).isEqualTo(id)), collectionName) } catch (e: Exception) { @@ -217,6 +235,34 @@ class FileReferenceCleanupJob( return bf } + /** + * 是否在映射存储中存在相同sha256的引用 + */ + private fun existsRefOfMappingStorage(ref: FileReferenceData, collectionName: String): Boolean { + // 查询是否存在映射的key + val mappingKeys = storageKeyMapping[ref.credentialsKey ?: DEFAULT_STORAGE_KEY] + if (mappingKeys.isNullOrEmpty()) { + return false + } + + mappingKeys.forEach { mappingKey -> + // 兼容默认存储 + val mappingStorageKey = if (mappingKey == DEFAULT_STORAGE_KEY) { + null + } else { + mappingKey + } + + // 查询映射存储中是否存在对应的引用 + val criteria = Criteria.where(SHA256).isEqualTo(ref.sha256).and(CREDENTIALS).isEqualTo(mappingStorageKey) + if (mongoTemplate.exists(Query(criteria), collectionName)) { + return true + } + } + + return false + } + private fun getCredentials(key: String): StorageCredentials? { return cacheMap.getOrPut(key) { storageCredentialService.findByKey(key) ?: return null diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/OauthTokenCleanupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/OauthTokenCleanupJob.kt new file mode 100644 index 0000000000..3d4367fdd6 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/OauthTokenCleanupJob.kt @@ -0,0 +1,77 @@ +package com.tencent.bkrepo.job.batch.task.clean + +import com.tencent.bkrepo.common.metadata.constant.ID +import com.tencent.bkrepo.job.batch.base.DefaultContextMongoDbJob +import com.tencent.bkrepo.job.batch.base.JobContext +import com.tencent.bkrepo.job.batch.utils.TimeUtils +import com.tencent.bkrepo.job.config.properties.OauthTokenCleanupJobProperties +import org.slf4j.LoggerFactory +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.data.mongodb.core.query.where +import org.springframework.stereotype.Component +import java.time.Duration +import java.time.Instant +import java.time.ZoneId +import kotlin.reflect.KClass + + +@Component +@EnableConfigurationProperties(OauthTokenCleanupJobProperties::class) +class OauthTokenCleanupJob( + private val properties: OauthTokenCleanupJobProperties +) : DefaultContextMongoDbJob(properties) { + override fun collectionNames(): List { + return listOf(COLLECTION_NAME) + } + + override fun buildQuery(): Query { + return Query(where(OauthToken::expireSeconds).gt(0)) + } + + override fun mapToEntity(row: Map): OauthToken { + return OauthToken(row) + } + + override fun entityClass(): KClass { + return OauthToken::class + } + + override fun run(row: OauthToken, collectionName: String, context: JobContext) { + val accessTokenExpireTime = row.issuedAt.plusSeconds(row.expireSeconds!!) + val reservedTime = accessTokenExpireTime.plusSeconds(properties.reservedDuration.seconds) + if (Instant.now().isAfter(reservedTime)) { + val query = Query(Criteria.where(ID).isEqualTo(row.id)) + mongoTemplate.remove(query, COLLECTION_NAME) + context.success.incrementAndGet() + logger.info("clean up oauth token: ${row.id}, ${row.issuedAt.epochSecond}, ${row.expireSeconds}") + } + context.total.incrementAndGet() + } + + override fun getLockAtMostFor(): Duration { + return Duration.ofDays(7) + } + + data class OauthToken( + var id: String, + var issuedAt: Instant, + val expireSeconds: Long?, + ) { + constructor(map: Map) : this( + map[OauthToken::id.name].toString(), + TimeUtils.parseMongoDateTimeStr(map[OauthToken::issuedAt.name].toString()) + ?.atZone(ZoneId.systemDefault()) + ?.toInstant() ?: Instant.now(), + map[OauthToken::expireSeconds.name]?.toString()?.toLong(), + ) + } + + companion object { + private const val COLLECTION_NAME = "oauth_token" + private val logger = LoggerFactory.getLogger(OauthTokenCleanupJob::class.java) + } + +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/OauthTokenCleanupJobProperties.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/OauthTokenCleanupJobProperties.kt new file mode 100644 index 0000000000..a0b7fe49b5 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/OauthTokenCleanupJobProperties.kt @@ -0,0 +1,11 @@ +package com.tencent.bkrepo.job.config.properties + +import org.springframework.boot.context.properties.ConfigurationProperties +import java.time.Duration + +@ConfigurationProperties(value = "job.oauth-token-clean-up") +class OauthTokenCleanupJobProperties( + override var enabled: Boolean = true, + override var cron: String = "0 0 10 1/3 * ?", + var reservedDuration: Duration = Duration.ofDays(7), +) : MongodbJobProperties() diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/BaseTaskExecutor.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/BaseTaskExecutor.kt index 084b4d232e..e1af17781f 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/BaseTaskExecutor.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/BaseTaskExecutor.kt @@ -191,14 +191,16 @@ abstract class BaseTaskExecutor( transferData(context, node) // FileReferenceCleanupJob 会定期清理引用为0的文件数据，所以不需要删除文件数据 - // old引用计数 -1 - if (!fileReferenceService.decrement(sha256, srcStorageKey)) { - logger.error("Failed to decrement file reference[$sha256] on storage[$srcStorageKey].") - } + // 迁移前后使用的存储相同时，如果加引用失败减引用成功，可能导致文件误删，因此先增后减 // new引用计数 +1 if (!fileReferenceService.increment(sha256, dstStorageKey)) { logger.error("Failed to increment file reference[$sha256] on storage[$dstStorageKey].") } + + // old引用计数 -1 + if (!fileReferenceService.decrement(sha256, srcStorageKey)) { + logger.error("Failed to decrement file reference[$sha256] on storage[$srcStorageKey].") + } } open fun close(timeout: Long, unit: TimeUnit) {} diff --git a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/FileReferenceCleanupJobTest.kt b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/FileReferenceCleanupJobTest.kt index a220209de8..f264dd7130 100644 --- a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/FileReferenceCleanupJobTest.kt +++ b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/FileReferenceCleanupJobTest.kt @@ -31,6 +31,7 @@ import com.tencent.bkrepo.archive.api.ArchiveClient import com.tencent.bkrepo.archive.constant.DEFAULT_KEY import com.tencent.bkrepo.auth.api.ServiceBkiamV3ResourceClient import com.tencent.bkrepo.auth.api.ServicePermissionClient +import com.tencent.bkrepo.common.artifact.constant.DEFAULT_STORAGE_KEY import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.pojo.configuration.local.LocalConfiguration @@ -40,6 +41,8 @@ import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.credentials.InnerCosCredentials import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier +import com.tencent.bkrepo.job.CREDENTIALS +import com.tencent.bkrepo.job.SHA256 import com.tencent.bkrepo.job.SHARDING_COUNT import com.tencent.bkrepo.job.batch.task.clean.FileReferenceCleanupJob import com.tencent.bkrepo.job.batch.utils.NodeCommonUtils @@ -51,12 +54,15 @@ import com.tencent.bkrepo.router.api.RouterControllerClient import org.bson.Document import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.DisplayName import org.junit.jupiter.api.Test import org.mockito.ArgumentMatchers.any import org.mockito.ArgumentMatchers.anyString import org.mockito.Mockito +import org.mockito.kotlin.whenever import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.autoconfigure.data.mongo.DataMongoTest import org.springframework.boot.test.mock.mockito.MockBean @@ -262,6 +268,59 @@ class FileReferenceCleanupJobTest : JobBaseTest() { Assertions.assertEquals("key2", finds.first()["credentialsKey"]) } + @Test + fun mappingStorageKeyTest() { + whenever(storageCredentialService.getStorageKeyMapping()).thenReturn( + mapOf( + "key1" to setOf("key2", "key4", DEFAULT_STORAGE_KEY), + "key2" to setOf("key1", "key4", DEFAULT_STORAGE_KEY), + "key4" to setOf("key1", "key2", DEFAULT_STORAGE_KEY), + DEFAULT_STORAGE_KEY to setOf("key1", "key2", "key4"), + ) + ) + val collectionName = fileReferenceCleanupJob.collectionNames().first() + val sha2561 = "688787d8ff144c502c7f5cffaafe2cc588d86079f9de88304c26b0cb99ce91c1" + val sha2562 = "688787d8ff144c502c7f5cffaafe2cc588d86079f9de88304c26b0cb99ce91c2" + val sha2563 = "688787d8ff144c502c7f5cffaafe2cc588d86079f9de88304c26b0cb99ce91c3" + insertOne(sha2561, "key1", 0, collectionName) + insertOne(sha2561, "key2", 1, collectionName) + insertOne(sha2563, "key3", 0, collectionName) + insertOne(sha2562, "key4", 0, collectionName) + insertOne(sha2562, null, 1, collectionName) + + val deleted = HashSet() + whenever(storageService.delete(anyString(), any())).then { + deleted.add(it.getArgument(0)) + } + fileReferenceCleanupJob.start() + + // assert + assertTrue(deleted.size == 1 && deleted.contains(sha2563)) + + assertFalse(existsRef(sha2561, "key1", collectionName)) + assertTrue(existsRef(sha2561, "key2", collectionName)) + assertFalse(existsRef(sha2563, "key3", collectionName)) + assertFalse(existsRef(sha2562, "key4", collectionName)) + assertTrue(existsRef(sha2562, null, collectionName)) + } + + private fun existsRef(sha256: String, key: String?, collectionName: String): Boolean { + val criteria = Criteria.where(SHA256).isEqualTo(sha256).and(CREDENTIALS).isEqualTo(key) + return mongoTemplate.exists(Query(criteria), collectionName) + } + + private fun insertOne(sha256: String, key: String?, count: Int, collectionName: String) { + mongoTemplate.insert( + Document( + mutableMapOf( + "sha256" to sha256, + "credentialsKey" to key, + "count" to count, + ) as Map?, + ), collectionName + ) + } + private fun insertMany(num: Int, collectionName: String) { (0 until num).forEach { val doc = Document( diff --git a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/task/storage/StorageReconcileJobTest.kt b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/task/storage/StorageReconcileJobTest.kt index 84e4a05b8d..702a66d195 100644 --- a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/task/storage/StorageReconcileJobTest.kt +++ b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/task/storage/StorageReconcileJobTest.kt @@ -16,7 +16,6 @@ import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier import com.tencent.bkrepo.job.batch.JobBaseTest import com.tencent.bkrepo.job.batch.utils.NodeCommonUtils import com.tencent.bkrepo.job.migrate.MigrateRepoStorageService -import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.router.api.RouterControllerClient import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assertions @@ -64,9 +63,6 @@ class StorageReconcileJobTest @Autowired constructor( @MockBean lateinit var fileReferenceService: FileReferenceService - @MockBean - lateinit var repositoryClient: RepositoryClient - @MockBean lateinit var operateLogService: OperateLogService diff --git a/src/backend/lfs/biz-lfs/src/main/kotlin/com/tencent/bkrepo/lfs/controller/ObjectController.kt b/src/backend/lfs/biz-lfs/src/main/kotlin/com/tencent/bkrepo/lfs/controller/ObjectController.kt index cfa79cb0d1..9e10f86806 100644 --- a/src/backend/lfs/biz-lfs/src/main/kotlin/com/tencent/bkrepo/lfs/controller/ObjectController.kt +++ b/src/backend/lfs/biz-lfs/src/main/kotlin/com/tencent/bkrepo/lfs/controller/ObjectController.kt @@ -27,8 +27,16 @@ package com.tencent.bkrepo.lfs.controller +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_DOWNLOAD_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE +import com.tencent.bkrepo.common.artifact.audit.NODE_CREATE_ACTION import com.tencent.bkrepo.lfs.artifact.LfsArtifactInfo import com.tencent.bkrepo.lfs.pojo.BatchRequest import com.tencent.bkrepo.lfs.pojo.BatchResponse @@ -53,11 +61,45 @@ class ObjectController( return objectService.batch(projectId, repoName, request) } + @AuditEntry( + actionId = NODE_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#lfsArtifactInfo?.getArtifactFullPath()", + instanceNames = "#lfsArtifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#lfsArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#lfsArtifactInfo?.repoName") + ], + scopeId = "#lfsArtifactInfo?.projectId", + content = ActionAuditContent.NODE_UPLOAD_CONTENT + ) @PutMapping("/{projectId}/{repoName}/**") fun upload(@ArtifactPathVariable lfsArtifactInfo: LfsArtifactInfo, file: ArtifactFile) { objectService.upload(lfsArtifactInfo, file) } + @AuditEntry( + actionId = NODE_DOWNLOAD_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DOWNLOAD_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#lfsArtifactInfo?.getArtifactFullPath()", + instanceNames = "#lfsArtifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#lfsArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#lfsArtifactInfo?.repoName") + ], + scopeId = "#lfsArtifactInfo?.projectId", + content = ActionAuditContent.NODE_DOWNLOAD_CONTENT + ) @GetMapping("/{projectId}/{repoName}/**") fun download(@ArtifactPathVariable lfsArtifactInfo: LfsArtifactInfo) { objectService.download(lfsArtifactInfo) diff --git a/src/backend/lfs/biz-lfs/src/main/kotlin/com/tencent/bkrepo/lfs/service/ObjectService.kt b/src/backend/lfs/biz-lfs/src/main/kotlin/com/tencent/bkrepo/lfs/service/ObjectService.kt index 121dc72d91..d3ff332fe1 100644 --- a/src/backend/lfs/biz-lfs/src/main/kotlin/com/tencent/bkrepo/lfs/service/ObjectService.kt +++ b/src/backend/lfs/biz-lfs/src/main/kotlin/com/tencent/bkrepo/lfs/service/ObjectService.kt @@ -51,7 +51,7 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadConte import com.tencent.bkrepo.common.artifact.repository.core.ArtifactService import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HeaderUtils import com.tencent.bkrepo.common.service.util.HttpContextHolder diff --git a/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/controller/MavenResourceController.kt b/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/controller/MavenResourceController.kt index 087c86ccab..7f147cc050 100644 --- a/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/controller/MavenResourceController.kt +++ b/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/controller/MavenResourceController.kt @@ -27,8 +27,17 @@ package com.tencent.bkrepo.maven.controller +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_DELETE_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_DOWNLOAD_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE +import com.tencent.bkrepo.common.artifact.audit.NODE_CREATE_ACTION import com.tencent.bkrepo.maven.artifact.MavenArtifactInfo import com.tencent.bkrepo.maven.service.MavenService import org.springframework.http.MediaType @@ -41,6 +50,24 @@ import org.springframework.web.bind.annotation.RestController class MavenResourceController( private val mavenService: MavenService ) { + + @AuditEntry( + actionId = NODE_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#mavenArtifactInfo?.getArtifactFullPath()", + instanceNames = "#mavenArtifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#mavenArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#mavenArtifactInfo?.repoName") + ], + scopeId = "#mavenArtifactInfo?.projectId", + content = ActionAuditContent.NODE_UPLOAD_CONTENT + ) @PutMapping(MavenArtifactInfo.MAVEN_MAPPING_URI, produces = [MediaType.APPLICATION_JSON_VALUE]) fun deploy( @ArtifactPathVariable mavenArtifactInfo: MavenArtifactInfo, @@ -49,11 +76,45 @@ class MavenResourceController( return mavenService.deploy(mavenArtifactInfo, file) } + @AuditEntry( + actionId = NODE_DOWNLOAD_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DOWNLOAD_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#mavenArtifactInfo?.getArtifactFullPath()", + instanceNames = "#mavenArtifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#mavenArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#mavenArtifactInfo?.repoName") + ], + scopeId = "#mavenArtifactInfo?.projectId", + content = ActionAuditContent.NODE_DOWNLOAD_CONTENT + ) @GetMapping(MavenArtifactInfo.MAVEN_MAPPING_URI, produces = [MediaType.APPLICATION_JSON_VALUE]) fun dependency(@ArtifactPathVariable mavenArtifactInfo: MavenArtifactInfo) { mavenService.dependency(mavenArtifactInfo) } + @AuditEntry( + actionId = NODE_DELETE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DELETE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#mavenArtifactInfo?.getArtifactFullPath()", + instanceNames = "#mavenArtifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#mavenArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#mavenArtifactInfo?.repoName") + ], + scopeId = "#mavenArtifactInfo?.projectId", + content = ActionAuditContent.NODE_DELETE_CONTENT + ) @DeleteMapping(MavenArtifactInfo.MAVEN_MAPPING_URI, produces = [MediaType.APPLICATION_JSON_VALUE]) fun deleteDependency(@ArtifactPathVariable mavenArtifactInfo: MavenArtifactInfo) { mavenService.deleteDependency(mavenArtifactInfo) diff --git a/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/controller/MavenWebController.kt b/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/controller/MavenWebController.kt index 987566ab8d..093a62cd13 100644 --- a/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/controller/MavenWebController.kt +++ b/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/controller/MavenWebController.kt @@ -31,8 +31,15 @@ package com.tencent.bkrepo.maven.controller +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.REPO_EDIT_ACTION +import com.tencent.bkrepo.common.artifact.audit.REPO_RESOURCE import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.maven.api.MavenWebResource import com.tencent.bkrepo.maven.artifact.MavenArtifactInfo @@ -48,11 +55,47 @@ class MavenWebController( private val mavenExtService: MavenExtService ) : MavenWebResource { + @AuditEntry( + actionId = REPO_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = REPO_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#mavenArtifactInfo?.repoName", + instanceNames = "#mavenArtifactInfo?.repoName" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#mavenArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.NAME_TEMPLATE, value = "#packageKey") + ], + scopeId = "#mavenArtifactInfo?.projectId", + content = ActionAuditContent.REPO_PACKAGE_DELETE_CONTENT + ) override fun deletePackage(mavenArtifactInfo: MavenDeleteArtifactInfo, packageKey: String): Response { mavenService.delete(mavenArtifactInfo, packageKey, null) return ResponseBuilder.success() } + @AuditEntry( + actionId = REPO_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = REPO_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#mavenArtifactInfo?.repoName", + instanceNames = "#mavenArtifactInfo?.repoName" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#mavenArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.NAME_TEMPLATE, value = "#packageKey"), + AuditAttribute(name = ActionAuditContent.VERSION_TEMPLATE, value = "#version") + + ], + scopeId = "#mavenArtifactInfo?.projectId", + content = ActionAuditContent.REPO_PACKAGE_VERSION_DELETE_CONTENT + ) override fun deleteVersion( mavenArtifactInfo: MavenDeleteArtifactInfo, packageKey: String, diff --git a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/controller/UserStreamController.kt b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/controller/UserStreamController.kt index bd93064c6e..48bdb5ab48 100644 --- a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/controller/UserStreamController.kt +++ b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/controller/UserStreamController.kt @@ -4,7 +4,7 @@ import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.media.artifact.MediaArtifactInfo diff --git a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/TokenService.kt b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/TokenService.kt index 6d58a9daf8..a35226c8d6 100644 --- a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/TokenService.kt +++ b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/TokenService.kt @@ -11,7 +11,7 @@ import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.path.PathUtils -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HttpContextHolder import org.springframework.stereotype.Service diff --git a/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/artifact/repository/NpmLocalRepository.kt b/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/artifact/repository/NpmLocalRepository.kt index 7746863ed3..a03f98f4b9 100644 --- a/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/artifact/repository/NpmLocalRepository.kt +++ b/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/artifact/repository/NpmLocalRepository.kt @@ -188,7 +188,7 @@ class NpmLocalRepository( metadata["description"] as? String, metadata["maintainers"] as? List> ?: emptyList(), metadata["version"] as? String, - it["lastModifiedDate"] as String, + (it["lastModifiedDate"] as LocalDateTime).toString(), metadata["keywords"] as? List ?: emptyList(), metadata["author"] as? Map ?: emptyMap() ) diff --git a/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/controller/NpmClientController.kt b/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/controller/NpmClientController.kt index 409e22a2f4..d7acf11e99 100644 --- a/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/controller/NpmClientController.kt +++ b/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/controller/NpmClientController.kt @@ -31,8 +31,15 @@ package com.tencent.bkrepo.npm.controller +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.common.api.constant.MediaTypes import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_DOWNLOAD_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE import com.tencent.bkrepo.common.service.util.HeaderUtils import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.npm.artifact.NpmArtifactInfo @@ -150,6 +157,23 @@ class NpmClientController( /** * download tgz file */ + @AuditEntry( + actionId = NODE_DOWNLOAD_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DOWNLOAD_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_DOWNLOAD_CONTENT + ) @GetMapping("/{projectId}/{repoName}/**/*.tgz") fun download( @ArtifactPathVariable artifactInfo: NpmArtifactInfo diff --git a/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/controller/UserNpmController.kt b/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/controller/UserNpmController.kt index d320010a9a..01c82d8902 100644 --- a/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/controller/UserNpmController.kt +++ b/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/controller/UserNpmController.kt @@ -31,11 +31,18 @@ package com.tencent.bkrepo.npm.controller +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable import com.tencent.bkrepo.common.artifact.util.PackageKeys +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.REPO_EDIT_ACTION +import com.tencent.bkrepo.common.artifact.audit.REPO_RESOURCE import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.npm.artifact.NpmArtifactInfo @@ -62,6 +69,7 @@ class UserNpmController( private val npmWebService: NpmWebService ) { + @Permission(ResourceType.REPO, PermissionAction.READ) @ApiOperation("查询包的版本详情") @GetMapping("/version/detail/{projectId}/{repoName}") @@ -77,6 +85,23 @@ class UserNpmController( return ResponseBuilder.success(npmWebService.detailVersion(artifactInfo, packageKey, version)) } + @AuditEntry( + actionId = REPO_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = REPO_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#artifactInfo?.repoName", + instanceNames = "#artifactInfo?.repoName" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.NAME_TEMPLATE, value = "#packageKey") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.REPO_PACKAGE_DELETE_CONTENT + ) @Permission(ResourceType.REPO, PermissionAction.DELETE) @ApiOperation("删除仓库下的包") @DeleteMapping("/package/delete/{projectId}/{repoName}") @@ -97,6 +122,25 @@ class UserNpmController( } } + @AuditEntry( + actionId = REPO_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = REPO_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#artifactInfo?.repoName", + instanceNames = "#artifactInfo?.repoName" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.NAME_TEMPLATE, value = "#packageKey"), + AuditAttribute(name = ActionAuditContent.VERSION_TEMPLATE, value = "#version") + + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.REPO_PACKAGE_VERSION_DELETE_CONTENT + ) @Permission(ResourceType.REPO, PermissionAction.DELETE) @ApiOperation("删除仓库下的包版本") @DeleteMapping("/version/delete/{projectId}/{repoName}") diff --git a/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/service/impl/NpmClientServiceImpl.kt b/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/service/impl/NpmClientServiceImpl.kt index a850f65ed4..208baf6b5c 100644 --- a/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/service/impl/NpmClientServiceImpl.kt +++ b/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/service/impl/NpmClientServiceImpl.kt @@ -31,6 +31,10 @@ package com.tencent.bkrepo.npm.service.impl +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.api.util.JsonUtils.objectMapper @@ -42,6 +46,9 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactSearchConte import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadContext import com.tencent.bkrepo.common.artifact.resolve.file.ArtifactFileFactory import com.tencent.bkrepo.common.artifact.util.PackageKeys +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE +import com.tencent.bkrepo.common.artifact.audit.NODE_CREATE_ACTION import com.tencent.bkrepo.common.metadata.service.metadata.MetadataService import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.common.service.util.HttpContextHolder @@ -318,6 +325,23 @@ class NpmClientServiceImpl( } } + @AuditEntry( + actionId = NODE_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_UPLOAD_CONTENT + ) private fun handlerPackagePublish( userId: String, artifactInfo: NpmArtifactInfo, diff --git a/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/service/impl/NpmFixToolServiceImpl.kt b/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/service/impl/NpmFixToolServiceImpl.kt index e1308abb4a..d43e62e419 100644 --- a/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/service/impl/NpmFixToolServiceImpl.kt +++ b/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/service/impl/NpmFixToolServiceImpl.kt @@ -367,9 +367,9 @@ class NpmFixToolServiceImpl( private fun resolveNode(record: Map): NodeInfo { return NodeInfo( createdBy = record["createdBy"] as String, - createdDate = record["createdDate"] as String, + createdDate = (record["createdDate"] as LocalDateTime).toString(), lastModifiedBy = record["lastModifiedBy"] as String, - lastModifiedDate = record["lastModifiedDate"] as String, + lastModifiedDate = (record["lastModifiedDate"] as LocalDateTime).toString(), folder = record["folder"] as Boolean, path = record["path"] as String, name = record["name"] as String, diff --git a/src/backend/nuget/biz-nuget/src/main/kotlin/com/tencent/bkrepo/nuget/controller/NugetClientController.kt b/src/backend/nuget/biz-nuget/src/main/kotlin/com/tencent/bkrepo/nuget/controller/NugetClientController.kt index 0de819c911..4cda2d6f94 100644 --- a/src/backend/nuget/biz-nuget/src/main/kotlin/com/tencent/bkrepo/nuget/controller/NugetClientController.kt +++ b/src/backend/nuget/biz-nuget/src/main/kotlin/com/tencent/bkrepo/nuget/controller/NugetClientController.kt @@ -31,11 +31,20 @@ package com.tencent.bkrepo.nuget.controller +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.api.constant.MediaTypes.APPLICATION_JSON import com.tencent.bkrepo.common.api.constant.MediaTypes.APPLICATION_XML import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_DELETE_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_DOWNLOAD_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE +import com.tencent.bkrepo.common.artifact.audit.NODE_CREATE_ACTION import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.nuget.artifact.NugetArtifactInfo import com.tencent.bkrepo.nuget.artifact.NugetArtifactInfo.Companion.DELETE_V2 @@ -73,6 +82,23 @@ class NugetClientController( * Content-Type multipart/form-data * A package with the provided ID and version already exists, status code 409 */ + @AuditEntry( + actionId = NODE_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#publishInfo?.getArtifactFullPath()", + instanceNames = "#publishInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#publishInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#publishInfo?.repoName") + ], + scopeId = "#publishInfo?.projectId", + content = ActionAuditContent.NODE_UPLOAD_CONTENT + ) @PutMapping(PUBLISH_V2) @Permission(ResourceType.REPO, PermissionAction.WRITE) fun publish( @@ -82,6 +108,23 @@ class NugetClientController( nugetClientService.publish(userId, publishInfo) } + @AuditEntry( + actionId = NODE_DOWNLOAD_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DOWNLOAD_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_DOWNLOAD_CONTENT + ) @GetMapping(DOWNLOAD_V2) @Permission(ResourceType.REPO, PermissionAction.READ) fun download( @@ -102,6 +145,23 @@ class NugetClientController( /** * nuget delete [ options ] */ + @AuditEntry( + actionId = NODE_DELETE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DELETE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_DELETE_CONTENT + ) @DeleteMapping(DELETE_V2) @Permission(ResourceType.REPO, PermissionAction.DELETE) fun delete( diff --git a/src/backend/nuget/biz-nuget/src/main/kotlin/com/tencent/bkrepo/nuget/controller/NugetPackageContentController.kt b/src/backend/nuget/biz-nuget/src/main/kotlin/com/tencent/bkrepo/nuget/controller/NugetPackageContentController.kt index 19330508d3..308c80df5a 100644 --- a/src/backend/nuget/biz-nuget/src/main/kotlin/com/tencent/bkrepo/nuget/controller/NugetPackageContentController.kt +++ b/src/backend/nuget/biz-nuget/src/main/kotlin/com/tencent/bkrepo/nuget/controller/NugetPackageContentController.kt @@ -1,7 +1,14 @@ package com.tencent.bkrepo.nuget.controller +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_DOWNLOAD_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.nuget.artifact.NugetArtifactInfo import com.tencent.bkrepo.nuget.artifact.NugetArtifactInfo.Companion.DOWNLOAD_MANIFEST @@ -33,6 +40,23 @@ class NugetPackageContentController( * * return binary stream */ + @AuditEntry( + actionId = NODE_DOWNLOAD_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DOWNLOAD_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_DOWNLOAD_CONTENT + ) @GetMapping(DOWNLOAD_V3) @Permission(ResourceType.REPO, PermissionAction.READ) fun downloadPackageContent( @@ -48,6 +72,23 @@ class NugetPackageContentController( * * return xml document */ + @AuditEntry( + actionId = NODE_DOWNLOAD_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DOWNLOAD_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_DOWNLOAD_CONTENT + ) @GetMapping(DOWNLOAD_MANIFEST) @Permission(ResourceType.REPO, PermissionAction.READ) fun downloadPackageManifest( diff --git a/src/backend/nuget/biz-nuget/src/main/kotlin/com/tencent/bkrepo/nuget/controller/NugetWebController.kt b/src/backend/nuget/biz-nuget/src/main/kotlin/com/tencent/bkrepo/nuget/controller/NugetWebController.kt index 0195910f72..859ff34d4e 100644 --- a/src/backend/nuget/biz-nuget/src/main/kotlin/com/tencent/bkrepo/nuget/controller/NugetWebController.kt +++ b/src/backend/nuget/biz-nuget/src/main/kotlin/com/tencent/bkrepo/nuget/controller/NugetWebController.kt @@ -1,8 +1,15 @@ package com.tencent.bkrepo.nuget.controller +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.REPO_EDIT_ACTION +import com.tencent.bkrepo.common.artifact.audit.REPO_RESOURCE import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.nuget.artifact.NugetArtifactInfo @@ -30,6 +37,24 @@ import org.springframework.web.bind.annotation.RestController class NugetWebController( private val nugetWebService: NugetWebService ) { + + @AuditEntry( + actionId = REPO_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = REPO_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#artifactInfo?.repoName", + instanceNames = "#artifactInfo?.repoName" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.NAME_TEMPLATE, value = "#packageKey") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.REPO_PACKAGE_DELETE_CONTENT + ) @Permission(ResourceType.REPO, PermissionAction.DELETE) @ApiOperation("删除仓库下的包") @DeleteMapping(NUGET_EXT_DELETE_PACKAGE) @@ -43,6 +68,25 @@ class NugetWebController( return ResponseBuilder.success() } + @AuditEntry( + actionId = REPO_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = REPO_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#artifactInfo?.repoName", + instanceNames = "#artifactInfo?.repoName" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.NAME_TEMPLATE, value = "#packageKey"), + AuditAttribute(name = ActionAuditContent.VERSION_TEMPLATE, value = "#version") + + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.REPO_PACKAGE_VERSION_DELETE_CONTENT + ) @Permission(ResourceType.REPO, PermissionAction.DELETE) @ApiOperation("删除仓库下的包版本") @DeleteMapping(NUGET_EXT_DELETE_VERSION) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt index 9497ce4268..650be3f3c8 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt @@ -31,6 +31,10 @@ package com.tencent.bkrepo.oci.artifact.repository +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.common.api.constant.HttpStatus import com.tencent.bkrepo.common.api.constant.MediaTypes import com.tencent.bkrepo.common.artifact.api.ArtifactInfo @@ -44,6 +48,9 @@ import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream import com.tencent.bkrepo.common.artifact.util.PackageKeys +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE +import com.tencent.bkrepo.common.artifact.audit.NODE_CREATE_ACTION import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.common.storage.innercos.http.HttpMethod import com.tencent.bkrepo.common.storage.message.StorageErrorException @@ -188,6 +195,23 @@ class OciRegistryLocalRepository( * blob 上传，直接使用post * Pushing a blob monolithically ：A single POST request */ + @AuditEntry( + actionId = NODE_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#context.artifactInfo?.getArtifactFullPath()", + instanceNames = "#context.artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#context?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#context?.repoName") + ], + scopeId = "#context?.projectId", + content = ActionAuditContent.NODE_UPLOAD_CONTENT + ) private fun postUpload(context: ArtifactUploadContext): ResponseProperty? { val artifactFile = context.getArtifactFile() val digest = OciDigest.fromSha256(artifactFile.getFileSha256()) @@ -230,6 +254,23 @@ class OciRegistryLocalRepository( * 1 blob POST with PUT 上传的put模块处理 * 2 blob POST PATCH with PUT 上传的put模块处理 */ + @AuditEntry( + actionId = NODE_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#context.artifactInfo?.getArtifactFullPath()", + instanceNames = "#context.artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#context?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#context?.repoName") + ], + scopeId = "#context?.projectId", + content = ActionAuditContent.NODE_UPLOAD_CONTENT + ) private fun putUploadBlob(context: ArtifactUploadContext): ResponseProperty { val artifactInfo = context.artifactInfo as OciBlobArtifactInfo val sha256 = artifactInfo.getDigestHex() diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciBlobController.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciBlobController.kt index 0f0f3a8ac9..a345b88113 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciBlobController.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciBlobController.kt @@ -31,9 +31,17 @@ package com.tencent.bkrepo.oci.controller.user +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_DELETE_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_DOWNLOAD_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.BOLBS_UPLOAD_FIRST_STEP_URL import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.BOLBS_UPLOAD_SECOND_STEP_URL @@ -97,6 +105,23 @@ class OciBlobController( /** * 获取Blob文件 */ + @AuditEntry( + actionId = NODE_DOWNLOAD_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DOWNLOAD_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_DOWNLOAD_CONTENT + ) @GetMapping(BOLBS_URL) @Permission(type = ResourceType.REPO, action = PermissionAction.READ) fun downloadBlob( @@ -109,6 +134,23 @@ class OciBlobController( * 删除blob文件 * 只能通过digest删除 */ + @AuditEntry( + actionId = NODE_DELETE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DELETE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_DELETE_CONTENT + ) @DeleteMapping(BOLBS_URL) @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) fun deleteBlob( diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciManifestController.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciManifestController.kt index 6c6f3f4b53..fde0405fda 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciManifestController.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciManifestController.kt @@ -31,9 +31,18 @@ package com.tencent.bkrepo.oci.controller.user +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_CREATE_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_DELETE_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_DOWNLOAD_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.MANIFEST_URL import com.tencent.bkrepo.oci.pojo.artifact.OciManifestArtifactInfo @@ -56,6 +65,23 @@ class OciManifestController( * 上传manifest文件 * 可以通过digest或者tag去上传 */ + @AuditEntry( + actionId = NODE_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_UPLOAD_CONTENT + ) @PutMapping(MANIFEST_URL) @Permission(ResourceType.REPO, PermissionAction.WRITE) fun uploadManifests( @@ -69,6 +95,23 @@ class OciManifestController( * 下载manifest文件 * 可以通过digest或者tag去下载 */ + @AuditEntry( + actionId = NODE_DOWNLOAD_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DOWNLOAD_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_DOWNLOAD_CONTENT + ) @GetMapping(MANIFEST_URL) @Permission(type = ResourceType.REPO, action = PermissionAction.READ) fun downloadManifests( @@ -81,6 +124,23 @@ class OciManifestController( * 删除manifest文件 * 可以通过digest和tag删除 */ + @AuditEntry( + actionId = NODE_DELETE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DELETE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_DELETE_CONTENT + ) @DeleteMapping(MANIFEST_URL) @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) fun deleteManifests( diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt index 3d24080a5f..a8982cfd93 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt @@ -27,10 +27,17 @@ package com.tencent.bkrepo.oci.controller.user +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.REPO_EDIT_ACTION +import com.tencent.bkrepo.common.artifact.audit.REPO_RESOURCE import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.oci.constant.OCI_PACKAGE_NAME @@ -91,6 +98,23 @@ class UserOciController( ) } + @AuditEntry( + actionId = REPO_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = REPO_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#artifactInfo?.repoName", + instanceNames = "#artifactInfo?.repoName" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.NAME_TEMPLATE, value = "#packageKey") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.REPO_PACKAGE_DELETE_CONTENT + ) @ApiOperation("删除仓库下的包") @DeleteMapping(OCI_PACKAGE_DELETE_URL) @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) @@ -104,6 +128,25 @@ class UserOciController( return ResponseBuilder.success() } + @AuditEntry( + actionId = REPO_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = REPO_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#artifactInfo?.repoName", + instanceNames = "#artifactInfo?.repoName" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.NAME_TEMPLATE, value = "#packageKey"), + AuditAttribute(name = ActionAuditContent.VERSION_TEMPLATE, value = "#version") + + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.REPO_PACKAGE_VERSION_DELETE_CONTENT + ) @ApiOperation("删除仓库下的包版本") @DeleteMapping(OCI_VERSION_DELETE_URL) @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/exception/OciExceptionHandler.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/exception/OciExceptionHandler.kt index 88aac6b016..104ef8da19 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/exception/OciExceptionHandler.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/exception/OciExceptionHandler.kt @@ -49,6 +49,7 @@ import com.tencent.bkrepo.oci.constant.UNAUTHORIZED_DESCRIPTION import com.tencent.bkrepo.oci.constant.UNAUTHORIZED_MESSAGE import com.tencent.bkrepo.oci.pojo.response.OciErrorResponse import com.tencent.bkrepo.oci.pojo.response.OciResponse +import javax.servlet.http.HttpServletResponse import org.slf4j.LoggerFactory import org.springframework.core.Ordered import org.springframework.core.annotation.Order @@ -56,7 +57,6 @@ import org.springframework.http.HttpStatus import org.springframework.web.bind.annotation.ExceptionHandler import org.springframework.web.bind.annotation.ResponseStatus import org.springframework.web.bind.annotation.RestControllerAdvice -import javax.servlet.http.HttpServletResponse @Order(Ordered.HIGHEST_PRECEDENCE + 1) @RestControllerAdvice("com.tencent.bkrepo.oci") @@ -64,7 +64,7 @@ class OciExceptionHandler( private val ociProperties: OciProperties ) { -/** + /** * 单独处理认证失败异常，需要添加WWW_AUTHENTICATE响应头触发浏览器登录 */ @ExceptionHandler(AuthenticationException::class) @@ -122,12 +122,6 @@ class OciExceptionHandler( ociResponse(responseObject, exception) } - @ExceptionHandler(ErrorCodeException::class) - @ResponseStatus(HttpStatus.BAD_REQUEST) - fun handleException(exception: ErrorCodeException) { - ociResponse(exception) - } - @ExceptionHandler(PermissionException::class) @ResponseStatus(HttpStatus.FORBIDDEN) fun handleException(exception: PermissionException) { @@ -162,7 +156,7 @@ class OciExceptionHandler( val uri = HttpContextHolder.getRequest().requestURI logger.warn( "User[$userId] access oci resource[$uri] failed[${exception.javaClass.simpleName}]:" + - " ${responseObject.message}" + " ${responseObject.message}" ) } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt index 1a0ca90e21..338f5a1921 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt @@ -31,6 +31,7 @@ package com.tencent.bkrepo.oci.service.impl +import com.tencent.bk.audit.context.ActionAuditContext import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.common.api.constant.CharPool import com.tencent.bkrepo.common.api.constant.HttpStatus @@ -43,7 +44,7 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactRemoveConte import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadContext import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.oci.constant.BLOB_PATH_VERSION_KEY @@ -166,6 +167,7 @@ class OciBlobServiceImpl( md5 = nodeProperty.md5!!, metadata = metadata ) + ActionAuditContext.current().setInstance(nodeCreateRequest) nodeService.createNode(nodeCreateRequest) val blobLocation = OciLocationUtils.blobLocation(ociDigest, this) val responseProperty = ResponseProperty( diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index 007cc1d245..710ebf4073 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -27,6 +27,7 @@ package com.tencent.bkrepo.oci.service.impl +import com.tencent.bk.audit.context.ActionAuditContext import com.tencent.bkrepo.common.api.constant.DEFAULT_PAGE_NUMBER import com.tencent.bkrepo.common.api.constant.HttpHeaders import com.tencent.bkrepo.common.api.constant.StringPool @@ -407,6 +408,7 @@ class OciOperationServiceImpl( md5 = fileInfo.md5, sha256 = fileInfo.sha256 ) + ActionAuditContext.current().setInstance(newNodeRequest) nodeService.createNode(newNodeRequest) } else { storageManager.storeArtifactFile(request, artifactFile, storageCredentials) @@ -679,6 +681,7 @@ class OciOperationServiceImpl( md5 = nodeProperty.md5 ?: StringPool.UNKNOWN, userId = userId ) + ActionAuditContext.current().setInstance(nodeCreateRequest) nodeService.createNode(nodeCreateRequest) } val metadataMap = metadataService.listMetadata(projectId, repoName, fullPath) diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/controller/FsClientController.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/controller/FsClientController.kt index e3cb5d40a8..c85266499a 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/controller/FsClientController.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/controller/FsClientController.kt @@ -34,7 +34,7 @@ package com.tencent.bkrepo.opdata.controller import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.api.pojo.Response -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.security.permission.PrincipalType import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.fs.server.api.FsClientClient diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/controller/ProjectController.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/controller/ProjectController.kt index 2e3554c824..7682e00ac4 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/controller/ProjectController.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/controller/ProjectController.kt @@ -30,10 +30,10 @@ package com.tencent.bkrepo.opdata.controller import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.api.pojo.Response -import com.tencent.bkrepo.common.metadata.service.project.ProjectUsageStatisticsService +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.metadata.pojo.project.ProjectUsageStatistics import com.tencent.bkrepo.common.metadata.pojo.project.ProjectUsageStatisticsListOption -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.service.project.ProjectUsageStatisticsService import com.tencent.bkrepo.common.security.permission.Principal import com.tencent.bkrepo.common.security.permission.PrincipalType import com.tencent.bkrepo.common.security.util.SecurityUtils diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/controller/RateLimitController.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/controller/RateLimitController.kt new file mode 100644 index 0000000000..fe1468e306 --- /dev/null +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/controller/RateLimitController.kt @@ -0,0 +1,118 @@ +package com.tencent.bkrepo.opdata.controller + +import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.api.exception.NotFoundException +import com.tencent.bkrepo.common.api.message.CommonMessageCode +import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.api.util.toJsonString +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.model.RateLimitCreatOrUpdateRequest +import com.tencent.bkrepo.common.ratelimiter.model.TRateLimit +import com.tencent.bkrepo.common.ratelimiter.service.user.RateLimiterConfigService +import com.tencent.bkrepo.common.security.permission.Principal +import com.tencent.bkrepo.common.security.permission.PrincipalType +import com.tencent.bkrepo.common.service.util.ResponseBuilder +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController +import org.springframework.web.bind.annotation.DeleteMapping +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.RequestParam + +@RestController +@RequestMapping("/api/rateLimit") +@Principal(PrincipalType.ADMIN) +class RateLimitController( + private val rateLimiterConfigService: RateLimiterConfigService, + private val rateLimiterProperties: RateLimiterProperties +) { + + @GetMapping("/list") + fun list(): Response> { + return ResponseBuilder.success(rateLimiterConfigService.list()) + } + + @PostMapping("/update") + fun update(@RequestBody request: RateLimitCreatOrUpdateRequest): Response { + if (request.id.isNullOrBlank()) { + throw NotFoundException(CommonMessageCode.PARAMETER_EMPTY, "ID") + } + if (!rateLimiterConfigService.checkExist(request.id!!)) { + throw NotFoundException(CommonMessageCode.RESOURCE_NOT_FOUND, request.id!!) + } + val tRateLimit = rateLimiterConfigService.findByModuleNameAndLimitDimensionAndResource( + request.resource, + request.moduleName, + request.limitDimension + ) + if (tRateLimit == null || !tRateLimit.id.equals(request.id)) { + checkResource(request) + } + rateLimiterConfigService.update(request) + return ResponseBuilder.success() + } + + private fun checkResource(request: RateLimitCreatOrUpdateRequest) { + with(request) { + val tRateLimits = rateLimiterConfigService.findByResourceAndLimitDimension( + resource = resource, + limitDimension = limitDimension + ) + val modules = ArrayList() + tRateLimits.forEach { tRateLimit -> + if(id == null || !tRateLimit.id.equals(id)) { + modules.addAll(tRateLimit.moduleName) + } + } + if (modules.isNotEmpty()) { + modules.retainAll(moduleName) + if (modules.isNotEmpty()) { + throw ErrorCodeException( + CommonMessageCode.RESOURCE_EXISTED, + "resource:$resource,limitDimension:$limitDimension,module:${modules}" + ) + } + } + } + } + + // 新增 + @PostMapping("/create") + fun create(@RequestBody request:RateLimitCreatOrUpdateRequest): Response { + checkResource(request) + rateLimiterConfigService.create(request) + return ResponseBuilder.success() + } + + // 删除 + @DeleteMapping("/delete/{id}") + fun delete(@PathVariable id:String): Response { + if (!rateLimiterConfigService.checkExist(id)) { + throw NotFoundException(CommonMessageCode.RESOURCE_NOT_FOUND, id) + } + rateLimiterConfigService.delete(id) + return ResponseBuilder.success() + } + + // 获取配置中的属性 + @GetMapping("/config") + fun getConfig(): Response { + val config = rateLimiterProperties + return ResponseBuilder.success(config.rules.toJsonString()) + } + + // 获取数据库里面的模块名 + @PostMapping("/getExistModule") + fun getExistModule(@RequestParam resource:String,@RequestParam limitDimension:String): Response> { + val tRateLimits = rateLimiterConfigService.findByResourceAndLimitDimension( + resource = resource, + limitDimension = limitDimension + ) + val modules = ArrayList() + tRateLimits.forEach { tRateLimit -> modules.addAll(tRateLimit.moduleName) } + return ResponseBuilder.success(modules) + } + +} \ No newline at end of file diff --git a/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/repository/PypiLocalRepository.kt b/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/repository/PypiLocalRepository.kt index 92a4e89c8a..4f81f103db 100644 --- a/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/repository/PypiLocalRepository.kt +++ b/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/artifact/repository/PypiLocalRepository.kt @@ -27,6 +27,10 @@ package com.tencent.bkrepo.pypi.artifact.repository +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.api.ArtifactInfo @@ -43,6 +47,9 @@ import com.tencent.bkrepo.common.artifact.util.PackageKeys import com.tencent.bkrepo.common.metadata.service.packages.StageService import com.tencent.bkrepo.common.metadata.util.version.SemVersion import com.tencent.bkrepo.common.metadata.util.version.SemVersionParser +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_DELETE_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE import com.tencent.bkrepo.common.query.enums.OperationType import com.tencent.bkrepo.common.query.model.PageLimit import com.tencent.bkrepo.common.query.model.QueryModel @@ -228,6 +235,23 @@ class PypiLocalRepository( /** * pypi 产品删除接口 */ + @AuditEntry( + actionId = NODE_DELETE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DELETE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactFullPath", + instanceNames = "#artifactFullPath" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#context?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#context?.repoName") + ], + scopeId = "#context?.projectId", + content = ActionAuditContent.NODE_DELETE_CONTENT + ) override fun remove(context: ArtifactRemoveContext) { val packageKey = HttpContextHolder.getRequest().getParameter("packageKey") val name = PackageKeys.resolvePypi(packageKey) diff --git a/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/controller/PypiResourceController.kt b/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/controller/PypiResourceController.kt index 769a43e5ec..d9f177f60d 100644 --- a/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/controller/PypiResourceController.kt +++ b/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/controller/PypiResourceController.kt @@ -31,7 +31,15 @@ package com.tencent.bkrepo.pypi.controller +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.common.artifact.api.ArtifactFileMap +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_DOWNLOAD_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE +import com.tencent.bkrepo.common.artifact.audit.NODE_CREATE_ACTION import com.tencent.bkrepo.pypi.artifact.PypiArtifactInfo import com.tencent.bkrepo.pypi.artifact.PypiSimpleArtifactInfo import com.tencent.bkrepo.pypi.service.PypiService @@ -50,6 +58,24 @@ import org.springframework.web.bind.annotation.RestController class PypiResourceController( private var pypiService: PypiService ) { + + @AuditEntry( + actionId = NODE_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#pypiArtifactInfo?.getArtifactFullPath()", + instanceNames = "#pypiArtifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#pypiArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#pypiArtifactInfo?.repoName") + ], + scopeId = "#pypiArtifactInfo?.projectId", + content = ActionAuditContent.NODE_UPLOAD_CONTENT + ) @PostMapping(PypiArtifactInfo.PYPI_ROOT_POST_URI) fun upload(pypiArtifactInfo: PypiArtifactInfo, artifactFileMap: ArtifactFileMap) { logger.info("upload pypi package: $pypiArtifactInfo") @@ -75,6 +101,23 @@ class PypiResourceController( return pypiService.simple(artifactInfo) } + @AuditEntry( + actionId = NODE_DOWNLOAD_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DOWNLOAD_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_DOWNLOAD_CONTENT + ) @GetMapping(PypiArtifactInfo.PYPI_PACKAGES_MAPPING_URI) fun packages(artifactInfo: PypiArtifactInfo) { logger.info("packages pypi package: $artifactInfo") diff --git a/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/controller/PypiWebResourceController.kt b/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/controller/PypiWebResourceController.kt index fc4029bd5c..309a7532d2 100644 --- a/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/controller/PypiWebResourceController.kt +++ b/src/backend/pypi/biz-pypi/src/main/kotlin/com/tencent/bkrepo/pypi/controller/PypiWebResourceController.kt @@ -31,8 +31,15 @@ package com.tencent.bkrepo.pypi.controller +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.REPO_EDIT_ACTION +import com.tencent.bkrepo.common.artifact.audit.REPO_RESOURCE import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.pypi.artifact.PypiArtifactInfo import com.tencent.bkrepo.pypi.service.PypiWebService @@ -50,6 +57,24 @@ import org.springframework.web.bind.annotation.RestController class PypiWebResourceController( private val pypiWebService: PypiWebService ) { + + @AuditEntry( + actionId = REPO_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = REPO_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#pypiArtifactInfo?.repoName", + instanceNames = "#pypiArtifactInfo?.repoName" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#pypiArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.NAME_TEMPLATE, value = "#packageKey") + ], + scopeId = "#pypiArtifactInfo?.projectId", + content = ActionAuditContent.REPO_PACKAGE_DELETE_CONTENT + ) @ApiOperation("pypi包删除接口") @DeleteMapping(PypiArtifactInfo.PYPI_EXT_PACKAGE_DELETE) fun deletePackage(pypiArtifactInfo: PypiArtifactInfo, packageKey: String): Response { @@ -57,6 +82,25 @@ class PypiWebResourceController( return ResponseBuilder.success() } + @AuditEntry( + actionId = REPO_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = REPO_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#pypiArtifactInfo?.repoName", + instanceNames = "#pypiArtifactInfo?.repoName" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#pypiArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.NAME_TEMPLATE, value = "#packageKey"), + AuditAttribute(name = ActionAuditContent.VERSION_TEMPLATE, value = "#version") + + ], + scopeId = "#pypiArtifactInfo?.projectId", + content = ActionAuditContent.REPO_PACKAGE_VERSION_DELETE_CONTENT + ) @ApiOperation("pypi版本删除接口") @DeleteMapping(PypiArtifactInfo.PYPI_EXT_VERSION_DELETE) fun deleteVersion( diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/controller/api/EdgePullReplicaTaskController.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/controller/api/EdgePullReplicaTaskController.kt index 89c89256d8..ae3245e83f 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/controller/api/EdgePullReplicaTaskController.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/controller/api/EdgePullReplicaTaskController.kt @@ -30,7 +30,7 @@ package com.tencent.bkrepo.replication.controller.api import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.common.api.util.Preconditions -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.replication.pojo.request.ReplicaType import com.tencent.bkrepo.replication.pojo.task.ReplicaTaskInfo diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/controller/api/RemoteDistributionController.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/controller/api/RemoteDistributionController.kt index 6ad71220f6..acf4006a0c 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/controller/api/RemoteDistributionController.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/controller/api/RemoteDistributionController.kt @@ -27,9 +27,16 @@ package com.tencent.bkrepo.replication.controller.api +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.REPO_EDIT_ACTION +import com.tencent.bkrepo.common.artifact.audit.REPO_RESOURCE import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.replication.pojo.cluster.ClusterNodeInfo @@ -57,6 +64,7 @@ import org.springframework.web.bind.annotation.RestController class RemoteDistributionController( private val remoteNodeService: RemoteNodeService ) { + @ApiOperation("创建远端集群节点") @Permission(ResourceType.REPO, PermissionAction.WRITE) @PostMapping("/create/{projectId}/{repoName}") @@ -150,9 +158,27 @@ class RemoteDistributionController( return ResponseBuilder.success() } + /** * 创建一次性分发任务 */ + @AuditEntry( + actionId = REPO_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = REPO_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#repoName", + instanceNames = "#repoName" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#repoName"), + ], + scopeId = "#projectId", + content = ActionAuditContent.REPO_REPLICATION_CREATE_CONTENT + ) @Permission(ResourceType.REPO, PermissionAction.WRITE) @PostMapping("/create/runOnceTask/{projectId}/{repoName}") fun createRunOnceTask( @@ -166,9 +192,28 @@ class RemoteDistributionController( return ResponseBuilder.success() } + /** * 手动调用一次性执行任务 */ + @AuditEntry( + actionId = REPO_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = REPO_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#repoName", + instanceNames = "#repoName" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#repoName"), + AuditAttribute(name = ActionAuditContent.NAME_TEMPLATE, value = "#name"), + ], + scopeId = "#projectId", + content = ActionAuditContent.REPO_REPLICATION_EXECUTE_CONTENT + ) @Permission(ResourceType.REPO, PermissionAction.WRITE) @PostMapping("/execute/runOnceTask/{projectId}/{repoName}") fun executeRunOnceTask( diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/controller/service/ArtifactReplicaController.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/controller/service/ArtifactReplicaController.kt index 0f0b9109e7..7dc0b64bec 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/controller/service/ArtifactReplicaController.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/controller/service/ArtifactReplicaController.kt @@ -30,6 +30,7 @@ package com.tencent.bkrepo.replication.controller.service import com.tencent.bkrepo.auth.api.ServiceUserClient import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.metadata.service.metadata.MetadataService import com.tencent.bkrepo.common.metadata.service.node.NodeService @@ -37,7 +38,6 @@ import com.tencent.bkrepo.common.metadata.service.packages.PackageService import com.tencent.bkrepo.common.metadata.service.project.ProjectService import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.security.exception.PermissionException -import com.tencent.bkrepo.common.security.manager.PermissionManager import com.tencent.bkrepo.common.security.permission.Principal import com.tencent.bkrepo.common.security.permission.PrincipalType import com.tencent.bkrepo.common.service.util.ResponseBuilder diff --git a/src/backend/replication/biz-replication/src/test/kotlin/com/tencent/bkrepo/replication/fdtp/FdtpAFTTest.kt b/src/backend/replication/biz-replication/src/test/kotlin/com/tencent/bkrepo/replication/fdtp/FdtpAFTTest.kt index 8402ab5397..40ee61c8a4 100644 --- a/src/backend/replication/biz-replication/src/test/kotlin/com/tencent/bkrepo/replication/fdtp/FdtpAFTTest.kt +++ b/src/backend/replication/biz-replication/src/test/kotlin/com/tencent/bkrepo/replication/fdtp/FdtpAFTTest.kt @@ -36,6 +36,8 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactClient import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder import com.tencent.bkrepo.common.artifact.repository.proxy.ProxyRepository import com.tencent.bkrepo.common.artifact.resolve.file.ArtifactFileFactory +import com.tencent.bkrepo.common.ratelimiter.config.RateLimiterProperties +import com.tencent.bkrepo.common.ratelimiter.service.RequestLimitCheckService import com.tencent.bkrepo.common.security.http.core.HttpAuthSecurity import com.tencent.bkrepo.common.security.service.ServiceAuthManager import com.tencent.bkrepo.common.security.service.ServiceAuthProperties @@ -51,11 +53,7 @@ import io.micrometer.core.instrument.simple.SimpleMeterRegistry import io.mockk.every import io.mockk.mockk import io.mockk.mockkObject -import org.junit.jupiter.api.AfterAll -import org.junit.jupiter.api.Assertions -import org.junit.jupiter.api.BeforeAll -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.* import org.mockito.Mockito import org.springframework.cloud.loadbalancer.support.SimpleObjectProvider import org.springframework.cloud.sleuth.Tracer @@ -95,6 +93,7 @@ class FdtpAFTTest { val proxyRepository = Mockito.mock(ProxyRepository::class.java) val artifactClient = Mockito.mock(ArtifactClient::class.java) val httpAuthSecurity = SimpleObjectProvider(null) + val limitCheckService = RequestLimitCheckService(RateLimiterProperties()) ArtifactContextHolder( listOf(artifactConfigurer), compositeRepository, @@ -103,7 +102,7 @@ class FdtpAFTTest { httpAuthSecurity, ) val helper = StorageHealthMonitorHelper(ConcurrentHashMap()) - ArtifactFileFactory(StorageProperties(), helper) + ArtifactFileFactory(StorageProperties(), helper, limitCheckService) mockkObject(ArtifactMetrics) every { ArtifactMetrics.getUploadingCounters(any()) } returns emptyList() every { ArtifactMetrics.getUploadingTimer(any()) } returns Timer.builder(ARTIFACT_UPLOADING_TIME) diff --git a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/PipelineNodeClient.kt b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/PipelineNodeClient.kt index 656b7fa95a..c77f742ce9 100644 --- a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/PipelineNodeClient.kt +++ b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/PipelineNodeClient.kt @@ -40,6 +40,7 @@ import org.springframework.web.bind.annotation.RequestMapping @Api("流水线节点") @FeignClient(REPOSITORY_SERVICE_NAME, contextId = "PipelineNodeClient", primary = false) @RequestMapping("/service/pipeline") +@Deprecated("replace with PipelineNodeService") interface PipelineNodeClient { @GetMapping("/list/{projectId}/{repoName}") @ApiOperation("获取流水线制品目录") diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterMetadataController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterMetadataController.kt index 17a2a9e0b2..315e1573cc 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterMetadataController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterMetadataController.kt @@ -29,12 +29,12 @@ package com.tencent.bkrepo.repository.controller.cluster import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.common.api.pojo.Response -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.permission.PermissionManager +import com.tencent.bkrepo.common.metadata.service.metadata.MetadataService import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.repository.api.cluster.ClusterMetadataClient import com.tencent.bkrepo.repository.pojo.metadata.MetadataDeleteRequest import com.tencent.bkrepo.repository.pojo.metadata.MetadataSaveRequest -import com.tencent.bkrepo.common.metadata.service.metadata.MetadataService import org.springframework.web.bind.annotation.RestController @RestController diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterNodeController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterNodeController.kt index 91cf25648e..8b999777b8 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterNodeController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterNodeController.kt @@ -31,7 +31,7 @@ import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.path.PathUtils -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.repository.api.cluster.ClusterNodeClient import com.tencent.bkrepo.repository.pojo.node.NodeDeleteResult diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterPackageController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterPackageController.kt index fb4869ff69..fa7b85451d 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterPackageController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterPackageController.kt @@ -33,7 +33,7 @@ package com.tencent.bkrepo.repository.controller.cluster import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.common.api.pojo.Response -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.repository.api.cluster.ClusterPackageClient import com.tencent.bkrepo.repository.pojo.packages.request.PackageUpdateRequest diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterPackageDependentsController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterPackageDependentsController.kt index 9a5740bf06..a8da78e8a9 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterPackageDependentsController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterPackageDependentsController.kt @@ -33,7 +33,7 @@ package com.tencent.bkrepo.repository.controller.cluster import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.common.api.pojo.Response -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.repository.api.cluster.ClusterPackageDependentsClient import com.tencent.bkrepo.repository.pojo.dependent.PackageDependentsRelation diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterRepositoryController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterRepositoryController.kt index 04fc3a25df..ba7db7ffa3 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterRepositoryController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterRepositoryController.kt @@ -29,14 +29,14 @@ package com.tencent.bkrepo.repository.controller.cluster import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.common.api.pojo.Response -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.permission.PermissionManager +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.repository.api.cluster.ClusterRepositoryClient import com.tencent.bkrepo.repository.pojo.repo.RepoCreateRequest import com.tencent.bkrepo.repository.pojo.repo.RepoDeleteRequest import com.tencent.bkrepo.repository.pojo.repo.RepoUpdateRequest import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail -import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import org.springframework.web.bind.annotation.RestController @RestController diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterStageController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterStageController.kt index 4018e25219..d60a85ca9c 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterStageController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/cluster/ClusterStageController.kt @@ -33,7 +33,7 @@ package com.tencent.bkrepo.repository.controller.cluster import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.common.api.pojo.Response -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.repository.api.cluster.ClusterStageClient import com.tencent.bkrepo.repository.pojo.stage.StageUpgradeRequest diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/PipelineNodeController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/PipelineNodeController.kt index 4bc387d177..6f58179144 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/PipelineNodeController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/PipelineNodeController.kt @@ -32,7 +32,7 @@ import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.repository.api.PipelineNodeClient import com.tencent.bkrepo.repository.pojo.node.NodeInfo -import com.tencent.bkrepo.repository.service.node.PipelineNodeService +import com.tencent.bkrepo.common.metadata.service.node.PipelineNodeService import org.springframework.context.annotation.Primary import org.springframework.web.bind.annotation.RestController diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/FavoriteController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/FavoriteController.kt index 1b044b0f30..0aeeb57679 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/FavoriteController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/FavoriteController.kt @@ -36,13 +36,13 @@ import com.tencent.bkrepo.common.api.constant.ANONYMOUS_USER import com.tencent.bkrepo.common.api.constant.HttpStatus import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.api.pojo.Response -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.service.util.ResponseBuilder -import com.tencent.bkrepo.repository.pojo.favorite.FavoriteRequest -import com.tencent.bkrepo.repository.pojo.favorite.FavoriteType import com.tencent.bkrepo.repository.pojo.favorite.FavoriteCreateRequest import com.tencent.bkrepo.repository.pojo.favorite.FavoriteQueryRequest +import com.tencent.bkrepo.repository.pojo.favorite.FavoriteRequest import com.tencent.bkrepo.repository.pojo.favorite.FavoriteResult +import com.tencent.bkrepo.repository.pojo.favorite.FavoriteType import com.tencent.bkrepo.repository.service.favorites.FavoriteService import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation @@ -117,4 +117,4 @@ class FavoriteController( return ResponseBuilder.success(favoriteService.queryFavorite(userId, request)) } -} \ No newline at end of file +} diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserArtifactPreloadController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserArtifactPreloadController.kt index 166d8b4ea3..b4be322fa6 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserArtifactPreloadController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserArtifactPreloadController.kt @@ -42,8 +42,8 @@ import com.tencent.bkrepo.common.artifact.cache.pojo.ArtifactPreloadStrategyCrea import com.tencent.bkrepo.common.artifact.cache.pojo.ArtifactPreloadStrategyUpdateRequest import com.tencent.bkrepo.common.artifact.cache.service.ArtifactPreloadPlanService import com.tencent.bkrepo.common.artifact.cache.service.ArtifactPreloadStrategyService +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.mongo.dao.util.Pages -import com.tencent.bkrepo.common.security.manager.PermissionManager import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.ResponseBuilder diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserListViewController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserListViewController.kt index 790d350065..67f318bd32 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserListViewController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserListViewController.kt @@ -35,7 +35,7 @@ import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.api.DefaultArtifactInfo.Companion.DEFAULT_MAPPING_URI -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.common.security.permission.Principal import com.tencent.bkrepo.common.security.permission.PrincipalType diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserMetadataController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserMetadataController.kt index e0a6f332dd..7fccda0d93 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserMetadataController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserMetadataController.kt @@ -31,12 +31,21 @@ package com.tencent.bkrepo.repository.controller.user +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord +import com.tencent.bk.audit.context.ActionAuditContext import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable import com.tencent.bkrepo.common.artifact.api.DefaultArtifactInfo.Companion.DEFAULT_MAPPING_URI +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE +import com.tencent.bkrepo.common.artifact.audit.NODE_VIEW_ACTION +import com.tencent.bkrepo.common.artifact.audit.REPO_EDIT_ACTION import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.ResponseBuilder @@ -65,6 +74,23 @@ class UserMetadataController( private val metadataService: MetadataService ) { + @AuditEntry( + actionId = NODE_VIEW_ACTION + ) + @ActionAuditRecord( + actionId = NODE_VIEW_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName"), + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_METADATA_VIEW_CONTENT + ) @ApiOperation("查询元数据列表") @Permission(type = ResourceType.NODE, action = PermissionAction.READ) @GetMapping(DEFAULT_MAPPING_URI) @@ -77,6 +103,23 @@ class UserMetadataController( } } + @AuditEntry( + actionId = REPO_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = REPO_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName"), + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_METADATA_EDIT_CONTENT + ) @ApiOperation("创建/更新元数据列表") @Permission(type = ResourceType.NODE, action = PermissionAction.WRITE) @PostMapping(DEFAULT_MAPPING_URI) @@ -94,11 +137,29 @@ class UserMetadataController( nodeMetadata = metadataSaveRequest.nodeMetadata?.map { it.copy(system = false) }, operator = SecurityUtils.getUserId() ) + ActionAuditContext.current().setInstance(request) metadataService.saveMetadata(request) return ResponseBuilder.success() } } + @AuditEntry( + actionId = REPO_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = REPO_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName"), + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_METADATA_FORBID_CONTENT + ) @ApiOperation("创建/更新禁用元数据") @Permission(type = ResourceType.REPO, action = PermissionAction.UPDATE) @PostMapping("/forbid$DEFAULT_MAPPING_URI") @@ -114,11 +175,29 @@ class UserMetadataController( fullPath = getArtifactFullPath(), nodeMetadata = metadataSaveRequest.nodeMetadata ) + ActionAuditContext.current().setInstance(request) metadataService.addForbidMetadata(request) return ResponseBuilder.success() } } + @AuditEntry( + actionId = REPO_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = REPO_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName"), + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_METADATA_DELETE_CONTENT + ) @ApiOperation("删除元数据") @Permission(type = ResourceType.NODE, action = PermissionAction.DELETE) @DeleteMapping(DEFAULT_MAPPING_URI) @@ -135,6 +214,7 @@ class UserMetadataController( keyList = metadataDeleteRequest.keyList, operator = SecurityUtils.getUserId() ) + ActionAuditContext.current().setInstance(request) metadataService.deleteMetadata(request, false) return ResponseBuilder.success() } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserMetadataLabelController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserMetadataLabelController.kt index ba0ada74d7..b78b18d34b 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserMetadataLabelController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserMetadataLabelController.kt @@ -29,7 +29,7 @@ package com.tencent.bkrepo.repository.controller.user import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.common.api.pojo.Response -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.repository.pojo.metadata.label.MetadataLabelDetail import com.tencent.bkrepo.repository.pojo.metadata.label.MetadataLabelRequest diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserNodeController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserNodeController.kt index 6cabfe9e36..9f3d0f4790 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserNodeController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserNodeController.kt @@ -31,6 +31,11 @@ package com.tencent.bkrepo.repository.controller.user +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord +import com.tencent.bk.audit.context.ActionAuditContext import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.api.exception.ErrorCodeException @@ -39,15 +44,20 @@ import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable import com.tencent.bkrepo.common.artifact.api.DefaultArtifactInfo.Companion.DEFAULT_MAPPING_URI +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_CREATE_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_DELETE_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_EDIT_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE +import com.tencent.bkrepo.common.artifact.audit.NODE_VIEW_ACTION import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.path.PathUtils +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.query.model.QueryModel -import com.tencent.bkrepo.common.security.manager.PermissionManager import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.common.security.permission.Principal import com.tencent.bkrepo.common.security.permission.PrincipalType import com.tencent.bkrepo.common.service.util.ResponseBuilder -import com.tencent.bkrepo.repository.pojo.node.service.NodeArchiveRestoreRequest import com.tencent.bkrepo.repository.pojo.node.NodeDeleteResult import com.tencent.bkrepo.repository.pojo.node.NodeDeletedPoint import com.tencent.bkrepo.repository.pojo.node.NodeDetail @@ -56,6 +66,7 @@ import com.tencent.bkrepo.repository.pojo.node.NodeListOption import com.tencent.bkrepo.common.metadata.pojo.node.NodeRestoreOption import com.tencent.bkrepo.repository.pojo.node.NodeRestoreResult import com.tencent.bkrepo.repository.pojo.node.NodeSizeInfo +import com.tencent.bkrepo.repository.pojo.node.service.NodeArchiveRestoreRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeLinkRequest @@ -96,6 +107,23 @@ class UserNodeController( private val permissionManager: PermissionManager, ) { + @AuditEntry( + actionId = NODE_VIEW_ACTION + ) + @ActionAuditRecord( + actionId = NODE_VIEW_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_VIEW_CONTENT + ) @ApiOperation("根据路径查看节点详情") @Permission(type = ResourceType.NODE, action = PermissionAction.READ) @GetMapping(DEFAULT_MAPPING_URI/* Deprecated */, "/detail/$DEFAULT_MAPPING_URI") @@ -108,6 +136,23 @@ class UserNodeController( return ResponseBuilder.success(node) } + @AuditEntry( + actionId = NODE_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_CREATE_CONTENT + ) @ApiOperation("创建文件夹") @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) @PostMapping(DEFAULT_MAPPING_URI/* Deprecated */, "/mkdir/$DEFAULT_MAPPING_URI") @@ -124,11 +169,29 @@ class UserNodeController( overwrite = false, operator = userId, ) + ActionAuditContext.current().setInstance(createRequest) nodeService.createNode(createRequest) return ResponseBuilder.success() } } + @AuditEntry( + actionId = NODE_DELETE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DELETE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_DELETE_CONTENT + ) @ApiOperation("删除节点") @Permission(type = ResourceType.NODE, action = PermissionAction.DELETE) @DeleteMapping(DEFAULT_MAPPING_URI/* Deprecated */, "/delete/$DEFAULT_MAPPING_URI") @@ -143,10 +206,28 @@ class UserNodeController( fullPath = getArtifactFullPath(), operator = userId, ) + ActionAuditContext.current().setInstance(deleteRequest) return ResponseBuilder.success(nodeService.deleteNode(deleteRequest)) } } + @AuditEntry( + actionId = NODE_DELETE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DELETE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#fullPaths", + instanceNames = "#fullPaths" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#repoName") + ], + scopeId = "#projectId", + content = ActionAuditContent.NODE_DELETE_CONTENT + ) @ApiOperation("批量删除节点") @Permission(type = ResourceType.REPO, action = PermissionAction.DELETE) @DeleteMapping("/batch/{projectId}/{repoName}") @@ -164,6 +245,7 @@ class UserNodeController( fullPaths = fullPaths, operator = userId, ) + ActionAuditContext.current().setInstance(nodesDeleteRequest) return ResponseBuilder.success(nodeService.deleteNodes(nodesDeleteRequest)) } @@ -189,6 +271,25 @@ class UserNodeController( return ResponseBuilder.success(nodeService.countDeleteNodes(nodesDeleteRequest)) } + + @AuditEntry( + actionId = NODE_DELETE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DELETE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName"), + AuditAttribute(name = ActionAuditContent.DATE_TEMPLATE, value = "#date") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_CLEAN_CONTENT + ) @ApiOperation("清理最后访问时间早于{date}的文件节点") @Permission(type = ResourceType.NODE, action = PermissionAction.DELETE) @DeleteMapping("/clean/$DEFAULT_MAPPING_URI") @@ -237,6 +338,24 @@ class UserNodeController( } } + @AuditEntry( + actionId = NODE_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = NODE_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName"), + AuditAttribute(name = ActionAuditContent.EXPIRES_DYAS_TEMPLATE, value = "#request?.expires") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_EXPIRES_EDIT_CONTENT + ) @ApiOperation("更新节点") @Permission(type = ResourceType.NODE, action = PermissionAction.UPDATE) @PostMapping("/update/$DEFAULT_MAPPING_URI") @@ -258,6 +377,24 @@ class UserNodeController( } } + @AuditEntry( + actionId = NODE_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = NODE_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName"), + AuditAttribute(name = ActionAuditContent.NAME_TEMPLATE, value = "#newFullPath") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_RENAME_CONTENT + ) @ApiOperation("重命名节点") @Permission(type = ResourceType.NODE, action = PermissionAction.UPDATE) @PostMapping("/rename/$DEFAULT_MAPPING_URI") @@ -275,6 +412,7 @@ class UserNodeController( newFullPath = newFullPath, operator = userId, ) + ActionAuditContext.current().setInstance(renameRequest) nodeService.renameNode(renameRequest) return ResponseBuilder.success() } @@ -297,11 +435,35 @@ class UserNodeController( newFullPath = newFullPath, operator = userId, ) + ActionAuditContext.current().setInstance(renameRequest) nodeService.renameNode(renameRequest) return ResponseBuilder.success() } } + @AuditEntry( + actionId = NODE_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = NODE_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#request?.srcFullPath", + instanceNames = "#request?.srcFullPath" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#request?.srcProjectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#request?.srcRepoName"), + AuditAttribute(name = ActionAuditContent.NAME_TEMPLATE, value = "#request?.destFullPath"), + AuditAttribute( + name = ActionAuditContent.NEW_PROJECT_CODE_CONTENT_TEMPLATE, + value = "#request?.destProjectId" + ), + AuditAttribute(name = ActionAuditContent.NEW_REPO_NAME_CONTENT_TEMPLATE, value = "#request?.destRepoName"), + ], + scopeId = "#request?.srcProjectId", + content = ActionAuditContent.NODE_MOVE_CONTENT + ) @ApiOperation("移动节点") @PostMapping("/move") fun moveNode( @@ -320,11 +482,35 @@ class UserNodeController( overwrite = overwrite, operator = userId, ) + ActionAuditContext.current().setInstance(moveRequest) nodeService.moveNode(moveRequest) return ResponseBuilder.success() } } + @AuditEntry( + actionId = NODE_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#request?.srcFullPath", + instanceNames = "#request?.srcFullPath" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#request?.srcProjectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#request?.srcRepoName"), + AuditAttribute( + name = ActionAuditContent.NEW_PROJECT_CODE_CONTENT_TEMPLATE, + value = "#request?.destProjectId" + ), + AuditAttribute(name = ActionAuditContent.NEW_REPO_NAME_CONTENT_TEMPLATE, value = "#request?.destRepoName"), + AuditAttribute(name = ActionAuditContent.NAME_TEMPLATE, value = "#request?.destFullPath") + ], + scopeId = "#request?.srcProjectId", + content = ActionAuditContent.NODE_COPY_CONTENT + ) @ApiOperation("复制节点") @PostMapping("/copy") fun copyNode( @@ -343,6 +529,7 @@ class UserNodeController( overwrite = overwrite, operator = userId, ) + ActionAuditContext.current().setInstance(copyRequest) nodeService.copyNode(copyRequest) return ResponseBuilder.success() } @@ -399,6 +586,24 @@ class UserNodeController( return ResponseBuilder.success(nodeService.listDeletedPoint(artifactInfo)) } + @AuditEntry( + actionId = NODE_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = NODE_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName"), + AuditAttribute(name = ActionAuditContent.DATE_TEMPLATE, value = "#nodeRestoreOption?.deletedId") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_RESTORE_CONTENT + ) @ApiOperation("恢复被删除节点") @Permission(type = ResourceType.NODE, action = PermissionAction.WRITE) @PostMapping("/restore/$DEFAULT_MAPPING_URI") @@ -406,6 +611,7 @@ class UserNodeController( artifactInfo: ArtifactInfo, nodeRestoreOption: NodeRestoreOption, ): Response { + ActionAuditContext.current().setInstance(nodeRestoreOption) return ResponseBuilder.success(nodeService.restoreNode(artifactInfo, nodeRestoreOption)) } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserOperateLogController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserOperateLogController.kt index c5b5c7c3dc..31e244073c 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserOperateLogController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserOperateLogController.kt @@ -30,12 +30,12 @@ package com.tencent.bkrepo.repository.controller.user import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.api.pojo.Response -import com.tencent.bkrepo.common.metadata.service.log.OperateLogService +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.metadata.pojo.log.OpLogListOption import com.tencent.bkrepo.common.metadata.pojo.log.OperateLog import com.tencent.bkrepo.common.metadata.pojo.log.OperateLogResponse +import com.tencent.bkrepo.common.metadata.service.log.OperateLogService import com.tencent.bkrepo.common.security.exception.PermissionException -import com.tencent.bkrepo.common.security.manager.PermissionManager import com.tencent.bkrepo.common.security.permission.PrincipalType import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.ResponseBuilder diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserPackageDownloadsController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserPackageDownloadsController.kt index 5a4c33a046..a228f0509e 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserPackageDownloadsController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserPackageDownloadsController.kt @@ -33,7 +33,7 @@ package com.tencent.bkrepo.repository.controller.user import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.common.api.pojo.Response -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.repository.pojo.download.DetailsQueryRequest import com.tencent.bkrepo.repository.pojo.download.PackageDownloadsDetails diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserPipelineController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserPipelineController.kt index b92c2712d1..9b62dc4c6c 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserPipelineController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserPipelineController.kt @@ -34,7 +34,7 @@ package com.tencent.bkrepo.repository.controller.user import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.repository.pojo.node.NodeInfo -import com.tencent.bkrepo.repository.service.node.PipelineNodeService +import com.tencent.bkrepo.common.metadata.service.node.PipelineNodeService import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.PathVariable import org.springframework.web.bind.annotation.RequestAttribute diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserProjectController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserProjectController.kt index b503ee6652..bf4d2e06a8 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserProjectController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserProjectController.kt @@ -27,10 +27,20 @@ package com.tencent.bkrepo.repository.controller.user +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord +import com.tencent.bk.audit.annotations.AuditRequestBody +import com.tencent.bk.audit.context.ActionAuditContext +import com.tencent.bkrepo.common.artifact.audit.PROJECT_CREATE_ACTION +import com.tencent.bkrepo.common.artifact.audit.PROJECT_EDIT_ACTION +import com.tencent.bkrepo.common.artifact.audit.PROJECT_RESOURCE import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.api.pojo.Response -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.permission.PermissionManager +import com.tencent.bkrepo.common.metadata.service.project.ProjectService +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent import com.tencent.bkrepo.common.security.permission.Principal import com.tencent.bkrepo.common.security.permission.PrincipalType import com.tencent.bkrepo.common.service.util.ResponseBuilder @@ -41,7 +51,6 @@ import com.tencent.bkrepo.repository.pojo.project.ProjectMetricsInfo import com.tencent.bkrepo.repository.pojo.project.ProjectSearchOption import com.tencent.bkrepo.repository.pojo.project.ProjectUpdateRequest import com.tencent.bkrepo.repository.pojo.project.UserProjectCreateRequest -import com.tencent.bkrepo.common.metadata.service.project.ProjectService import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation import io.swagger.annotations.ApiParam @@ -62,11 +71,25 @@ class UserProjectController( private val permissionManager: PermissionManager, private val projectService: ProjectService ) { + @AuditEntry( + actionId = PROJECT_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = PROJECT_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = PROJECT_RESOURCE, + instanceIds = "#userProjectRequest?.name", + instanceNames = "#userProjectRequest?.displayName" + ), + scopeId = "#userProjectRequest?.name", + content = ActionAuditContent.PROJECT_CREATE_CONTENT + ) @ApiOperation("创建项目") @Principal(PrincipalType.GENERAL) @PostMapping("/create") fun createProject( @RequestAttribute userId: String, + @AuditRequestBody @RequestBody userProjectRequest: UserProjectCreateRequest ): Response { val createRequest = with(userProjectRequest) { @@ -79,6 +102,7 @@ class UserProjectController( metadata = metadata ) } + ActionAuditContext.current().setInstance(createRequest) projectService.createProject(createRequest) return ResponseBuilder.success() } @@ -106,14 +130,30 @@ class UserProjectController( return ResponseBuilder.success(projectService.checkProjectExist(name, displayName)) } + + @AuditEntry( + actionId = PROJECT_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = PROJECT_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = PROJECT_RESOURCE, + instanceIds = "#name", + instanceNames = "#name" + ), + scopeId = "#name", + content = ActionAuditContent.PROJECT_EDIT_CONTENT + ) @ApiOperation("编辑项目") @PutMapping("/{name}") fun updateProject( @RequestAttribute userId: String, @ApiParam(value = "项目ID", required = true) @PathVariable name: String, + @AuditRequestBody @RequestBody projectUpdateRequest: ProjectUpdateRequest ): Response { + ActionAuditContext.current().setInstance(projectUpdateRequest) permissionManager.checkProjectPermission(PermissionAction.UPDATE, name) return ResponseBuilder.success(projectService.updateProject(name, projectUpdateRequest)) } @@ -134,6 +174,19 @@ class UserProjectController( return ResponseBuilder.success(projectService.listPermissionProject(userId, option)) } + @AuditEntry( + actionId = PROJECT_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = PROJECT_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = PROJECT_RESOURCE, + instanceIds = "#userProjectRequest?.name", + instanceNames = "#userProjectRequest?.displayName" + ), + scopeId = "#userProjectRequest?.name", + content = ActionAuditContent.PROJECT_CREATE_CONTENT + ) @Deprecated("waiting kb-ci", replaceWith = ReplaceWith("createProject")) @ApiOperation("创建项目") @Principal(PrincipalType.PLATFORM) diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserRepositoryController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserRepositoryController.kt index a3a6c87556..95599a3c17 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserRepositoryController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserRepositoryController.kt @@ -27,13 +27,28 @@ package com.tencent.bkrepo.repository.controller.user +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord +import com.tencent.bk.audit.annotations.AuditRequestBody +import com.tencent.bk.audit.context.ActionAuditContext +import com.tencent.bkrepo.common.artifact.audit.PROJECT_RESOURCE +import com.tencent.bkrepo.common.artifact.audit.PROJECT_VIEW_ACTION +import com.tencent.bkrepo.common.artifact.audit.REPO_CREATE_ACTION +import com.tencent.bkrepo.common.artifact.audit.REPO_DELETE_ACTION +import com.tencent.bkrepo.common.artifact.audit.REPO_EDIT_ACTION +import com.tencent.bkrepo.common.artifact.audit.REPO_RESOURCE +import com.tencent.bkrepo.common.artifact.audit.REPO_VIEW_ACTION import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.metadata.permission.PermissionManager +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent import com.tencent.bkrepo.common.metadata.service.node.NodeService import com.tencent.bkrepo.common.metadata.service.repo.QuotaService -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.repository.pojo.repo.ArchiveInfo @@ -46,7 +61,6 @@ import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail import com.tencent.bkrepo.repository.pojo.repo.RepositoryInfo import com.tencent.bkrepo.repository.pojo.repo.UserRepoCreateRequest import com.tencent.bkrepo.repository.pojo.repo.UserRepoUpdateRequest -import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation import io.swagger.annotations.ApiParam @@ -70,6 +84,22 @@ class UserRepositoryController( private val nodeService: NodeService ) { + @AuditEntry( + actionId = REPO_VIEW_ACTION + ) + @ActionAuditRecord( + actionId = REPO_VIEW_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#repoName", + instanceNames = "#repoName" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#projectId"), + ], + scopeId = "#projectId", + content = ActionAuditContent.REPO_VIEW_CONTENT + ) @ApiOperation("根据名称类型查询仓库") @Permission(type = ResourceType.REPO, action = PermissionAction.READ) @GetMapping("/info/{projectId}/{repoName}", "/info/{projectId}/{repoName}/{type}") @@ -100,10 +130,30 @@ class UserRepositoryController( return ResponseBuilder.success(repositoryService.checkExist(projectId, repoName)) } + @AuditEntry( + actionId = REPO_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = REPO_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#userRepoCreateRequest?.name", + instanceNames = "#userRepoCreateRequest?.name" + ), + attributes = [ + AuditAttribute( + name = ActionAuditContent.PROJECT_CODE_TEMPLATE, + value = "#userRepoCreateRequest?.projectId" + ), + ], + scopeId = "#userRepoCreateRequest?.projectId", + content = ActionAuditContent.REPO_CREATE_CONTENT + ) @ApiOperation("创建仓库") @PostMapping("/create") fun createRepo( @RequestAttribute userId: String, + @AuditRequestBody @RequestBody userRepoCreateRequest: UserRepoCreateRequest, ): Response { val createRequest = with(userRepoCreateRequest) { @@ -126,9 +176,26 @@ class UserRepositoryController( display = display, ) } + ActionAuditContext.current().setInstance(createRequest) return ResponseBuilder.success(repositoryService.createRepo(createRequest)) } + @AuditEntry( + actionId = REPO_VIEW_ACTION + ) + @ActionAuditRecord( + actionId = REPO_VIEW_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#repoListOption?.name", + instanceNames = "#repoListOption?.name", + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#projectId"), + ], + scopeId = "#projectId", + content = ActionAuditContent.REPO_LIST_CONTENT + ) @ApiOperation("查询有权限的仓库列表") @GetMapping("/list/{projectId}") fun listUserRepo( @@ -141,6 +208,22 @@ class UserRepositoryController( return ResponseBuilder.success(repositoryService.listPermissionRepo(userId, projectId, repoListOption)) } + @AuditEntry( + actionId = PROJECT_VIEW_ACTION + ) + @ActionAuditRecord( + actionId = PROJECT_VIEW_ACTION, + instance = AuditInstanceRecord( + resourceType = PROJECT_RESOURCE, + instanceIds = "#projectId", + instanceNames = "#projectId", + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#projectId"), + ], + scopeId = "#projectId", + content = ActionAuditContent.REPO_LIST_CONTENT + ) @ApiOperation("分页查询有权限的仓库列表") @GetMapping("/page/{projectId}/{pageNumber}/{pageSize}") fun listUserRepoPage( @@ -160,6 +243,22 @@ class UserRepositoryController( return ResponseBuilder.success(page) } + @AuditEntry( + actionId = REPO_VIEW_ACTION + ) + @ActionAuditRecord( + actionId = REPO_VIEW_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#repoName", + instanceNames = "#repoName", + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#projectId"), + ], + scopeId = "#projectId", + content = ActionAuditContent.REPO_QUOTE_VIEW_CONTENT + ) @ApiOperation("查询仓库配额") @Permission(type = ResourceType.REPO, action = PermissionAction.READ) @GetMapping("/quota/{projectId}/{repoName}") @@ -174,6 +273,23 @@ class UserRepositoryController( return ResponseBuilder.success(quotaService.getRepoQuotaInfo(projectId, repoName)) } + + @AuditEntry( + actionId = REPO_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = REPO_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#repoName", + instanceNames = "#repoName", + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#projectId"), + ], + scopeId = "#projectId", + content = ActionAuditContent.REPO_QUOTE_EDIT_CONTENT + ) @ApiOperation("修改仓库配额") @Permission(type = ResourceType.REPO, action = PermissionAction.MANAGE) @PostMapping("/quota/{projectId}/{repoName}") @@ -195,10 +311,27 @@ class UserRepositoryController( quota = quota, operator = userId, ) + ActionAuditContext.current().setInstance(repoUpdateRequest) repositoryService.updateRepo(repoUpdateRequest) return ResponseBuilder.success() } + @AuditEntry( + actionId = REPO_DELETE_ACTION + ) + @ActionAuditRecord( + actionId = REPO_DELETE_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#repoName", + instanceNames = "#repoName", + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#projectId"), + ], + scopeId = "#projectId", + content = ActionAuditContent.REPO_DELETE_CONTENT + ) @ApiOperation("删除仓库") @Permission(type = ResourceType.REPO, action = PermissionAction.DELETE) @DeleteMapping("/delete/{projectId}/{repoName}") @@ -218,6 +351,22 @@ class UserRepositoryController( return ResponseBuilder.success() } + @AuditEntry( + actionId = REPO_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = REPO_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#repoName", + instanceNames = "#repoName", + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#projectId"), + ], + scopeId = "#projectId", + content = ActionAuditContent.REPO_EDIT_CONTENT + ) @ApiOperation("更新仓库") @Permission(type = ResourceType.REPO, action = PermissionAction.MANAGE) @PostMapping("/update/{projectId}/{repoName}") @@ -240,10 +389,29 @@ class UserRepositoryController( operator = userId, display = request.display ) + ActionAuditContext.current().setInstance(repoUpdateRequest) repositoryService.updateRepo(repoUpdateRequest) return ResponseBuilder.success() } + @AuditEntry( + actionId = REPO_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = REPO_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#userRepoCreateRequest?.name", + instanceNames = "#userRepoCreateRequest?.name" + ), + attributes = [ + AuditAttribute( + name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#userRepoCreateRequest?.projectId" + ), + ], + scopeId = "#userRepoCreateRequest?.projectId", + content = ActionAuditContent.REPO_CREATE_CONTENT + ) @Deprecated("waiting kb-ci and bk", replaceWith = ReplaceWith("createRepo")) @ApiOperation("创建仓库") @PostMapping diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserShareController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserShareController.kt index 41a3651790..6c7a04552a 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserShareController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserShareController.kt @@ -31,6 +31,14 @@ package com.tencent.bkrepo.repository.controller.user +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord +import com.tencent.bk.audit.context.ActionAuditContext +import com.tencent.bkrepo.common.artifact.audit.NODE_DOWNLOAD_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE +import com.tencent.bkrepo.common.artifact.audit.NODE_VIEW_ACTION import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.api.pojo.Response @@ -39,7 +47,8 @@ import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable import com.tencent.bkrepo.common.artifact.api.DefaultArtifactInfo import com.tencent.bkrepo.common.artifact.api.DefaultArtifactInfo.Companion.DEFAULT_MAPPING_URI import com.tencent.bkrepo.common.artifact.path.PathUtils -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.permission.PermissionManager +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent import com.tencent.bkrepo.common.security.permission.Permission import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.repository.pojo.share.BatchShareRecordCreateRequest @@ -67,6 +76,23 @@ class UserShareController( private val shareService: ShareService ) { + @AuditEntry( + actionId = NODE_VIEW_ACTION + ) + @ActionAuditRecord( + actionId = NODE_VIEW_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_SHARE_CREATE_CONTENT + ) @ApiOperation("创建分享链接") @Permission(type = ResourceType.NODE, action = PermissionAction.WRITE) @PostMapping(DEFAULT_MAPPING_URI) @@ -75,9 +101,33 @@ class UserShareController( @ArtifactPathVariable artifactInfo: ArtifactInfo, @RequestBody shareRecordCreateRequest: ShareRecordCreateRequest ): Response { + ActionAuditContext.current().setInstance(shareRecordCreateRequest) return ResponseBuilder.success(shareService.create(userId, artifactInfo, shareRecordCreateRequest)) } + @AuditEntry( + actionId = NODE_VIEW_ACTION + ) + @ActionAuditRecord( + actionId = NODE_VIEW_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#batchShareRecordCreateRequest?.fullPathList", + instanceNames = "#batchShareRecordCreateRequest?.fullPathList", + ), + attributes = [ + AuditAttribute( + name = ActionAuditContent.PROJECT_CODE_TEMPLATE, + value = "#batchShareRecordCreateRequest?.projectId" + ), + AuditAttribute( + name = ActionAuditContent.REPO_NAME_TEMPLATE, + value = "#batchShareRecordCreateRequest?.repoName" + ) + ], + scopeId = "#batchShareRecordCreateRequest?.projectId", + content = ActionAuditContent.NODE_SHARE_CREATE_CONTENT + ) @ApiOperation("批量创建分享链接") @PostMapping("/batch") fun batchShare( @@ -85,6 +135,7 @@ class UserShareController( @RequestBody batchShareRecordCreateRequest: BatchShareRecordCreateRequest ): Response> { with(batchShareRecordCreateRequest) { + ActionAuditContext.current().setInstance(batchShareRecordCreateRequest) val shareRecordCreateRequest = ShareRecordCreateRequest(authorizedUserList, authorizedIpList, expireSeconds) val recordInfoList = fullPathList.map { val fullPath = PathUtils.normalizeFullPath(it) @@ -96,6 +147,33 @@ class UserShareController( } } + @AuditEntry( + actionId = NODE_DOWNLOAD_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DOWNLOAD_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()", + ), + attributes = [ + AuditAttribute( + name = ActionAuditContent.PROJECT_CODE_TEMPLATE, + value = "#artifactInfo?.projectId" + ), + AuditAttribute( + name = ActionAuditContent.REPO_NAME_TEMPLATE, + value = "#artifactInfo?.repoName" + ), + AuditAttribute( + name = ActionAuditContent.TOKEN_TEMPLATE, + value = "#token" + ) + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_SHARE_DOWNLOAD_CONTENT + ) @ApiOperation("下载分享文件") @GetMapping(DEFAULT_MAPPING_URI) fun download( @@ -105,6 +183,7 @@ class UserShareController( @ArtifactPathVariable artifactInfo: ArtifactInfo ) { val downloadUser = downloadUserId ?: userId + ActionAuditContext.current().addExtendData("downloadUser", downloadUser) shareService.download(downloadUser, token, artifactInfo) } } diff --git a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt index 657dec0907..dc990601b3 100644 --- a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt +++ b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt @@ -48,6 +48,7 @@ import com.tencent.bkrepo.common.metadata.dao.node.NodeDao import com.tencent.bkrepo.common.metadata.dao.project.ProjectDao import com.tencent.bkrepo.common.metadata.dao.repo.RepositoryDao import com.tencent.bkrepo.common.metadata.listener.ResourcePermissionListener +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.metadata.service.log.OperateLogService import com.tencent.bkrepo.common.metadata.service.project.ProjectService import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService @@ -55,7 +56,6 @@ import com.tencent.bkrepo.common.metadata.service.repo.ResourceClearService import com.tencent.bkrepo.common.metadata.util.RepositoryServiceHelper import com.tencent.bkrepo.common.metadata.util.StorageCredentialHelper import com.tencent.bkrepo.common.security.http.core.HttpAuthProperties -import com.tencent.bkrepo.common.security.manager.PermissionManager import com.tencent.bkrepo.common.security.manager.ci.CIPermissionManager import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties import com.tencent.bkrepo.common.service.util.ResponseBuilder diff --git a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/StorageCredentialServiceTest.kt b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/StorageCredentialServiceTest.kt index 68f0bd0843..829e7c750f 100644 --- a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/StorageCredentialServiceTest.kt +++ b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/StorageCredentialServiceTest.kt @@ -34,6 +34,7 @@ package com.tencent.bkrepo.repository.service import com.tencent.bkrepo.common.api.exception.BadRequestException import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.exception.NotFoundException +import com.tencent.bkrepo.common.artifact.constant.DEFAULT_STORAGE_KEY import com.tencent.bkrepo.common.metadata.dao.repo.StorageCredentialsDao import com.tencent.bkrepo.common.metadata.service.project.ProjectService import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService @@ -49,6 +50,7 @@ import com.tencent.bkrepo.repository.pojo.credendials.StorageCredentialsUpdateRe import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.DisplayName import org.junit.jupiter.api.Test @@ -195,15 +197,33 @@ internal class StorageCredentialServiceTest @Autowired constructor( assertEquals(1, list.size) } + @Test + fun testGetStorageKeyMapping() { + val key1 = "key1" + val key2 = "key2" + val key3 = "key3" + createCredential(key1) + createCredential(key2, filePath = "test2") + createCredential(key3) + + val mapping = storageCredentialService.getStorageKeyMapping() + assertEquals(4, mapping.size) + assertTrue(mapping[key1]!!.size == 1 && mapping[key1]!!.contains(key3)) + assertTrue(mapping[key2]!!.isEmpty()) + assertTrue(mapping[key3]!!.size == 1 && mapping[key3]!!.contains(key1)) + assertTrue(mapping[DEFAULT_STORAGE_KEY]!!.isEmpty()) + } + private fun createCredential( key: String = UT_STORAGE_CREDENTIALS_KEY, region: String = UT_REGION, - type: String = FileSystemCredentials.type + type: String = FileSystemCredentials.type, + filePath: String = "test" ): StorageCredentials { val credential = when (type) { FileSystemCredentials.type -> { FileSystemCredentials().apply { - path = "test" + path = filePath } } else -> throw RuntimeException("Unknown credential type: $type") diff --git a/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/artifact/repository/RpmLocalRepository.kt b/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/artifact/repository/RpmLocalRepository.kt index 7eeb794d74..2439905e6d 100644 --- a/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/artifact/repository/RpmLocalRepository.kt +++ b/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/artifact/repository/RpmLocalRepository.kt @@ -239,7 +239,7 @@ class RpmLocalRepository( rpmMetadata.packages[0].format.changeLogs ) ), - 1L + packageNum = 1L ) stopWatch.start("storeOthers") storeIndexMarkFile( @@ -256,7 +256,7 @@ class RpmLocalRepository( rpmMetadata.packages[0].format.files ) ), - 1L + packageNum = 1L ) stopWatch.start("storeFilelists") storeIndexMarkFile( diff --git a/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/controller/RpmResourceController.kt b/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/controller/RpmResourceController.kt index 61d04c4cc3..e1b8b981b5 100644 --- a/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/controller/RpmResourceController.kt +++ b/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/controller/RpmResourceController.kt @@ -31,8 +31,20 @@ package com.tencent.bkrepo.rpm.controller +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord +import com.tencent.bk.audit.context.ActionAuditContext import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_DELETE_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_DOWNLOAD_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE +import com.tencent.bkrepo.common.artifact.audit.NODE_CREATE_ACTION +import com.tencent.bkrepo.common.artifact.audit.REPO_EDIT_ACTION +import com.tencent.bkrepo.common.artifact.audit.REPO_RESOURCE import com.tencent.bkrepo.rpm.api.RpmResource import com.tencent.bkrepo.rpm.artifact.RpmArtifactInfo import com.tencent.bkrepo.rpm.servcie.RpmService @@ -43,22 +55,108 @@ import org.springframework.web.bind.annotation.RestController class RpmResourceController( private val rpmService: RpmService ) : RpmResource { + + @AuditEntry( + actionId = NODE_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#rpmArtifactInfo?.getArtifactFullPath()", + instanceNames = "#rpmArtifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#rpmArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#rpmArtifactInfo?.repoName") + ], + scopeId = "#rpmArtifactInfo?.projectId", + content = ActionAuditContent.NODE_UPLOAD_CONTENT + ) override fun deploy(rpmArtifactInfo: RpmArtifactInfo, artifactFile: ArtifactFile) { rpmService.deploy(rpmArtifactInfo, artifactFile) } + @AuditEntry( + actionId = NODE_DOWNLOAD_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DOWNLOAD_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#rpmArtifactInfo?.getArtifactFullPath()", + instanceNames = "#rpmArtifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#rpmArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#rpmArtifactInfo?.repoName") + ], + scopeId = "#rpmArtifactInfo?.projectId", + content = ActionAuditContent.NODE_DOWNLOAD_CONTENT + ) override fun install(rpmArtifactInfo: RpmArtifactInfo) { rpmService.install(rpmArtifactInfo) } + @AuditEntry( + actionId = REPO_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = REPO_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#rpmArtifactInfo?.repoName", + instanceNames = "#rpmArtifactInfo?.repoName" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#rpmArtifactInfo?.projectId") + ], + scopeId = "#rpmArtifactInfo?.projectId", + content = ActionAuditContent.REPO_EDIT_CONTENT + ) override fun addGroups(rpmArtifactInfo: RpmArtifactInfo, groups: MutableSet) { + ActionAuditContext.current().setInstance(groups) rpmService.addGroups(rpmArtifactInfo, groups) } + @AuditEntry( + actionId = REPO_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = REPO_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#rpmArtifactInfo?.repoName", + instanceNames = "#rpmArtifactInfo?.repoName" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#rpmArtifactInfo?.projectId") + ], + scopeId = "#rpmArtifactInfo?.projectId", + content = ActionAuditContent.REPO_EDIT_CONTENT + ) override fun deleteGroups(rpmArtifactInfo: RpmArtifactInfo, groups: MutableSet) { + ActionAuditContext.current().setInstance(groups) rpmService.deleteGroups(rpmArtifactInfo, groups) } + @AuditEntry( + actionId = NODE_DELETE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DELETE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#rpmArtifactInfo?.getArtifactFullPath()", + instanceNames = "#rpmArtifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#rpmArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#rpmArtifactInfo?.repoName") + ], + scopeId = "#rpmArtifactInfo?.projectId", + content = ActionAuditContent.NODE_DELETE_CONTENT + ) @DeleteMapping(RpmArtifactInfo.RPM) fun delete(@ArtifactPathVariable rpmArtifactInfo: RpmArtifactInfo) { rpmService.delete(rpmArtifactInfo) diff --git a/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/controller/RpmResourceWebController.kt b/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/controller/RpmResourceWebController.kt index f455e6d7ac..0e49f8e4cf 100644 --- a/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/controller/RpmResourceWebController.kt +++ b/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/controller/RpmResourceWebController.kt @@ -31,8 +31,15 @@ package com.tencent.bkrepo.rpm.controller +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.REPO_EDIT_ACTION +import com.tencent.bkrepo.common.artifact.audit.REPO_RESOURCE import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.rpm.api.RpmWebResource import com.tencent.bkrepo.rpm.artifact.RpmArtifactInfo @@ -46,11 +53,48 @@ import org.springframework.web.bind.annotation.RestController class RpmResourceWebController( private val rpmWebService: RpmWebService ) : RpmWebResource { + + @AuditEntry( + actionId = REPO_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = REPO_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#rpmArtifactInfo?.repoName", + instanceNames = "#rpmArtifactInfo?.repoName" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#rpmArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.NAME_TEMPLATE, value = "#packageKey") + ], + scopeId = "#rpmArtifactInfo?.projectId", + content = ActionAuditContent.REPO_PACKAGE_DELETE_CONTENT + ) override fun deletePackage(rpmArtifactInfo: RpmArtifactInfo, packageKey: String): Response { rpmWebService.deletePackage(rpmArtifactInfo, packageKey) return ResponseBuilder.success() } + @AuditEntry( + actionId = REPO_EDIT_ACTION + ) + @ActionAuditRecord( + actionId = REPO_EDIT_ACTION, + instance = AuditInstanceRecord( + resourceType = REPO_RESOURCE, + instanceIds = "#rpmArtifactInfo?.repoName", + instanceNames = "#rpmArtifactInfo?.repoName" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#rpmArtifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.NAME_TEMPLATE, value = "#packageKey"), + AuditAttribute(name = ActionAuditContent.VERSION_TEMPLATE, value = "#version") + + ], + scopeId = "#rpmArtifactInfo?.projectId", + content = ActionAuditContent.REPO_PACKAGE_VERSION_DELETE_CONTENT + ) override fun deleteVersion(rpmArtifactInfo: RpmArtifactInfo, packageKey: String, version: String?): Response { rpmWebService.delete(rpmArtifactInfo, packageKey, version) return ResponseBuilder.success() diff --git a/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/job/JobService.kt b/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/job/JobService.kt index dcbb404234..c8844e2d76 100644 --- a/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/job/JobService.kt +++ b/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/job/JobService.kt @@ -86,6 +86,7 @@ import java.io.File import java.io.FileInputStream import java.io.FileOutputStream import java.io.RandomAccessFile +import java.time.LocalDateTime import java.util.Locale import java.util.concurrent.ConcurrentHashMap import javax.xml.parsers.SAXParserFactory @@ -492,8 +493,8 @@ class JobService( return NodeInfo( createdBy = mapData["createdBy"] as String, createdDate = mapData["lastModifiedBy"] as String, - lastModifiedBy = mapData["lastModifiedDate"] as String, - lastModifiedDate = mapData["lastModifiedDate"] as String, + lastModifiedBy = (mapData["lastModifiedDate"] as LocalDateTime).toString(), + lastModifiedDate = (mapData["lastModifiedDate"] as LocalDateTime).toString(), folder = mapData["folder"] as Boolean, path = mapData["path"] as String, name = mapData["name"] as String, diff --git a/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/util/rpm/RpmMetadataUtils.kt b/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/util/rpm/RpmMetadataUtils.kt index 9d6643287c..66a40e5a63 100644 --- a/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/util/rpm/RpmMetadataUtils.kt +++ b/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/util/rpm/RpmMetadataUtils.kt @@ -113,7 +113,7 @@ object RpmMetadataUtils { ) ) ), - 1L + packageNum = 1L ) } diff --git a/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/util/xStream/pojo/RpmMetadata.kt b/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/util/xStream/pojo/RpmMetadata.kt index aff92c6597..340bb8ce88 100644 --- a/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/util/xStream/pojo/RpmMetadata.kt +++ b/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/util/xStream/pojo/RpmMetadata.kt @@ -40,14 +40,14 @@ class RpmMetadata( @XStreamImplicit(itemFieldName = "package") override val packages: List, @XStreamAsAttribute - @XStreamAlias("packages") - override var packageNum: Long -) : RpmXmlMetadata(packages, packageNum) { - @XStreamAsAttribute - val xmlns: String = "http://linux.duke.edu/metadata/common" + val xmlns: String = "http://linux.duke.edu/metadata/common", @XStreamAsAttribute @XStreamAlias("xmlns:rpm") - val rpm: String = "http://linux.duke.edu/metadata/rpm" + val rpm: String = "http://linux.duke.edu/metadata/rpm", + @XStreamAsAttribute + @XStreamAlias("packages") + override var packageNum: Long, +) : RpmXmlMetadata(packages, packageNum) { fun filterRpmFileLists() { packages[0].format.files = packages[0].format.files.filter { diff --git a/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/util/xStream/pojo/RpmMetadataChangeLog.kt b/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/util/xStream/pojo/RpmMetadataChangeLog.kt index 7555364c59..31cefef05d 100644 --- a/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/util/xStream/pojo/RpmMetadataChangeLog.kt +++ b/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/util/xStream/pojo/RpmMetadataChangeLog.kt @@ -40,9 +40,8 @@ class RpmMetadataChangeLog( @XStreamImplicit(itemFieldName = "package") override val packages: List, @XStreamAsAttribute + val xmlns: String = "http://linux.duke.edu/metadata/other", + @XStreamAsAttribute @XStreamAlias("packages") override var packageNum: Long -) : RpmXmlMetadata(packages, packageNum) { - @XStreamAsAttribute - val xmlns: String = "http://linux.duke.edu/metadata/other" -} +) : RpmXmlMetadata(packages, packageNum) diff --git a/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/util/xStream/pojo/RpmMetadataFileList.kt b/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/util/xStream/pojo/RpmMetadataFileList.kt index 5e3ceb6cf6..e263d81a2e 100644 --- a/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/util/xStream/pojo/RpmMetadataFileList.kt +++ b/src/backend/rpm/biz-rpm/src/main/kotlin/com/tencent/bkrepo/rpm/util/xStream/pojo/RpmMetadataFileList.kt @@ -40,9 +40,8 @@ class RpmMetadataFileList( @XStreamImplicit(itemFieldName = "package") override val packages: List, @XStreamAsAttribute + val xmlns: String = "http://linux.duke.edu/metadata/filelists", + @XStreamAsAttribute @XStreamAlias("packages") override var packageNum: Long -) : RpmXmlMetadata(packages, packageNum) { - @XStreamAsAttribute - val xmlns: String = "http://linux.duke.edu/metadata/filelists" -} +) : RpmXmlMetadata(packages, packageNum) diff --git a/src/backend/s3/biz-s3/src/main/kotlin/com/tencent/bkrepo/s3/artifact/configuration/ResourceWriterConfigurer.kt b/src/backend/s3/biz-s3/src/main/kotlin/com/tencent/bkrepo/s3/artifact/configuration/ResourceWriterConfigurer.kt index 402419b406..5a964cc4c8 100644 --- a/src/backend/s3/biz-s3/src/main/kotlin/com/tencent/bkrepo/s3/artifact/configuration/ResourceWriterConfigurer.kt +++ b/src/backend/s3/biz-s3/src/main/kotlin/com/tencent/bkrepo/s3/artifact/configuration/ResourceWriterConfigurer.kt @@ -33,17 +33,23 @@ package com.tencent.bkrepo.s3.artifact.configuration import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResourceWriter import com.tencent.bkrepo.common.storage.config.StorageProperties +import com.tencent.bkrepo.common.ratelimiter.service.RequestLimitCheckService import com.tencent.bkrepo.s3.artifact.response.S3ArtifactResourceWriter import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import org.springframework.context.annotation.Primary @Configuration -class ResourceWriterConfigurer{ +class ResourceWriterConfigurer { @Primary @Bean - fun artifactResourceWriter(storageProperties: StorageProperties): ArtifactResourceWriter { - return S3ArtifactResourceWriter(storageProperties) + fun artifactResourceWriter( + storageProperties: StorageProperties, + requestLimitCheckService: RequestLimitCheckService + ): ArtifactResourceWriter { + return S3ArtifactResourceWriter( + storageProperties, requestLimitCheckService + ) } } diff --git a/src/backend/s3/biz-s3/src/main/kotlin/com/tencent/bkrepo/s3/artifact/response/S3ArtifactResourceWriter.kt b/src/backend/s3/biz-s3/src/main/kotlin/com/tencent/bkrepo/s3/artifact/response/S3ArtifactResourceWriter.kt index 7c9eec2736..8fd9134352 100644 --- a/src/backend/s3/biz-s3/src/main/kotlin/com/tencent/bkrepo/s3/artifact/response/S3ArtifactResourceWriter.kt +++ b/src/backend/s3/biz-s3/src/main/kotlin/com/tencent/bkrepo/s3/artifact/response/S3ArtifactResourceWriter.kt @@ -38,6 +38,8 @@ import com.tencent.bkrepo.common.artifact.exception.ArtifactResponseException import com.tencent.bkrepo.common.artifact.resolve.response.AbstractArtifactResourceHandler import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.ratelimiter.service.RequestLimitCheckService import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.common.storage.config.StorageProperties import com.tencent.bkrepo.common.storage.monitor.Throughput @@ -51,13 +53,17 @@ import javax.servlet.http.HttpServletResponse /** * S3协议的响应输出 */ -class S3ArtifactResourceWriter ( - storageProperties: StorageProperties -) : AbstractArtifactResourceHandler(storageProperties) { +class S3ArtifactResourceWriter( + storageProperties: StorageProperties, + requestLimitCheckService: RequestLimitCheckService +) : AbstractArtifactResourceHandler( + storageProperties, requestLimitCheckService +) { - @Throws(ArtifactResponseException::class) + @Throws(ArtifactResponseException::class, OverloadException::class) override fun write(resource: ArtifactResource): Throughput { responseRateLimitCheck() + downloadRateLimitCheck(resource) return writeArtifact(resource) } diff --git a/src/backend/s3/biz-s3/src/main/kotlin/com/tencent/bkrepo/s3/service/S3ObjectService.kt b/src/backend/s3/biz-s3/src/main/kotlin/com/tencent/bkrepo/s3/service/S3ObjectService.kt index 2b9a4f9b0c..dd150e898d 100644 --- a/src/backend/s3/biz-s3/src/main/kotlin/com/tencent/bkrepo/s3/service/S3ObjectService.kt +++ b/src/backend/s3/biz-s3/src/main/kotlin/com/tencent/bkrepo/s3/service/S3ObjectService.kt @@ -31,6 +31,11 @@ package com.tencent.bkrepo.s3.service +import com.tencent.bk.audit.annotations.ActionAuditRecord +import com.tencent.bk.audit.annotations.AuditAttribute +import com.tencent.bk.audit.annotations.AuditEntry +import com.tencent.bk.audit.annotations.AuditInstanceRecord +import com.tencent.bk.audit.context.ActionAuditContext import com.tencent.bkrepo.common.api.constant.HttpStatus import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.api.constant.StringPool.SLASH @@ -42,6 +47,12 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHold import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadContext import com.tencent.bkrepo.common.artifact.repository.core.ArtifactService +import com.tencent.bkrepo.common.artifact.audit.ActionAuditContent +import com.tencent.bkrepo.common.artifact.audit.NODE_CREATE_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_DOWNLOAD_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_RESOURCE +import com.tencent.bkrepo.common.artifact.audit.NODE_VIEW_ACTION +import com.tencent.bkrepo.common.artifact.audit.NODE_CREATE_ACTION import com.tencent.bkrepo.common.generic.configuration.AutoIndexRepositorySettings import com.tencent.bkrepo.common.metadata.service.metadata.MetadataService import com.tencent.bkrepo.common.metadata.service.node.NodeSearchService @@ -80,6 +91,23 @@ class S3ObjectService( private val metadataService: MetadataService ) : ArtifactService() { + @AuditEntry( + actionId = NODE_DOWNLOAD_ACTION + ) + @ActionAuditRecord( + actionId = NODE_DOWNLOAD_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_DOWNLOAD_CONTENT + ) fun getObject(artifactInfo: S3ArtifactInfo) { val node = ArtifactContextHolder.getNodeDetail(artifactInfo) ?: throw S3NotFoundException( @@ -103,11 +131,46 @@ class S3ObjectService( repository.download(context) } + + @AuditEntry( + actionId = NODE_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_UPLOAD_CONTENT + ) fun putObject(artifactInfo: S3ArtifactInfo, file: ArtifactFile) { val context = ArtifactUploadContext(file) repository.upload(context) } + @AuditEntry( + actionId = NODE_VIEW_ACTION + ) + @ActionAuditRecord( + actionId = NODE_VIEW_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#prefix", + instanceNames = "#prefix" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_VIEW_CONTENT + ) fun listObjects( artifactInfo: S3ArtifactInfo, marker: Int, @@ -162,6 +225,24 @@ class S3ObjectService( return ListBucketResult(repoName, data, maxKeys, prefix, folders, delimiter, currentNode) } + @AuditEntry( + actionId = NODE_CREATE_ACTION + ) + @ActionAuditRecord( + actionId = NODE_CREATE_ACTION, + instance = AuditInstanceRecord( + resourceType = NODE_RESOURCE, + instanceIds = "#artifactInfo?.getArtifactFullPath()", + instanceNames = "#artifactInfo?.getArtifactFullPath()" + ), + attributes = [ + AuditAttribute(name = ActionAuditContent.PROJECT_CODE_TEMPLATE, value = "#artifactInfo?.projectId"), + AuditAttribute(name = ActionAuditContent.REPO_NAME_TEMPLATE, value = "#artifactInfo?.repoName"), + AuditAttribute(name = ActionAuditContent.NAME_TEMPLATE, value = "#artifactInfo?.getArtifactFullPath()") + ], + scopeId = "#artifactInfo?.projectId", + content = ActionAuditContent.NODE_COPY_CONTENT + ) fun copyObject(artifactInfo: S3ArtifactInfo): CopyObjectResult { val source = HeaderUtils.getHeader(X_AMZ_COPY_SOURCE) ?: throw IllegalArgumentException(X_AMZ_COPY_SOURCE) val delimiterIndex = source.indexOf(SLASH) @@ -177,6 +258,7 @@ class S3ObjectService( overwrite = true, operator = SecurityUtils.getUserId() ) + ActionAuditContext.current().setInstance(copyRequest) var dstNode = nodeService.copyNode(copyRequest) dstNode = replaceMetadata(dstNode) return CopyObjectResult( diff --git a/src/backend/settings.gradle.kts b/src/backend/settings.gradle.kts index 8792746a34..23128e5dcc 100644 --- a/src/backend/settings.gradle.kts +++ b/src/backend/settings.gradle.kts @@ -62,6 +62,7 @@ includeAll(":common:common-storage") includeAll(":common:common-query") includeAll(":common:common-artifact") includeAll(":common:common-notify") +includeAll(":common:common-ratelimiter") includeAll(":composer") includeAll(":generic") includeAll(":helm") diff --git a/src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/service/impl/WebHookServiceImpl.kt b/src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/service/impl/WebHookServiceImpl.kt index fcc74e953b..2a204cf073 100644 --- a/src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/service/impl/WebHookServiceImpl.kt +++ b/src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/service/impl/WebHookServiceImpl.kt @@ -33,7 +33,7 @@ import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.util.Preconditions import com.tencent.bkrepo.common.api.util.readJsonString import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.common.security.permission.PrincipalType import com.tencent.bkrepo.webhook.constant.AssociationType import com.tencent.bkrepo.webhook.constant.WebHookRequestStatus diff --git a/src/backend/webhook/biz-webhook/src/test/kotlin/com/tencent/bkrepo/webhook/service/ServiceBaseTest.kt b/src/backend/webhook/biz-webhook/src/test/kotlin/com/tencent/bkrepo/webhook/service/ServiceBaseTest.kt index b9a51826d4..26716f90cc 100644 --- a/src/backend/webhook/biz-webhook/src/test/kotlin/com/tencent/bkrepo/webhook/service/ServiceBaseTest.kt +++ b/src/backend/webhook/biz-webhook/src/test/kotlin/com/tencent/bkrepo/webhook/service/ServiceBaseTest.kt @@ -29,7 +29,7 @@ package com.tencent.bkrepo.webhook.service import com.tencent.bkrepo.auth.api.ServicePermissionClient import com.tencent.bkrepo.auth.api.ServiceUserClient -import com.tencent.bkrepo.common.security.manager.PermissionManager +import com.tencent.bkrepo.common.metadata.permission.PermissionManager import com.tencent.bkrepo.webhook.config.WebHookProperties import com.tencent.bkrepo.webhook.dao.WebHookDao import com.tencent.bkrepo.webhook.dao.WebHookLogDao diff --git a/src/frontend/devops-op/src/api/rateLimit.js b/src/frontend/devops-op/src/api/rateLimit.js new file mode 100644 index 0000000000..95917ceab1 --- /dev/null +++ b/src/frontend/devops-op/src/api/rateLimit.js @@ -0,0 +1,51 @@ +import request from '@/utils/request' + +const PREFIX_SERVICES = '/opdata/api/rateLimit' + +export function queryRateLimits() { + return request({ + url: `${PREFIX_SERVICES}/list/`, + method: 'get' + }) +} + +export function deleteRateLimit(id) { + return request({ + url: `${PREFIX_SERVICES}/delete/${id}`, + method: 'delete' + }) +} + +export function createRateLimit(data) { + return request({ + url: `${PREFIX_SERVICES}/create/`, + method: 'post', + data: data + }) +} + +export function updateRateLimit(data) { + return request({ + url: `${PREFIX_SERVICES}/update/`, + method: 'post', + data: data + }) +} + +export function getRateLimitConfig() { + return request({ + url: `${PREFIX_SERVICES}/config/`, + method: 'get' + }) +} + +export function getExistModule(resource, limitDimension) { + return request({ + url: `${PREFIX_SERVICES}/getExistModule`, + method: 'post', + params: { + resource: resource, + limitDimension: limitDimension + } + }) +} diff --git a/src/frontend/devops-op/src/router/index.js b/src/frontend/devops-op/src/router/index.js index e84ed35e23..e5e859f09c 100644 --- a/src/frontend/devops-op/src/router/index.js +++ b/src/frontend/devops-op/src/router/index.js @@ -24,6 +24,7 @@ export const ROUTER_NAME_FILE_SYSTEM = 'FileSystem' export const ROUTER_NAME_FILE_CACHE = 'FileCache' export const ROUTER_NAME_FILE_SYSTEM_RECORD = 'FileSystemRecord' export const ROUTER_NAME_REPO_CONFIG = 'RepoConfig' +export const ROUTER_NAME_RATE_LIMITER_CONFIG = 'RateLimiterConfig' export const ROUTER_NAME_PRELOAD_CONFIG = 'PreloadConfig' Vue.use(Router) @@ -314,6 +315,18 @@ export const asyncRoutes = [ } ] }, + { + path: '/rateLimiter', + component: Layout, + children: [ + { + path: '/', + name: ROUTER_NAME_RATE_LIMITER_CONFIG, + meta: { title: '限流管理', icon: 'permission' }, + component: () => import('@/views/rateLimitConfg/RateLimiter') + } + ] + }, { path: '/preload-config', component: Layout, diff --git a/src/frontend/devops-op/src/views/rateLimitConfg/RateLimiter.vue b/src/frontend/devops-op/src/views/rateLimitConfg/RateLimiter.vue new file mode 100644 index 0000000000..6f5e419fce --- /dev/null +++ b/src/frontend/devops-op/src/views/rateLimitConfg/RateLimiter.vue @@ -0,0 +1,246 @@ + + + + + diff --git a/src/frontend/devops-op/src/views/rateLimitConfg/components/CreateOrUpdateRateLimitDialog.vue b/src/frontend/devops-op/src/views/rateLimitConfg/components/CreateOrUpdateRateLimitDialog.vue new file mode 100644 index 0000000000..6caec34c72 --- /dev/null +++ b/src/frontend/devops-op/src/views/rateLimitConfg/components/CreateOrUpdateRateLimitDialog.vue @@ -0,0 +1,577 @@ + + + + + + diff --git a/src/frontend/devops-repository/src/components/FilePreview/filePreview.vue b/src/frontend/devops-repository/src/components/FilePreview/filePreview.vue index c05062cbb9..e50c93d7e2 100644 --- a/src/frontend/devops-repository/src/components/FilePreview/filePreview.vue +++ b/src/frontend/devops-repository/src/components/FilePreview/filePreview.vue @@ -103,6 +103,7 @@ || this.filePath.endsWith('bat') || this.filePath.endsWith('json') || this.filePath.endsWith('yaml') + || this.filePath.endsWith('yml') || this.filePath.endsWith('xml') || this.filePath.endsWith('log') || this.filePath.endsWith('ini') diff --git a/src/frontend/devops-repository/src/components/IamDenyDialog/IamDenyDialog.vue b/src/frontend/devops-repository/src/components/IamDenyDialog/IamDenyDialog.vue index e31fffb690..fcc3fb458f 100644 --- a/src/frontend/devops-repository/src/components/IamDenyDialog/IamDenyDialog.vue +++ b/src/frontend/devops-repository/src/components/IamDenyDialog/IamDenyDialog.vue @@ -9,10 +9,10 @@ :outer-border="false" :row-border="false" size="small"> - + - + @@ -94,7 +94,7 @@ diff --git a/src/frontend/devops-repository/src/views/repoGeneric/index.vue b/src/frontend/devops-repository/src/views/repoGeneric/index.vue index e1c9444eab..e0d64d7cef 100644 --- a/src/frontend/devops-repository/src/views/repoGeneric/index.vue +++ b/src/frontend/devops-repository/src/views/repoGeneric/index.vue @@ -811,6 +811,7 @@ || row.fullPath.endsWith('bat') || row.fullPath.endsWith('json') || row.fullPath.endsWith('yaml') + || row.fullPath.endsWith('yml') || row.fullPath.endsWith('xml') || row.fullPath.endsWith('log') || row.fullPath.endsWith('ini') @@ -1063,6 +1064,11 @@ }) } }) + } else if (e.status === 429) { + this.$bkMessage({ + theme: 'error', + message: e.message + }) } else { const message = this.$t('fileError') this.$bkMessage({ @@ -1419,6 +1425,7 @@ || name.endsWith('bat') || name.endsWith('json') || name.endsWith('yaml') + || name.endsWith('yml') || name.endsWith('xml') || name.endsWith('log') || name.endsWith('ini')