Merge remote-tracking branch 'fork/issue_2683' into issue_2683

# Conflicts: # src/frontend/devops-op/src/router/index.js
TencentBlueKing · Nov 6, 2024 · 29d1774 · 29d1774
2 parents 84aac7d + 4401758
commit 29d1774
Show file tree

Hide file tree

Showing 291 changed files with 18,663 additions and 460 deletions.
diff --git a/...yst/src/main/kotlin/com/tencent/bkrepo/analyst/component/ScannerPermissionCheckHandler.kt b/...yst/src/main/kotlin/com/tencent/bkrepo/analyst/component/ScannerPermissionCheckHandler.kt
@@ -32,13 +32,13 @@ import com.tencent.bkrepo.auth.pojo.enums.ResourceType
 import com.tencent.bkrepo.common.artifact.constant.PROJECT_ID
 import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder
 import com.tencent.bkrepo.common.security.exception.PermissionException
-import com.tencent.bkrepo.common.security.manager.PermissionManager
 import com.tencent.bkrepo.common.security.permission.Permission
 import com.tencent.bkrepo.common.security.permission.PermissionCheckHandler
 import com.tencent.bkrepo.common.security.permission.Principal
 import com.tencent.bkrepo.common.security.util.SecurityUtils
 import com.tencent.bkrepo.common.service.util.HttpContextHolder
 import com.tencent.bkrepo.analyst.model.SubScanTaskDefinition
+import com.tencent.bkrepo.common.metadata.permission.PermissionManager
 import com.tencent.bkrepo.common.artifact.pojo.RepositoryId
 import com.tencent.bkrepo.common.security.permission.PrincipalType
 import org.springframework.context.annotation.Primary

diff --git a/.../biz-archive/src/test/kotlin/com/tencent/bkrepo/archive/core/compress/BDZipManagerTest.kt b/.../biz-archive/src/test/kotlin/com/tencent/bkrepo/archive/core/compress/BDZipManagerTest.kt
@@ -13,7 +13,6 @@ import com.tencent.bkrepo.common.bksync.file.BkSyncDeltaSource.Companion.toBkSyn
 import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService
 import com.tencent.bkrepo.common.storage.StorageAutoConfiguration
 import com.tencent.bkrepo.common.storage.core.StorageService
-import com.tencent.bkrepo.repository.api.RepositoryClient
 import org.junit.jupiter.api.AfterEach
 import org.junit.jupiter.api.Assertions
 import org.junit.jupiter.api.BeforeEach
@@ -44,9 +43,6 @@ class BDZipManagerTest @Autowired constructor(
     @MockBean
     lateinit var fileReferenceService: FileReferenceService
 
-    @MockBean
-    lateinit var repositoryClient: RepositoryClient
-
     private val timeout = Duration.ofSeconds(10)
 
     @BeforeEach

diff --git a/...auth/api-auth/src/main/kotlin/com/tencent/bkrepo/auth/pojo/enums/ResourceActionMapping.kt b/...auth/api-auth/src/main/kotlin/com/tencent/bkrepo/auth/pojo/enums/ResourceActionMapping.kt
@@ -28,26 +28,32 @@
 package com.tencent.bkrepo.auth.pojo.enums
 
 enum class ResourceActionMapping(val resourceType: String, val actions: List<String>) {
-    PROJECT_ACTIONS(ResourceType.PROJECT.id(), listOf(
+    PROJECT_ACTIONS(
+        ResourceType.PROJECT.id(), listOf(
         ActionTypeMapping.PROJECT_VIEW.id(),
         ActionTypeMapping.PROJECT_EDIT.id(),
         ActionTypeMapping.PROJECT_MANAGE.id(),
         ActionTypeMapping.REPO_CREATE.id()
-    )),
-    REPO_ACTIONS(ResourceType.REPO.id(),
-                 listOf(
-                     ActionTypeMapping.REPO_VIEW.id(),
-                     ActionTypeMapping.REPO_EDIT.id(),
-                     ActionTypeMapping.REPO_MANAGE.id(),
-                     ActionTypeMapping.REPO_DELETE.id(),
-                     ActionTypeMapping.NODE_CREATE.id()
-                 )),
-    NODE_ACTIONS(ResourceType.NODE.id(),
-                 listOf(
-                     ActionTypeMapping.NODE_DELETE.id(),
-                     ActionTypeMapping.NODE_DOWNLOAD.id(),
-                     ActionTypeMapping.NODE_EDIT.id(),
-                     ActionTypeMapping.NODE_WRITE.id(),
-                     ActionTypeMapping.NODE_VIEW.id()
-                 ));
+    )
+    ),
+    REPO_ACTIONS(
+        ResourceType.REPO.id(),
+        listOf(
+            ActionTypeMapping.REPO_VIEW.id(),
+            ActionTypeMapping.REPO_EDIT.id(),
+            ActionTypeMapping.REPO_MANAGE.id(),
+            ActionTypeMapping.REPO_DELETE.id(),
+            ActionTypeMapping.NODE_CREATE.id()
+        )
+    ),
+    NODE_ACTIONS(
+        ResourceType.NODE.id(),
+        listOf(
+            ActionTypeMapping.NODE_DELETE.id(),
+            ActionTypeMapping.NODE_DOWNLOAD.id(),
+            ActionTypeMapping.NODE_EDIT.id(),
+            ActionTypeMapping.NODE_WRITE.id(),
+            ActionTypeMapping.NODE_VIEW.id()
+        )
+    );
 }
diff --git a/...h/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/dao/repository/OauthTokenRepository.kt b/...h/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/dao/repository/OauthTokenRepository.kt
@@ -33,7 +33,6 @@ import org.springframework.stereotype.Repository
 
 @Repository
 interface OauthTokenRepository : MongoRepository<TOauthToken, String> {
-    fun findFirstByAccountIdAndUserId(accountId: String, userId: String): TOauthToken?
     fun findFirstByAccessToken(accessToken: String): TOauthToken?
     fun findByUserId(userId: String): List<TOauthToken>
     fun findFirstByAccountIdAndRefreshToken(accountId: String, refreshToken: String): TOauthToken?

diff --git a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/model/TOauthToken.kt b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/model/TOauthToken.kt
@@ -27,13 +27,28 @@
 
 package com.tencent.bkrepo.auth.model
 
+import com.tencent.bkrepo.auth.model.TOauthToken.Companion.ACCESS_TOKEN_IDX
+import com.tencent.bkrepo.auth.model.TOauthToken.Companion.ACCESS_TOKEN_IDX_DEF
+import com.tencent.bkrepo.auth.model.TOauthToken.Companion.ACCOUNT_ID_ACCESS_TOKEN_IDX
+import com.tencent.bkrepo.auth.model.TOauthToken.Companion.ACCOUNT_ID_ACCESS_TOKEN_IDX_DEF
+import com.tencent.bkrepo.auth.model.TOauthToken.Companion.ACCOUNT_ID_USER_ID_IDX
+import com.tencent.bkrepo.auth.model.TOauthToken.Companion.ACCOUNT_ID_USER_ID_IDX_DEF
+import com.tencent.bkrepo.auth.model.TOauthToken.Companion.USER_IDX
+import com.tencent.bkrepo.auth.model.TOauthToken.Companion.USER_IDX_DEF
 import com.tencent.bkrepo.auth.pojo.enums.ResourceType
 import com.tencent.bkrepo.auth.pojo.oauth.IdToken
+import org.springframework.data.mongodb.core.index.CompoundIndex
+import org.springframework.data.mongodb.core.index.CompoundIndexes
 import org.springframework.data.mongodb.core.mapping.Document
 import java.time.Instant
 
 @Document("oauth_token")
-data class TOauthToken(
+@CompoundIndexes(
+    CompoundIndex(name = ACCESS_TOKEN_IDX, def = ACCESS_TOKEN_IDX_DEF, background = true),
+    CompoundIndex(name = USER_IDX, def = USER_IDX_DEF, background = true),
+    CompoundIndex(name = ACCOUNT_ID_ACCESS_TOKEN_IDX, def = ACCOUNT_ID_ACCESS_TOKEN_IDX_DEF, background = true),
+    CompoundIndex(name = ACCOUNT_ID_USER_ID_IDX, def = ACCOUNT_ID_USER_ID_IDX_DEF, background = true),
+)data class TOauthToken(
     val id: String? = null,
     var accessToken: String,
     var refreshToken: String?,
@@ -44,4 +59,15 @@ data class TOauthToken(
     var scope: Set<ResourceType>?,
     var issuedAt: Instant,
     var idToken: IdToken?
-)
+) {
+    companion object {
+        const val ACCESS_TOKEN_IDX = "access_token"
+        const val ACCESS_TOKEN_IDX_DEF = "{'accessToken': 1}"
+        const val USER_IDX = "user_id"
+        const val USER_IDX_DEF = "{'userId': 1}"
+        const val ACCOUNT_ID_ACCESS_TOKEN_IDX = "account_id_access_token"
+        const val ACCOUNT_ID_ACCESS_TOKEN_IDX_DEF = "{'accountId': 1, 'access_token': 1}"
+        const val ACCOUNT_ID_USER_ID_IDX = "account_id_user_id"
+        const val ACCOUNT_ID_USER_ID_IDX_DEF = "{'accountId': 1, 'userId': 1}"
+    }
+}
diff --git a/...nd/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/impl/ProxyServiceImpl.kt b/...nd/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/impl/ProxyServiceImpl.kt
@@ -27,6 +27,7 @@
 
 package com.tencent.bkrepo.auth.service.impl
 
+import com.tencent.bkrepo.auth.dao.ProxyDao
 import com.tencent.bkrepo.auth.message.AuthMessageCode
 import com.tencent.bkrepo.auth.model.TProxy
 import com.tencent.bkrepo.auth.pojo.enums.PermissionAction
@@ -37,15 +38,14 @@ import com.tencent.bkrepo.auth.pojo.proxy.ProxyListOption
 import com.tencent.bkrepo.auth.pojo.proxy.ProxyStatus
 import com.tencent.bkrepo.auth.pojo.proxy.ProxyStatusRequest
 import com.tencent.bkrepo.auth.pojo.proxy.ProxyUpdateRequest
-import com.tencent.bkrepo.auth.dao.ProxyDao
 import com.tencent.bkrepo.auth.service.ProxyService
 import com.tencent.bkrepo.common.api.constant.StringPool
 import com.tencent.bkrepo.common.api.exception.ErrorCodeException
 import com.tencent.bkrepo.common.api.pojo.Page
 import com.tencent.bkrepo.common.api.util.Preconditions
 import com.tencent.bkrepo.common.api.util.UrlFormatter
+import com.tencent.bkrepo.common.metadata.permission.PermissionManager
 import com.tencent.bkrepo.common.mongo.dao.util.Pages
-import com.tencent.bkrepo.common.security.manager.PermissionManager
 import com.tencent.bkrepo.common.security.util.AESUtils
 import com.tencent.bkrepo.common.security.util.SecurityUtils
 import com.tencent.bkrepo.common.service.util.HttpContextHolder

diff --git a/...th/src/main/kotlin/com/tencent/bkrepo/auth/service/oauth/OauthAuthorizationServiceImpl.kt b/...th/src/main/kotlin/com/tencent/bkrepo/auth/service/oauth/OauthAuthorizationServiceImpl.kt
@@ -199,29 +199,19 @@ class OauthAuthorizationServiceImpl(
         client: TAccount,
         openId: Boolean
     ): TOauthToken {
-        var tOauthToken = oauthTokenRepository.findFirstByAccountIdAndUserId(client.id!!, userId)
-        val idToken = generateOpenIdToken(client.id, userId, nonce)
-        if (tOauthToken == null) {
-            tOauthToken = TOauthToken(
-                accessToken = idToken.toJwtToken(),
-                refreshToken = OauthUtils.generateRefreshToken(),
-                expireSeconds = oauthProperties.expiredDuration.seconds,
-                type = "Bearer",
-                accountId = client.id,
-                userId = userId,
-                scope = client.scope,
-                issuedAt = Instant.now(Clock.systemDefaultZone()),
-                idToken = if (openId) idToken else null
-            )
-        }
-        if (client.scope != tOauthToken.scope) {
-            tOauthToken.scope = client.scope!!
-        }
-        tOauthToken.userId = userId
-        tOauthToken.accessToken = idToken.toJwtToken()
-        tOauthToken.idToken = if (openId) idToken else null
-        tOauthToken.issuedAt = Instant.now(Clock.systemDefaultZone())
-        oauthTokenRepository.save(tOauthToken)
+        val idToken = generateOpenIdToken(client.id!!, userId, nonce)
+        val tOauthToken = TOauthToken(
+            accessToken = idToken.toJwtToken(),
+            refreshToken = OauthUtils.generateRefreshToken(),
+            expireSeconds = oauthProperties.expiredDuration.seconds,
+            type = "Bearer",
+            accountId = client.id,
+            userId = userId,
+            scope = client.scope,
+            issuedAt = Instant.now(Clock.systemDefaultZone()),
+            idToken = if (openId) idToken else null,
+        )
+        oauthTokenRepository.insert(tOauthToken)
         return tOauthToken
     }
 
@@ -261,17 +251,11 @@ class OauthAuthorizationServiceImpl(
     }
 
     override fun validateToken(accessToken: String): String? {
-        val token = oauthTokenRepository.findFirstByAccessToken(accessToken)
-            ?: throw ErrorCodeException(CommonMessageCode.RESOURCE_NOT_FOUND, "access_token[$accessToken]")
-        if (token.expireSeconds == null) {
-            return token.userId
-        }
-
-        val expiredInstant = Instant.ofEpochSecond(token.issuedAt.epochSecond + token.expireSeconds)
-        if (expiredInstant.isBefore(Instant.now())) {
-            throw ErrorCodeException(CommonMessageCode.RESOURCE_EXPIRED, "access_token[$accessToken]")
-        }
-        return token.userId
+        val claims = JwtUtils.validateToken(
+            signingKey = RsaUtils.stringToPrivateKey(cryptoProperties.privateKeyStr2048PKCS8),
+            token = accessToken
+        )
+        return claims.body.subject
     }
 
     override fun deleteToken(clientId: String, clientSecret: String, accessToken: String) {

diff --git a/src/backend/build.gradle.kts b/src/backend/build.gradle.kts
@@ -76,6 +76,7 @@ allprojects {
             dependency("com.playtika.reactivefeign:feign-reactor-spring-cloud-starter:${Versions.ReactiveFeign}")
             dependency("com.tencent.bk.sdk:crypto-java-sdk:${Versions.CryptoJavaSdk}")
             dependency("org.apache.tika:tika-core:${Versions.TiKa}")
+            dependency("com.tencent.bk.sdk:spring-boot-bk-audit-starter:${Versions.Audit}")
             dependency("com.tencent.devops:devops-schedule-common:${Versions.DevopsBootSNAPSHOT}")
             dependency("com.tencent.devops:devops-schedule-model:${Versions.DevopsBootSNAPSHOT}")
             dependency("com.tencent.devops:devops-schedule-server:${Versions.DevopsBootSNAPSHOT}")

diff --git a/src/backend/buildSrc/src/main/kotlin/Versions.kt b/src/backend/buildSrc/src/main/kotlin/Versions.kt
@@ -71,4 +71,5 @@ object Versions {
     const val JavaCpp = "1.5.9"
     const val Notice = "1.0.0"
     const val SpringCloudFunction = "3.2.11"
+    const val Audit = "1.0.8"
 }
diff --git a/...n/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/exception/OverloadException.kt b/...n/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/exception/OverloadException.kt
@@ -0,0 +1,38 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.common.api.exception
+
+import com.tencent.bkrepo.common.api.constant.HttpStatus
+import com.tencent.bkrepo.common.api.message.CommonMessageCode
+
+/**
+ * 超过限流配置异常
+ */
+class OverloadException(
+    val resource: String
+) : ErrorCodeException(CommonMessageCode.RATE_LIMITER_OVERLOAD, resource, status = HttpStatus.TOO_MANY_REQUESTS)
diff --git a/...mon/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/message/CommonMessageCode.kt b/...mon/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/message/CommonMessageCode.kt
@@ -61,6 +61,9 @@ enum class CommonMessageCode(private val key: String) : MessageCode {
     MEDIA_TYPE_UNACCEPTABLE("system.media-type.unacceptable"),
     TOO_MANY_REQUESTS("too.many.requests"),
     PIPELINE_NOT_RUNNING("pipeline.not-running"),
+    INVALID_CONFIG("system.config.invalid"),
+    ACQUIRE_LOCK_FAILED("acquire.lock.failed"),
+    RATE_LIMITER_OVERLOAD("rate.limiter.overload")
     ;
 
     override fun getBusinessCode() = ordinal + 1

diff --git a/src/backend/common/common-api/src/main/resources/i18n/messages_en.properties b/src/backend/common/common-api/src/main/resources/i18n/messages_en.properties
@@ -55,3 +55,6 @@ operation.cross-cluster.not-allowed=Cross location operation is not allowed
 system.media-type.unacceptable=Unacceptable Media Type
 too.many.requests=Too Many Requests: {0}
 pipeline.not-running=Pipeline[{0}] is not running status
+system.config.invalid=Config [{0}] is invalid
+acquire.lock.failed=acquire lock failed:[{0}]
+rate.limiter.overload=resource requests reached rate limit:[{0}]
diff --git a/src/backend/common/common-api/src/main/resources/i18n/messages_zh_CN.properties b/src/backend/common/common-api/src/main/resources/i18n/messages_zh_CN.properties
@@ -55,3 +55,6 @@ operation.cross-cluster.not-allowed=不允许跨地点操作
 system.media-type.unacceptable=不接受的Media Type
 too.many.requests=请求过多: {0}
 pipeline.not-running=流水线[{0}]不是运行状态
+system.config.invalid=配置[{0}]无效
+acquire.lock.failed=获取锁失败: [{0}]
+rate.limiter.overload=资源请求量超过限流值: [{0}]
diff --git a/src/backend/common/common-api/src/main/resources/i18n/messages_zh_TW.properties b/src/backend/common/common-api/src/main/resources/i18n/messages_zh_TW.properties
@@ -55,3 +55,6 @@ operation.cross-cluster.not-allowed=不允許跨地點操作
 system.media-type.unacceptable=不接受的Media Type
 too.many.requests=請求過多: {0}
 pipeline.not-running=流水線[{0}]不是運行狀態
+system.config.invalid=配置[{0}]無效
+acquire.lock.failed=獲取鎖失敗: [{0}]
+rate.limiter.overload=資源請求量超過限流值: [{0}]
diff --git a/src/backend/common/common-artifact/artifact-service/build.gradle.kts b/src/backend/common/common-artifact/artifact-service/build.gradle.kts
@@ -40,6 +40,7 @@ dependencies {
     api(project(":common:common-security"))
     api(project(":common:common-artifact:artifact-api"))
     api(project(":common:common-storage:storage-service"))
+    api(project(":common:common-ratelimiter"))
     api(project(":common:common-stream"))
     api(project(":common:common-metrics-push"))
     api(project(":common:common-metadata:metadata-service"))
@@ -49,6 +50,7 @@ dependencies {
     api("io.micrometer:micrometer-registry-prometheus")
     api("org.influxdb:influxdb-java")
     api("org.apache.commons:commons-text")
+    api("com.tencent.bk.sdk:spring-boot-bk-audit-starter")
 
     testImplementation("org.mockito.kotlin:mockito-kotlin")
     testImplementation("io.mockk:mockk")

diff --git a/...t-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/ArtifactAutoConfiguration.kt b/...t-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/ArtifactAutoConfiguration.kt
@@ -31,6 +31,7 @@
 
 package com.tencent.bkrepo.common.artifact
 
+import com.tencent.bkrepo.common.artifact.audit.BkAuditConfiguration
 import com.tencent.bkrepo.common.artifact.cluster.ArtifactClusterConfiguration
 import com.tencent.bkrepo.common.artifact.cns.CnsConfiguration
 import com.tencent.bkrepo.common.artifact.event.ArtifactEventConfiguration
@@ -70,5 +71,6 @@ import org.springframework.context.annotation.PropertySource
     ArtifactClusterConfiguration::class,
     CnsConfiguration::class,
     ArtifactRouterControllerConfiguration::class,
+    BkAuditConfiguration::class,
 )
 class ArtifactAutoConfiguration