generic类型制品支持S3协议获取 #1296

src/backend/auth/api-auth/src/main/kotlin/com/tencent/bkrepo/auth/api/ServiceUserClient.kt

@@ -97,4 +97,10 @@ interface ServiceUserClient {
     fun userInfoById(
         @PathVariable uid: String
     ): Response<UserInfo?>
+
+    @ApiOperation("获取用户pwd ")
+    @GetMapping("/userpwd/{uid}")
+    fun userPwdById(
+        @PathVariable uid: String
+    ): Response<String?>
 }

src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/controller/service/ServiceUserController.kt

@@ -85,4 +85,8 @@ class ServiceUserController @Autowired constructor(
     override fun userInfoById(uid: String): Response<UserInfo?> {
         return ResponseBuilder.success(userService.getUserInfoById(uid))
     }
+
+    override fun userPwdById(uid: String): Response<String?> {
+        return ResponseBuilder.success(userService.getUserPwdById(uid))
+    }
 }

src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/UserService.kt

@@ -79,6 +79,8 @@ interface UserService {
 
     fun getUserInfoById(userId: String): UserInfo?
 
+    fun getUserPwdById(userId: String): String?
+
     fun updatePassword(userId: String, oldPwd: String, newPwd: String): Boolean
 
     fun resetPassword(userId: String): Boolean

src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/local/UserServiceImpl.kt

@@ -371,6 +371,11 @@ class UserServiceImpl constructor(
         return UserRequestUtil.convToUserInfo(tUser)
     }
 
+    override fun getUserPwdById(userId: String): String? {
+        val tUser = userRepository.findFirstByUserId(userId) ?: return null
+        return tUser.pwd
+    }
+
     override fun updatePassword(userId: String, oldPwd: String, newPwd: String): Boolean {
         val query = UserQueryHelper.getUserByIdAndPwd(userId, oldPwd)
         val user = mongoTemplate.find(query, TUser::class.java)

src/backend/buildSrc/src/main/kotlin/Versions.kt

@@ -66,4 +66,5 @@ object Versions {
     const val Jasypt = "3.0.5"
     const val CryptoJavaSdk = "1.1.0"
     const val IamJavaSdk = "1.0.30-SNAPSHOT"
+    const val Dom4j = "2.1.0"
 }

src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/Constants.kt

@@ -102,6 +102,7 @@ const val BEARER_AUTH_PREFIX = "Bearer "
 const val AUTH_HEADER_UID = "X-BKREPO-UID"
 const val OAUTH_AUTH_PREFIX = "Oauth "
 const val TEMPORARY_TOKEN_AUTH_PREFIX = "Temporary "
+const val AWS4_AUTH_PREFIX = "AWS4-HMAC-SHA256 "
 
 /**
  * micro service header user id key

src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/message/CommonMessageCode.kt

@@ -56,7 +56,7 @@ enum class CommonMessageCode(private val key: String) : MessageCode {
     MODIFY_PASSWORD_FAILED("modify.password.failed"),
     OPERATION_CROSS_CLUSTER_NOT_ALLOWED("operation.cross-cluster.not-allowed"),
     MEDIA_TYPE_UNACCEPTABLE("system.media-type.unacceptable"),
-    TOO_MANY_REQUESTS("too.many.requests")
+    TOO_MANY_REQUESTS("too.many.requests"),
     ;
 
     override fun getBusinessCode() = ordinal + 1

src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/response/BaseArtifactResourceHandler.kt

@@ -0,0 +1,122 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.common.artifact.resolve.response
+
+import com.tencent.bkrepo.common.api.constant.HttpStatus
+import com.tencent.bkrepo.common.api.exception.TooManyRequestsException
+import com.tencent.bkrepo.common.artifact.exception.ArtifactResponseException
+import com.tencent.bkrepo.common.artifact.metrics.RecordAbleInputStream
+import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder
+import com.tencent.bkrepo.common.artifact.stream.STREAM_BUFFER_SIZE
+import com.tencent.bkrepo.common.artifact.stream.rateLimit
+import com.tencent.bkrepo.common.artifact.util.http.IOExceptionUtils
+import com.tencent.bkrepo.common.storage.core.StorageProperties
+import com.tencent.bkrepo.common.storage.monitor.Throughput
+import com.tencent.bkrepo.common.storage.monitor.measureThroughput
+import org.springframework.http.HttpMethod
+import org.springframework.util.unit.DataSize
+import java.io.IOException
+import javax.servlet.http.HttpServletRequest
+import javax.servlet.http.HttpServletResponse
+
+
+open class BaseArtifactResourceHandler(
+    private val storageProperties: StorageProperties
+) {
+    /**
+     * 获取动态buffer size
+     * @param totalSize 数据总大小
+     */
+    protected  fun getBufferSize(totalSize: Int): Int {
+        val bufferSize = storageProperties.response.bufferSize.toBytes().toInt()
+        return when {
+            bufferSize < 0 || totalSize < 0 -> STREAM_BUFFER_SIZE
+            totalSize < bufferSize -> totalSize
+            else -> bufferSize
+        }
+    }
+
+    /**
+     * 将仓库级别的限速配置导入
+     * 当同时存在全局限速配置以及仓库级别限速配置时，以仓库级别配置优先
+     */
+    protected fun responseRateLimitWrapper(rateLimit: DataSize): Long {
+        val rateLimitOfRepo = ArtifactContextHolder.getRateLimitOfRepo()
+        if (rateLimitOfRepo.responseRateLimit != DataSize.ofBytes(-1)) {
+            return rateLimitOfRepo.responseRateLimit.toBytes()
+        }
+        return rateLimit.toBytes()
+    }
+
+    /**
+     * 当仓库配置下载限速小于等于最低限速时则直接将请求断开, 避免占用过多连接
+     */
+    protected fun responseRateLimitCheck() {
+        val rateLimitOfRepo = ArtifactContextHolder.getRateLimitOfRepo()
+        if (rateLimitOfRepo.responseRateLimit != DataSize.ofBytes(-1) &&
+            rateLimitOfRepo.responseRateLimit <= storageProperties.response.circuitBreakerThreshold) {
+            throw TooManyRequestsException(
+                "The circuit breaker is activated when too many download requests are made to the service!"
+            )
+        }
+    }
+
+    /**
+     * 将数据流以Range方式写入响应
+     */
+    protected fun writeRangeStream(
+        resource: ArtifactResource,
+        request: HttpServletRequest,
+        response: HttpServletResponse
+    ): Throughput {
+        val inputStream = resource.getSingleStream()
+        if (request.method == HttpMethod.HEAD.name) {
+            return Throughput.EMPTY
+        }
+        val recordAbleInputStream = RecordAbleInputStream(inputStream)
+        try {
+            return measureThroughput {
+                recordAbleInputStream.rateLimit(responseRateLimitWrapper(storageProperties.response.rateLimit)).use {
+                    it.copyTo(
+                        out = response.outputStream,
+                        bufferSize = getBufferSize(inputStream.range.length.toInt())
+                    )
+                }
+            }
+        } catch (exception: IOException) {
+            // 直接向上抛IOException经过CglibAopProxy会抛java.lang.reflect.UndeclaredThrowableException: null
+            // 由于已经设置了Content-Type为application/octet-stream, spring找不到对应的Converter，导致抛
+            // org.springframework.http.converter.HttpMessageNotWritableException异常，会重定向到/error页面
+            // 又因为/error页面不存在，最终返回404，所以要对IOException进行包装，在上一层捕捉处理
+            val message = exception.message.orEmpty()
+            val status = if (IOExceptionUtils.isClientBroken(exception)) HttpStatus.BAD_REQUEST else HttpStatus.INTERNAL_SERVER_ERROR
+            throw ArtifactResponseException(message, status)
+        }
+    }
+
+}

src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/response/DefaultArtifactResourceWriter.kt

@@ -67,7 +67,7 @@ import javax.servlet.http.HttpServletResponse
  */
 open class DefaultArtifactResourceWriter(
     private val storageProperties: StorageProperties
-) : ArtifactResourceWriter {
+) : BaseArtifactResourceHandler(storageProperties), ArtifactResourceWriter {
 
     @Throws(ArtifactResponseException::class)
     override fun write(resource: ArtifactResource): Throughput {
@@ -174,39 +174,6 @@ open class DefaultArtifactResourceWriter(
         return localDateTime.atZone(ZoneOffset.systemDefault()).toInstant().toEpochMilli()
     }
 
-    /**
-     * 将数据流以Range方式写入响应
-     */
-    private fun writeRangeStream(
-        resource: ArtifactResource,
-        request: HttpServletRequest,
-        response: HttpServletResponse
-    ): Throughput {
-        val inputStream = resource.getSingleStream()
-        if (request.method == HttpMethod.HEAD.name) {
-            return Throughput.EMPTY
-        }
-        val recordAbleInputStream = RecordAbleInputStream(inputStream)
-        try {
-            return measureThroughput {
-                recordAbleInputStream.rateLimit(responseRateLimitWrapper(storageProperties.response.rateLimit)).use {
-                    it.copyTo(
-                        out = response.outputStream,
-                        bufferSize = getBufferSize(inputStream.range.length.toInt())
-                    )
-                }
-            }
-        } catch (exception: IOException) {
-            // 直接向上抛IOException经过CglibAopProxy会抛java.lang.reflect.UndeclaredThrowableException: null
-            // 由于已经设置了Content-Type为application/octet-stream, spring找不到对应的Converter，导致抛
-            // org.springframework.http.converter.HttpMessageNotWritableException异常，会重定向到/error页面
-            // 又因为/error页面不存在，最终返回404，所以要对IOException进行包装，在上一层捕捉处理
-            val message = exception.message.orEmpty()
-            val status = if (isClientBroken(exception)) HttpStatus.BAD_REQUEST else HttpStatus.INTERNAL_SERVER_ERROR
-            throw ArtifactResponseException(message, status)
-        }
-    }
-
     /**
      * 将数据流以ZipOutputStream方式写入响应
      */
@@ -248,31 +215,6 @@ open class DefaultArtifactResourceWriter(
         }
     }
 
-    /**
-     * 将仓库级别的限速配置导入
-     * 当同时存在全局限速配置以及仓库级别限速配置时，以仓库级别配置优先
-     */
-    private fun responseRateLimitWrapper(rateLimit: DataSize): Long {
-        val rateLimitOfRepo = ArtifactContextHolder.getRateLimitOfRepo()
-        if (rateLimitOfRepo.responseRateLimit != DataSize.ofBytes(-1)) {
-            return rateLimitOfRepo.responseRateLimit.toBytes()
-        }
-        return rateLimit.toBytes()
-    }
-
-    /**
-     * 当仓库配置下载限速小于等于最低限速时则直接将请求断开, 避免占用过多连接
-     */
-    private fun responseRateLimitCheck() {
-        val rateLimitOfRepo = ArtifactContextHolder.getRateLimitOfRepo()
-        if (rateLimitOfRepo.responseRateLimit != DataSize.ofBytes(-1) &&
-            rateLimitOfRepo.responseRateLimit <= storageProperties.response.circuitBreakerThreshold) {
-            throw TooManyRequestsException(
-                "The circuit breaker is activated when too many download requests are made to the service!"
-            )
-        }
-    }
-
     /**
      * 判断charset,一些媒体类型设置了charset会影响其表现，如application/vnd.android.package-archive
      * */
@@ -290,18 +232,6 @@ open class DefaultArtifactResourceWriter(
         return node.sha256!!
     }
 
-    /**
-     * 获取动态buffer size
-     * @param totalSize 数据总大小
-     */
-    private fun getBufferSize(totalSize: Int): Int {
-        val bufferSize = storageProperties.response.bufferSize.toBytes().toInt()
-        if (bufferSize < 0 || totalSize < 0) {
-            return STREAM_BUFFER_SIZE
-        }
-        return if (totalSize < bufferSize) totalSize else bufferSize
-    }
-
     /**
      * 根据[artifactName]生成ZipEntry
      */

src/backend/common/common-generic/build.gradle.kts

@@ -0,0 +1,34 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2020 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+dependencies {
+    api(project(":common:common-artifact:artifact-service"))
+}

src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/configuration/AutoIndexRepositorySettings.kt → src/backend/common/common-generic/src/main/kotlin/com/tencent/bkrepo/common/generic/configuration/AutoIndexRepositorySettings.kt

@@ -25,7 +25,7 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package com.tencent.bkrepo.generic.artifact.configuration
+package com.tencent.bkrepo.common.generic.configuration
 
 import com.tencent.bkrepo.common.artifact.pojo.configuration.RepositoryConfiguration
 

src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/manager/AuthenticationManager.kt

@@ -92,6 +92,13 @@ class AuthenticationManager(
     fun findUserAccount(userId: String): UserInfo? {
         return serviceUserClient.userInfoById(userId).data
     }
+    /**
+     * 根据用户id[userId]查询用户密码
+     * 当用户不存在时返回`null`
+     */
+    fun findUserPwd(userId: String): String? {
+        return serviceUserClient.userPwdById(userId).data
+    }
 
     fun findOauthToken(accessToken: String): OauthToken? {
         return serviceOauthAuthorizationClient.getToken(accessToken).data

src/backend/common/common-service/build.gradle.kts

@@ -64,4 +64,5 @@ dependencies {
     }
     api("org.springframework.retry:spring-retry")
     api("com.github.ulisesbocchio:jasypt-spring-boot-starter:${Versions.Jasypt}")
+    api("org.dom4j:dom4j:${Versions.Dom4j}")
 }

src/backend/common/common-service/src/main/resources/i18n/messages_en.properties

@@ -52,4 +52,7 @@ permission.project.denied = user[{0}] does not have [{1}] permission in project[
 permission.repo.denied = user[{0}] does not have [{1}] permission in project[{2}] repo[{3}]
 operation.cross-cluster.not-allowed=Cross location operation is not allowed
 system.media-type.unacceptable=Unacceptable Media Type
-too.many.requests=Too Many Requests: {0}
+too.many.requests=Too Many Requests: {0}
+system.request.sign.not-match=The Signature you specified is invalid
+s3.no.such.key=The specified key does not exist
+s3.no.such.bucket=The specified bucket does not exist

src/backend/common/common-service/src/main/resources/i18n/messages_zh_CN.properties

@@ -52,4 +52,7 @@ permission.project.denied=用户[{0}]没有[{1}]权限在[{2}]项目上
 permission.repo.denied = 用户[{0}]没有[{1}]权限在[{2}]项目的[{3}]仓库上
 operation.cross-cluster.not-allowed=不允许跨地点操作
 system.media-type.unacceptable=不接受的Media Type
-too.many.requests=请求过多: {0}
+too.many.requests=请求过多: {0}
+system.request.sign.not-match=指定的签名无效
+s3.no.such.key=指定的key不存在
+s3.no.such.bucket=指定的bucket不存在

src/backend/common/common-service/src/main/resources/i18n/messages_zh_TW.properties

@@ -52,4 +52,7 @@ permission.project.denied=用戶[{0}]沒有[{1}]權限在[{2}]項目上
 permission.repo.denied = 用戶[{0}]沒有[{1}]權限在[{2}]項目的[{3}]倉庫上
 operation.cross-cluster.not-allowed=不允許跨地點操作
 system.media-type.unacceptable=不接受的Media Type
-too.many.requests=請求過多: {0}
+too.many.requests=請求過多: {0}
+system.request.sign.not-match=指定的簽名無效
+s3.no.such.key=指定的key不存在
+s3.no.such.bucket=指定的bucket不存在

src/backend/generic/biz-generic/build.gradle.kts

@@ -33,5 +33,6 @@ dependencies {
     api(project(":generic:api-generic"))
     api(project(":common:common-redis"))
     api(project(":analyst:api-analyst"))
+    api(project(":common:common-generic"))
     api(project(":common:common-artifact:artifact-service"))
 }