Merge pull request #10530 from mingshewhe/feat_8130_var

【PAC】feat：开启PAC模式的代码库支持自动同步代码库YAML变更到蓝盾 #8130 当修改read-only var变量时输出告警日志
TencentBlueKing · Jun 19, 2024 · ce5de7e · ce5de7e
2 parents 78365d8 + f1569b5
commit ce5de7e
Show file tree

Hide file tree

Showing 7 changed files with 78 additions and 9 deletions.
diff --git a/...pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/TriggerTransfer.kt b/...pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/TriggerTransfer.kt
@@ -246,7 +246,7 @@ class TriggerTransfer @Autowired(required = false) constructor(
                 CodeEventType.PUSH -> nowExist.push = PushRule(
                     name = git.name.nullIfDefault(defaultName),
                     enable = git.enable.nullIfDefault(true),
-                    branches = git.branchName?.disjoin() ?: emptyList(),
+                    branches = git.branchName?.disjoin(),
                     branchesIgnore = git.excludeBranchName?.disjoin(),
                     paths = git.includePaths?.disjoin(),
                     pathsIgnore = git.excludePaths?.disjoin(),

diff --git a/...on-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/on/PushRule.kt b/...on-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/on/PushRule.kt
@@ -40,7 +40,7 @@ import io.swagger.v3.oas.annotations.media.Schema
 data class PushRule(
     val name: String? = null,
     val enable: Boolean? = true,
-    val branches: List<String>? = listOf("*"),
+    val branches: List<String>?,
 
     @get:Schema(title = "branches-ignore")
     @JsonProperty("branches-ignore")

diff --git a/...ess/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildVarDao.kt b/...ess/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildVarDao.kt
@@ -114,6 +114,36 @@ class PipelineBuildVarDao @Autowired constructor() {
         }
     }
 
+    fun getBuildVarMap(
+        dslContext: DSLContext,
+        projectId: String,
+        buildId: String,
+        keys: Set<String>? = null
+    ): Map<String, BuildParameters> {
+        with(T_PIPELINE_BUILD_VAR) {
+            val where = dslContext.selectFrom(this)
+                .where(BUILD_ID.eq(buildId).and(PROJECT_ID.eq(projectId)))
+            if (!keys.isNullOrEmpty()) {
+                where.and(KEY.`in`(keys))
+            }
+            return where.fetch().associateBy(
+                { it.key },
+                {
+                    if (it.varType != null) {
+                        BuildParameters(
+                            key = it.key,
+                            value = it.value,
+                            valueType = BuildFormPropertyType.valueOf(it.varType),
+                            readOnly = it.readOnly
+                        )
+                    } else {
+                        BuildParameters(key = it.key, value = it.value, readOnly = it.readOnly)
+                    }
+                }
+            )
+        }
+    }
+
     fun getVarsWithType(
         dslContext: DSLContext,
         projectId: String,

diff --git a/...ocess/biz-base/src/main/kotlin/com/tencent/devops/process/service/BuildVariableService.kt b/...ocess/biz-base/src/main/kotlin/com/tencent/devops/process/service/BuildVariableService.kt
@@ -40,6 +40,7 @@ import com.tencent.devops.process.utils.PipelineVarUtil
 import org.apache.commons.lang3.math.NumberUtils
 import org.jooq.DSLContext
 import org.jooq.impl.DSL
+import org.slf4j.LoggerFactory
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.stereotype.Service
 
@@ -52,6 +53,10 @@ class BuildVariableService @Autowired constructor(
     private val redisOperation: RedisOperation
 ) {
 
+    companion object {
+        private val logger = LoggerFactory.getLogger(BuildVariableService::class.java)
+    }
+
     /**
      * 获取构建执行次数（重试次数+1），如没有重试过，则为1
      */
@@ -277,13 +282,15 @@ class BuildVariableService @Autowired constructor(
             // 加锁防止数据被重复插入
             redisLock.lock()
             watch.start("getVars")
-            val buildVarMap = pipelineBuildVarDao.getVars(dslContext, projectId, buildId)
+            val buildVarMap = pipelineBuildVarDao.getBuildVarMap(dslContext, projectId, buildId)
             val insertBuildParameters = mutableListOf<BuildParameters>()
             val updateBuildParameters = mutableListOf<BuildParameters>()
             pipelineBuildParameters.forEach {
                 if (!buildVarMap.containsKey(it.key)) {
                     insertBuildParameters.add(it)
                 } else {
+                    // TODO PAC上线后删除, 打印流水线运行时覆盖只读变量,只在第一次运行时输出,重试不输出
+                    printReadOnlyVar(buildVarMap, variables, it, projectId, pipelineId, buildId)
                     updateBuildParameters.add(it)
                 }
             }
@@ -308,6 +315,25 @@ class BuildVariableService @Autowired constructor(
         }
     }
 
+    private fun printReadOnlyVar(
+        buildVarMap: Map<String, BuildParameters>,
+        variables: Map<String, BuildParameters>,
+        buildParameters: BuildParameters,
+        projectId: String,
+        pipelineId: String,
+        buildId: String
+    ) {
+        if (buildVarMap[PIPELINE_RETRY_COUNT] == null &&
+            variables[PIPELINE_RETRY_COUNT] == null &&
+            buildVarMap[buildParameters.key]?.readOnly == true
+        ) {
+            logger.warn(
+                "BKSystemErrorMonitor|$projectId|$pipelineId|$buildId|${buildParameters.key}| " +
+                        "build var read-only cannot be modified"
+            )
+        }
+    }
+
     // #10082 查询Agent复用互斥使用的AgentId
     fun fetchAgentReuseMutexVar(
         projectId: String,

diff --git a/.../biz-process/src/main/kotlin/com/tencent/devops/process/yaml/PipelineYamlFacadeService.kt b/.../biz-process/src/main/kotlin/com/tencent/devops/process/yaml/PipelineYamlFacadeService.kt
@@ -172,7 +172,6 @@ class PipelineYamlFacadeService @Autowired constructor(
                 val repository = client.get(ServiceRepositoryPacResource::class).getPacRepository(
                     externalId = externalId, scmType = scmType
                 ).data ?: run {
-                    logger.info("pipeline yaml trigger|repository not enable pac|$externalId|$scmType")
                     return
                 }
                 val setting = PacRepoSetting(repository = repository)

diff --git a/...-process/src/main/kotlin/com/tencent/devops/process/yaml/PipelineYamlRepositoryService.kt b/...-process/src/main/kotlin/com/tencent/devops/process/yaml/PipelineYamlRepositoryService.kt
@@ -365,10 +365,17 @@ class PipelineYamlRepositoryService @Autowired constructor(
         val defaultBranch = action.data.context.defaultBranch
         val ref = yamlFile.ref
         logger.info("deleteYamlPipeline|$userId|$projectId|$repoHashId|yamlFile:$yamlFile")
+        if (ref.isNullOrBlank()) {
+            return
+        }
+        val lock = PipelineYamlTriggerLock(
+            redisOperation = redisOperation,
+            projectId = projectId,
+            repoHashId = repoHashId,
+            filePath = filePath
+        )
         try {
-            if (ref.isNullOrBlank()) {
-                return
-            }
+            lock.lock()
             pipelineYamlService.deleteBranchFile(
                 projectId = projectId,
                 repoHashId = repoHashId,
@@ -387,7 +394,8 @@ class PipelineYamlRepositoryService @Autowired constructor(
                 ref = ref,
                 defaultBranch = defaultBranch
             )
-            if (needUnbindYamlPipeline) {
+            // 只有删除分支或者删除文件时才能解绑
+            if (releaseBranch == false && needUnbindYamlPipeline) {
                 unBindYamlPipeline(
                     userId = userId,
                     projectId = projectId,
@@ -424,6 +432,8 @@ class PipelineYamlRepositoryService @Autowired constructor(
         } catch (ignored: Exception) {
             logger.warn("Failed to delete pipeline yaml|$projectId|${action.format()}", ignored)
             throw ignored
+        } finally {
+            lock.unlock()
         }
     }
 

diff --git a/...ker-common/src/main/kotlin/com/tencent/devops/worker/common/task/market/MarketAtomTask.kt b/...ker-common/src/main/kotlin/com/tencent/devops/worker/common/task/market/MarketAtomTask.kt
@@ -68,6 +68,7 @@ import com.tencent.devops.process.utils.PIPELINE_ATOM_VERSION
 import com.tencent.devops.process.utils.PIPELINE_START_USER_ID
 import com.tencent.devops.process.utils.PIPELINE_STEP_ID
 import com.tencent.devops.process.utils.PIPELINE_TASK_NAME
+import com.tencent.devops.process.utils.PipelineVarUtil
 import com.tencent.devops.store.pojo.atom.AtomEnv
 import com.tencent.devops.store.pojo.atom.enums.AtomStatusEnum
 import com.tencent.devops.store.pojo.common.ATOM_POST_ENTRY_PARAM
@@ -191,6 +192,10 @@ open class MarketAtomTask : ITask() {
                 acrossProjectId = acrossInfo?.targetProjectId
             )
         }.toMap()
+        // 如果开启PAC,插件入参增加旧变量，防止开启PAC后,插件获取参数失败
+        if (asCodeEnabled) {
+            variables = PipelineVarUtil.mixOldVarAndNewVar(variables.toMutableMap())
+        }
 
         // 解析输入输出字段模板
         val props = JsonUtil.toMutableMap(atomData.props!!)
@@ -679,7 +684,6 @@ open class MarketAtomTask : ITask() {
         workspace: File,
         inputVariables: Map<String, Any>
     ) {
-//        logger.info("runtimeVariables is:$runtimeVariables") // 有敏感信息
         val inputFileFile = File(workspace, inputFile)
         inputFileFile.writeText(JsonUtil.toJson(inputVariables))
     }