feat(CI): Add multi component packages integrate support

通过obs的project的父子关系管理组件间的依赖关系,默认main为父项目,dde和community为子项目;后者依赖前者,
同时dde和community组件间默认没有依赖关系,同时原则上不允许有,特殊情况协商后手动在obs上添加

.github/workflows/01-parsec-integration.yml

@@ -3,21 +3,27 @@ name: parsec integration
 on:
   workflow_call:
     outputs:
-      build_matrix:
-        description: build order 0, build tasks
-        value: ${{ jobs.parsec_integration.outputs.build_matrix }}
+      build_matrix_main:
+        description: deepin main build tasks
+        value: ${{ jobs.parsec_integration.outputs.build_matrix_main }}
+      build_matrix_dde:
+        description: build tasks
+        value: ${{ jobs.parsec_integration.outputs.build_matrix_dde }}
+      build_matrix_community:
+        description: build tasks
+        value: ${{ jobs.parsec_integration.outputs.build_matrix_community }}
       all_build_task:
         description: all build tasks used for
         value: ${{ jobs.parsec_integration.outputs.all_build_task }}
+      testrepoaddr:
+        description: testing repositories addresses
+        value: ${{ jobs.parsec_integration.outputs.testrepoaddr }}
       deploy:
         description: deploy way AUTO or MANUALLY
         value: ${{ jobs.parsec_integration.outputs.deploy }}
       integration_message:
         description: integration message at integration.yml
         value: ${{ jobs.parsec_integration.outputs.integration_message }}
-      component:
-        description: integration repos component
-        value: ${{ jobs.parsec_integration.outputs.component }}
       integrationinfo:
         description: integration repos list
         value: ${{ jobs.parsec_integration.outputs.integrationinfo }}
@@ -39,11 +45,13 @@ jobs:
     name: parsec integration
     runs-on: ubuntu-latest
     outputs:
-      build_matrix: ${{ steps.parsec_integration.outputs.build_matrix }}
+      build_matrix_main: ${{ steps.parsec_integration.outputs.build_matrix_main }}
+      build_matrix_dde: ${{ steps.parsec_integration.outputs.build_matrix_dde }}
+      build_matrix_community: ${{ steps.parsec_integration.outputs.build_matrix_community }}
       all_build_task: ${{ steps.parsec_integration.outputs.all_build_task }}
+      testrepoaddr: ${{ steps.parsec_integration.outputs.testrepoaddr }}
       deploy: ${{ steps.illegal_repo_tags.outputs.DEPLOY }}
       integration_message: ${{ steps.parsec_integration.outputs.integration_message }}
-      component: ${{ steps.illegal_repo_tags.outputs.component }}
       integrationinfo: ${{ steps.parsec_integration.outputs.integrationinfo }}
       changelog: ${{ steps.parsec_integration.outputs.changelog }}
       influence: ${{ steps.parsec_integration.outputs.influence }}
@@ -73,7 +81,6 @@ jobs:
           import json
           from requests.utils import requote_uri
 
-          maxSize = os.environ.get("MAX_TASKS", 10)
           illegalTags = []
           illegalRepos = []
           noChangelogs = []
@@ -163,8 +170,25 @@ jobs:
             print(name, version, changelog, influence)
             return name, version, "\n".join(changelog), "\n".join(influence)
 
+          def get_component(repo):
+            print("get component")
+            component = "main"
+            if "linuxdeepin/" in repo:
+              component = "dde"
+            else:
+              rawurl = "https://raw.githubusercontent.com/{repo}/master/debian/deepin/workflows.yml".format(repo=repo)
+              res = requests.get(rawurl.replace("+", "%2B"))
+              if res.status_code == 200:
+                if "deepin:Develop:community" in str(res.content):
+                  component = "community"
+              else:
+                print("Warn: get workflows.yml content failed!!!")
+                component = "community"
+            return component
+
           try:
-              allBuildTasks = [{"order": j,"data":{"payload": []}} for j in range(maxSize)]
+              components = ["main", "dde", "community"]
+              allBuildTasks = [{"component": j,"data":{"payload": []}} for j in components]
               data = read_integration()
 
               message = data.get("message", "Integrated for V23-Beta3").replace("\n", "")
@@ -180,9 +204,14 @@ jobs:
               changelogs = []
               influences = []
               for repo in data.get("repos"):
-                order = int(repo.get("order",0))
-                order = max(0, order) and min(order , maxSize - 1)
                 fullreponame = str(repo.get('repo', ""))
+                component = get_component(fullreponame)
+                component_index = 0
+                if component == "dde":
+                  component_index = 1
+                if component == "community":
+                  component_index = 2
+                print("component: %s" % component)
                 tagsha = repo.get('tagsha', "")
                 tag = repo.get('tag', "")
                 if tag and tag != "" and len(tag) > 0:
@@ -204,7 +233,7 @@ jobs:
                     noChangelogs.append({"repo": fullreponame, "tag": tagsha, "tagsha": tagsha})
                 if tagsha == "" and len(tagsha) <= 0 and tag == "" and len(tag) <= 0:
                   illegalTags.append({"repo": fullreponame, "tag": tag, "tagsha": tagsha})
-                allBuildTasks[order].get("data").get("payload").append(repo)
+                allBuildTasks[component_index].get("data").get("payload").append(repo)
               integrationinfo = str("\n".join(integrationInfos))
               print("integrationinfo: " + integrationinfo)
               changelog = str("\n".join(changelogs))
@@ -238,12 +267,27 @@ jobs:
                   set_output("no_changelogs", "undefined")
 
               allRepos = []
-              for order in allBuildTasks:
-                  for repo in order.get("data").get("payload"):
+              testrepoaddr = ""
+              for c in allBuildTasks:
+                  print(c)
+                  print(c.get("component"))
+                  for repo in c.get("data").get("payload"):
                       allRepos.append(repo)
-                  if len(order.get("data").get("payload")) > 0:
-                      set_output("build_matrix", str(order.get("data")))
+                  if len(c.get("data").get("payload")) > 0:
+                      set_output("build_matrix_" + str(c.get("component")), str(c.get("data")))
+                      if str(c.get("component")) == "main":
+                        if testrepoaddr != "":
+                          testrepoaddr = testrepoaddr + "\n    deb [trusted=yes] https://ci.deepin.com/repo/obs/deepin:/CI:/TestingIntegration:/test-integration-pr-${{ github.event.number }}/testing/ ./"
+                        else:
+                          testrepoaddr ="    deb [trusted=yes] https://ci.deepin.com/repo/obs/deepin:/CI:/TestingIntegration:/test-integration-pr-${{ github.event.number }}/testing/ ./"
+                      else:
+                        if testrepoaddr != "":
+                          testrepoaddr = testrepoaddr + "\n    deb [trusted=yes] https://ci.deepin.com/repo/obs/deepin:/CI:/TestingIntegration:/test-integration-pr-${{ github.event.number }}:/" + str(c.get("component")) + "/testing/ ./"
+                        else:
+                          testrepoaddr ="    deb [trusted=yes] https://ci.deepin.com/repo/obs/deepin:/CI:/TestingIntegration:/test-integration-pr-${{ github.event.number }}:/" + str(c.get("component")) + "/testing/ ./"
               set_output("all_build_task", str(allRepos))
+              print(testrepoaddr)
+              set_output("testrepoaddr", str(testrepoaddr))
 
           except BaseException as e:
               logging.error(e)
@@ -318,87 +362,6 @@ jobs:
               core.setFailed("some tags are not invalid!")
             }
 
-            // component 检查
-            all_build_task = ${{ steps.parsec_integration.outputs.all_build_task }}
-            let component = ""
-            for (repo of all_build_task) {
-              console.log(repo)
-              obsWorkflowsPath = ".obs/workflows.yml"
-              if (repo.repo.includes("deepin-community")) {
-                obsWorkflowsPath = "debian/deepin/workflows.yml"
-              }
-              repoinfo = repo.repo.split('/')
-              if (repoinfo.length == 2) {
-                const {data} = await github.rest.repos.getContent({
-                  owner: repoinfo[0],
-                  repo: repoinfo[1],
-                  path: obsWorkflowsPath,
-                });
-                datas = Buffer.from(data.content, data.encoding)
-                obsinfo = datas.toString()
-                //console.log(obsinfo)
-                if (obsinfo.length > 0) {
-                  if (obsinfo.includes("source_project: deepin:Develop:dde")) {
-                    if (component == "") {
-                      component = "dde"
-                    }
-
-                    if (component != "dde") {
-                      core.setFailed("intergrate multi components repo are not allowed!")
-                      break
-                    }
-
-                  } else if (obsinfo.includes("source_project: deepin:Develop:main")) {
-                    if (component == "") {
-                      component = "main"
-                    }
-
-                    if (component != "main") {
-                      core.setFailed("intergrate multi components repo are not allowed!")
-                      break
-                    }
-
-                  } else if (obsinfo.includes("source_project: deepin:Develop:community")) {
-                    if (component == "") {
-                      component = "community"
-                    }
-
-                    if (component != "community") {
-                      core.setFailed("intergrate multi components repo are not allowed!")
-                      break
-                    }
-
-                  } else if (obsinfo.includes("source_project: deepin:Develop:commercial")) {
-                    if (component == "") {
-                      component = "commercial"
-                    }
-
-                    if (component != "commercial") {
-                      core.setFailed("intergrate multi components repo are not allowed!")
-                      break
-                    }
-                  }
-                } else {
-                  // 默认没有配置obs工作流程的项目组件为community
-                  if (component == "") {
-                    component = "community"
-                  }
-
-                  if (component != "community") {
-                    core.setFailed("intergrate multi components repo are not allowed!")
-                    break
-                  }
-                }
-              }
-            }
-
-            if (component == "") {
-              // 默认community
-              component = "community"
-            }
-            //console.log(component)
-            core.setOutput('component', component)
-
       - name: upload integration detail
         uses: actions/upload-artifact@v3
         with:

.github/workflows/02-build-obs.yml

@@ -13,6 +13,9 @@ on:
       topic:
         required: true
         type: string
+      component:
+        type: string
+        default: "main"
   workflow_dispatch:
     inputs:
       repo:
@@ -25,9 +28,13 @@ on:
       topic:
         required: true
         type: string
+      component:
+        type: string
+        default: "main"
 
 env:
   TOPIC: ${{ inputs.topic }}
+  COMPONENT: ${{ inputs.component }}
   TAG: ${{ inputs.tag }}
   REPO: ${{ inputs.repo }}
   TAGSHA: ${{ inputs.tagsha }}
@@ -64,6 +71,17 @@ jobs:
             curl -X PUT -u golf66:$OSCPASS -H "Content-type: text/xml" -d @meta.xml "https://build.deepin.com/source/deepin:CI:TestingIntegration:$TOPIC/_meta"
           fi
 
+          # create obs component project
+          if [ "$COMPONENT" != "main" ]; then
+            result=$(curl -u golf66:$OSCPASS "https://build.deepin.com/source/deepin:CI:TestingIntegration:$TOPIC:$COMPONENT/_meta"|grep "unknown_project")
+            if [ "$result" != "" ];then
+              curl -o meta1.xml https://raw.githubusercontent.com/deepin-community/Repository-Integration/master/.github/workflows/obs-proj-component-meta.tpl
+              sed -i "s#TOPIC#${TOPIC}#g" meta1.xml
+              sed -i "s#COMPONENT#${COMPONENT}#g" meta1.xml
+              curl -X PUT -u golf66:$OSCPASS -H "Content-type: text/xml" -d @meta1.xml "https://build.deepin.com/source/deepin:CI:TestingIntegration:$TOPIC:$COMPONENT/_meta"
+            fi
+          fi
+
       - name: get tag info
         id: tagsha
         run: |
@@ -108,10 +126,16 @@ jobs:
 
           curl -o meta.xml https://raw.githubusercontent.com/deepin-community/Repository-Integration/master/.github/workflows/obs-pkg-meta.tpl
           sed -i "s#PKGNAME#${pkgname}#g" meta.xml
-          sed -i "s#Testing:COMPONENT#CI:TestingIntegration:${TOPIC}#g" meta.xml
-          curl -X PUT -u golf66:$OSCPASS -H "Content-type: text/xml" -d @meta.xml "https://build.deepin.com/source/deepin:CI:TestingIntegration:${TOPIC}/$pkgname/_meta"
+          if [ "$COMPONENT" = "main" ]; then
+            sed -i "s#Testing:COMPONENT#CI:TestingIntegration:${TOPIC}#g" meta.xml
+            curl -X PUT -u golf66:$OSCPASS -H "Content-type: text/xml" -d @meta.xml "https://build.deepin.com/source/deepin:CI:TestingIntegration:${TOPIC}/$pkgname/_meta"
+            osc co deepin:CI:TestingIntegration:${TOPIC}/${pkgname} && cd $_
+          else
+            sed -i "s#Testing:COMPONENT#CI:TestingIntegration:${TOPIC}:${COMPONENT}#g" meta.xml
+            curl -X PUT -u golf66:$OSCPASS -H "Content-type: text/xml" -d @meta.xml "https://build.deepin.com/source/deepin:CI:TestingIntegration:${TOPIC}:${COMPONENT}/$pkgname/_meta"
+            osc co deepin:CI:TestingIntegration:${TOPIC}:${COMPONENT}/${pkgname} && cd $_
+          fi
 
-          osc co deepin:CI:TestingIntegration:${TOPIC}/${pkgname} && cd $_
           needbuild=true
           if [ -f _branch_request ];then
             cat _branch_request |grep "${TAGSHA}" && needbuild=false ||echo true
@@ -147,7 +171,14 @@ jobs:
           mkdir -p ~/.config/osc
           echo "${{ secrets.OSCRC }}" > ~/.config/osc/oscrc
           pkgname="${{ needs.build.outputs.pkgname }}"
-          osc co deepin:CI:TestingIntegration:${TOPIC}/${pkgname} && cd $_
-          if [ -z $(osc buildinfo testing ${ARCH} |grep "<error>excluded</error>") ];then osc buildlog testing ${ARCH}; else echo "${ARCH} build excluded"; fi
-          if [ "$(osc api -X GET /build/deepin:CI:TestingIntegration:${TOPIC}/testing/${ARCH}/${pkgname}/_status |grep 'succeeded')" = "" -a "$(osc api -X GET /build/deepin:CI:TestingIntegration:${TOPIC}/testing/${ARCH}/${pkgname}/_status |grep 'finished')" = "" ];\
-          then echo "${ARCH} build failed" && -1; else echo "${ARCH} build succeeded"; fi
+          if [ "$COMPONENT" = "main" ]; then
+            osc co deepin:CI:TestingIntegration:${TOPIC}/${pkgname} && cd $_
+            if [ -z $(osc buildinfo testing ${ARCH} |grep "<error>excluded</error>") ];then osc buildlog testing ${ARCH}; else echo "${ARCH} build excluded"; fi
+            if [ "$(osc api -X GET /build/deepin:CI:TestingIntegration:${TOPIC}/testing/${ARCH}/${pkgname}/_status |grep 'succeeded')" = "" -a "$(osc api -X GET /build/deepin:CI:TestingIntegration:${TOPIC}/testing/${ARCH}/${pkgname}/_status |grep 'finished')" = "" ];\
+            then echo "${ARCH} build failed" && -1; else echo "${ARCH} build succeeded"; fi
+          else
+            osc co deepin:CI:TestingIntegration:${TOPIC}:${COMPONENT}/${pkgname} && cd $_
+            if [ -z $(osc buildinfo testing ${ARCH} |grep "<error>excluded</error>") ];then osc buildlog testing ${ARCH}; else echo "${ARCH} build excluded"; fi
+            if [ "$(osc api -X GET /build/deepin:CI:TestingIntegration:${TOPIC}:${COMPONENT}/testing/${ARCH}/${pkgname}/_status |grep 'succeeded')" = "" -a "$(osc api -X GET /build/deepin:CI:TestingIntegration:${TOPIC}/testing/${ARCH}/${pkgname}/_status |grep 'finished')" = "" ];\
+            then echo "${ARCH} build failed" && -1; else echo "${ARCH} build succeeded"; fi
+          fi

.github/workflows/auto-integration.yml

@@ -90,11 +90,15 @@ jobs:
       - parsec_integration
       - ouput_message
       - build_project_prepare
-      - builds
+      - build_main
+      - build_dde
+      - build_community
     if: |
       always() && !cancelled() &&
       (needs.parsec_integration.result == 'success') && (needs.ouput_message.result != 'failure' ) &&
-      (needs.builds.result == 'success' || needs.builds.result == 'skipped')
+      (needs.build_main.result == 'success' || needs.build_main.result == 'skipped') &&
+      (needs.build_dde.result == 'success' || needs.build_dde.result == 'skipped') &&
+      (needs.build_community.result == 'success' || needs.build_community.result == 'skipped')
     uses: deepin-community/Repository-Integration/.github/workflows/issue-project-manager.yml@master
     secrets: inherit
     with:
@@ -109,7 +113,7 @@ jobs:
       integrationinfo: ${{ needs.parsec_integration.outputs.integrationinfo }}
       changelog: ${{ needs.parsec_integration.outputs.changelog }}
       influence: ${{ needs.parsec_integration.outputs.influence }}
-      testrepoaddr: "    deb [trusted=yes] https://ci.deepin.com/repo/obs/deepin:/CI:/TestingIntegration:/test-integration-pr-${{ github.event.number }}/testing/ ./"
+      testrepoaddr: ${{ needs.parsec_integration.outputs.testrepoaddr }}
 
   pass_integration:
     name: pass integration
@@ -142,19 +146,55 @@ jobs:
       integrations: ${{ needs.parsec_integration.outputs.all_build_task }}
 
   # build task
-  builds:
-    name: builds
-    if: always() && !cancelled() && (needs.parsec_integration.outputs.build_matrix != '') && (needs.parsec_integration.result == 'success')
+  build_main:
+    name: build_main
+    if: always() && !cancelled() && (needs.parsec_integration.outputs.build_matrix_main != '') && (needs.parsec_integration.result == 'success')
     needs:
       - parsec_integration
       - build_project_prepare
     strategy:
       fail-fast: false
-      matrix: ${{ fromJson(needs.parsec_integration.outputs.build_matrix) }}
+      matrix: ${{ fromJson(needs.parsec_integration.outputs.build_matrix_main) }}
     uses: deepin-community/Repository-Integration/.github/workflows/02-build-obs.yml@master
     secrets: inherit
     with:
       repo: ${{ matrix.payload.repo }}
       tag: ${{ matrix.payload.tag }}
       tagsha: ${{ matrix.payload.tagsha }}
       topic: test-integration-pr-${{ github.event.number }}
+
+  build_dde:
+    name: build_dde
+    if: always() && !cancelled() && (needs.parsec_integration.outputs.build_matrix_dde != '') && (needs.parsec_integration.result == 'success')
+    needs:
+      - parsec_integration
+      - build_project_prepare
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.parsec_integration.outputs.build_matrix_dde) }}
+    uses: deepin-community/Repository-Integration/.github/workflows/02-build-obs.yml@master
+    secrets: inherit
+    with:
+      repo: ${{ matrix.payload.repo }}
+      tag: ${{ matrix.payload.tag }}
+      tagsha: ${{ matrix.payload.tagsha }}
+      topic: test-integration-pr-${{ github.event.number }}
+      component: "dde"
+
+  build_community:
+    name: build_community
+    if: always() && !cancelled() && (needs.parsec_integration.outputs.build_matrix_community != '') && (needs.parsec_integration.result == 'success')
+    needs:
+      - parsec_integration
+      - build_project_prepare
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.parsec_integration.outputs.build_matrix_community) }}
+    uses: deepin-community/Repository-Integration/.github/workflows/02-build-obs.yml@master
+    secrets: inherit
+    with:
+      repo: ${{ matrix.payload.repo }}
+      tag: ${{ matrix.payload.tag }}
+      tagsha: ${{ matrix.payload.tagsha }}
+      topic: test-integration-pr-${{ github.event.number }}
+      component: "community"