From a8536632be1ff804127a7f2b108a7f3c509253bf Mon Sep 17 00:00:00 2001 From: Yuxiang Zhu Date: Wed, 2 Aug 2023 17:42:52 +0800 Subject: [PATCH] Update gen-assembly tekton job for ART OSD - Bump base image and dependencies for artcd image - Update gen-assembly job to use pyartcd - Add externalsecrets for secrets managed on AWS Secrets Manager --- tekton-pipelines/config/artcd-config.yaml | 6 +- tekton-pipelines/config/doozer-config.yaml | 5 +- tekton-pipelines/config/kerberos-config.yaml | 10 +- tekton-pipelines/config/ssh-config.yaml | 4 +- .../externalsecrets/art-bot-github-token.yaml | 26 ++ .../art-bot-slack-api-token.yaml | 24 ++ .../art-publish-ci-dockerconfigjson.yaml | 23 ++ .../art_quay_dev-dockerconfigjson.yaml | 23 ++ .../exd-ocp-buildvm-bot-prod-keytab.yaml | 29 ++ .../openshift-bot-ssh-private-key.yaml | 34 ++ tekton-pipelines/images/artcd/Containerfile | 31 +- .../artcd/files/etc/yum.repos.d/pulp.repo | 4 +- .../files/etc/yum.repos.d/rcm-tools.repo | 16 +- tekton-pipelines/infra/artcd-is.yaml | 2 +- tekton-pipelines/pipelines/gen-assembly.yaml | 329 ++++++++++++++---- .../serviceaccounts/pipeline.yaml | 8 + 16 files changed, 465 insertions(+), 109 deletions(-) create mode 100644 tekton-pipelines/externalsecrets/art-bot-github-token.yaml create mode 100644 tekton-pipelines/externalsecrets/art-bot-slack-api-token.yaml create mode 100644 tekton-pipelines/externalsecrets/art-publish-ci-dockerconfigjson.yaml create mode 100644 tekton-pipelines/externalsecrets/art_quay_dev-dockerconfigjson.yaml create mode 100644 tekton-pipelines/externalsecrets/exd-ocp-buildvm-bot-prod-keytab.yaml create mode 100644 tekton-pipelines/externalsecrets/openshift-bot-ssh-private-key.yaml create mode 100644 tekton-pipelines/serviceaccounts/pipeline.yaml diff --git a/tekton-pipelines/config/artcd-config.yaml b/tekton-pipelines/config/artcd-config.yaml index 3be360ab73..f9ee05f0e2 100644 --- a/tekton-pipelines/config/artcd-config.yaml +++ b/tekton-pipelines/config/artcd-config.yaml @@ -18,7 +18,7 @@ data: smtp_server = "smtp.corp.redhat.com" from = "aos-art-automation@redhat.com" reply_to = "aos-team-art@redhat.com" - cc = [] + cc = ["aos-art-automation@redhat.com"] prepare_release_notification_recipients_ocp4=["multi-arch-zstream-testing@redhat.com", "aos-qe@redhat.com", "mp-entitlement-qe@redhat.com"] prepare_release_notification_recipients_ocp3=["wzheng@redhat.com", "gpei@redhat.com", "aos-qe@redhat.com", "mp-entitlement-qe@redhat.com", "openshift-lp-test@redhat.com", "pprakash@redhat.com"] promote_image_list_recipients = ["openshift-ccs@redhat.com"] @@ -26,5 +26,5 @@ data: [jira] url = "https://issues.redhat.com" [jira.templates] - ocp4 = "OCPPLAN-4756" - ocp3 = "OCPPLAN-1373" + ocp4 = "ART-4223" + ocp3 = "ART-4234" diff --git a/tekton-pipelines/config/doozer-config.yaml b/tekton-pipelines/config/doozer-config.yaml index 66a9ff17e6..e075702bef 100644 --- a/tekton-pipelines/config/doozer-config.yaml +++ b/tekton-pipelines/config/doozer-config.yaml @@ -15,13 +15,16 @@ data: group: #Username for running rhpkg / brew / tito - user: ocp-build + user: exd-ocp-buildvm-bot-prod # Pointer to relational db to store info in. Possible values: prod, stage, empty # datastore: prod # cache_dir: /mnt/workspace/jenkins/doozer_cache + hosts: + prodsec_git: git.prodsec.redhat.com + global_opts: # num of concurrent distgit pull/pushes distgit_threads: 20 diff --git a/tekton-pipelines/config/kerberos-config.yaml b/tekton-pipelines/config/kerberos-config.yaml index 51b5cc573f..b787b8cd6c 100644 --- a/tekton-pipelines/config/kerberos-config.yaml +++ b/tekton-pipelines/config/kerberos-config.yaml @@ -32,9 +32,13 @@ data: # by storing krb5 credential cache into a file rather than kernel keyring. # See https://blog.tomecek.net/post/kerberos-in-a-container/ default_ccache_name = FILE:/tmp/krb5cc_%{uid} - rdns = false - default_realm = IPA.REDHAT.COM dns_lookup_realm = true dns_lookup_kdc = true - allow_weak_crypto = yes + ticket_lifetime = 24h + renew_lifetime = 7d + forwardable = true + rdns = false + dns_canonicalize_hostname = false + allow_weak_crypto = no udp_preference_limit = 0 + default_realm = IPA.REDHAT.COM diff --git a/tekton-pipelines/config/ssh-config.yaml b/tekton-pipelines/config/ssh-config.yaml index d908927ede..d501915909 100644 --- a/tekton-pipelines/config/ssh-config.yaml +++ b/tekton-pipelines/config/ssh-config.yaml @@ -5,5 +5,7 @@ metadata: name: ssh-config data: known_hosts: | - pkgs.devel.redhat.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAplqWKs26qsoaTxvWn3DFcdbiBxqRLhFngGiMYhbudnAj4li9/VwAJqLm1M6YfjOoJrj9dlmuXhNzkSzvyoQODaRgsjCG5FaRjuN8CSM/y+glgCYsWX1HFZSnAasLDuW0ifNLPR2RBkmWx61QKq+TxFDjASBbBywtupJcCsA5ktkjLILS+1eWndPJeSUJiOtzhoN8KIigkYveHSetnxauxv1abqwQTk5PmxRgRt20kZEFSRqZOJUlcl85sZYzNC/G7mneptJtHlcNrPgImuOdus5CW+7W49Z/1xqqWI/iRjwipgEMGusPMlSzdxDX4JzIx6R53pDpAwSAQVGDz4F9eQ== + pkgs.devel.redhat.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDT8I6l839M7tb6V/Le8x3pGo3sTo6SG/kMrVwPQ6kUtxuaWKBLCmI1HVawfRbBz4fO+8AifdKjtOKUHcI6iPr8= github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ== + github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl + github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg= diff --git a/tekton-pipelines/externalsecrets/art-bot-github-token.yaml b/tekton-pipelines/externalsecrets/art-bot-github-token.yaml new file mode 100644 index 0000000000..95c7c5c846 --- /dev/null +++ b/tekton-pipelines/externalsecrets/art-bot-github-token.yaml @@ -0,0 +1,26 @@ +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: art-bot-github-token +spec: + data: + - remoteRef: + key: art-bot-github-token + property: token-for-rate-limiting + secretKey: token-for-rate-limiting + - remoteRef: + key: art-bot-github-token + property: powerful + secretKey: powerful + refreshInterval: 1h + secretStoreRef: + kind: ClusterSecretStore + name: main-secret-store + target: + creationPolicy: Owner + deletionPolicy: Retain + name: synced-art-bot-github-token + template: + engineVersion: v2 + mergePolicy: Replace + type: Opaque diff --git a/tekton-pipelines/externalsecrets/art-bot-slack-api-token.yaml b/tekton-pipelines/externalsecrets/art-bot-slack-api-token.yaml new file mode 100644 index 0000000000..34c2fd7db9 --- /dev/null +++ b/tekton-pipelines/externalsecrets/art-bot-slack-api-token.yaml @@ -0,0 +1,24 @@ +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: art-bot-slack-api-token +spec: + data: + - remoteRef: + conversionStrategy: Default + decodingStrategy: None + key: art/prod/art-bot-slack-api-token + property: api_token + secretKey: api_token + refreshInterval: 1h + secretStoreRef: + kind: ClusterSecretStore + name: main-secret-store + target: + creationPolicy: Owner + deletionPolicy: Retain + name: synced-art-bot-slack-api-token + template: + engineVersion: v2 + mergePolicy: Replace + type: Opaque diff --git a/tekton-pipelines/externalsecrets/art-publish-ci-dockerconfigjson.yaml b/tekton-pipelines/externalsecrets/art-publish-ci-dockerconfigjson.yaml new file mode 100644 index 0000000000..17d2ee76b0 --- /dev/null +++ b/tekton-pipelines/externalsecrets/art-publish-ci-dockerconfigjson.yaml @@ -0,0 +1,23 @@ +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: art-publish-ci-dockerconfigjson +spec: + data: + - remoteRef: + conversionStrategy: Default + decodingStrategy: None + key: art/prod/art-publish@ci-dockerconfigjson + secretKey: .dockerconfigjson + refreshInterval: 1h + secretStoreRef: + kind: ClusterSecretStore + name: main-secret-store + target: + creationPolicy: Owner + deletionPolicy: Retain + name: synced-art-publish-ci-dockerconfigjson + template: + engineVersion: v2 + mergePolicy: Replace + type: kubernetes.io/dockerconfigjson diff --git a/tekton-pipelines/externalsecrets/art_quay_dev-dockerconfigjson.yaml b/tekton-pipelines/externalsecrets/art_quay_dev-dockerconfigjson.yaml new file mode 100644 index 0000000000..6a4aa8058e --- /dev/null +++ b/tekton-pipelines/externalsecrets/art_quay_dev-dockerconfigjson.yaml @@ -0,0 +1,23 @@ +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: art-quay-dev-dockerconfigjson +spec: + data: + - remoteRef: + conversionStrategy: Default + decodingStrategy: None + key: art/prod/openshift-release-dev+art_quay_dev@quay.io-dockerconfigjson + secretKey: .dockerconfigjson + refreshInterval: 1h + secretStoreRef: + kind: ClusterSecretStore + name: main-secret-store + target: + creationPolicy: Owner + deletionPolicy: Retain + name: synced-art-quay-dev-dockerconfigjson + template: + engineVersion: v2 + mergePolicy: Replace + type: kubernetes.io/dockerconfigjson diff --git a/tekton-pipelines/externalsecrets/exd-ocp-buildvm-bot-prod-keytab.yaml b/tekton-pipelines/externalsecrets/exd-ocp-buildvm-bot-prod-keytab.yaml new file mode 100644 index 0000000000..3986ab533c --- /dev/null +++ b/tekton-pipelines/externalsecrets/exd-ocp-buildvm-bot-prod-keytab.yaml @@ -0,0 +1,29 @@ +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: exd-ocp-buildvm-bot-prod-keytab +spec: + data: + - remoteRef: + conversionStrategy: Default + decodingStrategy: None + key: art/prod/exd-ocp-buildvm-bot-prod-keytab-principal + property: principal + secretKey: principal + - remoteRef: + conversionStrategy: Default + decodingStrategy: None + key: art/prod/exd-ocp-buildvm-bot-prod-keytab + secretKey: keytab + refreshInterval: 1h + secretStoreRef: + kind: ClusterSecretStore + name: main-secret-store + target: + creationPolicy: Owner + deletionPolicy: Retain + name: synced-exd-ocp-buildvm-bot-prod-keytab + template: + engineVersion: v2 + mergePolicy: Replace + type: Opaque diff --git a/tekton-pipelines/externalsecrets/openshift-bot-ssh-private-key.yaml b/tekton-pipelines/externalsecrets/openshift-bot-ssh-private-key.yaml new file mode 100644 index 0000000000..1ed0e65fc7 --- /dev/null +++ b/tekton-pipelines/externalsecrets/openshift-bot-ssh-private-key.yaml @@ -0,0 +1,34 @@ +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: openshift-bot-ssh-private-key +spec: + data: + - remoteRef: + conversionStrategy: Default + decodingStrategy: None + key: art/prod/openshift-bot-ssh-private-key + secretKey: ssh_privatekey + refreshInterval: 1h + secretStoreRef: + kind: ClusterSecretStore + name: main-secret-store + target: + creationPolicy: Owner + deletionPolicy: Retain + name: synced-openshift-bot-ssh-private-key + template: + engineVersion: v2 + mergePolicy: Replace + type: kubernetes.io/ssh-auth + metadata: + annotations: + tekton.dev/git-0: github.com + tekton.dev/git-1: pkgs.devel.redhat.com + data: + known_hosts: | + pkgs.devel.redhat.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDT8I6l839M7tb6V/Le8x3pGo3sTo6SG/kMrVwPQ6kUtxuaWKBLCmI1HVawfRbBz4fO+8AifdKjtOKUHcI6iPr8= + github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ== + github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl + github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg= + ssh-privatekey: "{{ .ssh_privatekey }}" diff --git a/tekton-pipelines/images/artcd/Containerfile b/tekton-pipelines/images/artcd/Containerfile index 9d962b6db6..92c8fcc3af 100644 --- a/tekton-pipelines/images/artcd/Containerfile +++ b/tekton-pipelines/images/artcd/Containerfile @@ -1,39 +1,36 @@ -FROM registry.access.redhat.com/ubi8/ubi:8.5 AS builder +FROM registry.access.redhat.com/ubi9/ubi:9.2 AS builder # Install build dependencies WORKDIR /usr/local/src USER root -RUN dnf -y module enable python36 \ - && dnf -y install python3 python3-wheel python3-devel gcc krb5-devel wget tar gzip \ - && python3 -m pip install "pip >= 21" +RUN dnf -y install python3 python3-pip python3-wheel python3-devel gcc krb5-devel wget tar gzip git # Download oc ARG OC_VERSION=latest -RUN wget -O openshift-client-linux-"$OC_VERSION".tar.gz https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/"$OC_VERSION"/openshift-client-linux.tar.gz \ - && tar -xzvf openshift-client-linux-"$OC_VERSION".tar.gz oc kubectl +RUN wget -O "openshift-client-linux-${OC_VERSION}.tar.gz" "https://mirror.openshift.com/pub/openshift-v4/$(arch)/clients/ocp/${OC_VERSION}/openshift-client-linux.tar.gz" \ + && tar -xzvf "openshift-client-linux-$OC_VERSION.tar.gz" oc kubectl # Build pyartcd, elliott, and doozer -COPY art-tools . COPY pyartcd pyartcd -RUN python3 -m pip wheel --wheel-dir /usr/local/src/wheels --use-deprecated=legacy-resolver \ - ./elliott ./doozer ./pyartcd +COPY art-tools art-tools +RUN python3 -m pip wheel --wheel-dir /usr/local/src/wheels \ + ./art-tools/elliott ./art-tools/doozer ./pyartcd - -FROM registry.access.redhat.com/ubi8/ubi:8.5 +FROM registry.access.redhat.com/ubi9/ubi:9.2 LABEL name="openshift-art/art-cd" \ maintainer="OpenShift Team Automated Release Tooling " -# Trust Red Hat IT Root CA -RUN curl -o /etc/pki/ca-trust/source/anchors/RH-IT-Root-CA.crt --fail -L \ - https://password.corp.redhat.com/RH-IT-Root-CA.crt \ +# Trust Red Hat IT Root CA certificates +RUN curl -fLo /etc/pki/ca-trust/source/anchors/2022-IT-Root-CA.pem https://certs.corp.redhat.com/certs/2022-IT-Root-CA.pem \ + && curl -fLo /etc/pki/ca-trust/source/anchors/2015-IT-Root-CA.pem https://certs.corp.redhat.com/certs/2015-IT-Root-CA.pem \ + && curl -fLo /etc/pki/ca-trust/source/anchors/RH-IT-Root-CA.crt https://certs.corp.redhat.com/certs/RH-IT-Root-CA.crt \ && update-ca-trust extract # Install runtime dependencies COPY ./tekton-pipelines/images/artcd/files/ / RUN \ # Configure Python environment - dnf -y module enable python36 && dnf -y install python3 \ - && python3 -m pip install "pip >= 21" \ + dnf -y install python3 python3-pip \ # Other tools && dnf -y install git brewkoji rhpkg krb5-workstation \ # Clean up @@ -47,7 +44,7 @@ COPY --from=builder /usr/local/src/wheels /usr/local/src/wheels RUN python3 -m pip install --ignore-installed --no-index --find-links=/usr/local/src/wheels pyartcd rh-elliott rh-doozer \ && rm /usr/local/src/wheels/*.whl \ # Make python-certifi trust system CA certificates - && python3 -m pip --no-cache-dir install certifi && ln -sf /etc/pki/tls/cert.pem /usr/local/lib/python3.6/site-packages/certifi/cacert.pem + && python3 -m pip install pip_system_certs # Set up user RUN useradd -m -d /home/dev -u 1000 dev diff --git a/tekton-pipelines/images/artcd/files/etc/yum.repos.d/pulp.repo b/tekton-pipelines/images/artcd/files/etc/yum.repos.d/pulp.repo index 7202fc70e6..5300e541e3 100644 --- a/tekton-pipelines/images/artcd/files/etc/yum.repos.d/pulp.repo +++ b/tekton-pipelines/images/artcd/files/etc/yum.repos.d/pulp.repo @@ -1,13 +1,13 @@ [pulp-dist-baseos] name = Dist repo - BaseOS -baseurl = http://rhsm-pulp.corp.redhat.com/content/dist/rhel8/8/$basearch/baseos/os +baseurl = http://rhsm-pulp.corp.redhat.com/content/dist/rhel$releasever/$releasever/$basearch/baseos/os enabled = 1 gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release gpgcheck = 1 [pulp-dist-appstream] name = Dist repo - AppStream -baseurl = http://rhsm-pulp.corp.redhat.com/content/dist/rhel8/8/$basearch/appstream/os/ +baseurl = http://rhsm-pulp.corp.redhat.com/content/dist/rhel$releasever/$releasever/$basearch/appstream/os/ enabled = 1 gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release gpgcheck = 1 diff --git a/tekton-pipelines/images/artcd/files/etc/yum.repos.d/rcm-tools.repo b/tekton-pipelines/images/artcd/files/etc/yum.repos.d/rcm-tools.repo index 099756bad1..679c6a912b 100644 --- a/tekton-pipelines/images/artcd/files/etc/yum.repos.d/rcm-tools.repo +++ b/tekton-pipelines/images/artcd/files/etc/yum.repos.d/rcm-tools.repo @@ -1,27 +1,27 @@ -[rcm-tools-rhel-8-baseos-rpms] +[rcm-tools-rhel-$releasever-baseos-rpms] name=RCM Tools for Red Hat Enterprise Linux $releasever BaseOS (RPMs) -baseurl=http://download.devel.redhat.com/rel-eng/RCMTOOLS/latest-RCMTOOLS-2-RHEL-8/compose/BaseOS/$basearch/os/ +baseurl=http://download.devel.redhat.com/rel-eng/RCMTOOLS/latest-RCMTOOLS-2-RHEL-$releasever/compose/BaseOS/$basearch/os/ enabled=1 gpgcheck=1 gpgkey=http://download.devel.redhat.com/rel-eng/RCMTOOLS/RPM-GPG-KEY-rcminternal -[rcm-tools-rhel-8-baseos-source-rpms] +[rcm-tools-rhel-$releasever-baseos-source-rpms] name=RCM Tools for Red Hat Enterprise Linux $releasever BaseOS (Source RPMs) -baseurl=http://download.devel.redhat.com/rel-eng/RCMTOOLS/latest-RCMTOOLS-2-RHEL-8/compose/BaseOS/source/tree/ +baseurl=http://download.devel.redhat.com/rel-eng/RCMTOOLS/latest-RCMTOOLS-2-RHEL-$releasever/compose/BaseOS/source/tree/ enabled=0 gpgcheck=1 gpgkey=http://download.devel.redhat.com/rel-eng/RCMTOOLS/RPM-GPG-KEY-rcminternal -[rcm-tools-rhel-8-server-optional-rpms] +[rcm-tools-rhel-$releasever-server-optional-rpms] name=RCM Tools for Red Hat Enterprise Linux $releasever BaseOS - Optional (RPMs) -baseurl=http://download.devel.redhat.com/rel-eng/RCMTOOLS/latest-RCMTOOLS-2-RHEL-8/compose/BaseOS-optional/$basearch/os/ +baseurl=http://download.devel.redhat.com/rel-eng/RCMTOOLS/latest-RCMTOOLS-2-RHEL-$releasever/compose/BaseOS-optional/$basearch/os/ enabled=1 gpgcheck=1 gpgkey=http://download.devel.redhat.com/rel-eng/RCMTOOLS/RPM-GPG-KEY-rcminternal -[rcm-tools-rhel-8-server-optional-source-rpms] +[rcm-tools-rhel-$releasever-server-optional-source-rpms] name=RCM Tools for Red Hat Enterprise Linux $releasever BaseOS - Optional (Source RPMs) -baseurl=http://download.devel.redhat.com/rel-eng/RCMTOOLS/latest-RCMTOOLS-2-RHEL-8/compose/BaseOS-optional/source/tree/ +baseurl=http://download.devel.redhat.com/rel-eng/RCMTOOLS/latest-RCMTOOLS-2-RHEL-$releasever/compose/BaseOS-optional/source/tree/ enabled=0 gpgcheck=1 gpgkey=http://download.devel.redhat.com/rel-eng/RCMTOOLS/RPM-GPG-KEY-rcminternal diff --git a/tekton-pipelines/infra/artcd-is.yaml b/tekton-pipelines/infra/artcd-is.yaml index c1031f3432..495f79ad57 100644 --- a/tekton-pipelines/infra/artcd-is.yaml +++ b/tekton-pipelines/infra/artcd-is.yaml @@ -1,5 +1,5 @@ --- -apiVersion: v1 +apiVersion: image.openshift.io/v1 kind: ImageStream metadata: name: "artcd" diff --git a/tekton-pipelines/pipelines/gen-assembly.yaml b/tekton-pipelines/pipelines/gen-assembly.yaml index e23841fe3e..53fad94b28 100644 --- a/tekton-pipelines/pipelines/gen-assembly.yaml +++ b/tekton-pipelines/pipelines/gen-assembly.yaml @@ -7,40 +7,103 @@ metadata: app: "artcd" spec: description: >- - Generate a recommended definition for an assembly based on a set of nightlies. - This are no side effects from running this job. It is the responsibility of the Artist to check the results into git / releases.yml. + Generate a recommended definition for an assembly based on a set of + nightlies. Find nightlies ready for release and define an assembly to add to + `releases.yml`. A pull request will be automatically created to add the + generated assembly definition to releases.yml. It is the responsibility of + the ARTist to review and merge the PR. params: - - name: group - description: Group name. e.g. openshift-4.9 - - name: assembly - description: The name of an assembly; must be defined in releases.yml (e.g. 4.9.1) - - name: custom - description: Use "true" to generate an assembly definition for a custom release. Custom assemblies are not for official release. They can, for example, not have all required arches for the group. - default: "false" - - name: nightlies - description: List of nightlies for each arch. For custom releases you do not need a nightly for each arch. - - name: in_flight_prev - description: (Optional for custom release) This is the in-flight release version of previous minor version of OCP. If there is no in-flight release, use "none". - - name: previous - description: '(Optional) List of OCP releases that can upgrade to the current release. Leave empty to use suggested value. Otherwise, follow item #6 "PREVIOUS" of the following doc for instructions on how to fill this field: https://mojo.redhat.com/docs/DOC-1201843#jive_content_id_Completing_a_4yz_release' - default: "" + - description: Group name. e.g. openshift-4.9 + name: group + type: string + - description: The name of an assembly; must be defined in releases.yml (e.g. 4.9.1) + name: assembly + type: string + - default: 'false' + description: >- + Use "true" to generate an assembly definition for a custom release. + Custom assemblies are not for official release. They can, for example, + not have all required arches for the group. + name: custom + type: string + - default: '' + description: >- + (Optional) List of nightlies for each arch. For custom releases you do + not need a nightly for each arch. + name: nightlies + type: string + - description: >- + This is the in-flight release version of previous minor version of OCP. + If there is no in-flight release, use "none". + name: in-flight-prev + type: string + - default: '' + description: >- + (Optional) List of OCP releases that can upgrade to the current release. + Leave empty to use suggested value. Otherwise, follow item #6 "PREVIOUS" + of the following doc for instructions on how to fill this field: + https://mojo.redhat.com/docs/DOC-1201843#jive_content_id_Completing_a_4yz_release + name: previous + type: string + - default: 'false' + description: Match nightlies that have not completed tests + name: allow-pending + type: string + - default: 'false' + description: Match nightlies that have failed their tests + name: allow-rejected + type: string + - default: 'false' + description: >- + Allow matching nightlies built from matching commits but with + inconsistent RPMs + name: allow-inconsistency + type: string + - default: '' + description: (Optional) Limit included arches. Only applicable to a custom release. + name: limit-arches + type: string + - default: '' + description: >- + (Optional) ocp-build-data fork to use (e.g. assembly definition in your + own fork) + name: data-path + type: string + - default: 'false' + description: 'Take no action, just echo what the job would have done.' + name: dry-run + type: string tasks: - name: gen-assembly - taskRef: - name: gen-assembly params: - name: group - value: "$(params.group)" + value: $(params.group) - name: assembly - value: "$(params.assembly)" + value: $(params.assembly) - name: nightlies - value: "$(params.nightlies)" + value: $(params.nightlies) - name: custom - value: "$(params.custom)" - - name: in_flight_prev - value: "$(params.in_flight_prev)" + value: $(params.custom) + - name: in-flight-prev + value: $(params.in-flight-prev) - name: previous - value: "$(params.previous)" + value: $(params.previous) + - name: allow-pending + value: $(params.allow-pending) + - name: allow-rejected + value: $(params.allow-rejected) + - name: allow-inconsistency + value: $(params.allow-inconsistency) + - name: limit-arches + value: $(params.limit-arches) + - name: data-path + value: $(params.data-path) + - name: dry-run + value: $(params.dry-run) + taskRef: + kind: Task + name: gen-assembly + --- apiVersion: tekton.dev/v1beta1 @@ -51,75 +114,195 @@ metadata: app: "artcd" spec: params: - - name: group - description: Group name. e.g. openshift-4.9 - - name: assembly - description: The name of an assembly; must be defined in releases.yml (e.g. 4.9.1) - - name: custom - description: Use "true" to generate an assembly definition for a custom release. Custom assemblies are not for official release. They can, for example, not have all required arches for the group. - default: "false" - - name: nightlies - description: List of nightlies for each arch. For custom releases you do not need a nightly for each arch. - - name: in_flight_prev - description: This is the in-flight release version of previous minor version of OCP. If there is no in-flight release, use "none". - - name: previous - description: '(Optional) List of OCP releases that can upgrade to the current release. Leave empty to use suggested value. Otherwise, follow item #6 "PREVIOUS" of the following doc for instructions on how to fill this field: https://mojo.redhat.com/docs/DOC-1201843#jive_content_id_Completing_a_4yz_release' - default: "" + - description: Group name. e.g. openshift-4.9 + name: group + type: string + - description: The name of an assembly; must be defined in releases.yml (e.g. 4.9.1) + name: assembly + type: string + - default: 'false' + description: >- + Use "true" to generate an assembly definition for a custom release. + Custom assemblies are not for official release. They can, for example, + not have all required arches for the group. + name: custom + type: string + - default: '' + description: >- + (Optional) List of nightlies for each arch. For custom releases you do + not need a nightly for each arch. + name: nightlies + type: string + - description: >- + This is the in-flight release version of previous minor version of OCP. + If there is no in-flight release, use "none". + name: in-flight-prev + type: string + - default: '' + description: >- + (Optional) List of OCP releases that can upgrade to the current release. + Leave empty to use suggested value. Otherwise, follow item #6 "PREVIOUS" + of the following doc for instructions on how to fill this field: + https://mojo.redhat.com/docs/DOC-1201843#jive_content_id_Completing_a_4yz_release + name: previous + type: string + - default: 'false' + description: Match nightlies that have not completed tests + name: allow-pending + type: string + - default: 'false' + description: Match nightlies that have failed their tests + name: allow-rejected + type: string + - default: 'false' + description: >- + Allow matching nightlies built from matching commits but with + inconsistent RPMs + name: allow-inconsistency + type: string + - default: '' + description: (Optional) Limit included arches. Only applicable to a custom release. + name: limit-arches + type: string + - default: '' + description: >- + (Optional) ocp-build-data fork to use (e.g. assembly definition in your + own fork) + name: data-path + type: string + - default: 'false' + description: 'Take no action, just echo what the job would have done.' + name: dry-run + type: string steps: - - name: gen-assembly - image: image-registry.openshift-image-registry.svc:5000/art-cd/artcd:latest - script: | + - env: + - name: SLACK_BOT_TOKEN + valueFrom: + secretKeyRef: + key: api_token + name: synced-art-bot-slack-api-token + - name: GITHUB_TOKEN + valueFrom: + secretKeyRef: + key: powerful + name: synced-art-bot-github-token + image: >- + image-registry.openshift-image-registry.svc:5000/hackspace-yuxzhu/artcd:latest + name: gen-assembly + resources: + limits: + cpu: '1' + memory: 2Gi + requests: + cpu: 500m + memory: 512Mi + script: > #!/usr/bin/env python3 + import re + import os + import subprocess + from pathlib import Path - cmd = ["doozer", "--group=$(params.group)", "--assembly=stream", "release:gen-assembly", "--name=$(params.assembly)", "from-releases"] + cmd = [ + "artcd", + "-v", + "--config=/etc/artcd/artcd.toml", + ] + + + if "$(params.dry-run)" == "true": + cmd.append("--dry-run") + + cmd.extend([ + "gen-assembly", + "--group", "$(params.group)", + "--assembly", "$(params.assembly)", + ]) + + + if "$(params.data-path)": + cmd.append(f"--data-path=$(params.data-path)") + + limit_arches = [p for p in re.split(r'[\s,]', "$(params.limit-arches)") + if p] + + if limit_arches: + cmd.extend([f"--arch={p}" for p in limit_arches]) + if "$(params.custom)" == "true": cmd.append("--custom") + + if "$(params.allow-pending)" == "true": + cmd.append("--allow-pending") + + if "$(params.allow-rejected)" == "true": + cmd.append("--allow-rejected") + + if "$(params.allow-inconsistency)" == "true": + cmd.append("--allow-inconsistency") + + if "$(params.in-flight-prev)" and "$(params.in-flight-prev)" != "none": + cmd.append("--in-flight=$(params.in-flight-prev)") + + previous_list = [p for p in re.split(r'[\s,]', "$(params.previous)") if + p] + + if previous_list: + cmd.extend([f"--previous={p}" for p in previous_list]) else: - if "$(params.in_flight_prev)" and "$(params.in_flight_prev)" != "none": - cmd.append("--in-flight=$(params.in_flight_prev)") - previous_list = [p for p in re.split(r'[\s,]', "$(params.previous)") if p] - if previous_list: - cmd.extend([f"--previous={p}" for p in previous_list]) - else: - cmd.append("--auto-previous") + cmd.append("--auto-previous") + nightlies = [n for n in re.split(r'[\s,]', "$(params.nightlies)") if n] + cmd.extend([f"--nightly={n}" for n in nightlies]) - subprocess.run(["kinit", "-f", "-k", "-t", "/etc/kerberos-keytab/jenkins-buildvm-keytab", "ocp-build/buildvm.openshift.eng.bos.redhat.com@IPA.REDHAT.COM"], check=True, universal_newlines=True) + + print(f"Running kinit...") + + subprocess.run(["kinit", "-f", "-k", "-t", + "/etc/kerberos-keytab/keytab", + "exd-ocp-buildvm-bot-prod@IPA.REDHAT.COM"], check=True, + universal_newlines=True) + print(f"Running {cmd}...") - env=os.environ.copy() - subprocess.run(cmd, check=True, universal_newlines=True, env=env) - env: - # https://github.com/tektoncd/pipeline/issues/2013 - - name: HOME - value: /home/dev + + subprocess.run(cmd, check=True, universal_newlines=True, + env=os.environ.copy()) volumeMounts: - - name: doozer-config - mountPath: /home/dev/.config/doozer/ - - name: kerberos-keytab - mountPath: /etc/kerberos-keytab - - name: kerberos-config - mountPath: /etc/krb5.conf.d/krb5-redhat.conf + - mountPath: /etc/artcd/ + name: artcd-config + - mountPath: /home/dev/.config/doozer/ + name: doozer-config + - mountPath: /etc/kerberos-keytab + name: kerberos-keytab + - mountPath: /etc/krb5.conf.d/krb5-redhat.conf + name: kerberos-config subPath: krb5-redhat.conf - - name: registry-cred - mountPath: /home/dev/.docker/config.json - subPath: .dockerconfigjson + - mountPath: /home/dev/.gitconfig + name: git-config + subPath: .gitconfig volumes: - - name: doozer-config - configMap: + - configMap: + name: artcd-config + name: artcd-config + - configMap: name: doozer-config - - name: kerberos-config - configMap: + name: doozer-config + - configMap: name: kerberos-config + name: kerberos-config - name: kerberos-keytab secret: - secretName: jenkins-buildvm-keytab - - name: registry-cred + secretName: synced-exd-ocp-buildvm-bot-prod-keytab + - name: art-bot-slack-api-token secret: - secretName: registry-cred + secretName: synced-exd-ocp-buildvm-bot-prod-keytab + - configMap: + name: git-config + name: git-config diff --git a/tekton-pipelines/serviceaccounts/pipeline.yaml b/tekton-pipelines/serviceaccounts/pipeline.yaml new file mode 100644 index 0000000000..23ee7655b6 --- /dev/null +++ b/tekton-pipelines/serviceaccounts/pipeline.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: pipeline +secrets: +- name: synced-art-publish-ci-dockerconfigjson +- name: synced-art-quay-dev-dockerconfigjson +- name: synced-openshift-bot-ssh-private-key