fix: update rhel build (#813)

* fix: update rhel build Signed-off-by: Valeriy Svydenko <[email protected]> * update license Signed-off-by: Valeriy Svydenko <[email protected]> --------- Signed-off-by: Valeriy Svydenko <[email protected]>
eclipse-che · Jan 31, 2024 · b90f17b · b90f17b
1 parent 130103f
commit b90f17b
Show file tree

Hide file tree

Showing 5 changed files with 174 additions and 17 deletions.
diff --git a/build/dockerfiles/content_sets_centos8.repo b/build/dockerfiles/content_sets_centos8.repo
diff --git a/build/dockerfiles/content_sets_rhel8.repo b/build/dockerfiles/content_sets_rhel8.repo
@@ -0,0 +1,13 @@
+[rhel-8-for-appstream-rpms-pulp]
+name=rhel-8-for-appstream-rpms-pulp
+baseurl=http://rhsm-pulp.corp.redhat.com/content/dist/rhel8/8/$basearch/appstream/os
+enabled=0
+gpgcheck=0
+skip_if_unavailable=True
+
+[rhel-8-for-baseos-rpms-pulp]
+name=rhel-8-for-baseos-rpms-pulp
+baseurl=http://rhsm-pulp.corp.redhat.com/content/dist/rhel8/8/$basearch/baseos/os
+enabled=0
+gpgcheck=0
+skip_if_unavailable=True
diff --git a/build/dockerfiles/rhel.Dockerfile b/build/dockerfiles/rhel.Dockerfile
@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2018-2021 Red Hat, Inc.
+# Copyright (c) 2018-2024 Red Hat, Inc.
 # This program and the accompanying materials are made
 # available under the terms of the Eclipse Public License 2.0
 # which is available at https://www.eclipse.org/legal/epl-2.0/
@@ -12,8 +12,8 @@
 #
 
 # Builder: check meta.yamls and create index.json
-# https://access.redhat.com/containers/?tab=tags#/registry.access.redhat.com/ubi8/nodejs-16-minimal
-FROM registry.access.redhat.com/ubi8/nodejs-16-minimal:1-155 as builder
+# https://access.redhat.com/containers/?tab=tags#/registry.access.redhat.com/ubi8/nodejs-18-minimal
+FROM registry.access.redhat.com/ubi8/nodejs-18-minimal:1-91 as builder
 USER 0
 
 ################# 
@@ -23,6 +23,8 @@ USER 0
 ARG BOOTSTRAP=false
 ENV BOOTSTRAP=${BOOTSTRAP}
 
+ARG VERSION
+
 # to get all the python deps pre-fetched so we can build in Brew:
 # 1. extract files in the container to your local filesystem
 #    find v3 -type f -exec dos2unix {} \;
@@ -50,8 +52,11 @@ RUN ./check_mandatory_fields.sh devfiles
 
 RUN ./index.sh > /build/devfiles/index.json
 RUN ./list_referenced_images.sh devfiles > /build/devfiles/external_images.txt
-RUN ./generate_devworkspace_templates.sh
+RUN ./generate_devworkspace_templates.sh $VERSION
+RUN ./update_devworkspace_templates.sh $VERSION
+RUN ./list_referenced_images.sh devfiles > /build/devfiles/external_images.txt
 RUN chmod -R g+rwX /build/devfiles
+RUN chmod -R g+rwX /build/resources
 
 ################# 
 # PHASE TWO: configure registry image
@@ -77,6 +82,8 @@ RUN sed -i /etc/httpd/conf/httpd.conf \
     -e "s,logs/error_log,/dev/stderr," \
     -e "s,logs/access_log,/dev/stdout," \
     -e "s,AllowOverride None,AllowOverride All," && \
+    echo "ServerName localhost" >> /etc/httpd/conf/httpd.conf && \
+    echo "LimitRequestFieldSize 32768" >> /etc/httpd/conf/httpd.conf && \
     chmod a+rwX /etc/httpd/conf /run/httpd /etc/httpd/logs/
 STOPSIGNAL SIGWINCH
 
@@ -85,6 +92,7 @@ WORKDIR /var/www/html
 RUN mkdir -m 777 /var/www/html/devfiles
 COPY .htaccess README.md /var/www/html/
 COPY --from=builder /build/devfiles /var/www/html/devfiles
+COPY --from=builder /build/resources /var/www/html/resources
 COPY --from=builder /build/devfiles/index.json /var/www/html/index
 COPY ./images /var/www/html/images
 COPY ./build/dockerfiles/rhel.entrypoint.sh ./build/dockerfiles/entrypoint.sh /usr/local/bin/

diff --git a/build/dockerfiles/rhel.entrypoint.sh b/build/dockerfiles/rhel.entrypoint.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2018-2021 Red Hat, Inc.
+# Copyright (c) 2018-2024 Red Hat, Inc.
 # This program and the accompanying materials are made
 # available under the terms of the Eclipse Public License 2.0
 # which is available at https://www.eclipse.org/legal/epl-2.0/
@@ -14,7 +14,152 @@ if ! whoami &> /dev/null; then
   fi
 fi
 
-set -x
+set -e
+
+REGISTRY=${CHE_DEVFILE_IMAGES_REGISTRY_URL}
+ORGANIZATION=${CHE_DEVFILE_IMAGES_REGISTRY_ORGANIZATION}
+TAG=${CHE_DEVFILE_IMAGES_REGISTRY_TAG}
+PUBLIC_URL=${CHE_DEVFILE_REGISTRY_URL}
+INTERNAL_URL=${CHE_DEVFILE_REGISTRY_INTERNAL_URL}
+
+DEFAULT_DEVFILES_DIR="/var/www/html/devfiles"
+DEVFILES_DIR="${DEVFILES_DIR:-${DEFAULT_DEVFILES_DIR}}"
+INDEX_JSON="${DEVFILES_DIR}/index.json"
+
+# Regex used to break an image reference into groups:
+#   \1 - Whitespace and (optional) quotation preceding image reference
+#   \2 - Registry portion of image, e.g. (quay.io)/che-incubator/che-code:tag
+#   \3 - Organization portion of image, e.g. quay.io/(che-incubator)/che-code:tag
+#   \4 - Image name portion of image, e.g. quay.io/che-incubator/(che-code):tag
+#   \5 - Optional image digest identifier (empty for tags), e.g. quay.io/che-incubator/che-code(@sha256):digest
+#   \6 - Tag of image or digest, e.g. quay.io/che-incubator/che-code:(tag)
+#   \7 - Optional quotation following image reference
+IMAGE_REGEX='([[:space:]]*"?)([._:a-zA-Z0-9-]*)/([._a-zA-Z0-9-]*)/([._a-zA-Z0-9-]*)(@sha256)?:([._a-zA-Z0-9-]*)("?)'
+
+# Extract and use env variables with image digest information.
+# Env variable name format:
+# RELATED_IMAGES_(Image_name)_(Image_label)_(Encoded_base32_image_tag)
+# Where are:
+# "Image_name" - image name. Not valid chars for env variable name replaced to '_'.
+# "Image_label" - image target, for example 'devfile_registry_image'.
+# "Encoded_base32_image_tag_" - original image tag encoded to base32, to avoid invalid for env name chars. base32 alphabet has only
+# one invalid character for env name: '='. That's why it was replaced to '_'.
+# INFO: "=" for base32 it is pad character. If encoded string contains this char(s), then it is always located at the end of the string.
+# Env value it is image with digest to use.
+# Example env variable:
+# RELATED_IMAGE_che_rust_1_39_devfile_registry_image_G4XDCMZOGIFA____=quay.io/eclipse/che-rust-1.39@sha256:3d9f36e6b3ed99c7a9959ac9476778ef5019add15b7c0f0b5f27b55587db3def
+if env | grep -q ".*devfile_registry_image.*"; then
+  declare -A imageMap
+  readarray -t ENV_IMAGES < <(env | grep ".*devfile_registry_image.*")
+  for imageEnv in "${ENV_IMAGES[@]}"; do
+    tagOrDigest=$(echo "${imageEnv}" | sed -e 's;.*registry_image_\(.*\)=.*;\1;' | tr _ = | base32 -d)
+    if [[ ${tagOrDigest} == *"@"* ]]; then
+      # Well, image was "freezed", because it already has got digest, so do nothing.
+      continue
+    fi
+    imageWithDigest=${imageEnv#*=};
+    if [[ -n "${tagOrDigest}" ]]; then
+      imageToReplace="${imageWithDigest%@*}:${tagOrDigest}"
+    else
+      imageToReplace="${imageWithDigest%@*}"
+    fi
+    digest="@${imageWithDigest#*@}"
+    imageMap["${imageToReplace}"]="${digest}"
+  done
+
+  echo "--------------------------Digest map--------------------------"
+  for KEY in "${!imageMap[@]}"; do
+    echo "Key: $KEY Value: ${imageMap[${KEY}]}"
+  done
+  echo "--------------------------------------------------------------"
+
+  readarray -t devfiles < <(find "${DEVFILES_DIR}" -name 'devworkspace-che-*.yaml')
+  for devfile in "${devfiles[@]}"; do
+    # Put an actual image on the same line with 'image' field
+    # It allows to grep image with digest in the next step
+    sed -i -E ':a;N;$!ba;s|image: >-\n[ ]+|image: |g' "${devfile}"
+
+    readarray -t images < <(grep "image:" "${devfile}" | sed -r "s;.*image:[[:space:]]*'?\"?([._:a-zA-Z0-9-]*/?[._a-zA-Z0-9-]*/[._a-zA-Z0-9-]*(@sha256)?:?[._a-zA-Z0-9-]*)'?\"?[[:space:]]*;\1;")
+    for image in "${images[@]}"; do
+      separators="${image//[^\/]}"
+      # Warning, keep in mind: image without registry name is it possible case. It's mean, that image comes from private registry, where is we have organization name, but no registry name...
+      digest="${imageMap[${image}]}"
+
+      if [[ -z "${digest}" ]] && [ "${#separators}" == "1" ]; then
+        imageWithDefaultRegistry="docker.io/${image}"
+        digest="${imageMap[${imageWithDefaultRegistry}]}"
+      fi
+
+      if [[ -n "${digest}" ]]; then
+        if [[ ${image} == *":"* ]]; then
+          imageWithoutTag="${image%:*}"
+          tag="${image#*:}"
+        else
+          imageWithoutTag=${image}
+          tag=""
+        fi
+
+        REGEX="([[:space:]]*\"?'?)(${imageWithoutTag}):?(${tag})(\"?'?)"
+        sed -i -E "s|image:${REGEX}|image:\1\2${digest}\4|" "${devfile}"
+      fi
+    done
+  done
+fi
+
+# We can't use the `-d` option for readarray because
+# registry.centos.org/centos/httpd-24-centos7 ships with Bash 4.2
+# The below command will fail if any path contains whitespace
+readarray -t devfiles < <(find "${DEVFILES_DIR}" -name 'devworkspace-che-*.yaml')
+readarray -t metas < <(find "${DEVFILES_DIR}" -name 'meta.yaml')
+readarray -t templates < <(find "${DEVFILES_DIR}" -name 'devworkspace-che-*.yaml')
+for devfile in "${devfiles[@]}"; do
+  echo "Checking devfile $devfile"
+  # Need to update each field separately in case they are not defined.
+  # Defaults don't work because registry and tags may be different.
+  if [ -n "$REGISTRY" ]; then
+    echo "    Updating image registry to $REGISTRY"
+    sed -i -E "s|image:$IMAGE_REGEX|image:\1${REGISTRY}/\3/\4\5:\6\7|" "$devfile"
+  fi
+  if [ -n "$ORGANIZATION" ]; then
+    echo "    Updating image organization to $ORGANIZATION"
+    sed -i -E "s|image:$IMAGE_REGEX|image:\1\2/${ORGANIZATION}/\4\5:\6\7|" "$devfile"
+  fi
+  if [ -n "$TAG" ]; then
+    echo "    Updating image tag to $TAG"
+    sed -i -E "s|image:$IMAGE_REGEX|image:\1\2/\3/\4:${TAG}\7|" "$devfile"
+  fi
+done
+
+if [ -n "$INTERNAL_URL" ]; then
+  INTERNAL_URL=${INTERNAL_URL%/}
+  echo "Updating internal URL in files to ${INTERNAL_URL}"
+  sed -i "s|{{ INTERNAL_URL }}|${INTERNAL_URL}|" "${devfiles[@]}" "${metas[@]}" "${templates[@]}" "$INDEX_JSON"
+fi
+
+if [ -n "$PUBLIC_URL" ]; then
+  echo "Updating devfiles to point at internal project zip files"
+  PUBLIC_URL=${PUBLIC_URL%/}
+  sed -i "s|{{ DEVFILE_REGISTRY_URL }}|${PUBLIC_URL}|" "${devfiles[@]}" "${metas[@]}" "${templates[@]}" "$INDEX_JSON"
+
+  # Add PUBLIC_URL at the begining of 'icon' field and links ('self', 'che-incubator/che-code/latest')
+  sed -i "s|\"icon\": \"/images/|\"icon\": \"${PUBLIC_URL}/images/|" "$INDEX_JSON"
+  sed -i "s|\"self\": \"/devfiles/|\"self\": \"${PUBLIC_URL}/devfiles/|" "$INDEX_JSON"
+  sed -i "s|\"che-incubator/che-code/insiders\": \"/devfiles/|\"che-incubator/che-code/insiders\": \"${PUBLIC_URL}/devfiles/|" "$INDEX_JSON"
+  sed -i "s|\"che-incubator/che-code/latest\": \"/devfiles/|\"che-incubator/che-code/latest\": \"${PUBLIC_URL}/devfiles/|" "$INDEX_JSON"
+  sed -i "s|\"che-incubator/che-idea/next\": \"/devfiles/|\"che-incubator/che-idea/next\": \"${PUBLIC_URL}/devfiles/|" "$INDEX_JSON"
+else
+  if grep -q '{{ DEVFILE_REGISTRY_URL }}' "${devfiles[@]}"; then
+    echo "WARNING: environment variable 'CHE_DEVFILE_REGISTRY_URL' not configured" \
+         "This may cause issues with importing projects in a workspace."
+    # Experimental workaround -- detect service IP for che-devfile-registry
+    # Depends on service used being named 'che-devfile-registry' and only works
+    # within the cluster (i.e. browser-side retrieval won't work)
+    SERVICE_HOST=$(env | grep DEVFILE_REGISTRY_SERVICE_HOST= | cut -d '=' -f 2)
+    SERVICE_PORT=$(env | grep DEVFILE_REGISTRY_SERVICE_PORT= | cut -d '=' -f 2)
+    URL="http://${SERVICE_HOST}:${SERVICE_PORT}"
+    sed -i "s|{{ DEVFILE_REGISTRY_URL }}|${URL}|" "${devfiles[@]}" "${metas[@]}" "${templates[@]}" "$INDEX_JSON"
+  fi
+fi
 
 # start httpd
 if [[ -x /usr/sbin/httpd ]]; then

diff --git a/build/scripts/generate_devworkspace_templates.sh b/build/scripts/generate_devworkspace_templates.sh
@@ -10,6 +10,8 @@
 
 set -e
 
+npm install -g [email protected]
+
 VERSION="${1%/}"
 if [[ -z "$VERSION" || "$VERSION" == *"-next" ]]; then
   VERSION="main"