Merge pull request #2 from balinorLiang/CICD_updating

CI/CD init & chmod 777 all shell scripts

.github/workflows/ci.yml

@@ -0,0 +1,50 @@
+name: GitHub CI
+
+on:
+  pull_request:
+  push:
+  schedule:
+    - cron: 0 0 * * 0
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: 'bash -Eeuo pipefail -x {0}'
+
+jobs:
+
+  generate-jobs:
+    name: Generate Jobs
+    runs-on: ubuntu-latest
+    outputs:
+      strategy: ${{ steps.generate-jobs.outputs.strategy }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: docker-library/bashbrew@HEAD
+      - id: generate-jobs
+        name: Generate Jobs
+        run: |
+          strategy="$("$BASHBREW_SCRIPTS/github-actions/generate.sh")"
+          strategy="$(.github/workflows/munge.sh -c <<<"$strategy")"
+          echo "strategy=$strategy" >> "$GITHUB_OUTPUT"
+          jq . <<<"$strategy" # sanity check / debugging aid
+
+  test:
+    needs: generate-jobs
+    strategy: ${{ fromJson(needs.generate-jobs.outputs.strategy) }}
+    name: ${{ matrix.name }}
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Prepare Environment
+        run: ${{ matrix.runs.prepare }}
+      - name: Pull Dependencies
+        run: ${{ matrix.runs.pull }}
+      - name: Build ${{ matrix.name }}
+        run: ${{ matrix.runs.build }}
+      - name: History ${{ matrix.name }}
+        run: ${{ matrix.runs.history }}
+      - name: Test ${{ matrix.name }}
+        run: ${{ matrix.runs.test }}
+      - name: '"docker images"'
+        run: ${{ matrix.runs.images }}

.github/workflows/munge.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+# copy all the Debian build jobs into "force deb build" jobs which build like architectures upstream doesn't publish for will
+jq \
+	--arg prefix '[ "$(dpkg --print-architecture)" = "amd64" ]' \
+	--arg dfMunge 'grep -qE "amd64 [|] " "$df"; sed -ri -e "s/amd64 [|] //g" "$df"; ! grep -qE "amd64 [|] " "$df"' \
+	'
+		.matrix.include += [
+			.matrix.include[]
+			| select(.name | test(" (.+)") | not) # ignore any existing munged builds
+			| select(.meta.froms[] | test("^debian:|^ubuntu:"))
+			| .name += " (force deb build)"
+			| .runs.build = (
+				[
+					"# force us to build debs instead of downloading them",
+					$prefix,
+					("for df in " + ([ .meta.dockerfiles[] | @sh ] | join(" ")) + "; do " + $dfMunge + "; done"),
+					.runs.build
+				] | join ("\n")
+			)
+		]
+	' "$@"

.github/workflows/verify-templating.yml

@@ -0,0 +1,23 @@
+name: Verify Templating
+
+on:
+  pull_request:
+  push:
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: 'bash -Eeuo pipefail -x {0}'
+
+jobs:
+  apply-templates:
+    name: Check For Uncomitted Changes
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Apply Templates
+        run: ./apply-templates.sh
+      - name: Check Git Status
+        run: |
+          status="$(git status --short)"
+          [ -z "$status" ]

.gitignore

@@ -0,0 +1 @@
+.jq-template.awk

3/ubi-8/DockerFile → 3/ubi8/Dockerfile

@@ -1,7 +1,14 @@
+#
+# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh"
+#
+# PLEASE DO NOT EDIT IT DIRECTLY.
+#
+
 FROM redhat/ubi8:latest as builder
 
+# explicitly set user/group IDs
 RUN groupadd -g 1000 ivorysql; \
-        useradd -u 1000 -g ivorysql -d /var/local/ivorysql -s /bin/sh ivorysql; \
+		useradd -u 1000 -g ivorysql -d /var/local/ivorysql -s /bin/sh ivorysql; \
         mkdir -p /var/local/ivorysql; \
         mkdir -p /usr/src/ivorysql; \
         mkdir -p /var/lib/ivorysql; \
@@ -89,7 +96,7 @@ RUN chown -R ivorysql:ivorysql /usr/local/bin
 ENV PGDATA /var/local/ivorysql/ivorysql-$IVORY_MAJOR/data
 # this 1777 will be replaced by 0700 at runtime (allows semi-arbitrary "--user" values)
 RUN mkdir -p "$PGDATA" && chown -R ivorysql:ivorysql "$PGDATA" && chmod 1777 "$PGDATA"
-RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 3777 /var/run/postgresql
+RUN mkdir -p /var/run/postgresql && chown -R ivorysql:ivorysql /var/run/postgresql && chmod 3777 /var/run/postgresql
 
 VOLUME /var/lib/ivorysql/data
 
@@ -104,4 +111,4 @@ STOPSIGNAL SIGINT
 
 EXPOSE 5432 5866
 
-CMD ["postgres"]
+CMD ["postgres"]

3/ubi-8/docker-entrypoint.sh → 3/ubi8/docker-entrypoint.sh

@@ -90,7 +90,7 @@ docker_init_database_dir() {
 
 	# --pwfile refuses to handle a properly-empty file (hence the "\n"): https://github.com/docker-library/postgres/issues/1025
 	eval 'initdb --username="$IVORYSQL_USER" --pwfile=<(printf "%s\n" "$IVORYSQL_PASSWORD") '"$IVORYSQL_INITDB_ARGS"' "$@"'
-
+	sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" $PGDATA/postgresql.conf
 	# unset/cleanup "nss_wrapper" bits
 	if [[ "${LD_PRELOAD:-}" == */libnss_wrapper.so ]]; then
 		rm -f "$NSS_WRAPPER_PASSWD" "$NSS_WRAPPER_GROUP"
@@ -347,7 +347,7 @@ _main() {
 			EOM
 		fi
 	fi
-	sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" $PGDATA/postgresql.conf
+
 	exec "$@"
 }
 

Dockerfile-redhat.template

@@ -0,0 +1,108 @@
+FROM redhat/{{ env.variant }}:latest as builder
+
+# explicitly set user/group IDs
+RUN groupadd -g 1000 ivorysql; \
+		useradd -u 1000 -g ivorysql -d /var/local/ivorysql -s /bin/sh ivorysql; \
+        mkdir -p /var/local/ivorysql; \
+        mkdir -p /usr/src/ivorysql; \
+        mkdir -p /var/lib/ivorysql; \
+        chown -R ivorysql:ivorysql /var/local/ivorysql; \
+        chown -R ivorysql:ivorysql /usr/src/ivorysql; \
+        chown -R ivorysql:ivorysql /var/lib/ivorysql
+
+RUN mkdir /docker-entrypoint-initdb.d
+
+ENV IVORY_MAJOR {{ env.version }}
+ENV IVORY_VERSION {{ .[env.variant].version }}
+
+RUN curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-8.repo; \
+    sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo
+
+RUN mkdir -p /usr/src/ivorysql; \
+    dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm; \
+    dnf install -y epel-release;\
+    dnf install -y \
+    make \
+    readline-devel \
+    zlib-devel \
+    openssl-devel \
+    flex \
+    wget \
+    dpkg; \
+    dnf groupinstall -y "Development Tools";\
+    dnf install -y \
+    llvm-toolset llvm-devel lz4 lz4-devel lz4-libs clang-devel \
+    krb5-devel libselinux-devel libxml2-devel cyrus-sasl-gssapi \
+    libicu-devel e2fsprogs-devel selinux-policy systemd-devel \
+    libxslt-devel openldap-devel openssl-devel pam-devel \
+    libuuid-devel python3-devel readline-devel tcl-devel zlib-devel \
+    perl perl-devel perl-ExtUtils-Embed;\
+    dnf install -y --enablerepo=*ower*ools perl-IPC-Run perl-Time-HiRes perl-Test-Simple uuid-devel;\
+    wget -O ivorysql.tar.gz "https://github.com/IvorySQL/IvorySQL/archive/refs/tags/IvorySQL_$IVORY_VERSION.tar.gz"; \
+    tar \
+	    --extract \
+		--file ivorysql.tar.gz \
+		--directory /usr/src/ivorysql \
+		--strip-components 1 \
+	; \
+    rm ivorysql.tar.gz; \
+    cd /usr/src/ivorysql; \
+    wget http://mirror.centos.org/centos/7/os/x86_64/Packages/bison-devel-3.0.4-2.el7.x86_64.rpm; \
+    dnf install -y bison-devel-3.0.4-2.el7.x86_64.rpm; \
+    wget https://www.rpmfind.net/linux/centos/8-stream/AppStream/x86_64/os/Packages/bison-3.0.4-10.el8.x86_64.rpm; \
+    dnf install -y bison-3.0.4-10.el8.x86_64.rpm; \
+    ./configure \
+            --prefix=/var/local/ivorysql/ivorysql-$IVORY_MAJOR \
+            --enable-cassert --enable-debug --enable-rpath --with-tcl \
+            --with-python --with-gssapi --with-pam --with-ldap \
+            --with-openssl --with-libedit-preferred --with-uuid=e2fs \
+            --with-ossp-uuid  --with-libxml --with-libxslt  --with-perl \
+            --with-icu \
+    ; \
+    make && make install; \
+    rm -rf \
+		/usr/src/ivorysql \
+		/usr/local/share/doc \
+		/usr/local/share/man
+
+RUN chown -R ivorysql:ivorysql /var/local/ivorysql/ivorysql-$IVORY_MAJOR
+RUN curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/1.11/gosu-amd64"
+RUN chmod +x /usr/local/bin/gosu
+
+FROM redhat/{{ env.variant }}:latest
+
+COPY --from=builder /var/local/ivorysql /var/local/ivorysql/
+COPY --from=builder /usr/local/bin/gosu /usr/local/bin/gosu
+
+ENV IVORY_MAJOR {{ env.version}}
+ENV IVORY_VERSION {{ .[env.variant].version }}
+
+RUN mkdir /docker-entrypoint-initdb.d
+
+RUN useradd -u 1000 ivorysql
+
+RUN dnf install -y \
+    lz4 lz4-devel lz4-libs krb5-devel libxslt-devel libicu-devel && dnf -y clean all 
+
+RUN chown -R ivorysql:ivorysql /var/local/ivorysql
+RUN chown -R ivorysql:ivorysql /usr/local/bin
+
+ENV PGDATA /var/local/ivorysql/ivorysql-$IVORY_MAJOR/data
+# this 1777 will be replaced by 0700 at runtime (allows semi-arbitrary "--user" values)
+RUN mkdir -p "$PGDATA" && chown -R ivorysql:ivorysql "$PGDATA" && chmod 1777 "$PGDATA"
+RUN mkdir -p /var/run/postgresql && chown -R ivorysql:ivorysql /var/run/postgresql && chmod 3777 /var/run/postgresql
+
+VOLUME /var/lib/ivorysql/data
+
+ENV PATH $PATH:/var/local/ivorysql/ivorysql-$IVORY_MAJOR/bin
+
+COPY docker-entrypoint.sh docker-ensure-initdb.sh /usr/local/bin/
+RUN ln -sT docker-ensure-initdb.sh /usr/local/bin/docker-enforce-initdb.sh
+
+ENTRYPOINT ["docker-entrypoint.sh"]
+
+STOPSIGNAL SIGINT
+
+EXPOSE 5432 5866
+
+CMD ["postgres"]

apply-templates.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+[ -f versions.json ] # run "versions.sh" first
+
+cd "$(dirname "$(readlink -f "$BASH_SOURCE")")"
+
+jqt='.jq-template.awk'
+if [ -n "${BASHBREW_SCRIPTS:-}" ]; then
+	jqt="$BASHBREW_SCRIPTS/jq-template.awk"
+elif [ "$BASH_SOURCE" -nt "$jqt" ]; then
+	# https://github.com/docker-library/bashbrew/blob/master/scripts/jq-template.awk
+	wget -qO "$jqt" 'https://github.com/docker-library/bashbrew/raw/9f6a35772ac863a0241f147c820354e4008edf38/scripts/jq-template.awk'
+fi
+
+if [ "$#" -eq 0 ]; then
+	versions="$(jq -r 'keys | map(@sh) | join(" ")' versions.json)"
+	eval "set -- $versions"
+fi
+
+generated_warning() {
+	cat <<-EOH
+		#
+		# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh"
+		#
+		# PLEASE DO NOT EDIT IT DIRECTLY.
+		#
+
+	EOH
+}
+
+for version; do
+	export version
+
+	major="$(jq -r '.[env.version].major' versions.json)"
+
+	variants="$(jq -r '.[env.version].variants | map(@sh) | join(" ")' versions.json)"
+	eval "variants=( $variants )"
+
+	rm -rf "$version"
+
+	for variant in "${variants[@]}"; do
+		export variant
+
+		dir="$version/$variant"
+		mkdir -p "$dir"
+
+		echo "processing $dir ..."
+
+		cp -a docker-entrypoint.sh docker-ensure-initdb.sh "$dir/"
+
+		case "$variant" in
+			*)
+				template='Dockerfile-redhat.template'
+				;;
+		esac
+
+		{
+			generated_warning
+			gawk -f "$jqt" "$template"
+		} > "$dir/Dockerfile"
+	done
+done

docker-ensure-initdb.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+#
+# This script is intended for three main use cases:
+#
+#  1. (most importantly) as an example of how to use "docker-entrypoint.sh" to extend/reuse the initialization behavior
+#
+#  2. ("docker-ensure-initdb.sh") as a Kubernetes "init container" to ensure the provided database directory is initialized; see also "startup probes" for an alternative solution
+#       (no-op if database is already initialized)
+#
+#  3. ("docker-enforce-initdb.sh") as part of CI to ensure the database is fully initialized before use
+#       (error if database is already initialized)
+#
+
+source /usr/local/bin/docker-entrypoint.sh
+
+# arguments to this script are assumed to be arguments to the "postgres" server (same as "docker-entrypoint.sh"), and most "docker-entrypoint.sh" functions assume "postgres" is the first argument (see "_main" over there)
+if [ "$#" -eq 0 ] || [ "$1" != 'postgres' ]; then
+	set -- postgres "$@"
+fi
+
+# see also "_main" in "docker-entrypoint.sh"
+
+docker_setup_env
+# setup data directories and permissions (when run as root)
+docker_create_db_directories
+if [ "$(id -u)" = '0' ]; then
+	# then restart script as postgres user
+	exec su-exec postgres "$BASH_SOURCE" "$@"
+fi
+
+# only run initialization on an empty data directory
+if [ -z "$DATABASE_ALREADY_EXISTS" ]; then
+	docker_verify_minimum_env
+
+	# check dir permissions to reduce likelihood of half-initialized database
+	ls /docker-entrypoint-initdb.d/ > /dev/null
+
+	docker_init_database_dir
+	pg_setup_hba_conf "$@"
+
+	# PGPASSWORD is required for psql when authentication is required for 'local' connections via pg_hba.conf and is otherwise harmless
+	# e.g. when '--auth=md5' or '--auth-local=md5' is used in POSTGRES_INITDB_ARGS
+	export PGPASSWORD="${PGPASSWORD:-$POSTGRES_PASSWORD}"
+	docker_temp_server_start "$@"
+
+	docker_setup_db
+	docker_process_init_files /docker-entrypoint-initdb.d/*
+
+	docker_temp_server_stop
+	unset PGPASSWORD
+else
+	self="$(basename "$0")"
+	case "$self" in
+		docker-ensure-initdb.sh)
+			echo >&2 "$self: note: database already initialized in '$PGDATA'!"
+			exit 0
+			;;
+
+		docker-enforce-initdb.sh)
+			echo >&2 "$self: error: (unexpected) database found in '$PGDATA'!"
+			exit 1
+			;;
+
+		*)
+			echo >&2 "$self: error: unknown file name: $self"
+			exit 99
+			;;
+	esac
+fi