Release 2023.4.1

README.md

@@ -2,9 +2,9 @@
 
 This repository contains orchestration files and documentation for deploying Black Duck Docker containers.
 
-## Location of Black Duck 2023.4.0 archive:
+## Location of Black Duck 2023.4.1 archive:
 
-https://github.com/blackducksoftware/hub/archive/v2023.4.0.tar.gz
+https://github.com/blackducksoftware/hub/archive/v2023.4.1.tar.gz
 
 NOTE:
 

docker-swarm/bin/hub_add_replication_user.sh

@@ -3,7 +3,7 @@
 set -e
 
 TIMEOUT=${TIMEOUT:-10}
-HUB_POSTGRES_VERSION=${HUB_POSTGRES_VERSION:-13-2.22}
+HUB_POSTGRES_VERSION=${HUB_POSTGRES_VERSION:-13-2.24}
 HUB_DATABASE_IMAGE_NAME=${HUB_DATABASE_IMAGE_NAME:-postgres}
 
 function fail() {

docker-swarm/bin/hub_create_data_dump.sh

@@ -5,15 +5,18 @@
 #  2. The database container has been properly initialized.
 
 HUB_DATABASE_IMAGE_NAME=${HUB_DATABASE_IMAGE_NAME:-postgres}
-HUB_POSTGRES_VERSION=${HUB_POSTGRES_VERSION:-13-2.22}
+HUB_POSTGRES_VERSION=${HUB_POSTGRES_VERSION:-13-2.24}
+HUB_VERSION=${HUB_VERSION:-2023.4.1}
 OPT_FORCE=
+OPT_LIVE_SYSTEM=
 OPT_MAX_CPU=${MAX_CPU:-1}
 OPT_NO_STORAGE=${NO_STORAGE:-}
 TIMEOUT=${TIMEOUT:-10}
 
 database_name=
 local_destination=
 typeset -a container_id
+typeset -a storage_id
 
 function fail() {
     local -r code=$1
@@ -35,6 +38,7 @@ Supported options are:
     --force               Overwrite existing files in the destination
     --no-storage          Do not attempt to backup the storage service
     --max-cpu <n>         Number of parallel database jobs (see below!)
+    --live-system         Backup even if not in dbMigrate mode.
 
 This script tries to backup the running Black Duck database to a local
 directory.  The database must be running in a local docker container.
@@ -50,10 +54,16 @@ container /tmp partition to store the entire dump temporarily,
 typically around 10% of the size of the full database. By default
 dumps are streamed directly to the local destination.
 
+Unless '--live-system' is supplied this script will refuse to run if
+system appears to be live, rather than in dbMigrate mode.  Backing up
+a live system is discouraged; it will impact performance and might not
+produce a fully self-consistent dump.
+
 Command line options take precedence over environment variables.
 Recognized environment variables:
     HUB_DATABASE_IMAGE_NAME  Expected postgres image name [$HUB_DATABASE_IMAGE_NAME]
     HUB_POSTGRES_VERSION     Expected postgres image version [$HUB_POSTGRES_VERSION]
+    HUB_VERSION              Expected storage image version [$HUB_VERSION]
     MAX_CPU                  Number of parallel database threads to use [$OPT_MAX_CPU]
     NO_STORAGE               Skip storage service backup when non-empty [$OPT_NO_STORAGE]
     TIMEOUT                  Seconds to wait for postgresql startup [$TIMEOUT]
@@ -74,7 +84,9 @@ function process_args() {
             '--help' | '-h' )
                 usage ;;
             '--force' | '-f' )
-		OPT_FORCE=1 ;;
+                OPT_FORCE=1 ;;
+            '--live-system' )
+                OPT_LIVE_SYSTEM=1 ;;
             *)
                 if [[ -z "${arg_1}" ]]; then
                     arg_1="$1"
@@ -105,6 +117,7 @@ function process_args() {
 function set_container_id() {
     container_id=( $(docker ps -q -f "label=com.blackducksoftware.hub.version=${HUB_POSTGRES_VERSION}" \
                                   -f "label=com.blackducksoftware.hub.image=${HUB_DATABASE_IMAGE_NAME}") )
+    storage_id=( $(docker ps -q -f "volume=/tmp/uploads") $(docker ps -q -f "volume=/opt/blackduck/hub/uploads") )
     return 0
 }
 
@@ -198,11 +211,12 @@ function manage_database() {
 
 function manage_storage() {
     local id="$1"
-    local dir=$2
+    local mnt="$2"
+    local dir="$3"
 
-    if [[ -n $(docker exec "$id" ls "/opt/blackduck/hub/$dir/") ]]; then
-        echo "Attempting to backup storage [Container: $id | Mount: $dir]."
-        docker exec "$id" tar czf - -C "/opt/blackduck/hub/$dir" . > "$local_absolute_path/$dir.tgz"
+    if [[ -n $(docker exec "$id" ls "$mnt/$dir/" 2>/dev/null) ]]; then
+        echo "Attempting to backup storage [Container: $id | Mount: $mnt | Volume: $dir]."
+        docker exec "$id" tar czf - -C "$mnt/$dir" . > "$local_absolute_path/$dir.tgz"
     fi
 }
 
@@ -235,6 +249,12 @@ until docker exec "${container_id[0]}" pg_isready -U postgres -q ; do
     sleep 1
 done
 
+# Check that we're not accidentally trying to dump a live system.
+if [[ -n "$(docker ps -q -f 'label=com.blackducksoftware.hub.image=webserver')" ]] && [[ -z "${OPT_LIVE_SYSTEM}" ]]; then
+    echo "* This appears to be a live system -- re-invoke with '--live-system' to proceed anyway." 1>&2
+    exit 1
+fi
+
 # Create an absolute path to copy to, adds support for symbolic links
 if [ ! -d "$local_destination" ]; then
     cd "$(dirname "$local_destination")" || exit 1
@@ -266,15 +286,11 @@ if [[ -z "$OPT_NO_STORAGE" ]]; then
     echo
     echo "Attempting to save storage service file provider uploads."
 
-    storage_id=$(docker ps --format "{{.ID}} {{.Image}}" | grep -F "blackduck-storage:" | cut -d' ' -f1)
-    if [[ -z "$storage_id" ]]; then
-        fail 20 "No storage container is running."
-    elif [[ "$(echo "$storage_id" | wc -l)" -gt 1 ]]; then
-        fail 21 "Multiple storage containers are running."
-    fi
-
     # Backup each directory separately
-    for dir in $(docker exec "$storage_id" ls /opt/blackduck/hub/ | grep -F uploads); do
-        manage_storage "$storage_id" "$dir"
+    for dir in $(docker exec "${storage_id[0]}" ls /tmp/ | grep -F uploads); do
+        manage_storage "${storage_id[0]}" "/tmp" "$dir"
+    done
+    for dir in $(docker exec "${storage_id[0]}" ls /opt/blackduck/hub/ | grep -F uploads); do
+        manage_storage "${storage_id[0]}" "/opt/blackduck/hub" "$dir"
     done
 fi

docker-swarm/bin/hub_db_migrate.sh

@@ -14,7 +14,7 @@
 set -o errexit
 
 HUB_DATABASE_IMAGE_NAME=${HUB_DATABASE_IMAGE_NAME:-postgres}
-HUB_POSTGRES_VERSION=${HUB_POSTGRES_VERSION:-13-2.22}
+HUB_POSTGRES_VERSION=${HUB_POSTGRES_VERSION:-13-2.24}
 OPT_MAX_CPU=${MAX_CPU:-1}
 OPT_NO_DATABASE=${NO_DATABASE:-}
 OPT_NO_STORAGE=${NO_STORAGE:-}
@@ -243,6 +243,14 @@ function determine_database_emptiness() {
     echo "Determined database emptiness [Container: ${container} | Database: ${database}]."
 }
 
+function determine_dbmigrate_mode() {
+    # Check that we're not trying to restore to a live system.
+    if [[ -n "$(docker ps -q -f 'label=com.blackducksoftware.hub.image=webserver')" ]] && [[ -z "${OPT_LIVE_SYSTEM}" ]]; then
+        echo "* This appears to be a live system -- cannot proceed." 1>&2
+        exit 1
+    fi
+}
+
 function restore_globals() {
     local container=$1
     local sqlfile=$2
@@ -265,13 +273,15 @@ function restore_database() {
     if [ -d "${dump}" ]; then
         # Restoring directory format dumps requires a copy inside the container.
         docker cp "${dump}" "${container}:/tmp/${database}"
+        docker exec -u 0 "${container}" chmod -R a+rx "/tmp/${database}"
         docker exec -i "${container}" pg_restore -U postgres -Fd "-j${OPT_MAX_CPU}" --verbose --clean --if-exists -d "${database}" "/tmp/${database}" || true
-        docker exec "${container}" rm -rf "/tmp/${database}"
+        docker exec -u 0 "${container}" rm -rf "/tmp/${database}"
     elif [ "${OPT_MAX_CPU}" -gt 1 ]; then
         # Parallel restore of file format dumps requires a copy inside the container.
         docker cp "${dump}" "${container}:/tmp/${database}"
+        docker exec -u 0 "${container}" chmod -R a+rx "/tmp/${database}"
         docker exec "${container}" pg_restore -U postgres -Fc "-j${OPT_MAX_CPU}" --verbose --clean --if-exists -d "${database}" "/tmp/${database}" || true
-        docker exec "${container}" rm -rf "/tmp/${database}"
+        docker exec -u 0 "${container}" rm -rf "/tmp/${database}"
     else
         # Single-threaded restore of a dump file can be streamed.
         docker exec -i "${container}" pg_restore -U postgres -Fc --verbose --clean --if-exists -d "${database}" < "$dump" || true
@@ -316,6 +326,10 @@ function manage_storage_provider() {
     docker exec "$id" ls -d "/tmp/$dir" >/dev/null 2>&1 || \
         fail 30 "$dir is not a mount point -- check the provider configurations."
 
+    # Check that the desired target directory is empty.
+    [[ -z "$(docker exec "$id" ls "/tmp/$dir/" 2>&1)" ]] || \
+        fail 31 "$dir is not empty -- check the provider configurations."
+
     docker exec -u 0 -i "$id" tar xz -C "/tmp/$dir" -f - < "$dump"
 
     echo "Restored uploaded files [Container: $id | Mount: $dir | Dump: $dump]."
@@ -334,6 +348,7 @@ if [[ -n "$directory_path" ]]; then
         determine_container_readiness
         determine_singular_container
         determine_postgresql_readiness "${container_id[0]}"
+        determine_dbmigrate_mode
 
         echo "Attempting to manage all databases [Container: ${container_id[0]} | Directory path: ${directory_path}]." 
         determine_file_validity "${directory_path}/globals.sql"
@@ -351,6 +366,7 @@ if [[ -n "$directory_path" ]]; then
         if [[ "${#container_id[*]}" -eq 0 ]]; then
             determine_container_readiness
             determine_singular_container
+            determine_dbmigrate_mode
         fi
         for tar in "${directory_path}"/upload*.tgz; do
             manage_storage_provider "${container_id[0]}" "$(basename "$tar" .tgz)" "$tar"
@@ -367,6 +383,7 @@ elif [[ -n "$database_name" ]] && [[ -n "$dump_file" ]]; then
     determine_container_readiness
     determine_singular_container
     determine_postgresql_readiness "${container_id[0]}"
+    determine_dbmigrate_mode
 
     manage_database "${container_id[0]}" "${database_name}" "${dump_file}"
 
@@ -375,6 +392,7 @@ elif [[ -n "$mount" ]] && [[ -n "$tar_file" ]]; then
     if [[ "${#container_id[*]}" -eq 0 ]]; then
         determine_container_readiness
         determine_singular_container
+        determine_dbmigrate_mode
     fi
     manage_storage_provider "${container_id[0]}" "$mount" "$tar_file"
 

docker-swarm/bin/hub_replication_changepassword.sh

@@ -3,7 +3,7 @@
 set -e
 
 TIMEOUT=${TIMEOUT:-10}
-HUB_POSTGRES_VERSION=${HUB_POSTGRES_VERSION:-13-2.22}
+HUB_POSTGRES_VERSION=${HUB_POSTGRES_VERSION:-13-2.24}
 HUB_DATABASE_IMAGE_NAME=${HUB_DATABASE_IMAGE_NAME:-postgres}
 
 function fail() {

docker-swarm/bin/hub_reportdb_changepassword.sh

@@ -3,7 +3,7 @@
 set -e
 
 TIMEOUT=${TIMEOUT:-10}
-HUB_POSTGRES_VERSION=${HUB_POSTGRES_VERSION:-13-2.22}
+HUB_POSTGRES_VERSION=${HUB_POSTGRES_VERSION:-13-2.24}
 HUB_DATABASE_IMAGE_NAME=${HUB_DATABASE_IMAGE_NAME:-postgres}
 
 function fail() {

docker-swarm/bin/system_check.sh

@@ -41,7 +41,7 @@ set -o noglob
 
 readonly NOW="$(date +"%Y%m%dT%H%M%S%z")"
 readonly NOW_ZULU="$(date -u +"%Y%m%dT%H%M%SZ")"
-readonly HUB_VERSION="${HUB_VERSION:-2023.4.0}"
+readonly HUB_VERSION="${HUB_VERSION:-2023.4.1}"
 readonly OUTPUT_FILE="${SYSTEM_CHECK_OUTPUT_FILE:-system_check_${NOW}.txt}"
 readonly PROPERTIES_FILE="${SYSTEM_CHECK_PROPERTIES_FILE:-${OUTPUT_FILE%.txt}.properties}"
 readonly SUMMARY_FILE="${SYSTEM_CHECK_SUMMARY_FILE:-${OUTPUT_FILE%.txt}_summary.properties}"

docker-swarm/blackduck-config.env

@@ -24,7 +24,7 @@ BLACKDUCK_CORS_ALLOW_CREDENTIALS_PROP_NAME=
 
 # Do not change
 HUB_PRODUCT_NAME=BLACK_DUCK
-HUB_VERSION=2023.4.0
+HUB_VERSION=2023.4.1
 
 # Specify any property-specific overrides here
 #

docker-swarm/docker-compose.bdba.yml

@@ -5,7 +5,7 @@ version: '3.6'
 
 services:
   binaryscanner:
-    image: sigsynopsys/bdba-worker:2023.3.0
+    image: sigsynopsys/bdba-worker:2023.3.1
     env_file: [hub-bdba.env]
     entrypoint: /docker-entrypoint.sh
     healthcheck:

docker-swarm/docker-compose.dbmigrate.yml

@@ -23,7 +23,7 @@ services:
     user: 'logstash:root'
 
   postgres:
-    image: blackducksoftware/blackduck-postgres:13-2.22
+    image: blackducksoftware/blackduck-postgres:13-2.24
     volumes:
       - postgres96-data-volume:/bitnami/postgresql
       - postgres-conf-volume:/opt/bitnami/postgresql/conf
@@ -41,7 +41,7 @@ services:
         condition: on-failure
 
   postgres-upgrader:
-    image: blackducksoftware/blackduck-postgres-upgrader:13-1.9
+    image: blackducksoftware/blackduck-postgres-upgrader:13-1.11
     volumes:
       - postgres96-data-volume:/bitnami/postgresql
       - postgres-conf-volume:/opt/bitnami/postgresql/conf

docker-swarm/docker-compose.externaldb.ubi.yml

@@ -13,7 +13,7 @@ x-long-start-period: &long-start-period
 services:
   authentication:
     user: authentication:root
-    image: blackducksoftware/blackduck-authentication:2023.4.0_ubi8.7
+    image: blackducksoftware/blackduck-authentication:2023.4.1_ubi8.7
     volumes:
     - authentication-volume:/opt/blackduck/hub/hub-authentication/ldap
     - {type: tmpfs, target: /opt/blackduck/hub/hub-authentication/security}
@@ -35,7 +35,7 @@ services:
       restart_policy: {condition: on-failure, delay: 5s, window: 60s}
   webapp:
     user: webapp:root
-    image: blackducksoftware/blackduck-webapp:2023.4.0_ubi8.7
+    image: blackducksoftware/blackduck-webapp:2023.4.1_ubi8.7
     volumes:
     - log-volume:/opt/blackduck/hub/logs
     - {type: tmpfs, target: /opt/blackduck/hub/hub-webapp/security}
@@ -57,7 +57,7 @@ services:
       restart_policy: {condition: on-failure, delay: 5s, window: 60s}
   scan:
     user: scan:root
-    image: blackducksoftware/blackduck-scan:2023.4.0_ubi8.7
+    image: blackducksoftware/blackduck-scan:2023.4.1_ubi8.7
     env_file: [blackduck-config.env , hub-postgres.env]
     healthcheck:
       test: [CMD, /usr/local/bin/docker-healthcheck.sh, 'https://localhost:8443/api/health-checks/liveness',
@@ -78,7 +78,7 @@ services:
       restart_policy: {condition: on-failure, delay: 5s, window: 60s}
   storage:
     user: storage:root
-    image: blackducksoftware/blackduck-storage:2023.4.0_ubi8.7
+    image: blackducksoftware/blackduck-storage:2023.4.1_ubi8.7
     env_file: [blackduck-config.env , hub-postgres.env]
     healthcheck:
       test: [CMD, /usr/local/bin/docker-healthcheck.sh, 'https://localhost:8443/api/health-checks/liveness',
@@ -100,7 +100,7 @@ services:
       restart_policy: {condition: on-failure, delay: 5s, window: 60s}
   jobrunner:
     user: jobrunner:root
-    image: blackducksoftware/blackduck-jobrunner:2023.4.0_ubi8.7
+    image: blackducksoftware/blackduck-jobrunner:2023.4.1_ubi8.7
     env_file: [blackduck-config.env , hub-postgres.env]
     healthcheck:
       test: [CMD, /usr/local/bin/docker-healthcheck.sh, 'https://localhost:8443/health-checks/liveness',
@@ -152,7 +152,7 @@ services:
       mode: replicated
       restart_policy: {condition: on-failure, delay: 5s, window: 60s}
   registration:
-    image: blackducksoftware/blackduck-registration:2023.4.0_ubi8.7
+    image: blackducksoftware/blackduck-registration:2023.4.1_ubi8.7
     volumes:
     - config-volume:/opt/blackduck/hub/hub-registration/config
     - {type: tmpfs, target: /opt/blackduck/hub/hub-registration/security}
@@ -191,7 +191,7 @@ services:
       mode: replicated
       restart_policy: {condition: on-failure, delay: 15s, window: 60s}
   webui:
-    image: blackducksoftware/blackduck-webui:2023.4.0_ubi8.7
+    image: blackducksoftware/blackduck-webui:2023.4.1_ubi8.7
     healthcheck:
       test: [CMD, /usr/local/bin/docker-healthcheck.sh, 'https://localhost:8443/health-checks/liveness',
         /opt/blackduck/hub/hub-ui/security/root.crt]
@@ -206,7 +206,7 @@ services:
       restart_policy: {condition: on-failure, delay: 15s, window: 60s}
 
   documentation:
-    image: blackducksoftware/blackduck-documentation:2023.4.0_ubi8.7
+    image: blackducksoftware/blackduck-documentation:2023.4.1_ubi8.7
     env_file: [blackduck-config.env]
     user: documentation:root
     environment:
@@ -223,7 +223,7 @@ services:
       mode: replicated
       restart_policy: {condition: on-failure, delay: 5s, window: 60s}
   uploadcache:
-      image: blackducksoftware/blackduck-upload-cache:1.0.40_ubi8.7
+      image: blackducksoftware/blackduck-upload-cache:1.0.41_ubi8.7
       env_file: [blackduck-config.env]
       environment:
         HUB_JOBRUNNER_HOST: 'tasks.jobrunner.'
@@ -241,7 +241,7 @@ services:
         mode: replicated
         restart_policy: {condition: on-failure, delay: 5s, window: 60s}
   redis:
-    image: blackducksoftware/blackduck-redis:2023.4.0_ubi8.7
+    image: blackducksoftware/blackduck-redis:2023.4.1_ubi8.7
     env_file: [blackduck-config.env]
     environment:
       HUB_JOBRUNNER_HOST: 'tasks.jobrunner.'
@@ -260,7 +260,7 @@ services:
     deploy:
       restart_policy: {condition: any}
   bomengine:
-    image: blackducksoftware/blackduck-bomengine:2023.4.0_ubi8.7
+    image: blackducksoftware/blackduck-bomengine:2023.4.1_ubi8.7
     env_file: [blackduck-config.env , hub-postgres.env]
     environment:
       << : *pg-usage-settings
@@ -281,7 +281,7 @@ services:
         mode: replicated
         restart_policy: {condition: on-failure, delay: 5s, window: 60s}
   matchengine:
-    image: blackducksoftware/blackduck-matchengine:2023.4.0_ubi8.7
+    image: blackducksoftware/blackduck-matchengine:2023.4.1_ubi8.7
     user: matchengine:root
     healthcheck:
       test: [ CMD, /usr/local/bin/docker-healthcheck.sh, 'https://localhost:8443/api/health-checks/liveness',