SVCPLAN-4857 new script to run generic SQL on a remote PG server

remote_control replaces jiractl
update mk_test_server for confluence
rename scripts for confluence
update fix_network for confluence server
fix_app_config fixed for Confluence
add rcp remote-copy
clean_xml.sh
start/stop services; upgrade_app
fix_java_heap_size.sh
set_web_access
move get_installer to utils
add elapsed time in upgrade
systemd wants=postgresql
move confluence sql to its own cleanup foldeer
split cleanup into jira & wiki specific dirs
update versions for workflow actions

remove COPY cleanup from Dockerfile
add workflow env
use version main in env

deactivate user working
quiet grep & puppet resource disable
use lvmdevices to deldev

HeapDumpOnOutOfMemoryError

update for production PM

update .gitignore

Complete sso_bypass.sh

Add option to specify key and additional errors

SVCPLAN-4131 migrate to JSM

go_log_rotate.sh go_mk_atlassian_supportzip.sh
add check for cmdline action in go_upgrade_app
attachments tarpipe script
migrate attachments working
enable service after installing service file
update api functions to work with netrc
script to enable/disable mail delivery

.github/workflows/env.yml

@@ -0,0 +1,7 @@
+name: "Call Env Vars"
+
+on: [ "push", "workflow_dispatch" ]
+
+jobs:
+  call-envs:
+    uses: andylytical/actions/.github/workflows/envs.yml@main

.github/workflows/ghcr-publish.yml

@@ -35,16 +35,16 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Docker buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       # Login against a Docker registry except on PR
       # https://github.com/docker/login-action
       - name: Log into registry ${{ env.REGISTRY }}
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -54,7 +54,7 @@ jobs:
       # https://github.com/docker/metadata-action
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |
@@ -68,7 +68,7 @@ jobs:
       # https://github.com/docker/build-push-action
       - name: Build and push Docker image
         id: build-and-push
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: "{{defaultContext}}"
           push: ${{ github.event_name != 'pull_request' }}

.gitignore

@@ -1,11 +1,25 @@
-#Binary installers
+#Binary files
 *.bin
+*.jar
 
 # password files
 *.pwd
 *.cnf
 
 # test files
-*.input
+*.csv
 *.ids
+*.input
+*.sample
+*.tsv
+*.txt
+NOTES
 output
+
+# log files
+*.log
+
+# working files
+*.swp
+*.tar
+attachments/

Dockerfile

@@ -4,9 +4,14 @@ ENV TZ=America/Chicago
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
 WORKDIR /srv
-COPY cleanup/* /srv/
+COPY bin bin/
+COPY conf conf/
+COPY lib lib/
+COPY requirements.txt ./
+COPY jira_cleanup jira_cleanup/
+COPY wiki_cleanup wiki_cleanup/
 RUN python -m pip install -r /srv/requirements.txt
-RUN ln -s /home/.netrc /root/.netrc
-RUN ln -s /home/.atlassian-tools-config.ini /root/.atlassian-tools-config.ini
+#RUN ln -s /home/.netrc /root/.netrc
+#RUN ln -s /home/.atlassian-tools-config.ini /root/.atlassian-tools-config.ini
 
 CMD ["bash"]

README.md

@@ -9,7 +9,7 @@ export QS_GIT_REPO=https://github.com/ncsa/atlassian-tools.git
 curl https://raw.githubusercontent.com/andylytical/quickstart/main/quickstart.sh | bash
 ```
 
-# Make a test instance from a clone of prod
+# Create a test instance on a clone of prod
 Create a copy of prod for testing
 1. Prod server:
    1. Install this repo via the instructions above
@@ -20,21 +20,23 @@ Create a copy of prod for testing
    1. Disconnect the network on the test server
    1. Start the test server
    1. Login as root on the console of the test server
-      1. `/root/atlassian-tools/bin/jira_mktest.sh`
-      1. `/root/atlassian-tools/bin/jira_validate.sh`
+      1. `/root/atlassian-tools/bin/go_mk_test_server.sh`
+      1. `/root/atlassian-tools/bin/go_validate_test_server.sh`
       1. `shutdown -h now`
    1. Ensure test VM is powered off
    1. Re-enable the network
    1. Power-on the test VM
-1. root@aesculus
+1. root@aesculus (only if DB is still remote)
    1. `bash /home/aloftus/mk-jira-test-db.sh`
-1. root@jira-test
-   1. `bash /root/atlassian-tools/bin/jira_fix_web.sh`
-   1. `/usr/services/jira-standalone/bin/start-jira.sh`
-   1. `tail -f /usr/services/jira-standalone/logs/catalina.out`
+1. root@jira-test (or root@wiki-test)
+   1. `bash /root/atlassian-tools/bin/go_fix_web.sh`
+   1. `/usr/services/jira-standalone/bin/start-jira.sh` (OR
+      `/usr/services/confluence/bin/start-confluence.sh`)
+   1. `tail -f /usr/services/jira-standalone/logs/catalina.out` (OR
+      `/usr/services/confluence/logs/catalina.out`)
 
 # Install / Upgrade Jira
-This should also work for Confluence, but is untested as of 6 Oct 2023.
+This also works for Confluence, tested Feb 2024.
 1. Follow the instructions above to create a test VM cloned from prod
 1. Clone this repo to a local machine
 1. `cd atlassian-tools/install-upgrade`

bin/backup_config_files.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -e
+
+BASE=${HOME}/atlassian-tools
+
+. ${BASE}/conf/config.sh
+. ${BASE}/lib/utils.sh
+
+[[ $VERBOSE -eq $YES ]] && set -x
+
+FILES=( 
+  "$APP_INSTALL_DIR"/conf/server.xml
+  "$APP_INSTALL_DIR"/bin/setenv.sh
+)
+case "$APP_NAME" in
+  jira)
+    DB_CONF="$APP_HOME_DIR"/dbconfig.xml
+    WEB_XML="$APP_INSTALL_DIR"/atlassian-jira/WEB-INF/web.xml
+    ;;
+  confluence)
+    DB_CONF="$APP_HOME_DIR"/confluence.cfg.xml
+    WEB_XML="$APP_INSTALL_DIR"/confluence/WEB-INF/web.xml
+    ;;
+esac
+FILES+=( "$DB_CONF" "$WEB_XML" )
+
+mkdir -p "$BACKUP_DIR" #BACKUP_DIR defined in config.sh
+
+tar -c -f - "${FILES[@]}" | tar -x -f - -C "$BACKUP_DIR" \
+  && success "Config files backed up to '$BACKUP_DIR'"

bin/clean_vgs.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/bash
+
+set -x 
+
+YES=0
+NO=1
+VERBOSE=$YES
+DEBUG=$YES
+
+VGS=( vg_pgsql vg_backups vg_confluence )
+
+ECHO=
+[[ $DEBUG -eq $YES ]] && ECHO=echo
+
+get_mountpoint() {
+  local _vg="$1"
+  mount | grep $_vg | cut -d' ' -f1
+}
+
+mp_destroy() {
+  local _mp="$1"
+  [[ -n "$_mp" ]] && {
+    $ECHO umount "$_mp"
+    $ECHO wipefs -a -f "$_mp"
+  }
+}
+
+vg_delete() {
+  local _vg="$1"
+  local _count=$( vgs | grep -c "$_vg" )
+  [[ $_count -gt 0 ]] && $ECHO vgremove -y $_vg
+}
+
+get_pv() {
+  local _vg="$1"
+  vgs | grep -qF "$_vg" \
+  && vgs -o pv_name --noheadings $_vg
+}
+
+pv_wipe() {
+  local _pv="$1"
+  # $ECHO dd if=/dev/zero of=$pv bs=1M count=1
+  $ECHO wipefs -a -f "$_pv"
+}
+
+get_lv() {
+  local _vg="$1"
+  vgs | grep -qF "$_vg" \
+  && vgs -o lv_name --noheadings "$_vg"
+}
+
+for vg in "${VGS[@]}" ; do
+
+  lv=$( get_lv "$vg" )
+  pv=$( get_pv "$vg" )
+  mountp=$( get_mountpoint "$vg" )
+  mp_destroy "$mountp"
+#  vg_delete "$vg"
+#
+#  for pv in "${peevees[@]}" ; do
+#    echo "PV REMOVE $pv"
+#    $ECHO pvremove -y "$pv"
+#    pv_wipe "$pv"
+#  done
+
+done
+

bin/disable_mail.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-BASE=/root/atlassian-tools
+BASE=${HOME}/atlassian-tools
 
 . ${BASE}/conf/config.sh
 . ${BASE}/lib/utils.sh

bin/fix_app_config.sh

@@ -1,41 +1,81 @@
 #!/bin/bash
 
-BASE=/root/atlassian-tools
+set -e
+
+BASE=${HOME}/atlassian-tools
 
 . ${BASE}/conf/config.sh
 . ${BASE}/lib/utils.sh
 
 [[ $VERBOSE -eq $YES ]] && set -x
 
+sed_opts=( '-i' )
+[[ $DEBUG -eq $YES ]] && unset sed_opts
 
 # Fix server.xml
-XML="$APP_INSTALL_DIR"/conf/server.xml
-[[ -z "$XML" ]] && die "File not found: '$XML'"
+SERVER_XML="$APP_INSTALL_DIR"/conf/server.xml
+[[ -f "$SERVER_XML" ]] || die "File not found: '$SERVER_XML'"
 
+# ... HOSTNAME
 TMP=$(mktemp)
-sed -e 's/\(proxyName="\)[^"]\+"/\1xxxyyyzzz"/' "$XML" \
+sed -e 's/\(proxyName="\)[^"]\+"/\1xxxyyyzzz"/' "$SERVER_XML" \
 | sed -e "s/xxxyyyzzz/$HOSTNAME_NEW/" >$TMP
 
+chown --reference="$SERVER_XML" "$TMP"
+chmod --reference="$SERVER_XML" "$TMP"
 if [[ $DEBUG -eq $YES ]] ; then
   cat $TMP
   rm $TMP
 else
-  mv $TMP "$XML"
+  mv $TMP "$SERVER_XML"
 fi
+success "Fixed hostname in server.xml"
+
+
+# ... Upgrade recovery (confluence only)
+[[ $APP_NAME == 'confluence' ]] && {
+  SETENV="${APP_INSTALL_DIR}"/bin/setenv.sh
+  [[ -f "$SETENV" ]] || die "File not found: '$SETENV'"
+  sed "${sed_opts[@]}" \
+    -e '/upgrade.recovery.file.enabled=false/ s/^[ #]*//' \
+    "$SETENV"
+  success "disabled confluence upgrade recovery file creation"
+}
 
 
 # Fix DB config
-DB_CONF="$APP_HOME_DIR"/dbconfig.xml
-sed_opts=( '-i' )
-[[ $DEBUG -eq $YES ]] && unset sed_opts
-sed "${sed_opts[@]}" -e "s/$DB_NAME_OLD/$DB_NAME_NEW/g" "$DB_CONF"
+if [[ "$DB_NAME_OLD" == "$DB_NAME_NEW" ]] ; then
+  echo 'DB_NAME_OLD == DB_NAME_NEW, no changes needed ... skipping "Fix DB Config" step'
+else
+  case "$APP_NAME" in
+    jira)
+      DB_CONF="$APP_HOME_DIR"/dbconfig.xml
+      PTRN="<url>jdbc:"
+      ;;
+    confluence)
+      DB_CONF="$APP_HOME_DIR"/confluence.cfg.xml
+      PTRN='hibernate\.connection\.url'
+      ;;
+  esac
+  sed "${sed_opts[@]}" -e "s/$DB_NAME_OLD/$DB_NAME_NEW/g" "$DB_CONF"
+  success "Updated DB config"
+fi
 
 
-# Fix web.xml
-XML="$APP_INSTALL_DIR"/atlassian-jira/WEB-INF/web.xml
-[[ -z "$XML" ]] && die "File not found: '$XML'"
-sed_opts=( '-i' )
-[[ $DEBUG -eq $YES ]] && unset sed_opts
+# Fix session timeout in web.xml
+# Historically, this was set to 12000 (8.3 days) ... why?
+# See also:
+# https://confluence.atlassian.com/confkb/how-to-adjust-the-session-timeout-for-confluence-126910597.html
+# https://confluence.atlassian.com/jirakb/change-the-default-session-timeout-to-avoid-user-logout-in-jira-server-604209887.html
+# TODO - Is this needed anymore after change to SSO via CiLogon?
+WEB_XML=''
+if [[ $APP_NAME == 'jira' ]] ; then
+  WEB_XML="$APP_INSTALL_DIR"/atlassian-jira/WEB-INF/web.xml
+elif [[ $APP_NAME == 'confluence' ]] ; then
+  WEB_XML="$APP_INSTALL_DIR"/confluence/WEB-INF/web.xml
+fi
+[[ -f "$WEB_XML" ]] || die "File not found: '$WEB_XML'"
 sed "${sed_opts[@]}" \
-  -e 's/\(<session-timeout>\)[0-9]\+\(<\/session-timeout>\)/\112000\2/' \
-  "$XML"
+  -e 's/\(<session-timeout>\)[0-9]\+\(<\/session-timeout>\)/\11200\2/' \
+  "$WEB_XML"
+success "Fix session timeout in web.xml"

bin/fix_cron.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-BASE=/root/atlassian-tools
+BASE=${HOME}/atlassian-tools
 
 . ${BASE}/conf/config.sh
 . ${BASE}/lib/utils.sh

bin/fix_hostname.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-BASE=/root/atlassian-tools
+BASE=${HOME}/atlassian-tools
 
 . ${BASE}/conf/config.sh
 . ${BASE}/lib/utils.sh