diff --git a/programs/ziti-edge-tunnel/package/CPackPackage.cmake b/programs/ziti-edge-tunnel/package/CPackPackage.cmake index 63d6d0237..20183ed18 100644 --- a/programs/ziti-edge-tunnel/package/CPackPackage.cmake +++ b/programs/ziti-edge-tunnel/package/CPackPackage.cmake @@ -139,7 +139,7 @@ install(FILES "${INSTALL_OUT_DIR}/${ZITI_POLKIT_RULES_FILE}.sample" COMPONENT "${COMPONENT_NAME}" RENAME "${ZITI_POLKIT_RULES_FILE}") -install(FILES "${CMAKE_SOURCE_DIR}/scripts/debug.bash" +install(FILES "${CMAKE_SOURCE_DIR}/scripts/ziti-edge-tunnel-debug.bash" COMPONENT "${COMPONENT_NAME}" DESTINATION "${CPACK_BIN_DIR}" PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ diff --git a/scripts/debug.bash b/scripts/debug.bash deleted file mode 100755 index 0cc77f2e1..000000000 --- a/scripts/debug.bash +++ /dev/null @@ -1,169 +0,0 @@ -#!/usr/bin/env bash -# -# create a debug bundle for systemd service unit ziti-edge-tunnel.service -# -# usage: sudo /opt/openziti/bin/debug.bash [UPLOAD_URL] - -set -o errexit -set -o nounset -set -o pipefail - -checkCommand() { - if ! command -v "$1" &>/dev/null; then - echo "NOTICE: install '$1' and try again." >&2 - $1 - fi -} - -main() { - # require commands - declare -a BINS=(sed gdb strace tar timeout /opt/openziti/bin/ziti-edge-tunnel systemctl journalctl) - for BIN in "${BINS[@]}"; do - checkCommand "$BIN" - done - - cd "$(mktemp -d)" - mkdir ./dump ./stack ./backtrace - chgrp -R ziti "${PWD}" - chmod -R g+rwX "${PWD}" - - echo -n "(estimated runtime 60s) [." - - NOW=$(date -u +'%Y-%m-%dT%H:%MZ') - ZITI_VERSION=$(/opt/openziti/bin/ziti-edge-tunnel version) - PREFIX=ziti-edge-tunnel-${ZITI_VERSION#v}-${NOW} - LOG_FILE=${PREFIX}.log - STRACE_FILE=${PREFIX}.strace - TUNNEL_STATUS_FILE=${PREFIX}.tunnel_status.json - SYSTEMD_RESOLVED_FILE=${PREFIX}.systemd-resolved - HOST_INFO_FILE=${PREFIX}.host - TARBALL=${PREFIX}.tgz - - # get the PID from systemd - ZET_PID="$(systemctl show -p MainPID --value ziti-edge-tunnel.service)" - if [[ -z "${ZET_PID}" ]] || [[ "${ZET_PID}" == "0" ]]; - then - echo "ERROR: failed to get ziti-edge-tunnel.service PID" >&2 - exit 1 - fi - - # save the systemd-resolved state - ( - set +e - set -x - systemctl status systemd-resolved.service - resolvectl status - resolvectl dns - resolvectl domain - ls -l /etc/resolv.conf - cat /etc/resolv.conf - ) &> "${SYSTEMD_RESOLVED_FILE}" - echo -n "." - - # save host info - ( - set +e - set -x - hostnamectl; hostname - ip link show; ip addr show; ip route show - cat /etc/hosts /etc/*-release - ) &> "${HOST_INFO_FILE}" - echo -n "." - - # save the current service unit invocation's log messages - journalctl _SYSTEMD_INVOCATION_ID="$(systemctl show -p InvocationID --value ziti-edge-tunnel.service)" -l --no-pager \ - &> "${LOG_FILE}" - echo -n "." - - # save the call stack at intervals - BTRACE_COUNT=1 - # allow parent env to override max count - : "${BTRACE_MAX:=10}" - BTRACE_MAX_LEN=$(( $(wc -c <<< "${BTRACE_MAX}") - 1 )) - # compare decimal form of iterator to decimal max - until [[ "10#${BTRACE_COUNT}" -gt "${BTRACE_MAX}" ]] - do - # save the threads and backtrace - timeout --kill-after=1s 3s \ - gdb /opt/openziti/bin/ziti-edge-tunnel \ - --pid "${ZET_PID}" \ - --batch \ - --ex "set sysroot /" \ - --ex "set verbose on" \ - --ex "set pagination off" \ - --ex "info threads" \ - --ex "thread apply all backtrace" \ - --ex "quit" \ - &> "./backtrace/${BTRACE_COUNT}_of_${BTRACE_MAX}-$(date -u +'%Y-%m-%dT%H:%M:%SZ').backtrace" \ - || echo "WARN: gdb backtrace timed out" >&2 - echo -n "." - sleep 1 - # increment decimal form of iterator - BTRACE_COUNT=$((10#${BTRACE_COUNT} + 1)) - # pad the decimal form of iterator for filename sorting - BTRACE_COUNT=$(printf "%0${BTRACE_MAX_LEN}d" "${BTRACE_COUNT}") - done - - # save 10s of strace calls - timeout --kill-after=1s 10s \ - strace --attach "${ZET_PID}" \ - --follow-forks \ - --absolute-timestamps=format:unix,precision:us \ - --syscall-times \ - --string-limit 4096 \ - &> "${STRACE_FILE}" \ - || true # catch the expected timeout exit code - # --trace=%network \ - # --trace=%process \ - # --trace=%signal \ - # --trace=%ipc \ - # --trace=%desc \ - # --trace=%memory \ - echo -n "." - - # save the call stack at intervals - STACK_COUNT=1 - # allow parent env to override max count - : "${STACK_MAX:=3}" - # find width of decimal max - STACK_MAX_LEN=$(( $(wc -c <<< "${STACK_MAX}") - 1 )) - # compare decimal form of iterator to decimal max - until [[ "10#${STACK_COUNT}" -gt "${STACK_MAX}" ]] - do - cat "/proc/${ZET_PID}/stack" \ - > "./stack/${STACK_COUNT}_of_${STACK_MAX}-$(date -u +'%Y-%m-%dT%H:%M:%SZ').stack" - echo -n "." - # shellcheck disable=SC2034 # iterator is unused - for i in {1..10}; do sleep 1; echo -n "."; done - # increment decimal form of iterator - STACK_COUNT=$((10#${STACK_COUNT} + 1)) - # pad the decimal form of iterator for filename sorting - STACK_COUNT=$(printf "%0${STACK_MAX_LEN}d" "${STACK_COUNT}") - done - - # save the identity status dumps - timeout --kill-after=1s 3s \ - ziti-edge-tunnel dump -p ./dump >/dev/null \ - || echo "WARN: failed to create dumps" >&2 - echo -n "." - - # save the tunnel_status JSON - timeout --kill-after=1s 3s \ - ziti-edge-tunnel tunnel_status \ - | sed -E "s/(^received\sresponse\s<|>$)//g" > "${TUNNEL_STATUS_FILE}" \ - || echo "WARN: failed to get tunnel_status" >&2 - echo -n "." - - tar -czf "/tmp/${TARBALL}" . - echo -en ".]\n" - - echo "INFO: debug bundle created at /tmp/${TARBALL} from files in ${PWD}" - - if [[ -n "${1:-}" ]] - then - echo "INFO: uploading debug bundle to ${1}" - curl -sSf -T "/tmp/${TARBALL}" "${1}" - fi -} - -main "$@" diff --git a/scripts/ziti-edge-tunnel-debug.bash b/scripts/ziti-edge-tunnel-debug.bash new file mode 100755 index 000000000..0e4900049 --- /dev/null +++ b/scripts/ziti-edge-tunnel-debug.bash @@ -0,0 +1,190 @@ +#!/usr/bin/env bash +# +# create a debug bundle for systemd service unit ziti-edge-tunnel.service +# +# usage: sudo /opt/openziti/bin/ziti-edge-tunnel-debug.bash [UPLOAD_URL] + +set -o errexit +set -o nounset +set -o pipefail + +checkCommand() { + if ! command -v "$1" &>/dev/null; then + echo "NOTICE: install '$1' and try again." >&2 + $1 + fi +} + +main() { + # require commands + declare -a BINS=(sed tar timeout /opt/openziti/bin/ziti-edge-tunnel systemctl journalctl) + for BIN in "${BINS[@]}"; do + checkCommand "$BIN" + done + + cd "$(mktemp -d)" + mkdir ./dump + chgrp -R ziti "${PWD}" + chmod -R g+rwX "${PWD}" + + echo -n "(estimated runtime 60s) [." + + NOW=$(date -u +'%Y-%m-%dT%H:%MZ') + ZITI_VERSION=$(/opt/openziti/bin/ziti-edge-tunnel version) + PREFIX=ziti-edge-tunnel-${ZITI_VERSION#v}-${NOW} + LOG_FILE=${PREFIX}.log + STRACE_FILE=${PREFIX}.strace + TUNNEL_STATUS_FILE=${PREFIX}.tunnel_status.json + SYSTEMD_RESOLVED_FILE=${PREFIX}.systemd-resolved + HOST_INFO_FILE=${PREFIX}.host + TARBALL=${PREFIX}.tgz + + # get the PID from systemd + ZET_PID="$(systemctl show -p MainPID --value ziti-edge-tunnel.service)" + if [[ -z "${ZET_PID}" ]] || [[ "${ZET_PID}" == "0" ]]; + then + echo "ERROR: failed to get ziti-edge-tunnel.service PID" >&2 + exit 1 + fi + + # save the systemd-resolved state + ( + set +e + set -x + systemctl status systemd-resolved.service + resolvectl status + resolvectl dns + resolvectl domain + ls -l /etc/resolv.conf + cat /etc/resolv.conf + ) &> "${SYSTEMD_RESOLVED_FILE}" + echo -n "." + + # save host info + ( + set +e + set -x + hostnamectl; hostname + ip link show; ip addr show; ip route show + cat /etc/hosts /etc/*-release + ) &> "${HOST_INFO_FILE}" + echo -n "." + + # save the current service unit invocation's log messages + journalctl _SYSTEMD_INVOCATION_ID="$(systemctl show -p InvocationID --value ziti-edge-tunnel.service)" -l --no-pager \ + &> "${LOG_FILE}" + echo -n "." + + # if stack then save it; else try backtrace + if [[ -f "/proc/${ZET_PID}/stack" ]]; then + mkdir ./stack + # save the call stack at intervals + STACK_COUNT=1 + # allow parent env to override max count + : "${STACK_MAX:=3}" + # find width of decimal max + STACK_MAX_LEN=$(( $(wc -c <<< "${STACK_MAX}") - 1 )) + # compare decimal form of iterator to decimal max + until [[ "10#${STACK_COUNT}" -gt "${STACK_MAX}" ]] + do + STACK_TIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ') + STACK_DIR="./stack/${STACK_COUNT}_of_${STACK_MAX}-${STACK_TIME}" + mkdir "${STACK_DIR}" + cat "/proc/${ZET_PID}/stack" > "${STACK_DIR}/stack" + + # Capture threads' stacks if task directory exists + if [[ -d "/proc/${ZET_PID}/task" ]]; then + mkdir -p "${STACK_DIR}/task" + for TID_DIR in "/proc/${ZET_PID}/task/"*; do + TID=$(basename "${TID_DIR}") + cat "${TID_DIR}/stack" > "${STACK_DIR}/task/${TID}.stack" + done + fi + echo -n "." + # shellcheck disable=SC2034 # iterator is unused + for i in {1..10}; do sleep 1; echo -n "."; done + # increment decimal form of iterator + STACK_COUNT=$((10#${STACK_COUNT} + 1)) + # pad the decimal form of iterator for filename sorting + STACK_COUNT=$(printf "%0${STACK_MAX_LEN}d" "${STACK_COUNT}") + done + + elif command -v gdb &>/dev/null; then + mkdir ./backtrace + # save the call stack at intervals + BTRACE_COUNT=1 + # allow parent env to override max count + : "${BTRACE_MAX:=10}" + BTRACE_MAX_LEN=$(( $(wc -c <<< "${BTRACE_MAX}") - 1 )) + # compare decimal form of iterator to decimal max + until [[ "10#${BTRACE_COUNT}" -gt "${BTRACE_MAX}" ]] + do + # save the threads and backtrace + timeout --kill-after=1s 3s \ + gdb /opt/openziti/bin/ziti-edge-tunnel \ + --pid "${ZET_PID}" \ + --batch \ + --ex "set sysroot /" \ + --ex "set verbose on" \ + --ex "set pagination off" \ + --ex "info threads" \ + --ex "thread apply all backtrace" \ + --ex "quit" \ + &> "./backtrace/${BTRACE_COUNT}_of_${BTRACE_MAX}-$(date -u +'%Y-%m-%dT%H:%M:%SZ').backtrace" \ + || echo "WARN: gdb backtrace timed out" >&2 + echo -n "." + sleep 1 + # increment decimal form of iterator + BTRACE_COUNT=$((10#${BTRACE_COUNT} + 1)) + # pad the decimal form of iterator for filename sorting + BTRACE_COUNT=$(printf "%0${BTRACE_MAX_LEN}d" "${BTRACE_COUNT}") + done + else + echo "WARN: install gdb to include a backtrace in the debug bundle" >&2 + fi + + if command -v strace &>/dev/null; then + # save 10s of strace calls + timeout --kill-after=1s 10s \ + strace --attach "${ZET_PID}" \ + --follow-forks \ + --absolute-timestamps=format:unix,precision:us \ + --syscall-times \ + --string-limit 4096 \ + &> "${STRACE_FILE}" \ + || true # catch the expected timeout exit code + # --trace=%network \ + # --trace=%process \ + # --trace=%signal \ + # --trace=%ipc \ + # --trace=%desc \ + # --trace=%memory \ + echo -n "." + fi + + # save the identity status dumps + timeout --kill-after=1s 3s \ + ziti-edge-tunnel dump -p ./dump >/dev/null \ + || echo "WARN: failed to create dumps" >&2 + echo -n "." + + # save the tunnel_status JSON + timeout --kill-after=1s 3s \ + ziti-edge-tunnel tunnel_status \ + | sed -E "s/(^received\sresponse\s<|>$)//g" > "${TUNNEL_STATUS_FILE}" \ + || echo "WARN: failed to get tunnel_status" >&2 + echo -n "." + + tar -czf "/tmp/${TARBALL}" . + echo -en ".]\n" + + echo "INFO: debug bundle created at /tmp/${TARBALL} from files in ${PWD}" + + if [[ -n "${1:-}" ]] + then + echo "INFO: uploading debug bundle to ${1}" + curl -sSf -T "/tmp/${TARBALL}" "${1}" + fi +} + +main "$@"