From 77f7288b9e8f92ce2a4be7be5294db1cbab7a426 Mon Sep 17 00:00:00 2001 From: illuminatus Date: Wed, 11 Oct 2023 00:59:06 +0200 Subject: [PATCH] RING-44448 - Apply suggestions from code review Co-authored-by: scality-fno <84861109+scality-fno@users.noreply.github.com> --- scripts/S3_FSCK/export_s3_keys.sh | 46 +++++++++++++++---------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/scripts/S3_FSCK/export_s3_keys.sh b/scripts/S3_FSCK/export_s3_keys.sh index 33449c0..b8b53fd 100644 --- a/scripts/S3_FSCK/export_s3_keys.sh +++ b/scripts/S3_FSCK/export_s3_keys.sh @@ -50,10 +50,10 @@ function is_valid_ip() { function init() { if ! [[ -d ${WORKDIR} ]]; then if ! mkdir -pv "${WORKDIR}"; then - echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") ERROR: Failed to create ${WORKDIR}" + echo "$(date -u +"%FT%TZ") ERROR: Failed to create ${WORKDIR}" exit "${RC_FILE_PERMISSIONS}" else - echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") INFO: Created ${WORKDIR}" | tee -a "${WORKDIR}/RID_${RID}.log" + echo "$(date -u +"%FT%TZ") INFO: Created ${WORKDIR}" | tee -a "${WORKDIR}/RID_${RID}.log" fi fi } @@ -65,14 +65,14 @@ function find_bucketd_instance() { # If the SS_OUTPUT is equal to the UNBOUND_BUCKETD string then set BUCKETD_HOST to 127.0.0.1 if [[ ${SS_OUTPUT} == "${UNBOUND_BUCKETD[0]}" ]] || [[ ${SS_OUTPUT} == "${UNBOUND_BUCKETD[1]}" ]]; then BUCKETD_HOST="127.0.0.1" - echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") INFO: Found bucketd instance matched UNBOUND_BUCKET. Using BUCKETD_HOST: ${BUCKETD_HOST}" | tee -a "${WORKDIR}/RID_${RID}.log" + echo "$(date -u +"%FT%TZ") INFO: Found bucketd instance matched UNBOUND_BUCKET. Using BUCKETD_HOST: ${BUCKETD_HOST}" | tee -a "${WORKDIR}/RID_${RID}.log" else BUCKETD_HOST=$(echo "${SS_OUTPUT}" | cut -d: -f1) if [[ -z ${BUCKETD_HOST} ]] || [[ $(is_valid_ip "${BUCKETD_HOST}") == 0 ]]; then - echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") ERROR: Failed to find bucketd instance" | tee -a "${WORKDIR}/RID_${RID}.log" + echo "$(date -u +"%FT%TZ") ERROR: Failed to find bucketd instance" | tee -a "${WORKDIR}/RID_${RID}.log" exit "${RC_NO_BUCKETD_FOUND}" else - echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") INFO: Found bucketd instance ${BUCKETD_HOST}" | tee -a "${WORKDIR}/RID_${RID}.log" + echo "$(date -u +"%FT%TZ") INFO: Found bucketd instance ${BUCKETD_HOST}" | tee -a "${WORKDIR}/RID_${RID}.log" fi fi } @@ -80,12 +80,12 @@ function find_bucketd_instance() { # shellcheck disable=SC2312 function list_buckets_of_rid() { RAFT_OUTPUT=$(curl --silent "http://${BUCKETD_HOST}:9000/_/raft_sessions/${RID}/bucket") - mapfile -t BUCKETS < <(echo "${RAFT_OUTPUT}" | jq -r '.[] | select (. | contains("mpuShadowBucket") | not) | select (. | contains("users..bucket") | not)') + mapfile -t BUCKETS < <(echo "${RAFT_OUTPUT}" | jq -r '.[] | select((contains("mpuShadowBucket")|not) and (contains("users..bucket")|not))') if [[ ${#BUCKETS[@]} -eq 0 ]]; then - echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") WARNING: No buckets found in raft session ${RID}" | tee -a "${WORKDIR}/RID_${RID}.log" + echo "$(date -u +"%FT%TZ") WARNING: No buckets found in raft session ${RID}" | tee -a "${WORKDIR}/RID_${RID}.log" exit "${RC_NO_BUCKETS_FOUND_IN_RID}" fi - echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") INFO: Found ${#BUCKETS[@]} buckets in raft session ${RID}" | tee -a "${WORKDIR}/RID_${RID}.log" + echo "$(date -u +"%FT%TZ") INFO: Found ${#BUCKETS[@]} buckets in raft session ${RID}" | tee -a "${WORKDIR}/RID_${RID}.log" UNPROCESSED_BUCKETS=("${BUCKETS[@]}") } @@ -103,8 +103,8 @@ function export_bucket_contents() { registry.scality.com/s3utils/s3utils:1.14.0 \ verifyBucketSproxydKeys.js \ >"${WORKDIR}"/raw_"${bucket}"_keys.txt; then - echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") ERROR: Failed to export keys for ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log - echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") ERROR: Unprocessed buckets: ${UNPROCESSED_BUCKETS[*]}" | tee -a "${WORKDIR}/RID_${RID}".log + echo "$(date -u +"%FT%TZ") ERROR: Failed to export keys for ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log + echo "$(date -u +"%FT%TZ") ERROR: Unprocessed buckets: ${UNPROCESSED_BUCKETS[*]}" | tee -a "${WORKDIR}/RID_${RID}".log exit "${RC_EXPORT_S3_KEYS_FAILURE}" fi } @@ -114,19 +114,19 @@ function check_for_completed_scan() { # See if the raw file contains the message "completed scan" SCAN_COMPLETE=$(jq -r '. | select(.message | contains("completed scan"))' "${WORKDIR}/raw_${bucket}_keys.txt" | jq -s length) if [[ ${SCAN_COMPLETE} -eq 0 ]]; then - echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") WARNING: verifyBucketSproxydKeys scan did not complete for ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log + echo "$(date -u +"%FT%TZ") WARNING: verifyBucketSproxydKeys scan did not complete for ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log exit "${RC_EXPORT_S3_KEYS_FAILURE}" else - echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") INFO: verifyBucketSproxydKeys scan completed for ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log + echo "$(date -u +"%FT%TZ") INFO: verifyBucketSproxydKeys scan completed for ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log fi } # shellcheck disable=SC2312 function process_bucket_contents() { - echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") INFO: Processing output of RID ${RID} - bucket ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log + echo "$(date -u +"%FT%TZ") INFO: Processing output of RID ${RID} - bucket ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log if ! jq -r '. | select(.message | contains("sproxyd key")) + {"bucket": .objectUrl } | .bucket |= sub("s3://(?.*)/.*"; .bname) | .objectUrl |= sub("s3://.*/(?.*)$"; .oname) | [.bucket, .objectUrl, .sproxydKey] | @csv' "${WORKDIR}/raw_${bucket}_keys.txt" >"${WORKDIR}/${bucket}_keys.txt"; then - echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") ERROR: Failed to process keys for ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log - echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") ERROR: Unprocessed buckets: ${UNPROCESSED_BUCKETS[*]}" | tee -a "${WORKDIR}/RID_${RID}".log + echo "$(date -u +"%FT%TZ") ERROR: Failed to process keys for ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log + echo "$(date -u +"%FT%TZ") ERROR: Unprocessed buckets: ${UNPROCESSED_BUCKETS[*]}" | tee -a "${WORKDIR}/RID_${RID}".log exit "${RC_PROCESS_KEYS_FAILURE}" fi } @@ -136,14 +136,14 @@ function compare_raw_and_processed_counts() { RAW_COUNT=$(jq -r '. | select(.message | contains("sproxyd key"))' "${WORKDIR}/raw_${bucket}_keys.txt" | jq -s length) PROCESSED_COUNT=$(wc -l <"${WORKDIR}/${bucket}_keys.txt") if [[ ${RAW_COUNT} -ne ${PROCESSED_COUNT} ]]; then - echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") WARNING: Raw and processed counts do not match for ${bucket} - RAW_COUNT: ${RAW_COUNT} != PROCESED_COUNT: ${PROCESSED_COUNT}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log - echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") ERROR: Unprocessed buckets: ${UNPROCESSED_BUCKETS[*]}" | tee -a "${WORKDIR}/RID_${RID}".log + echo "$(date -u +"%FT%TZ") WARNING: Raw and processed counts do not match for ${bucket} - RAW_COUNT: ${RAW_COUNT} != PROCESED_COUNT: ${PROCESSED_COUNT}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log + echo "$(date -u +"%FT%TZ") ERROR: Unprocessed buckets: ${UNPROCESSED_BUCKETS[*]}" | tee -a "${WORKDIR}/RID_${RID}".log exit "${RC_COUNT_MISMATCH}" else - echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") INFO: Raw and processed counts match for ${bucket} - RAW_COUNT: ${RAW_COUNT} == PROCESED_COUNT: ${PROCESSED_COUNT}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log + echo "$(date -u +"%FT%TZ") INFO: Raw and processed counts match for ${bucket} - RAW_COUNT: ${RAW_COUNT} == PROCESED_COUNT: ${PROCESSED_COUNT}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log # if the raw and processed counts match but are both 0 then print a warning that the bucket is empty if [[ ${RAW_COUNT} -eq 0 ]]; then - echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") WARNING: Bucket ${bucket} appears to be empty: RAW_COUNT: ${RAW_COUNT} == PROCESED_COUNT: ${PROCESSED_COUNT}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log + echo "$(date -u +"%FT%TZ") WARNING: Bucket ${bucket} appears to be empty: RAW_COUNT: ${RAW_COUNT} == PROCESED_COUNT: ${PROCESSED_COUNT}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log fi fi @@ -152,9 +152,9 @@ function compare_raw_and_processed_counts() { # shellcheck disable=SC2312 function cleanup() { if [[ ${KEEP_RAW_FILES} =~ ^[Yy]$ ]]; then - echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") INFO: Keeping raw files for ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log + echo "$(date -u +"%FT%TZ") INFO: Keeping raw files for ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log else - echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") INFO: Removing raw files for ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log + echo "$(date -u +"%FT%TZ") INFO: Removing raw files for ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log rm -f "${WORKDIR}/raw_${bucket}_keys.txt" fi } @@ -162,7 +162,7 @@ function cleanup() { # shellcheck disable=SC2312 function export_buckets() { for bucket in "${BUCKETS[@]}"; do - echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") INFO: Starting export of ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log + echo "$(date -u +"%FT%TZ") INFO: Starting export of ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log # Export the bucket contents from the Raft Session export_bucket_contents # Check for scan completed @@ -172,7 +172,7 @@ function export_buckets() { # Check the raw and processed counts match compare_raw_and_processed_counts echo - echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") INFO: Completed export of ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log + echo "$(date -u +"%FT%TZ") INFO: Completed export of ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log # Pop the bucket out of the UNPROCESSED_BUCKETS array UNPROCESSED_BUCKETS=("${UNPROCESSED_BUCKETS[@]/${bucket}/}") done