Skip to content

Commit

Permalink
RING-44448 - Apply suggestions from code review
Browse files Browse the repository at this point in the history 
Co-authored-by: scality-fno <[email protected]>
  • Loading branch information
@TrevorBenson @scality-fno
TrevorBenson and scality-fno authored Oct 10, 2023
1 parent 4d009b6 commit 77f7288
Showing 1 changed file with 23 additions and 23 deletions.
46 changes: 23 additions & 23 deletions scripts/S3_FSCK/export_s3_keys.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,10 @@ function is_valid_ip() {
function init() {
if ! [[ -d ${WORKDIR} ]]; then
if ! mkdir -pv "${WORKDIR}"; then
echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") ERROR: Failed to create ${WORKDIR}"
echo "$(date -u +"%FT%TZ") ERROR: Failed to create ${WORKDIR}"
exit "${RC_FILE_PERMISSIONS}"
else
echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") INFO: Created ${WORKDIR}" | tee -a "${WORKDIR}/RID_${RID}.log"
echo "$(date -u +"%FT%TZ") INFO: Created ${WORKDIR}" | tee -a "${WORKDIR}/RID_${RID}.log"
fi
fi
}
Expand All @@ -65,27 +65,27 @@ function find_bucketd_instance() {
# If the SS_OUTPUT is equal to the UNBOUND_BUCKETD string then set BUCKETD_HOST to 127.0.0.1
if [[ ${SS_OUTPUT} == "${UNBOUND_BUCKETD[0]}" ]] || [[ ${SS_OUTPUT} == "${UNBOUND_BUCKETD[1]}" ]]; then
BUCKETD_HOST="127.0.0.1"
echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") INFO: Found bucketd instance matched UNBOUND_BUCKET. Using BUCKETD_HOST: ${BUCKETD_HOST}" | tee -a "${WORKDIR}/RID_${RID}.log"
echo "$(date -u +"%FT%TZ") INFO: Found bucketd instance matched UNBOUND_BUCKET. Using BUCKETD_HOST: ${BUCKETD_HOST}" | tee -a "${WORKDIR}/RID_${RID}.log"
else
BUCKETD_HOST=$(echo "${SS_OUTPUT}" | cut -d: -f1)
if [[ -z ${BUCKETD_HOST} ]] || [[ $(is_valid_ip "${BUCKETD_HOST}") == 0 ]]; then
echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") ERROR: Failed to find bucketd instance" | tee -a "${WORKDIR}/RID_${RID}.log"
echo "$(date -u +"%FT%TZ") ERROR: Failed to find bucketd instance" | tee -a "${WORKDIR}/RID_${RID}.log"
exit "${RC_NO_BUCKETD_FOUND}"
else
echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") INFO: Found bucketd instance ${BUCKETD_HOST}" | tee -a "${WORKDIR}/RID_${RID}.log"
echo "$(date -u +"%FT%TZ") INFO: Found bucketd instance ${BUCKETD_HOST}" | tee -a "${WORKDIR}/RID_${RID}.log"
fi
fi
}

# shellcheck disable=SC2312
function list_buckets_of_rid() {
RAFT_OUTPUT=$(curl --silent "http://${BUCKETD_HOST}:9000/_/raft_sessions/${RID}/bucket")
mapfile -t BUCKETS < <(echo "${RAFT_OUTPUT}" | jq -r '.[] | select (. | contains("mpuShadowBucket") | not) | select (. | contains("users..bucket") | not)')
mapfile -t BUCKETS < <(echo "${RAFT_OUTPUT}" | jq -r '.[] | select((contains("mpuShadowBucket")|not) and (contains("users..bucket")|not))')
if [[ ${#BUCKETS[@]} -eq 0 ]]; then
echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") WARNING: No buckets found in raft session ${RID}" | tee -a "${WORKDIR}/RID_${RID}.log"
echo "$(date -u +"%FT%TZ") WARNING: No buckets found in raft session ${RID}" | tee -a "${WORKDIR}/RID_${RID}.log"
exit "${RC_NO_BUCKETS_FOUND_IN_RID}"
fi
echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") INFO: Found ${#BUCKETS[@]} buckets in raft session ${RID}" | tee -a "${WORKDIR}/RID_${RID}.log"
echo "$(date -u +"%FT%TZ") INFO: Found ${#BUCKETS[@]} buckets in raft session ${RID}" | tee -a "${WORKDIR}/RID_${RID}.log"
UNPROCESSED_BUCKETS=("${BUCKETS[@]}")
}

Expand All @@ -103,8 +103,8 @@ function export_bucket_contents() {
registry.scality.com/s3utils/s3utils:1.14.0 \
verifyBucketSproxydKeys.js \
>"${WORKDIR}"/raw_"${bucket}"_keys.txt; then
echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") ERROR: Failed to export keys for ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log
echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") ERROR: Unprocessed buckets: ${UNPROCESSED_BUCKETS[*]}" | tee -a "${WORKDIR}/RID_${RID}".log
echo "$(date -u +"%FT%TZ") ERROR: Failed to export keys for ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log
echo "$(date -u +"%FT%TZ") ERROR: Unprocessed buckets: ${UNPROCESSED_BUCKETS[*]}" | tee -a "${WORKDIR}/RID_${RID}".log
exit "${RC_EXPORT_S3_KEYS_FAILURE}"
fi
}
Expand All @@ -114,19 +114,19 @@ function check_for_completed_scan() {
# See if the raw file contains the message "completed scan"
SCAN_COMPLETE=$(jq -r '. | select(.message | contains("completed scan"))' "${WORKDIR}/raw_${bucket}_keys.txt" | jq -s length)
if [[ ${SCAN_COMPLETE} -eq 0 ]]; then
echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") WARNING: verifyBucketSproxydKeys scan did not complete for ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log
echo "$(date -u +"%FT%TZ") WARNING: verifyBucketSproxydKeys scan did not complete for ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log
exit "${RC_EXPORT_S3_KEYS_FAILURE}"
else
echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") INFO: verifyBucketSproxydKeys scan completed for ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log
echo "$(date -u +"%FT%TZ") INFO: verifyBucketSproxydKeys scan completed for ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log
fi
}

# shellcheck disable=SC2312
function process_bucket_contents() {
echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") INFO: Processing output of RID ${RID} - bucket ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log
echo "$(date -u +"%FT%TZ") INFO: Processing output of RID ${RID} - bucket ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log
if ! jq -r '. | select(.message | contains("sproxyd key")) + {"bucket": .objectUrl } | .bucket |= sub("s3://(?<bname>.*)/.*"; .bname) | .objectUrl |= sub("s3://.*/(?<oname>.*)$"; .oname) | [.bucket, .objectUrl, .sproxydKey] | @csv' "${WORKDIR}/raw_${bucket}_keys.txt" >"${WORKDIR}/${bucket}_keys.txt"; then
echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") ERROR: Failed to process keys for ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log
echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") ERROR: Unprocessed buckets: ${UNPROCESSED_BUCKETS[*]}" | tee -a "${WORKDIR}/RID_${RID}".log
echo "$(date -u +"%FT%TZ") ERROR: Failed to process keys for ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log
echo "$(date -u +"%FT%TZ") ERROR: Unprocessed buckets: ${UNPROCESSED_BUCKETS[*]}" | tee -a "${WORKDIR}/RID_${RID}".log
exit "${RC_PROCESS_KEYS_FAILURE}"
fi
}
Expand All @@ -136,14 +136,14 @@ function compare_raw_and_processed_counts() {
RAW_COUNT=$(jq -r '. | select(.message | contains("sproxyd key"))' "${WORKDIR}/raw_${bucket}_keys.txt" | jq -s length)
PROCESSED_COUNT=$(wc -l <"${WORKDIR}/${bucket}_keys.txt")
if [[ ${RAW_COUNT} -ne ${PROCESSED_COUNT} ]]; then
echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") WARNING: Raw and processed counts do not match for ${bucket} - RAW_COUNT: ${RAW_COUNT} != PROCESED_COUNT: ${PROCESSED_COUNT}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log
echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") ERROR: Unprocessed buckets: ${UNPROCESSED_BUCKETS[*]}" | tee -a "${WORKDIR}/RID_${RID}".log
echo "$(date -u +"%FT%TZ") WARNING: Raw and processed counts do not match for ${bucket} - RAW_COUNT: ${RAW_COUNT} != PROCESED_COUNT: ${PROCESSED_COUNT}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log
echo "$(date -u +"%FT%TZ") ERROR: Unprocessed buckets: ${UNPROCESSED_BUCKETS[*]}" | tee -a "${WORKDIR}/RID_${RID}".log
exit "${RC_COUNT_MISMATCH}"
else
echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") INFO: Raw and processed counts match for ${bucket} - RAW_COUNT: ${RAW_COUNT} == PROCESED_COUNT: ${PROCESSED_COUNT}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log
echo "$(date -u +"%FT%TZ") INFO: Raw and processed counts match for ${bucket} - RAW_COUNT: ${RAW_COUNT} == PROCESED_COUNT: ${PROCESSED_COUNT}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log
# if the raw and processed counts match but are both 0 then print a warning that the bucket is empty
if [[ ${RAW_COUNT} -eq 0 ]]; then
echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") WARNING: Bucket ${bucket} appears to be empty: RAW_COUNT: ${RAW_COUNT} == PROCESED_COUNT: ${PROCESSED_COUNT}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log
echo "$(date -u +"%FT%TZ") WARNING: Bucket ${bucket} appears to be empty: RAW_COUNT: ${RAW_COUNT} == PROCESED_COUNT: ${PROCESSED_COUNT}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log
fi
fi

Expand All @@ -152,17 +152,17 @@ function compare_raw_and_processed_counts() {
# shellcheck disable=SC2312
function cleanup() {
if [[ ${KEEP_RAW_FILES} =~ ^[Yy]$ ]]; then
echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") INFO: Keeping raw files for ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log
echo "$(date -u +"%FT%TZ") INFO: Keeping raw files for ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log
else
echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") INFO: Removing raw files for ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log
echo "$(date -u +"%FT%TZ") INFO: Removing raw files for ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log
rm -f "${WORKDIR}/raw_${bucket}_keys.txt"
fi
}

# shellcheck disable=SC2312
function export_buckets() {
for bucket in "${BUCKETS[@]}"; do
echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") INFO: Starting export of ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log
echo "$(date -u +"%FT%TZ") INFO: Starting export of ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log
# Export the bucket contents from the Raft Session
export_bucket_contents
# Check for scan completed
Expand All @@ -172,7 +172,7 @@ function export_buckets() {
# Check the raw and processed counts match
compare_raw_and_processed_counts
echo
echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") INFO: Completed export of ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log
echo "$(date -u +"%FT%TZ") INFO: Completed export of ${bucket}" | tee -a "${WORKDIR}/RID_${RID}".log | tee -a "${WORKDIR}/${bucket}".log
# Pop the bucket out of the UNPROCESSED_BUCKETS array
UNPROCESSED_BUCKETS=("${UNPROCESSED_BUCKETS[@]/${bucket}/}")
done
Expand Down

0 comments on commit 77f7288

Please sign in to comment.