-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathsidecar.sh
executable file
·190 lines (164 loc) · 7.56 KB
/
sidecar.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
function log(){
LOGFILE="/data/populate_log.txt"
if [[ ! -f ${LOGFILE} ]]; then
touch ${LOGFILE}
fi
TIMESTAMP=`date "+%Y-%m-%d %H:%M:%S"`
echo "${TIMESTAMP}: $1"
echo "${TIMESTAMP}: $1" >> ${LOGFILE}
}
populate_notebook() {
MANIFEST_FILE=$1
shift
FOLDER=$1
shift
manifest_pull="!gen3 drs-pull manifest manifest.json"
manifest_ls="!gen3 drs-pull ls manifest.json"
jq --arg cmd "$manifest_ls" '.cells[1].source |= $cmd' "$FOLDER/data.ipynb" > "$FOLDER/data.tmp" && mv "$FOLDER/data.tmp" "$FOLDER/data.ipynb"
jq --arg cmd "$manifest_pull" '.cells[3].source |= $cmd' "$FOLDER/data.ipynb" > "$FOLDER/data.tmp" && mv "$FOLDER/data.tmp" "$FOLDER/data.ipynb"
echo $MANIFEST_FILE | jq -c '.[]' | while read j; do
obj=$(echo $j | jq -r .object_id)
filename=$(echo $j | jq -r .file_name)
filesize=$(echo $j | jq -r .file_size)
# Need to add a literal newline character that's why the quote is ending on next line
drs_pull="!gen3 drs-pull object $obj"
# Need to add a literal newline character that's why the quote is ending on next line
jq --arg cmd "# File name: $filename - File size: $filesize" '.cells[5].source += [$cmd]' "$FOLDER/data.ipynb" > "$FOLDER/data.tmp"
mv "$FOLDER/data.tmp" "$FOLDER/data.ipynb"
jq --arg cmd "$drs_pull" '.cells[5].source += [$cmd]' "$FOLDER/data.ipynb" > "$FOLDER/data.tmp"
mv "$FOLDER/data.tmp" "$FOLDER/data.ipynb"
done
log "Done populating notebook"
}
function populate() {
log "querying manifest service at $GEN3_ENDPOINT/manifests"
MANIFESTS=$(curl -s -H "Authorization: Bearer ${ACCESS_TOKEN}" "https://$GEN3_ENDPOINT/manifests/")
log "querying manifest service at $GEN3_ENDPOINT/metadata"
METADATA=$(curl -s -H "Authorization: Bearer ${ACCESS_TOKEN}" "https://$GEN3_ENDPOINT/manifests/metadata")
while [ -z "$MANIFESTS" ] && [ -z "$METADATA" ]; do
if [ -z "$MANIFESTS" ]; then
log "Unable to get manifests from '$GEN3_ENDPOINT/manifests/'"
log $MANIFESTS
fi
if [ -z "$METADATA" ]; then
log "Unable to get metadata from '$GEN3_ENDPOINT/manifests/metadata'"
log $METADATA
fi
log "sleeping for 15 seconds before trying again.."
sleep 15
MANIFESTS=$(curl -s -H "Authorization: Bearer ${ACCESS_TOKEN}" "https://$GEN3_ENDPOINT/manifests/")
METADATA=$(curl -s -H "Authorization: Bearer ${ACCESS_TOKEN}" "https://$GEN3_ENDPOINT/manifests/metadata")
done
log "successfully retrieved manifests and metadata for user"
process_files() {
local base_dir=$1
local data=$2
echo $data | jq -c '.[]' | while read i; do
FILENAME=$(echo "${i}" | jq -r .filename)
FOLDERNAME=$(echo "${FILENAME%.*}")
FOLDER="/data/${GEN3_ENDPOINT}/exported-${base_dir}/exported-${FOLDERNAME}"
if [ ! -d "$FOLDER" ]; then
log "mkdir -p $FOLDER"
mkdir -p $FOLDER
# make sure folder can be written to by notebook
chown -R 1000:100 $FOLDER
if [[ "$base_dir" == "manifests" ]]; then
MANIFEST_FILE=$(curl -s -H "Authorization: Bearer ${ACCESS_TOKEN}" "https://$GEN3_ENDPOINT/manifests/file/$FILENAME")
echo "${MANIFEST_FILE}" > $FOLDER/manifest.json
log "Creating notebook for $FILENAME"
cp ./template_manifest.json $FOLDER/data.ipynb
populate_notebook "$MANIFEST_FILE" "$FOLDER"
elif [[ "$base_dir" == "metadata" ]]; then
METADATA_FILE=$(curl -s -H "Authorization: Bearer ${ACCESS_TOKEN}" "https://$GEN3_ENDPOINT/manifests/metadata/$FILENAME")
echo "${METADATA_FILE}" > $FOLDER/metadata.json
fi
fi
done
}
if [ -n "$MANIFESTS" ]; then
process_files "manifests" "$(echo $MANIFESTS | jq -c '.manifests')"
fi
if [ -n "$METADATA" ]; then
process_files "metadata" "$(echo $METADATA | jq -c '.external_file_metadata')"
fi
# Make sure notebook user has write access to the folders
chown -R 1000:100 /data
}
function apikeyfile() {
if [[ ! -d "/.gen3" ]]; then
log "Please mount shared docker volume under /.gen3. Gen3 SDK will not be configured correctly.."
mkdir /.gen3
fi
if [[ -z $API_KEY ]]; then
log '$API_KEY not set. Skipping writing api key to file. WARNING: Gen3 SDK will not be configured correctly.'
else
log "Writing apiKey to ~/.gen3/credentials.json"
apikey=$(jq --arg key0 'api_key' \
--arg value0 "${API_KEY}" \
'. | .[$key0]=$value0 ' \
<<<'{}')
echo "$apikey" > /.gen3/credentials.json
fi
}
function get_access_token() {
log "Getting access token using mounted API key from https://$GEN3_ENDPOINT/user/"
ACCESS_TOKEN=$(curl -s -H "Content-Type: application/json" -X POST "https://$GEN3_ENDPOINT/user/credentials/api/access_token/" -d "{ \"api_key\": \"${API_KEY}\" }" | jq -r .access_token)
while [ -z "$ACCESS_TOKEN" ]; do
log "Unable to get ACCESS TOKEN using API key."
log "sleeping for 15 seconds before trying again.."
sleep 15
ACCESS_TOKEN=$(curl -s -H "Content-Type: application/json" -X POST "https://$GEN3_ENDPOINT/user/credentials/api/access_token/" -d "{ \"api_key\": \"${API_KEY}\" }" | jq -r .access_token)
done
export ACCESS_TOKEN="$ACCESS_TOKEN"
}
function mount_hatchery_files() {
log "Mounting Hatchery files"
FOLDER="/data"
if [ ! -d "$FOLDER" ]; then
mkdir $FOLDER
fi
echo "Fetching files to mount..."
echo "This workspace flavor is '$WORKSPACE_FLAVOR'"
DATA=$(curl -s -H "Authorization: Bearer ${ACCESS_TOKEN}" "https://$GEN3_ENDPOINT/lw-workspace/mount-files")
echo $DATA | jq -c -r '.[]' | while read item; do
file_path=$(echo "${item}" | jq -r .file_path)
workspace_flavor=$(echo "${item}" | jq -r .workspace_flavor)
# mount the file if its workspace flavor is not set or if it matches the current workspace flavor
if [[ -z "${workspace_flavor}" || -z "${WORKSPACE_FLAVOR}" || $workspace_flavor == $WORKSPACE_FLAVOR ]]; then
echo "Mounting '$file_path'"
mkdir -p "$FOLDER/$(dirname "$file_path")"
curl -s -H "Authorization: Bearer ${ACCESS_TOKEN}" "https://$GEN3_ENDPOINT/lw-workspace/mount-files?file_path=$file_path" > $FOLDER/$file_path
else
echo "Not mounting '$file_path' because its workspace flavor '$workspace_flavor' does not match"
fi
done
# Make sure notebook user has write access to the folders
chown -R 1000:100 $FOLDER
}
function main() {
if [[ -z "${GEN3_ENDPOINT}" ]]; then
log "No base url set"
exit 1
fi
# Gen3SDK should work if $API_KEY is set
apikeyfile
get_access_token
mount_hatchery_files
if [[ ! -d "/data/${GEN3_ENDPOINT}" ]]; then
log "Creating /data/$GEN3_ENDPOINT/ directory"
mkdir "/data/${GEN3_ENDPOINT}/"
fi
log "Trying to populate data from Manifest Service..."
while true; do
populate
# If the access token expires, fetch a new access token and try again
if [[ $(echo "$MANIFESTS" | jq -r '.error') = "Please log in." ]]; then
echo "Session Expired. Trying again with new access token"
get_access_token
else
# log "Sleeping for 30 seconds before checking for new manifests."
sleep 30
fi
done
}
main