-
Notifications
You must be signed in to change notification settings - Fork 1
/
init.sh
295 lines (254 loc) · 12.4 KB
/
init.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
#!/usr/bin/env bash
# Build variables. This file is sourced in BASH by ./run_environment.sh script.
# Also it is sourced during continuous integration build in Cloud Build
# Auto-export all variables
set -a
echo
echo "Reading variables"
echo
GCP_CONFIG_DIR="${GCP_CONFIG_DIR:="/files/airflow-breeze-config/"}"
RANDOM_FILE="/random.txt"
if [[ ! -f "${RANDOM_FILE}" ]]; then
echo "${RANDOM}" > "${RANDOM_FILE}"
fi
RANDOM_POSTFIX=$(cat "${RANDOM_FILE}")
# Unique suffix to allow several builds in parallel
AIRFLOW_BREEZE_SHORT_SHA="${AIRFLOW_BREEZE_SHORT_SHA:="build"}"
AIRFLOW_BREEZE_TEST_SUITE="${AIRFLOW_BREEZE_TEST_SUITE:="test"}"
AIRFLOW_BREEZE_UNIQUE_SUFFIX=${AIRFLOW_BREEZE_TEST_SUITE}-${AIRFLOW_BREEZE_SHORT_SHA}-${RANDOM_POSTFIX}
AIRFLOW_BREEZE_UNIQUE_SUFFIX_UNDERSCORES=${AIRFLOW_BREEZE_TEST_SUITE}_${AIRFLOW_BREEZE_SHORT_SHA}_${RANDOM_POSTFIX}
PYTHON_VERSION=${PYTHON_VERSION:="3.6"}
# Version suffix is always major+major (36 for python 3.6.10)
VERSION_SUFFIX=${PYTHON_VERSION/\./}
VERSION_SUFFIX=${VERSION_SUFFIX:0:2}
#
# General variables
#
GCP_PROJECT_ID=${PROJECT_ID:=polidea-airflow}
GCP_LOCATION=${LOCATION:=europe-west1}
GCP_REGION=${REGION:=europe-west-1b}
GCP_VALIDATE_BODY=${VALIDATE_BODY:=True}
BUILD_BUCKET_SUFFIX=-builds
TEST_BUCKET_SUFFIX=-tests
AWS_ACCESS_KEY_ID_ENCRYPTED=${AWS_ACCESS_KEY_ID_ENCRYPTED:=CiQAckpnUGpAw/wDYelhRvl6LieY1iOCTfXtguAIwfzdmA2iVP8SUQCqyLOQF9iM9VYLIf8iZUXz0JegtvbS7ptdIvkgM4oWS/JK7dqkSDaRKpEcVvIPXBuOhtUwBwdDGD4X79OePUp6ewjjeXN5ShmbIiKjcBbOuw==}
AWS_SECRET_ACCESS_KEY_ENCRYPTED=${AWS_SECRET_ACCESS_KEY_ENCRYPTED:=CiQAckpnUJIvrqDc0eZL/YhhrBTeZDfoXSR4ZHPB5i9M0gKThToSzQEAqsizkKwckHTiUm/OYBI8Wcd9ysPWumB2Xlr8f7USrNceUFSLF8l9SrPGZ8OBJLSR7EbuNcHLpAHWpAvvAYE/+SYQNkZVkpmoKGhJpaK1gGrxyKx7r3E1o+7n/Dot+X1nXJDwLrcITDrXltOuVDl1eKJoWlqqCRc665TvvYcnndXn7iwVuvSj9SyVzxDJv7j66oxhaL3laOn177mz5Pzg1cDIAlr8CNY4tUPmSiOWB0phu4Qa88EBEm7Nli4bOIs5wsi7Di+w7G4lOXz6}
AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:=eu-central-1}
# Buckets
AIRFLOW_BREEZE_GCP_BUILD_BUCKET=polidea-airflow${BUILD_BUCKET_SUFFIX}
AIRFLOW_BREEZE_GCP_TEST_BUCKET=polidea-airflow${TEST_BUCKET_SUFFIX}-${VERSION_SUFFIX}
GCP_VIDEO_INTELLIGENCE_BUCKET_NAME=test-video-inteligence${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GMP_DISPLAY_VIDEO_BUCKET=test-display-video-bucket-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
MARKETING_BUCKET=test-marketing-bucket-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GMP_GCS_BUCKET=test-search-ads-bucket-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GCP_DATAPROC_BUCKET=dataproc-airflow-system-bucket-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GCP_SPEECH_TO_TEXT_TEST_BUCKET=gcp-speech-to-text-test-bucket-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GCP_TEXT_TO_SPEECH_BUCKET=gcp-text-to-speech-test-bucket-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GCP_TRANSLATE_SPEECH_TEST_BUCKET=gcp-translate-speech-test-bucket-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GCP_GCS_BUCKET_1=test-gcs-example-bucket-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GCP_GCS_BUCKET_2=test-gcs-example-bucket-2-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GCP_GCS_LIFE_SCIENCES_BUCKET=test-life-sciences-bucket-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GCP_MEMORYSTORE_BUCKET=test-memorystore-bucket-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GCP_DATAFUSION_BUCKET_1=test-datafusion-bucket-1-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GCP_DATAFUSION_BUCKET_2=test-datafusion-bucket-2-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
#
# Google Cloud BigQuery environment variables
#
GCP_DTS_BQ_DATASET=test_dts_${AIRFLOW_BREEZE_UNIQUE_SUFFIX_UNDERSCORES}
#
# Google Cloud Function environment variables
#
GCF_SOURCE_ARCHIVE_URL=
GCF_SOURCE_UPLOAD_URL=
GCF_ZIP_PATH=
GCF_ENTRYPOINT=helloWorld
GCF_SHORT_FUNCTION_NAME=hello-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GCF_RUNTIME=nodejs6
#
# Google Compute Engine environment variables
#
GCE_INSTANCE=testinstance-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GCE_ZONE=europe-west1-b
GCE_SHORT_MACHINE_TYPE_NAME=n1-standard-1
#
# Google Compute Engine IGM environment variables
#
GCE_INSTANCE_GROUP_MANAGER_NAME=instance-group-test-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GCE_TEMPLATE_NAME=instance-template-test-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GCE_NEW_TEMPLATE_NAME=instance-template-test-new-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GCE_NEW_DESCRIPTION="Test new description ${AIRFLOW_BREEZE_UNIQUE_SUFFIX}"
#
# Google Cloud SQL environment variables
#
GCSQL_POSTGRES_INSTANCE_NAME=test-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}-post
GCSQL_POSTGRES_INSTANCE_NAME2=test-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}-post2
GCSQL_POSTGRES_INSTANCE_NAME_QUERY=test-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}-post-query
GCSQL_POSTGRES_DATABASE_NAME=postgresdb
GCSQL_POSTGRES_USER=postgres_user
GCSQL_POSTGRES_PASSWORD_ENCRYPTED=CiQAckpnUAR5+CDb3oHiuwoqu44f4dxfw4AURNR4iXCYkjCQAJESOQCqyLOQRqX41WvUsSou1WkWbJs8x0w9thDni66MDnHjmakyz1DYi4UewncuW/X9VVhFqYqN0+YJzw==
GCSQL_POSTGRES_PUBLIC_PORT=5432
GCSQL_POSTGRES_CLIENT_CERT_FILE=/tmp/postgres-client-cert.pem
GCSQL_POSTGRES_CLIENT_KEY_FILE=/tmp/postgres-client-key.pem
GCSQL_POSTGRES_SERVER_CA_FILE=/tmp/postgres-server-ca.pem
GCSQL_POSTGRES_PUBLIC_IP_FILE=/tmp/postgres-ip.env
GCSQL_POSTGRES_EXPORT_URI=gs://${AIRFLOW_BREEZE_GCP_TEST_BUCKET}/postgres_export/${AIRFLOW_BREEZE_UNIQUE_SUFFIX}.txt
GCSQL_POSTGRES_IMPORT_URI=gs://${AIRFLOW_BREEZE_GCP_TEST_BUCKET}/postgres_export/${AIRFLOW_BREEZE_UNIQUE_SUFFIX}.txt
GCSQL_MYSQL_INSTANCE_NAME=test-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}-mysql
GCSQL_MYSQL_INSTANCE_NAME2=test-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}-mysql2
GCSQL_MYSQL_INSTANCE_NAME_QUERY=test-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}-mysql-query
GCSQL_MYSQL_DATABASE_NAME=mysqldb
GCSQL_MYSQL_USER=mysql_user
GCSQL_MYSQL_PASSWORD_ENCRYPTED=CiQAckpnUAR5+CDb3oHiuwoqu44f4dxfw4AURNR4iXCYkjCQAJESOQCqyLOQRqX41WvUsSou1WkWbJs8x0w9thDni66MDnHjmakyz1DYi4UewncuW/X9VVhFqYqN0+YJzw==
GCSQL_MYSQL_PUBLIC_PORT=3306
GCSQL_MYSQL_CLIENT_CERT_FILE=/tmp/mysql-client-cert.pem
GCSQL_MYSQL_CLIENT_KEY_FILE=/tmp/mysql-client-key.pem
GCSQL_MYSQL_SERVER_CA_FILE=/tmp/mysql-server-ca.pem
GCSQL_MYSQL_PUBLIC_IP_FILE=/tmp/mysql-ip.env
GCSQL_MYSQL_EXPORT_URI=gs://${AIRFLOW_BREEZE_GCP_TEST_BUCKET}/mysql_export/${AIRFLOW_BREEZE_UNIQUE_SUFFIX}.txt
GCSQL_MYSQL_IMPORT_URI=gs://${AIRFLOW_BREEZE_GCP_TEST_BUCKET}/mysql_export/${AIRFLOW_BREEZE_UNIQUE_SUFFIX}.txt
#
# Google Cloud BigTable environment variables
#
GCP_BIG_TABLE_INSTANCE_ID=test-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GCP_BIG_TABLE_INSTANCE_DISPLAY_NAME="My CBT instance"
GCP_BIG_TABLE_INSTANCE_TYPE_PROD=1 # PRODUCTION
GCP_BIG_TABLE_INSTANCE_LABELS='{"key":"value"}'
GCP_BIG_TABLE_CLUSTER_ID=cluster-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GCP_BIG_TABLE_CLUSTER_ZONE=europe-west1-b
GCP_BIG_TABLE_CLUSTER_NODES=3
GCP_BIG_TABLE_CLUSTER_NODES_UPDATED=5
GCP_BIG_TABLE_CLUSTER_STORAGE_TYPE=2 # HDD
GCP_BIG_TABLE_TABLE_ID=my-cbt-table
GCP_BIG_TABLE_POKE_INTERVAL=20
#
# Google Cloud Kubernetes environment variables
#
GCP_GKE_CLUSTER_NAME=cluster-name-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
#
# Google Cloud Spanner environment variables
#
GCP_SPANNER_INSTANCE_ID=testinstance-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GCP_SPANNER_CONFIG_NAME=projects/${GCP_PROJECT_ID}/instanceConfigs/regional-europe-west3
GCP_SPANNER_DATABASE_ID=testdatabase-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GCP_SPANNER_NODE_COUNT=1
GCP_SPANNER_DISPLAY_NAME=Test-Instance-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
#
# Google Cloud Storage environment variables
#
GCS_ACL_BUCKET=${AIRFLOW_BREEZE_GCP_TEST_BUCKET}
GCS_ACL_OBJECT="empty.txt"
GCS_ACL_ENTITY="user-gcp-storage-account@${GCP_PROJECT_ID}.iam.gserviceaccount.com"
GCS_ACL_BUCKET_ROLE="WRITER"
GCS_ACL_OBJECT_ROLE="READER"
#
# Google Cloud Dataproc environment variables
#
GCP_DATAPROC_CLUSTER_NAME=cluster-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
#
# Google Cloud Vision environment variables
#
GCP_VISION_LOCATION="europe-west1"
GCP_VISION_PRODUCT_SET_ID="product_set_explicit_id"
GCP_VISION_PRODUCT_ID="product_explicit_id"
GCP_VISION_REFERENCE_IMAGE_ID="reference_image_explicit_id"
GCP_VISION_BUCKET_NAME=${AIRFLOW_BREEZE_GCP_TEST_BUCKET}
GCP_VISION_ANNOTATE_IMAGE_URL=gs://${AIRFLOW_BREEZE_GCP_TEST_BUCKET}/image${AIRFLOW_BREEZE_UNIQUE_SUFFIX}.png
GCP_VISION_REFERENCE_IMAGE_URL=gs://${AIRFLOW_BREEZE_GCP_TEST_BUCKET}/image${AIRFLOW_BREEZE_UNIQUE_SUFFIX}.png
GCP_VIDEO_SOURCE_URL=http://mirrors.standaloneinstaller.com/video-sample/lion-sample.mp4
GCP_VISION_SOURCE_IMAGE_URL=https://upload.wikimedia.org/wikipedia/commons/thumb/2/2f/Google_2015_logo.svg/1200px-Google_2015_logo.svg.png
#
# Google Cloud Transfer storage buckets
#
GCP_TRANSFER_SOURCE_AWS_BUCKET="gcp-transfer-test-aws-source"
GCP_TRANSFER_FIRST_TARGET_BUCKET=gcp-transfer-test-first-target-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GCP_TRANSFER_SECOND_TARGET_BUCKET=gcp-transfer-test-second-target-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
#
# Google Cloud Build environment variables
#
GCP_CLOUD_BUILD_REPOSITORY_NAME=breeze-cloud-build-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}-repo
GCP_CLOUD_BUILD_ARCHIVE_URL=gs://${AIRFLOW_BREEZE_GCP_TEST_BUCKET}/source-code-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}-archive.tar.gz
#
# Google Cloud Memorystore environment variables
#
GCP_MEMORYSTORE_REDIS_INSTANCE_NAME=test-memorystore-redis-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GCP_MEMORYSTORE_REDIS_INSTANCE_NAME_2=test-memorystore-redis-2-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GCP_MEMORYSTORE_REDIS_INSTANCE_NAME_3=test-memorystore-redis-3-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GCP_MEMORYSTORE_MEMCACHED_INSTANCE_NAME=test-memorystore-memcached-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
#
# Google Marketing Platform
#
MARKETING_PROFILE_ID=${MARKETING_PROFILE_ID:="5396163"}
GMP_AGENCY_ID=${GMP_AGENCY_ID:="20700000000001916"}
GMP_ADVERTISER_ID=${GMP_ADVERTISER_ID:="21700000001673283"}
FLOODLIGHT_ACTIVITY_ID=${FLOODLIGHT_ACTIVITY_ID:="4327170"}
FLOODLIGHT_CONFIGURATION_ID=${FLOODLIGHT_CONFIGURATION_ID:="6440317"}
ENCRYPTION_ENTITY_ID=${ENCRYPTION_ENTITY_ID:="6440317"}
DEVICE_ID=${DEVICE_ID:="E95AE36E-A242-4A20-AB40-CA85BBBDA915"}
GA_ACCOUNT_ID=${GA_ACCOUNT_ID:="160844178"}
GA_WEB_PROPERTY=${GA_WEB_PROPERTY:="UA-160844178-1"}
GA_WEB_PROPERTY_AD_WORDS_LINK_ID=${GA_WEB_PROPERTY_AD_WORDS_LINK_ID:="Yb2-NDSfSEm-BpujtOjskA"}
#
# Google Suite
#
SPREADSHEET_ID=1B4z7aaILnOrudftcb5OEYE63ZAnsCah9b4x6x_srEJg
NEW_SPREADSHEET_ID=1UAwllpZYpO17KQIHM87rZNBKb9CiCRXm0o-rrLClc5k
export PYTHONPATH="/opt/airflow"
#
# Google Firebase
#
G_FIRESTORE_PROJECT_ID=polidea-airflow-firestore
GCP_FIRESTORE_ARCHIVE_URL=gs://airflow-firestore/namespace-${RANDOM_POSTFIX}
GCP_FIRESTORE_COLLECTION_ID=kotek
GCP_FIRESTORE_DATASET_NAME=test_firestore_to_bigquery_${RANDOM_POSTFIX}
#
# Google ML Engine
#
GCP_MLENGINE_MODEL_NAME="airflow_test_ci_model_name_${RANDOM_POSTFIX}"
GCP_MLENGINE_BUCKET_NAME_EPHEMERAL=${AIRFLOW_BREEZE_GCP_TEST_BUCKET}
GCP_MLENGINE_BUCKET_NAME_PERSISTENT=test-airflow-mlengine-persistent
GCP_MLENGINE_PREDICTION_INPUT="gs://${GCP_MLENGINE_BUCKET_NAME_PERSISTENT}/prediction_input.json"
GCP_MLENGINE_TRAINER_URI="gs://${GCP_MLENGINE_BUCKET_NAME_PERSISTENT}/trainer-0.0.0.tar.gz"
GCP_MLENGINE_SAVED_MODEL_PATH="gs://${GCP_MLENGINE_BUCKET_NAME_EPHEMERAL}/job-dir/keras_export/"
GCP_MLENGINE_JOB_DIR="gs://${GCP_MLENGINE_BUCKET_NAME_EPHEMERAL}/job-dir"
GCP_MLENGINE_PREDICTION_OUTPUT="gs://${GCP_MLENGINE_BUCKET_NAME_EPHEMERAL}/prediction_output/"
GCP_MLENGINE_DATAFLOW_TMP="gs://${GCP_MLENGINE_BUCKET_NAME_EPHEMERAL}/tmp/"
GCP_MLENGINE_DATAFLOW_STAGING="gs://${GCP_MLENGINE_BUCKET_NAME_EPHEMERAL}/staging/"
#
# Google PubSub
#
GCP_PUBSUB_SENSOR_TOPIC=PubSubSensorTestTopic-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GCP_PUBSUB_OPERATOR_TOPIC=PubSubOperatorTestTopic-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
#
# Google Datafusion
#
GCP_DATAFUSION_PIPELINE_NAME=airflow_test_${AIRFLOW_BREEZE_UNIQUE_SUFFIX_UNDERSCORES}
#
# Google Tasks
#
GCP_TASKS_QUEUE_ID=cloud-tasks-queue-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
#
# Google Dataflow
#
GCP_DATAFLOW_BQ_SQL_DATASET=airflow_dataflow_sql_samples_${AIRFLOW_BREEZE_UNIQUE_SUFFIX_UNDERSCORES}
GCP_DATAFLOW_SQL_JOB_NAME=dataflow-sql-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GCP_DATAFLOW_FLEX_TEMPLATE_JOB_NAME=dataflow-flex-template-${AIRFLOW_BREEZE_UNIQUE_SUFFIX_UNDERSCORES}
GCP_DATAFLOW_PUBSUB_FLEX_TEMPLATE_TOPIC=dataflow-flex-template-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GCP_DATAFLOW_BQ_FLEX_TEMPLATE_DATASET=airflow_dataflow_flex_samples_${AIRFLOW_BREEZE_UNIQUE_SUFFIX_UNDERSCORES}
#
# Google Secret Manager
#
GCP_SECRET_MANAGER_SECRET_ID=test-secret-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GCP_SECRET_MANAGER_SECRET_VALUE=test-secret-value-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GCP_SECRET_MANAGER_VALUE_UPDATED=test-secret-value-updated-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
GCP_SECRET_MANAGER_MISSING_SECRET_ID=test-missing-secret-${AIRFLOW_BREEZE_UNIQUE_SUFFIX}
if [[ -f "variables_decrypted.env" ]]; then
echo
echo "Reading decrypted variables"
echo
source "variables_decrypted.env"
fi
echo
echo "Variables read"
echo
AIRFLOW__PROVIDERS_GOOGLE__VERBOSE_LOGGING="true"
set -m
set +a