diff --git a/deploy/clowdapp.yaml b/deploy/clowdapp.yaml index 2dc6847..2f2b86c 100644 --- a/deploy/clowdapp.yaml +++ b/deploy/clowdapp.yaml @@ -13,7 +13,7 @@ objects: - apiVersion: v1 kind: ConfigMap metadata: - name: trino-scripts-${CONFIGMAP_HASH} + name: trino-scripts labels: app: trino data: @@ -56,24 +56,6 @@ objects: export PGSSLROOTCERT=$temp_file fi - export AWS_ACCESS_KEY_ID=$(jq -r '.objectStore.buckets[0].accessKey' ${ACG_CONFIG}) - export AWS_SECRET_ACCESS_KEY=$(jq -r '.objectStore.buckets[0].secretKey' ${ACG_CONFIG}) - export S3_BUCKET_NAME=$(jq -r '.objectStore.buckets[0].requestedName' ${ACG_CONFIG}) - - OBJECTSTORE_HOST=$(jq -r '.objectStore.hostname' ${ACG_CONFIG}) - OBJECTSTORE_PORT=$(jq -r '.objectStore.port' ${ACG_CONFIG}) - OBJECTSTORE_TLS=$(jq -r '.objectStore.tls' ${ACG_CONFIG}) - - export URI_PREFIX=https - if [[ $OBJECTSTORE_TLS == *"false"* ]]; then - export URI_PREFIX=http - fi - - S3_ENDPOINT="${URI_PREFIX}://${OBJECTSTORE_HOST}" - if [[ -n "${OBJECTSTORE_PORT}" ]] && [[ "${OBJECTSTORE_PORT}" != "null" ]]; then - S3_ENDPOINT="${S3_ENDPOINT}:${OBJECTSTORE_PORT}" - fi - export S3_ENDPOINT fi echo "Copy config files to ${TRINO_HOME}/" cp -v -L -r -f /etc/trino-init/* ${TRINO_HOME}/ @@ -108,12 +90,13 @@ objects: echo "" >> $HIVE_CATALOG_CONFIG if ! grep -q -F 's3.aws-access-key' "$HIVE_CATALOG_CONFIG"; then echo "Adding s3.aws-access-key and s3.aws-secret-key to $HIVE_CATALOG_CONFIG" + echo "s3.aws-access-key=$AWS_ACCESS_KEY_ID" >> "$HIVE_CATALOG_CONFIG" echo "s3.aws-secret-key=$AWS_SECRET_ACCESS_KEY" >> "$HIVE_CATALOG_CONFIG" - echo "s3.endpoint=$S3_ENDPOINT" >> "$HIVE_CATALOG_CONFIG" - if [[ $OBJECTSTORE_TLS == *"false"* ]]; then - echo "s3.sse.type=None" >> "$HIVE_CATALOG_CONFIG" - fi + + echo "hive.metastore.glue.aws-access-key=$AWS_ACCESS_KEY_ID" >> "$HIVE_CATALOG_CONFIG" + echo "hive.metastore.glue.aws-secret-key=$AWS_SECRET_ACCESS_KEY" >> "$HIVE_CATALOG_CONFIG" + fi # add UID to /etc/passwd if missing @@ -137,7 +120,7 @@ objects: - apiVersion: v1 kind: ConfigMap metadata: - name: trino-config-${CONFIGMAP_HASH} + name: trino-config labels: app: trino data: @@ -265,7 +248,7 @@ objects: - apiVersion: v1 kind: ConfigMap metadata: - name: trino-config-catalog-${CONFIGMAP_HASH} + name: trino-config-catalog labels: app: trino data: @@ -345,19 +328,24 @@ objects: type: GAUGE hive.properties: |- connector.name=hive + hive.metastore=glue hive.collect-column-statistics-on-write=true hive.compression-codec=SNAPPY - hive.metastore.authentication.type=NONE - hive.metastore.thrift.client.connect-timeout=${METASTORE_TIMEOUT} - hive.metastore.thrift.client.read-timeout=${METASTORE_READ_TIMEOUT} hive.max-partitions-per-scan=${HIVE_PARTITION_LIMIT} + hive.non-managed-table-writes-enabled=true hive.partition-statistics-sample-size=${HIVE_PARTITION_STATS_SAMPLE_SIZE} - hive.metastore.uri=thrift://hive-metastore:10000 hive.parquet.use-column-names=true + hive.recursive-directories=true hive.storage-format=Parquet fs.native-s3.enabled=true s3.path-style-access=true - s3.region=us-east-1 + s3.region=${S3_REGION} + s3.endpoint=${S3_ENDPOINT} + + hive.metastore.glue.default-warehouse-dir=s3://${S3_BUCKET_NAME}/${WAREHOUSE_DIR} + hive.metastore.glue.region=${S3_REGION} + hive.metastore.glue.catalogid=${AWS_CATALOG_ID} + blackhole.propeties: |- connector.name=blackhole jmx.properties: |- @@ -417,8 +405,18 @@ objects: fieldRef: apiVersion: v1 fieldPath: metadata.namespace - - name: S3_DATA_DIR - value: 'data' + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: aws-access-key-id + name: koku-aws + optional: false + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: aws-secret-access-key + name: koku-aws + optional: false - name: QUERY_MAX_MEMORY_PER_NODE value: ${QUERY_MAX_MEMORY_PER_NODE} - name: QUERY_MAX_MEMORY @@ -460,17 +458,17 @@ objects: volumes: - name: trino-scripts configMap: - name: trino-scripts-${CONFIGMAP_HASH} + name: trino-scripts items: - key: entrypoint.sh path: entrypoint.sh mode: 509 - name: trino-config configMap: - name: trino-config-${CONFIGMAP_HASH} + name: trino-config - name: trino-config-catalog configMap: - name: trino-config-catalog-${CONFIGMAP_HASH} + name: trino-config-catalog - name: trino-etc emptyDir: {} - name: trino-data @@ -532,8 +530,18 @@ objects: fieldRef: apiVersion: v1 fieldPath: metadata.namespace - - name: S3_DATA_DIR - value: 'data' + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: aws-access-key-id + name: koku-aws + optional: false + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: aws-secret-access-key + name: koku-aws + optional: false - name: QUERY_MAX_MEMORY_PER_NODE value: ${QUERY_MAX_MEMORY_PER_NODE} - name: QUERY_MAX_MEMORY @@ -581,17 +589,17 @@ objects: volumes: - name: trino-scripts configMap: - name: trino-scripts-${CONFIGMAP_HASH} + name: trino-scripts items: - key: entrypoint.sh path: entrypoint.sh mode: 509 - name: trino-config configMap: - name: trino-config-${CONFIGMAP_HASH} + name: trino-config - name: trino-config-catalog configMap: - name: trino-config-catalog-${CONFIGMAP_HASH} + name: trino-config-catalog - name: trino-etc emptyDir: {} - name: trino-data @@ -635,10 +643,8 @@ parameters: required: true - name: ENV_NAME required: false -- name: S3_BUCKET_NAME - value: 'hccm-s3' - name: NODE_ENV - value: 'production' + value: production - name: MACHINE_POOL_OPTION value: '' - name: TRINO_HISTORY_FILE @@ -704,6 +710,18 @@ parameters: name: MAX_HEAP_SIZE value: '3G' +# AWS params +- name: S3_ENDPOINT + value: https://s3.us-east-1.amazonaws.com +- name: S3_REGION + value: us-east-1 +- name: S3_BUCKET_NAME + value: hccm-s3 +- name: WAREHOUSE_DIR + value: data +- name: AWS_CATALOG_ID + value: '589173575009' + # Trino configruation Params - description: Max amount of user memory a query can use on a worker (Trino default - JVM max memory * 0.3) displayName: query.max-memory-per-node @@ -727,14 +745,6 @@ parameters: required: true # Trino Hive config -- description: Socket connect timeout for metastore client - displayName: hive.metastore.thrift.client.connect-timeout - name: METASTORE_READ_TIMEOUT - value: '300s' -- description: Socket read timeout for metastore client - displayName: hive.metastore.thrift.client.read-timeout - name: METASTORE_TIMEOUT - value: '300s' - description: Specifies the number of partitions to analyze when computing table statistics displayName: hive.partition-statistics-sample-size name: HIVE_PARTITION_STATS_SAMPLE_SIZE @@ -764,8 +774,3 @@ parameters: displayName: livenessPeriodSeconds name: LIVENESS_PROBE_PERIOD value: '120' - -# Configmap updater -- name: CONFIGMAP_HASH - description: "The random hash to change the configmap names" - value: '000001' diff --git a/pr_check.sh b/pr_check.sh index 7c651e1..2f1a8f5 100755 --- a/pr_check.sh +++ b/pr_check.sh @@ -23,11 +23,23 @@ curl -s $CICD_URL/bootstrap.sh > .cicd_bootstrap.sh && source .cicd_bootstrap.sh source $CICD_ROOT/build.sh -# source $CICD_ROOT/_common_deploy_logic.sh -# export NAMESPACE=$(bonfire namespace reserve) -# oc process --local -f deploy/clowdapp.yaml | oc apply -f - -n $NAMESPACE -source $CICD_ROOT/deploy_ephemeral_env.sh -#source $CICD_ROOT/smoke_test.sh +source ${CICD_ROOT}/_common_deploy_logic.sh +export NAMESPACE=$(bonfire namespace reserve) +SMOKE_NAMESPACE=$NAMESPACE +oc get secret koku-aws --namespace=ephemeral-base -oyaml | grep -v '^\s*namespace:\s' | oc apply --namespace=${NAMESPACE} -f - + +bonfire deploy \ + ${APP_NAME} \ + --source=appsre \ + --ref-env ${REF_ENV} \ + --set-image-tag ${IMAGE}=${IMAGE_TAG} \ + --namespace ${NAMESPACE} \ + --timeout ${DEPLOY_TIMEOUT} \ + --optional-deps-method hybrid \ + ${TEMPLATE_REF_ARG} \ + ${COMPONENTS_ARG} \ + ${COMPONENTS_RESOURCES_ARG} \ + ${EXTRA_DEPLOY_ARGS} mkdir -p $WORKSPACE/artifacts cat << EOF > ${WORKSPACE}/artifacts/junit-dummy.xml