diff --git a/Makefile b/Makefile index 291d3c3a..39242494 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,12 @@ docker-push-local-test: ## Push docker image with the manager to kind registry. .PHONY: deploy-testjob deploy-testjob: ## Run a wikipedia test pod kubectl create job wiki-test --image=${IMG_KIND}:${TEST_IMG_TAG} -- sh /wikipedia-test.sh - bash e2e/monitor-task.sh + JOB_ID="wiki-test" bash e2e/monitor-task.sh + +.PHONY: deploy-testingestionjob +deploy-testingestionjob: ## wait for the druidIngestion to complete and then verify dataset + kubectl create job ingestion-test --image=${IMG_KIND}:${TEST_IMG_TAG} -- sh /druid-ingestion-test.sh ${TASK_ID} + JOB_ID="ingestion-test" bash e2e/monitor-task.sh .PHONY: helm-install-druid-operator helm-install-druid-operator: ## Helm install to deploy the druid operator diff --git a/chart/templates/rbac_manager.yaml b/chart/templates/rbac_manager.yaml index 9136d9b8..327a54b7 100644 --- a/chart/templates/rbac_manager.yaml +++ b/chart/templates/rbac_manager.yaml @@ -126,6 +126,26 @@ rules: - get - patch - update +- apiGroups: + - druid.apache.org + resources: + - druidingestions + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - druid.apache.org + resources: + - druidingestions/status + verbs: + - get + - patch + - update - apiGroups: - networking.k8s.io resources: @@ -284,6 +304,26 @@ rules: - get - patch - update +- apiGroups: + - druid.apache.org + resources: + - druidingestions + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - druid.apache.org + resources: + - druidingestions/status + verbs: + - get + - patch + - update - apiGroups: - networking.k8s.io resources: diff --git a/controllers/ingestion/reconciler.go b/controllers/ingestion/reconciler.go index cdcecd4e..02276fa3 100644 --- a/controllers/ingestion/reconciler.go +++ b/controllers/ingestion/reconciler.go @@ -345,8 +345,8 @@ func (r *DruidIngestionReconciler) getRouterSvcUrl(namespace, druidClusterName s if svcName == "" { return "", errors.New("router svc discovery fail") } - // newName := "http://" + svcName + "." + namespace + ".svc.cluster.local:" + DruidRouterPort - newName := "http://localhost:" + DruidRouterPort + + newName := "http://" + svcName + "." + namespace + ".svc.cluster.local:" + DruidRouterPort return newName, nil } diff --git a/e2e/Dockerfile-testpod b/e2e/Dockerfile-testpod index 61263978..a3d17f6a 100644 --- a/e2e/Dockerfile-testpod +++ b/e2e/Dockerfile-testpod @@ -4,3 +4,4 @@ RUN apk add --update-cache \ && rm -rf /var/cache/apk/* ADD e2e/wikipedia-test.sh . +ADD e2e/druid-ingestion-test.sh . diff --git a/e2e/configs/druid-ingestion-cr.yaml b/e2e/configs/druid-ingestion-cr.yaml new file mode 100644 index 00000000..69fe4bb4 --- /dev/null +++ b/e2e/configs/druid-ingestion-cr.yaml @@ -0,0 +1,73 @@ +apiVersion: druid.apache.org/v1alpha1 +kind: DruidIngestion +metadata: + labels: + app.kubernetes.io/name: druidingestion + app.kubernetes.io/instance: druidingestion-sample + name: wikipedia-ingestion +spec: + suspend: false + druidCluster: tiny-cluster + ingestion: + type: native-batch + spec: |- + { + "type" : "index_parallel", + "spec" : { + "dataSchema" : { + "dataSource" : "wikipedia-2", + "timestampSpec": { + "column": "time", + "format": "iso" + }, + "dimensionsSpec" : { + "dimensions" : [ + "channel", + "cityName", + "comment", + "countryIsoCode", + "countryName", + "isAnonymous", + "isMinor", + "isNew", + "isRobot", + "isUnpatrolled", + "metroCode", + "namespace", + "page", + "regionIsoCode", + "regionName", + "user", + { "name": "added", "type": "long" }, + { "name": "deleted", "type": "long" }, + { "name": "delta", "type": "long" } + ] + }, + "metricsSpec" : [], + "granularitySpec" : { + "type" : "uniform", + "segmentGranularity" : "day", + "queryGranularity" : "none", + "intervals" : ["2015-09-12/2015-09-13"], + "rollup" : false + } + }, + "ioConfig" : { + "type" : "index_parallel", + "inputSource" : { + "type" : "local", + "baseDir" : "quickstart/tutorial/", + "filter" : "wikiticker-2015-09-12-sampled.json.gz" + }, + "inputFormat" : { + "type" : "json" + }, + "appendToExisting" : false + }, + "tuningConfig" : { + "type" : "index_parallel", + "maxRowsPerSegment" : 5000000, + "maxRowsInMemory" : 25000 + } + } + } diff --git a/e2e/druid-ingestion-test.sh b/e2e/druid-ingestion-test.sh new file mode 100644 index 00000000..54fcb58a --- /dev/null +++ b/e2e/druid-ingestion-test.sh @@ -0,0 +1,40 @@ +#!/bin/sh + +set -e + +TASK_ID=$1 + +echo "Checking Status for task $TASK_ID..." +STATUS=$(curl -s http://druid-tiny-cluster-coordinators.druid.svc:8088/druid/indexer/v1/task/${TASK_ID}/status | jq '.status.status' -r); +while [ $STATUS == "RUNNING" ] +do + sleep 8; + echo "TASK is "$STATUS "..." + STATUS=$(curl -s http://druid-tiny-cluster-coordinators.druid.svc:8088/druid/indexer/v1/task/${TASK_ID}/status | jq '.status.status' -r) +done + +if [ $STATUS == "SUCCESS" ] +then + echo "TASK $TASK_ID COMPLETED SUCCESSFULLY" + sleep 60 # need time for the segments to become queryable +else + echo "TASK $TASK_ID FAILED !!!!" + exit 1 +fi + +echo "Querying Data ... " +echo "Running query SELECT COUNT(*) AS \"Count\" FROM \"wikipedia-2\" WHERE isMinor = 'false'" + +cat > query.json <" ] then echo "Seems to be in progress ..." diff --git a/e2e/wikipedia-test.sh b/e2e/wikipedia-test.sh index c31cd654..9a0101ae 100644 --- a/e2e/wikipedia-test.sh +++ b/e2e/wikipedia-test.sh @@ -21,6 +21,7 @@ done if [ $STATUS == "SUCCESS" ] then echo "TASK $task_id COMPLETED SUCCESSFULLY" + sleep 60 # need time for the segments to become queryable else echo "TASK $task_id FAILED !!!!" fi