Merge pull request #3327 from magda-io/issue/3326

Issue/3326 Build OPA docker image with builtin policies & Run OPA as a side car

.gitlab-ci.yml

@@ -66,7 +66,6 @@ build-builder-image:
     cd magda-builder-scala && docker buildx build --push -t $CI_REGISTRY/magda-data/magda/data61/magda-builder-scala:$CI_COMMIT_REF_SLUG --platform linux/arm64,linux/amd64 -f Dockerfile . 
     cd .. 
 
-
 # Make sure sbt depenencies, plugins are in place, cached (only for this job) and pass to following stage as artifacts
 sbt-prebuild:
   stage: sbt-prebuild
@@ -368,18 +367,18 @@ buildtest:typescript-apis-with-pg:
     PGPASSWORD: password
     OPA_URL: "http://docker:8181/"
   script:
-    - cd deploy/helm/internal-charts/opa
-    - docker-compose up -d
-    - cd ../../../../
+    - cd magda-opa
+    - yarn dev -d
+    - cd ..
     - cd magda-typescript-common && yarn build && yarn test
     - cd ..
     - cd magda-minion-framework && yarn build && yarn test
     - cd ..
     - yarn run in-submodules -- -f language=typescript -f categories.api=true -f categories.uses-pg=true -- run build --include-filtered-dependencies
     - yarn run in-submodules -- -f language=typescript -f categories.api=true -f categories.uses-pg=true -- run test --include-filtered-dependencies
     - yarn run in-submodules -- -f categories.npmPackage=true -f categories.useAuthApi=true -- run build
-    - cd deploy/helm/internal-charts/opa
-    - docker-compose down
+    - cd magda-opa
+    - yarn dev-stop
   artifacts:
     paths:
       - "*/dist"
@@ -478,7 +477,8 @@ buildtest:opa-policies:
   services:
     - docker:dind
   script:
-    - docker run -v $PWD/deploy/helm/internal-charts/opa/policies:/policies openpolicyagent/opa:0.33.1 test -v ./policies
+    - cd magda-opa
+    - yarn test
 
 buildtest:helm-charts:
   stage: buildtest
@@ -610,6 +610,24 @@ dockerize:migrators:
     - ./gitlab-ci-buildx-setup.sh
     - yarn run in-submodules -- -f categories.migrator=true -- run docker-build-prod --include-filtered-dependencies -- -- --repository=$CI_REGISTRY/magda-data/magda --version=$CI_COMMIT_REF_SLUG  --platform linux/arm64,linux/amd64 
 
+dockerize:opa:
+  stage: buildtest
+  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:$BUILDER_IMG_TAG
+  retry: 1
+  needs: 
+    - yarn-install
+    - build-builder-image
+    - buildtest:opa-policies
+  cache:
+    paths: []
+  before_script:
+    - ./gitlab-ci-buildx-setup.sh
+  services:
+    - docker:dind
+  script:
+  - cd magda-opa
+  - yarn docker-build-prod --repository=$CI_REGISTRY/magda-data/magda --version=$CI_COMMIT_REF_SLUG  --platform linux/arm64,linux/amd64
+
 dockerize:dockerExtensions:
   stage: buildtest
   image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:$BUILDER_IMG_TAG
@@ -998,32 +1016,3 @@ Publish Helm Chart:
     - aws s3 sync sync_dir s3://magda-charts/
     - aws s3 cp index_dir/index.yaml s3://magda-charts/index.yaml
 
-# Update scripts:
-#   stage: release
-#   only:
-#     # Strict Semvar validation
-#     - /^v((([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)$/
-#   except:
-#     - branches
-#     - triggers
-#   image: registry.gitlab.com/magda-data/magda/data61/magda-builder-nodejs:$CI_COMMIT_REF_SLUG
-#   needs:
-#     - builders-and-yarn
-#     - pre-release:check-release-version
-#   dependencies:
-#     - builders-and-yarn
-#   script:
-#     - cd scripts
-#     - yarn pkg create-secrets/index.js --out-path create-secrets/build
-#     - cd ../../
-#     - if [ ! -d magda-config/ ]; then git clone https://github.com/magda-io/magda-config.git; else cd magda-config && git pull && cd ..; fi
-#     - cp magda/scripts/create-secrets/build/index-linux magda-config/create-secrets/index-linux
-#     - cp magda/scripts/create-secrets/build/index-macos magda-config/create-secrets/index-macos
-#     - cp magda/scripts/create-secrets/build/index-win.exe magda-config/create-secrets/index-win.exe
-#     - cd magda-config
-#     - cd create-secrets
-#     - git add index-linux index-macos index-win.exe
-#     - git config --global user.email "[email protected]"
-#     - git config --global user.name "magdabot"
-#     - git commit -m "Update create-secrets scripts `date`"
-#     - git push "https://x-access-token:[email protected]/magda-io/magda-config" master

CHANGES.md

@@ -13,6 +13,7 @@
 - Added new APIs for managing auth objects
 - #3308 Policy enforcement on auth objects APIs
 - related #3250: Rewrite decision enforcement logic for search API
+- #3326 Build OPA docker image with builtin policies & Run OPA as a side car
 
 ## 1.2.0
 

deploy/helm/internal-charts/authorization-api/README.md

@@ -16,14 +16,25 @@ Kubernetes: `>= 1.14.0-0`
 | autoscaler.maxReplicas | int | `3` |  |
 | autoscaler.minReplicas | int | `1` |  |
 | autoscaler.targetCPUUtilizationPercentage | int | `80` |  |
-| debug | bool | `false` | when set to true, auth API will print verbose debug info (e.g. auth decision process) to log |
+| debug | bool | `false` | when set to true, auth API will print verbose debug info (e.g. sql statements) to log |
 | defaultImage.pullPolicy | string | `"IfNotPresent"` |  |
 | defaultImage.pullSecrets | bool | `false` |  |
 | defaultImage.repository | string | `"docker.io/data61"` |  |
 | image.name | string | `"magda-authorization-api"` |  |
-| resources.limits.cpu | string | `"50m"` |  |
-| resources.requests.cpu | string | `"10m"` |  |
+| opa.customPolicyConfigMaps | list | `[]` | a list of names of the configMaps that contains custom policy files. the configMap must be created using magda helm chart template: [magda.filesToJson](https://github.com/magda-io/magda/blob/21499b75c7a7ee00d68886338713217d83ccb91f/deploy/helm/magda-core/templates/_helpers.tpl#L244). More info see [here](https://github.com/magda-io/magda-configmap-dir-loader). |
+| opa.image.name | string | `"magda-opa"` |  |
+| opa.loaderImage.name | string | `"magda-configmap-dir-loader"` |  |
+| opa.loaderImage.pullPolicy | string | `"IfNotPresent"` |  |
+| opa.loaderImage.pullSecrets | bool | `false` |  |
+| opa.loaderImage.repository | string | `"docker.io/data61"` |  |
+| opa.loaderImage.tag | string | `"1.0.0-alpha.0"` |  |
+| opa.resources.limits.cpu | string | `"500m"` |  |
+| opa.resources.requests.cpu | string | `"20m"` |  |
+| opa.resources.requests.memory | string | `"50Mi"` |  |
+| resources.limits.cpu | string | `"500m"` |  |
+| resources.requests.cpu | string | `"20m"` |  |
 | resources.requests.memory | string | `"50Mi"` |  |
+| skipAuth | bool | `false` | when set to true, API will not query policy engine for auth decision but assume it's always permitted.  It's for debugging only. |
 
 ----------------------------------------------
 Autogenerated from chart metadata using [helm-docs v1.5.0](https://github.com/norwoodj/helm-docs/releases/v1.5.0)

deploy/helm/internal-charts/authorization-api/templates/deployment.yaml

@@ -23,16 +23,20 @@ spec:
       - name: authorization-api
         image: {{ include "magda.image" . | quote }}
         imagePullPolicy: {{ include "magda.imagePullPolicy" . | quote }}
+        ports:
+        - containerPort: 80
         command: [
             "node",
             "/usr/src/app/component/dist/index.js",
             "--listenPort", "80",
             "--dbHost", "authorization-db",
             "--dbPort", "5432",
-            "--opaUrl", "http://opa/",
-            "--registryApiUrl", "http://registry-api/v0",
+            "--opaUrl", "http://localhost:8181/",
 {{- if .Values.debug }}
             "--debug", "true",
+{{- end }}
+{{- if .Values.skipAuth }}
+            "--skipAuth", "true",
 {{- end }}
             "--tenantId", "0"
         ]
@@ -63,3 +67,66 @@ spec:
               name: auth-secrets
               key: jwt-secret
         {{- include "magda.db-client-credential-env" (dict "dbName" "authorization-db" "root" .) | indent 8 }}
+      - name: opa
+        {{- $imageEnv := omit . "Values" }}
+        {{- $_ := set $imageEnv "Values" (omit .Values "image") }}
+        {{- $_ := set $imageEnv.Values "image" .Values.opa.image }}
+        image: {{ include "magda.image" $imageEnv | quote }}
+        imagePullPolicy: {{ include "magda.imagePullPolicy" $imageEnv | quote }}
+        ports:
+        - containerPort: 8181
+        readinessProbe:
+          httpGet:
+            path: "/health"
+            port: 8181
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          timeoutSeconds: 10
+{{- if .Values.global.enableLivenessProbes }}
+        livenessProbe:
+          httpGet:
+            path: "/health"
+            port: 8181
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          timeoutSeconds: 10
+{{- end }}
+        resources:
+{{ toYaml .Values.opa.resources | indent 10 }}
+{{- if .Values.opa.customPolicyConfigMaps | empty | not }}
+        volumeMounts:
+        - name: policy-dir
+          mountPath: /opa-data/policies
+{{- end }}
+{{- if .Values.opa.customPolicyConfigMaps | empty | not }}
+      initContainers:
+      - name: policy-files-loader
+        {{- $imageEnv := omit . "Values" }}
+        {{- $_ := set $imageEnv "Values" (omit .Values "image") }}
+        {{- $_ := set $imageEnv.Values "image" .Values.opa.loaderImage }}
+        image: {{ include "magda.image" $imageEnv | quote }}
+        imagePullPolicy: {{ include "magda.imagePullPolicy" $imageEnv | quote }}
+        env:
+          - name: DEFAULT_CFG_MAP_DIR
+            value: "/opa-raw-policy-data-files"
+          - name: TARGET_DIR
+            value: "/opa-data/policies"
+        volumeMounts:
+          - name: policy-dir
+            mountPath: /opa-data/policies
+          {{- range .Values.customPolicyConfigMaps }}
+          - name: {{.}}
+            mountPath: /opa-raw-policy-data-files
+            readOnly: true
+          {{- end }}
+{{- end }}
+{{- if .Values.opa.customPolicyConfigMaps | empty | not }}
+      volumes:
+      - name: policy-dir
+        emptyDir: {}
+      {{ range .Values.customPolicyConfigMaps -}}
+      - name: {{.}}
+        configMap:
+          name: {{.}}
+      {{ end }}
+{{- end }}

deploy/helm/internal-charts/authorization-api/values.yaml

@@ -17,10 +17,41 @@ autoscaler:
   targetCPUUtilizationPercentage: 80
 resources:
   requests:
-    cpu: 10m
+    cpu: 20m
     memory: 50Mi
   limits:
-    cpu: 50m
+    cpu: 500m
 
-# -- when set to true, auth API will print verbose debug info (e.g. auth decision process) to log
-debug: false
+# -- when set to true, auth API will print verbose debug info (e.g. sql statements) to log
+debug: false
+# -- when set to true, API will not query policy engine for auth decision but assume it's always permitted. 
+# It's for debugging only.
+skipAuth: false
+
+opa:
+  image: 
+    name: "magda-opa"
+    # repository: 
+    # tag: 
+    # pullPolicy: 
+    # pullSecrets:
+
+  loaderImage: 
+    # Github repo: https://github.com/magda-io/magda-configmap-dir-loader
+    repository: docker.io/data61
+    name: "magda-configmap-dir-loader"
+    tag: "1.0.0-alpha.0"
+    pullPolicy: IfNotPresent
+    pullSecrets: false
+
+  resources:
+    requests:
+      cpu: 20m
+      memory: 50Mi
+    limits:
+      cpu: 500m
+
+  # opa.customPolicyConfigMaps -- a list of names of the configMaps that contains custom policy files.
+  # the configMap must be created using magda helm chart template: [magda.filesToJson](https://github.com/magda-io/magda/blob/21499b75c7a7ee00d68886338713217d83ccb91f/deploy/helm/magda-core/templates/_helpers.tpl#L244).
+  # More info see [here](https://github.com/magda-io/magda-configmap-dir-loader).
+  customPolicyConfigMaps: []