From e454114b48180ebbe6c1255dc83083e4bb473b65 Mon Sep 17 00:00:00 2001 From: Johnathan Kupferer Date: Wed, 29 Jun 2022 16:08:07 -0500 Subject: [PATCH] Add cost-tracker (#410) --- .github/workflows/cost-tracker-publish.yaml | 63 ++++++++ cost-tracker/.gitignore | 3 + cost-tracker/Development.adoc | 134 ++++++++++++++++++ cost-tracker/Dockerfile | 18 +++ cost-tracker/build-template.yaml | 51 +++++++ cost-tracker/devfile.yaml | 30 ++++ cost-tracker/helm/Chart.yaml | 6 + cost-tracker/helm/templates/_helpers.tpl | 97 +++++++++++++ .../templates/aws-sandbox-manager-secret.yaml | 12 ++ cost-tracker/helm/templates/deployment.yaml | 62 ++++++++ cost-tracker/helm/templates/namespace.yaml | 6 + cost-tracker/helm/templates/rbac.yaml | 70 +++++++++ .../helm/templates/serviceaccount.yaml | 9 ++ cost-tracker/helm/values.yaml | 51 +++++++ cost-tracker/kopf-opt.sh | 3 + cost-tracker/operator/anarchy_subject.py | 35 +++++ cost-tracker/operator/aws_sandbox_cost.py | 43 ++++++ cost-tracker/operator/cost_tracker_state.py | 44 ++++++ .../operator/infinite_relative_backoff.py | 14 ++ cost-tracker/operator/operator.py | 99 +++++++++++++ cost-tracker/operator/resource_claim.py | 87 ++++++++++++ cost-tracker/requirements.txt | 2 + .../config/common/helm/babylon-cost-tracker | 1 + openshift/config/common/vars.yaml | 8 ++ 24 files changed, 948 insertions(+) create mode 100644 .github/workflows/cost-tracker-publish.yaml create mode 100644 cost-tracker/.gitignore create mode 100644 cost-tracker/Development.adoc create mode 100644 cost-tracker/Dockerfile create mode 100644 cost-tracker/build-template.yaml create mode 100644 cost-tracker/devfile.yaml create mode 100644 cost-tracker/helm/Chart.yaml create mode 100644 cost-tracker/helm/templates/_helpers.tpl create mode 100644 cost-tracker/helm/templates/aws-sandbox-manager-secret.yaml create mode 100644 cost-tracker/helm/templates/deployment.yaml create mode 100644 cost-tracker/helm/templates/namespace.yaml create mode 100644 cost-tracker/helm/templates/rbac.yaml create mode 100644 cost-tracker/helm/templates/serviceaccount.yaml create mode 100644 cost-tracker/helm/values.yaml create mode 100644 cost-tracker/kopf-opt.sh create mode 100644 cost-tracker/operator/anarchy_subject.py create mode 100644 cost-tracker/operator/aws_sandbox_cost.py create mode 100644 cost-tracker/operator/cost_tracker_state.py create mode 100644 cost-tracker/operator/infinite_relative_backoff.py create mode 100644 cost-tracker/operator/operator.py create mode 100644 cost-tracker/operator/resource_claim.py create mode 100644 cost-tracker/requirements.txt create mode 120000 openshift/config/common/helm/babylon-cost-tracker diff --git a/.github/workflows/cost-tracker-publish.yaml b/.github/workflows/cost-tracker-publish.yaml new file mode 100644 index 000000000..f153e4d65 --- /dev/null +++ b/.github/workflows/cost-tracker-publish.yaml @@ -0,0 +1,63 @@ +--- +name: cost-tracker-publish +on: + push: + branches-ignore: + - '*' + tags: + - 'cost-tracker-v[0-9]*' +jobs: + publish: + env: + IMAGE_NAME: babylon-cost-tracker + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@v2 + + - name: Get image tags + id: image_tags + run: | + # Version is a semantic version tag or semantic version with release number + # GITHUB_REF will be of the form "refs/tags/admin-v0.1.2" or "refs/tags/admin-v0.1.2-1" + # To determine RELEASE, strip off the leading "refs/tags/" + RELEASE=${GITHUB_REF#refs/tags/cost-tracker-} + # To determine VERSION, strip off any release number suffix + VERSION=${RELEASE/-*/} + echo "::set-output name=RELEASE::${RELEASE}" + echo "::set-output name=VERSION::${VERSION}" + + # Only build image if version tag without release number + # Releases indicate a change in the repository that should not trigger a new build. + if [[ "${VERSION}" == "${RELEASE}" ]]; then + # Publish to latest, minor, and patch tags + # Ex: latest,v0.1.2,v0.1 + IMAGE_TAGS=( + '${{ secrets.REGISTRY_URI }}/${{ secrets.GPTE_REGISTRY_REPOSITORY }}/${{ env.IMAGE_NAME }}:latest' + "${{ secrets.REGISTRY_URI }}/${{ secrets.GPTE_REGISTRY_REPOSITORY }}/${{ env.IMAGE_NAME }}:${VERSION%.*}" + "${{ secrets.REGISTRY_URI }}/${{ secrets.GPTE_REGISTRY_REPOSITORY }}/${{ env.IMAGE_NAME }}:${VERSION}" + ) + # Set IMAGE_TAGS output for use in next step + ( IFS=$','; echo "::set-output name=IMAGE_TAGS::${IMAGE_TAGS[*]}" ) + fi + + - name: Set up buildx + uses: docker/setup-buildx-action@v1 + if: steps.image_tags.outputs.IMAGE_TAGS + + - name: Login to image registry + uses: docker/login-action@v1 + if: steps.image_tags.outputs.IMAGE_TAGS + with: + registry: ${{ secrets.REGISTRY_URI }} + username: ${{ secrets.GPTE_REGISTRY_USERNAME }} + password: ${{ secrets.GPTE_REGISTRY_PASSWORD }} + + - name: Build and publish image + uses: docker/build-push-action@v2 + if: steps.image_tags.outputs.IMAGE_TAGS + with: + context: cost-tracker + file: cost-tracker/Dockerfile + push: true + tags: ${{ steps.image_tags.outputs.IMAGE_TAGS }} diff --git a/cost-tracker/.gitignore b/cost-tracker/.gitignore new file mode 100644 index 000000000..45042160b --- /dev/null +++ b/cost-tracker/.gitignore @@ -0,0 +1,3 @@ + +.odo/env +.odo/odo-file-index.json \ No newline at end of file diff --git a/cost-tracker/Development.adoc b/cost-tracker/Development.adoc new file mode 100644 index 000000000..f85dbe134 --- /dev/null +++ b/cost-tracker/Development.adoc @@ -0,0 +1,134 @@ +# Babylon Cost Tracker Development + +Development can be performed in the `odo` OpenShift developer CLI or building with OpenShift build configs. +An OpenShift cluster with cluster-admin is required for `odo` development. +https://developers.redhat.com/products/codeready-containers/overview[CodeReady Containers] is recommended for local development. +An Ansible test suite is available for functional testing. + +## Development with `odo` + +Use of `odo` is recommended for fast iterative development. +`odo` simplifies the build/deploy process and avoids creating unnecessary build artifacts during the development process. + +. Install the `odo` developer CLI as described in the OpenShift documentation: +https://docs.openshift.com/container-platform/latest/cli_reference/developer_cli_odo/installing-odo.html[Installing odo] + +. Create a project for development using `odo`: ++ +--------------------------------------- +odo project create babylon-cost-tracker-dev +--------------------------------------- + +. Create Babylon cost-tracker resources from the provided helm chart: ++ +------------------------------- +helm template helm \ +--set deploy=false \ +--set nameOverride=babylon-cost-tracker-dev \ +--set namespace.name=$(oc project --short) \ +| oc apply -f - +------------------------------- ++ +NOTE: Password `p4ssw0rd` is the default redis password value in `devfile.yaml`. + +. Grant privileges for cluster role `babylon-cost-tracker` to default service account: ++ +---------------------------------------------------------------------- +oc adm policy add-cluster-role-to-user babylon-cost-tracker-dev -z default +---------------------------------------------------------------------- + +. Setup `odo` from the provided `devfile.yaml`: ++ +--------------------------------- +odo create --devfile devfile.yaml +--------------------------------- + +. Set `AWS_SANDBOX_MANAGER_ACCESS_KEY_ID` and `AWS_SANDBOX_MANAGER_SECRET_ACCESS_KEY` values for `odo` deployment: ++ +--------------------------------------------------- +odo config set --env AWS_SANDBOX_MANAGER_ACCESS_KEY_ID="..." +odo config set --env AWS_SANDBOX_MANAGER_SECRET_ACCESS_KEY="..." +--------------------------------------------------- ++ +NOTE: Do not check the updated `devfile.yaml` into GitHub! + +. Use `odo push` to push code into the odo container: ++ +-------- +odo push +-------- + +. Cleanup ++ +Remove `odo` component ++ +--------------------------------------- +odo delete --force babylon-cost-tracker-dev +--------------------------------------- ++ +Remove the cluster-reader cluster role binding ++ +--------------------------------------------------------------------------- +oc adm policy remove-cluster-role-from-user babylon-cost-tracker-dev -z default +--------------------------------------------------------------------------- ++ +Remove resources created from the helm template ++ +------------------------------- +helm template helm \ +--set deploy=false \ +--set nameOverride=babylon-cost-tracker-dev \ +--set namespace.name=$(oc project --short) \ +| oc delete -f - +------------------------------- + + +## Development OpenShift Build + +The OpenShift build process is a bit slower for development but has the advantage of being a bit closer to a normal deployment of the babylon-cost-tracker. +It is often best to iterate development using `odo` and also test with an OpenShift build and deployment prior to opening a pull request. + +. Create a project for development using `oc`: ++ +--------------------------------------- +oc new-project babylon-cost-tracker-dev +--------------------------------------- + +. Process OpenShift build template to create BuildConfig and ImageStream: ++ +--------------------------------------------------------- +oc process --local -f build-template.yaml | oc apply -f - +--------------------------------------------------------- + +. Build babylon-cost-tracker image from local source: ++ +---------------------------------------------------------- +oc start-build babylon-cost-tracker --from-dir=.. --follow +---------------------------------------------------------- + +. Deploy Poolboy from build image ++ +-------------------------------------------------------------------------------- +helm template helm \ +--set nameOverride=babylon-cost-tracker-dev \ +--set namespace.create=false \ +--set=image.override="$(oc get imagestream babylon-cost-tracker -o jsonpath='{.status.tags[?(@.tag=="latest")].items[0].dockerImageReference}')" \ +| oc apply -f - +-------------------------------------------------------------------------------- + +. Cleanup ++ +Remove resources created from the helm template: ++ +--------------------------------------------- +helm template helm \ +--set nameOverride=babylon-cost-tracker-dev \ +--set namespace.create=false \ +| oc delete -f - +--------------------------------------------- ++ +Remove BuildConfig and ImageStream: ++ +---------------------------------------------------------- +oc process --local -f build-template.yaml | oc delete -f - +---------------------------------------------------------- diff --git a/cost-tracker/Dockerfile b/cost-tracker/Dockerfile new file mode 100644 index 000000000..024122ed2 --- /dev/null +++ b/cost-tracker/Dockerfile @@ -0,0 +1,18 @@ +FROM quay.io/redhat-cop/python-kopf-s2i:v1.35 + +USER root + +COPY . /tmp/src + +RUN dnf install -y @ruby && \ + gem install asciidoctor && \ + rm -rf /tmp/src/.git* && \ + chown -R 1001 /tmp/src && \ + chgrp -R 0 /tmp/src && \ + chmod -R g+w /tmp/src + +USER 1001 + +RUN /usr/libexec/s2i/assemble + +CMD ["/usr/libexec/s2i/run"] diff --git a/cost-tracker/build-template.yaml b/cost-tracker/build-template.yaml new file mode 100644 index 000000000..45d32f2a0 --- /dev/null +++ b/cost-tracker/build-template.yaml @@ -0,0 +1,51 @@ +--- +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + annotations: + description: babylon-cost-tracker deploy + name: babylon-cost-tracker-build + +parameters: +- name: NAME + value: babylon-cost-tracker +- name: GIT_REPO + value: https://github.com/redhat-cop/babylon.git +- name: GIT_REF + value: main +- name: KOPF_S2I_IMAGE + value: quay.io/redhat-cop/python-kopf-s2i:v1.35 + +objects: +- apiVersion: image.openshift.io/v1 + kind: ImageStream + metadata: + name: ${NAME} + spec: + lookupPolicy: + local: false + +- apiVersion: v1 + kind: BuildConfig + metadata: + name: ${NAME} + spec: + output: + to: + kind: ImageStreamTag + name: ${NAME}:latest + postCommit: {} + resources: {} + runPolicy: Serial + source: + contextDir: cost-tracker + git: + uri: ${GIT_REPO} + ref: ${GIT_REF} + strategy: + type: Docker + dockerStrategy: + from: + kind: DockerImage + name: ${KOPF_S2I_IMAGE} + triggers: [] diff --git a/cost-tracker/devfile.yaml b/cost-tracker/devfile.yaml new file mode 100644 index 000000000..954cecd90 --- /dev/null +++ b/cost-tracker/devfile.yaml @@ -0,0 +1,30 @@ +commands: +- exec: + commandLine: /usr/libexec/s2i/assemble + component: s2i-builder + group: + isDefault: true + kind: build + hotReloadCapable: false + workingDir: ${PROJECT_SOURCE} + id: s2i-assemble +- exec: + commandLine: /usr/libexec/s2i/run + component: s2i-builder + group: + isDefault: true + kind: run + hotReloadCapable: false + workingDir: ${PROJECT_SOURCE} + id: s2i-run +components: +- container: + dedicatedPod: false + image: quay.io/redhat-cop/python-kopf-s2i + mountSources: true + sourceMapping: /tmp/projects + name: s2i-builder +metadata: + name: babylon-cost-tracker + version: 1.0.0 +schemaVersion: 2.0.0 diff --git a/cost-tracker/helm/Chart.yaml b/cost-tracker/helm/Chart.yaml new file mode 100644 index 000000000..174bf35d2 --- /dev/null +++ b/cost-tracker/helm/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: babylon-cost-tracker +description: A Helm chart for the babylon cost tracker component. +type: application +version: 0.1.0 +appVersion: 0.1.0 diff --git a/cost-tracker/helm/templates/_helpers.tpl b/cost-tracker/helm/templates/_helpers.tpl new file mode 100644 index 000000000..58183808b --- /dev/null +++ b/cost-tracker/helm/templates/_helpers.tpl @@ -0,0 +1,97 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "babylon-cost-tracker.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "babylon-cost-tracker.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "babylon-cost-tracker.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "babylon-cost-tracker.labels" -}} +helm.sh/chart: {{ include "babylon-cost-tracker.chart" . }} +{{ include "babylon-cost-tracker.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "babylon-cost-tracker.selectorLabels" -}} +app.kubernetes.io/name: {{ include "babylon-cost-tracker.name" . }} +{{- if (ne .Release.Name "RELEASE-NAME") }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "babylon-cost-tracker.awsSandboxManagerSecretName" -}} +{{- include "babylon-cost-tracker.name" . -}}-aws-sandbox-manager +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "babylon-cost-tracker.serviceAccountName" -}} + {{- if .Values.serviceAccount.create -}} + {{- default (include "babylon-cost-tracker.name" .) .Values.serviceAccount.name -}} + {{- else -}} + {{- default "default" .Values.serviceAccount.name -}} + {{- end -}} +{{- end -}} + +{{/* +Create the name of the namespace to use +*/}} +{{- define "babylon-cost-tracker.namespaceName" -}} + {{- default (include "babylon-cost-tracker.name" .) .Values.namespace.name }} +{{- end -}} + + +{{/* +Define the image to deploy +*/}} +{{- define "babylon-cost-tracker.image" -}} + {{- if .Values.image.override -}} + {{- .Values.image.override -}} + {{- else -}} + {{- if eq .Values.image.tagOverride "-" -}} + {{- .Values.image.repository -}} + {{- else if .Values.image.tagOverride -}} + {{- printf "%s:%s" .Values.image.repository .Values.image.tagOverride -}} + {{- else -}} + {{- printf "%s:v%s" .Values.image.repository .Chart.AppVersion -}} + {{- end -}} + {{- end -}} +{{- end -}} diff --git a/cost-tracker/helm/templates/aws-sandbox-manager-secret.yaml b/cost-tracker/helm/templates/aws-sandbox-manager-secret.yaml new file mode 100644 index 000000000..b106f787b --- /dev/null +++ b/cost-tracker/helm/templates/aws-sandbox-manager-secret.yaml @@ -0,0 +1,12 @@ +{{ if .Values.awsSandboxManagerCredentials }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "babylon-cost-tracker.awsSandboxManagerSecretName" . }} + namespace: {{ include "babylon-cost-tracker.namespaceName" . }} + labels: + {{- include "babylon-cost-tracker.labels" . | nindent 4 }} +data: + aws_access_key_id: {{ .Values.awsSandboxManagerCredentials.awsAccessKeyId | b64enc }} + aws_secret_access_key: {{ .Values.awsSandboxManagerCredentials.awsSecretAccessKey | b64enc }} +{{ end }} diff --git a/cost-tracker/helm/templates/deployment.yaml b/cost-tracker/helm/templates/deployment.yaml new file mode 100644 index 000000000..307322a8a --- /dev/null +++ b/cost-tracker/helm/templates/deployment.yaml @@ -0,0 +1,62 @@ +{{ if .Values.deploy }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "babylon-cost-tracker.name" . }} + namespace: {{ include "babylon-cost-tracker.namespaceName" . }} + labels: + {{- include "babylon-cost-tracker.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "babylon-cost-tracker.selectorLabels" . | nindent 6 }} + strategy: + type: Recreate + template: + metadata: + labels: + {{- include "babylon-cost-tracker.selectorLabels" . | nindent 8 }} + spec: + containers: + - name: cost-tracker + env: + - name: AWS_SANDBOX_MANAGER_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ include "babylon-cost-tracker.awsSandboxManagerSecretName" . }} + key: aws_access_key_id + - name: AWS_SANDBOX_MANAGER_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ include "babylon-cost-tracker.awsSandboxManagerSecretName" . }} + key: aws_secret_access_key + image: {{ include "babylon-cost-tracker.image" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + livenessProbe: + initialDelaySeconds: 30 + tcpSocket: + port: 8080 + timeoutSeconds: 1 + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + restartPolicy: Always + serviceAccountName: {{ include "babylon-cost-tracker.serviceAccountName" . }} + terminationGracePeriodSeconds: 30 + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{ end }} diff --git a/cost-tracker/helm/templates/namespace.yaml b/cost-tracker/helm/templates/namespace.yaml new file mode 100644 index 000000000..a94306261 --- /dev/null +++ b/cost-tracker/helm/templates/namespace.yaml @@ -0,0 +1,6 @@ +{{ if .Values.namespace.create }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ include "babylon-cost-tracker.namespaceName" . }} +{{ end }} diff --git a/cost-tracker/helm/templates/rbac.yaml b/cost-tracker/helm/templates/rbac.yaml new file mode 100644 index 000000000..e3c7729f1 --- /dev/null +++ b/cost-tracker/helm/templates/rbac.yaml @@ -0,0 +1,70 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "babylon-cost-tracker.name" . }} + annotations: + description: >- + Access read cluster resources for babylon cost-tracker + labels: + {{- include "babylon-cost-tracker.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kopf.dev + resources: + - clusterkopfpeerings + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - {{ .Values.poolboy.domain }} + resources: + - resourceclaims + verbs: + - get + - list + - patch + - update + - watch + +{{ if .Values.deploy }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "babylon-cost-tracker.name" . }} + labels: + {{- include "babylon-cost-tracker.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "babylon-cost-tracker.name" . }} +subjects: +- kind: ServiceAccount + name: {{ include "babylon-cost-tracker.serviceAccountName" . }} + namespace: {{ include "babylon-cost-tracker.namespaceName" . }} +{{ end }} diff --git a/cost-tracker/helm/templates/serviceaccount.yaml b/cost-tracker/helm/templates/serviceaccount.yaml new file mode 100644 index 000000000..0ddc98c98 --- /dev/null +++ b/cost-tracker/helm/templates/serviceaccount.yaml @@ -0,0 +1,9 @@ +{{ if .Values.deploy }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "babylon-cost-tracker.serviceAccountName" . }} + namespace: {{ include "babylon-cost-tracker.namespaceName" . }} + labels: + {{- include "babylon-cost-tracker.labels" . | nindent 4 }} +{{ end }} diff --git a/cost-tracker/helm/values.yaml b/cost-tracker/helm/values.yaml new file mode 100644 index 000000000..e57ccc1ab --- /dev/null +++ b/cost-tracker/helm/values.yaml @@ -0,0 +1,51 @@ +deploy: true +replicaCount: 1 +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +namespace: + # Specifies whether a namespace should be created + create: true + # The name of the namespace to use. + # If not set and create is true, a name is generated using the name template + name: + +serviceAccount: + # Specifies whether a service account should be created + create: true + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: + +resources: + limits: + cpu: "1" + memory: 512Mi + requests: + cpu: 100m + memory: 512Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +anarchy: + domain: anarchy.gpte.redhat.com + +babylon: + domain: babylon.gpte.redhat.com + +poolboy: + domain: poolboy.gpte.redhat.com + +#awsSandboxManagerCredentials: +# awsAccessKeyId: '...' +# awsSecretAccessKey: '...' + +image: + repository: quay.io/redhat-gpte/babylon-cost-tracker + pullPolicy: Always + tagOverride: "" diff --git a/cost-tracker/kopf-opt.sh b/cost-tracker/kopf-opt.sh new file mode 100644 index 000000000..9ff38a7ff --- /dev/null +++ b/cost-tracker/kopf-opt.sh @@ -0,0 +1,3 @@ +#!/bin/sh +KOPF_OPTIONS="--log-format=json" +KOPF_PEERING="babylon-cost-tracker" diff --git a/cost-tracker/operator/anarchy_subject.py b/cost-tracker/operator/anarchy_subject.py new file mode 100644 index 000000000..8b0d77c41 --- /dev/null +++ b/cost-tracker/operator/anarchy_subject.py @@ -0,0 +1,35 @@ +class AnarchySubject: + def __init__(self, definition): + self.definition = definition + + @property + def aws_sandbox_account(self): + return self.job_vars.get('sandbox_account') + + @property + def guid(self): + return self.job_vars.get('guid') + + @property + def job_vars(self): + return self.vars.get('job_vars', {}) + + @property + def metadata(self): + return self.definition['metadata'] + + @property + def name(self): + return self.metadata['name'] + + @property + def namespace(self): + return self.metadata['namespace'] + + @property + def spec(self): + return self.definition['spec'] + + @property + def vars(self): + return self.spec.get('vars', {}) diff --git a/cost-tracker/operator/aws_sandbox_cost.py b/cost-tracker/operator/aws_sandbox_cost.py new file mode 100644 index 000000000..0f7294875 --- /dev/null +++ b/cost-tracker/operator/aws_sandbox_cost.py @@ -0,0 +1,43 @@ +import boto3 +import os + +from datetime import datetime, timedelta, timezone + +boto3_sts_client = boto3.client( + 'sts', + aws_access_key_id = os.environ.get('AWS_SANDBOX_MANAGER_ACCESS_KEY_ID'), + aws_secret_access_key = os.environ.get('AWS_SANDBOX_MANAGER_SECRET_ACCESS_KEY'), +) + +def get_aws_sandbox_cost(creation_datetime, sandbox_account): + sandbox_assumed_role = boto3_sts_client.assume_role( + RoleArn = f"arn:aws:iam::{sandbox_account}:role/OrganizationAccountAccessRole", + RoleSessionName = f"AssumeRole-{sandbox_account}", + ) + sandbox_credentials = sandbox_assumed_role['Credentials'] + boto3_ce_client = boto3.client( + 'ce', + aws_access_key_id = sandbox_credentials['AccessKeyId'], + aws_secret_access_key = sandbox_credentials['SecretAccessKey'], + aws_session_token = sandbox_credentials['SessionToken'], + ) + cost_and_usage = boto3_ce_client.get_cost_and_usage( + Filter = dict( + Dimensions = dict( + Key = 'RECORD_TYPE', + Values = ['SavingsPlanCoveredUsage', 'Usage'] + ) + ), + Granularity = 'DAILY', + Metrics = ['UnblendedCost'], + TimePeriod = dict( + Start = creation_datetime.strftime('%F'), + End = (datetime.now(timezone.utc) + timedelta(days=1)).strftime('%F'), + ) + ) + + total_cost = 0 + for result in cost_and_usage.get('ResultsByTime', []): + total_cost += float(result['Total']['UnblendedCost']['Amount']) + + return total_cost diff --git a/cost-tracker/operator/cost_tracker_state.py b/cost-tracker/operator/cost_tracker_state.py new file mode 100644 index 000000000..457676a3d --- /dev/null +++ b/cost-tracker/operator/cost_tracker_state.py @@ -0,0 +1,44 @@ +from datetime import datetime, timedelta, timezone +import json + +class CostTrackerState: + @staticmethod + def deserialize(json_string): + return CostTrackerState(**json.loads(json_string)) + + def __init__(self, estimatedCost=None, lastUpdate=None, lastRequest=None, **_): + self.estimated_cost = estimatedCost if estimatedCost else None + + self.last_update = datetime.strptime( + lastUpdate, '%Y-%m-%dT%H:%M:%SZ' + ).replace(tzinfo=timezone.utc) if lastUpdate else None + + self.last_request = datetime.strptime( + lastRequest, '%Y-%m-%dT%H:%M:%SZ' + ).replace(tzinfo=timezone.utc) if lastRequest else None + + @property + def update_is_requested(self): + if not self.last_request: + return False + if not self.last_update: + return True + return self.last_request > self.last_update + + def serialize(self): + data = {} + if self.estimated_cost: + data['estimatedCost'] = self.estimated_cost + if self.last_update: + data['lastUpdate'] = self.last_update.strftime('%Y-%m-%dT%H:%M:%SZ') + if self.last_request: + data['lastRequest'] = self.last_request.strftime('%Y-%m-%dT%H:%M:%SZ') + return json.dumps(data) + + def set_estimated_cost(self, estimated_cost): + self.estimated_cost = estimated_cost + self.last_update = datetime.now(timezone.utc) + # Set last request to last update if it was set to a future timestamp. + # This prevents the update to cause a busy loop of repeated updates. + if self.last_request > self.last_update: + self.last_request = self.last_update diff --git a/cost-tracker/operator/infinite_relative_backoff.py b/cost-tracker/operator/infinite_relative_backoff.py new file mode 100644 index 000000000..c30c8e814 --- /dev/null +++ b/cost-tracker/operator/infinite_relative_backoff.py @@ -0,0 +1,14 @@ +class InfiniteRelativeBackoff: + def __init__(self, initial_delay=0.1, scaling_factor=2, maximum=60): + self.initial_delay = initial_delay + self.scaling_factor = scaling_factor + self.maximum = maximum + + def __iter__(self): + delay = self.initial_delay + while True: + if delay > self.maximum: + yield self.maximum + else: + yield delay + delay *= self.scaling_factor diff --git a/cost-tracker/operator/operator.py b/cost-tracker/operator/operator.py new file mode 100644 index 000000000..4d4a9b289 --- /dev/null +++ b/cost-tracker/operator/operator.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 + +import kopf +import kubernetes +import logging +import os + +from cost_tracker_state import CostTrackerState +from infinite_relative_backoff import InfiniteRelativeBackoff +from resource_claim import ResourceClaim + +babylon_domain = os.environ.get('BABYLON_DOMAIN', 'babylon.gpte.redhat.com') +babylon_api_version = os.environ.get('BABYLON_API_VERSION', 'v1') +poolboy_domain = os.environ.get('POOLBOY_DOMAIN', 'poolboy.gpte.redhat.com') +poolboy_api_version = os.environ.get('POOLBOY_API_VERSION', 'v1') + +cost_tracker_annotation = f"{babylon_domain}/cost-tracker" + +if os.path.exists('/run/secrets/kubernetes.io/serviceaccount'): + kubernetes.config.load_incluster_config() +else: + kubernetes.config.load_kube_config() + +core_v1_api = kubernetes.client.CoreV1Api() +custom_objects_api = kubernetes.client.CustomObjectsApi() + +def set_cost_tracker_annotation(cost_tracker_state, resource_claim): + custom_objects_api.patch_namespaced_custom_object( + poolboy_domain, + poolboy_api_version, + resource_claim.namespace, + 'resourceclaims', + resource_claim.name, + { + "metadata": { + "annotations": { + cost_tracker_annotation: cost_tracker_state.serialize(), + } + } + } + ) + +@kopf.on.startup() +def configure(settings: kopf.OperatorSettings, **_): + # Never give up from network errors + settings.networking.error_backoffs = InfiniteRelativeBackoff() + + # Only create events for warnings and errors + settings.posting.level = logging.WARNING + + # Disable scanning for CustomResourceDefinitions + settings.scanning.disabled = True + +@kopf.on.event(poolboy_domain, poolboy_api_version, 'resourceclaims') +def resourceclaim_event(event, logger, **_): + resource_claim_definition = event.get('object') + if not resource_claim_definition \ + or resource_claim_definition.get('kind') != 'ResourceClaim': + logger.warning(event) + return + + resource_claim = ResourceClaim(definition=resource_claim_definition) + if not resource_claim.supports_cost_tracking: + return + + cost_tracker_json = resource_claim.annotations.get(cost_tracker_annotation) + if not cost_tracker_json: + set_cost_tracker_annotation(cost_tracker_state=CostTrackerState(), resource_claim=resource_claim) + logger.info("Created cost tracker annotation") + return + + cost_tracker_state = CostTrackerState.deserialize(cost_tracker_json) + if cost_tracker_state.update_is_requested: + resource_claim.update_cost_tracker_state(cost_tracker_state) + set_cost_tracker_annotation(cost_tracker_state=cost_tracker_state, resource_claim=resource_claim) + logger.info(f"Updated estimated cost ${cost_tracker_state.estimated_cost:.2f}") + + +# /usr/local/bin/aws --profile #{sandbox} ce get-cost-and-usage --time-period Start=#{startDate},End=#{endDate} --granularity DAILY --metrics UnblendedCost --output json --no-cli-pager --filter file:///usr/local/etc/cost_usage_filter.json +#[prutledg@demo00 ~]$ cat /usr/local/etc/cost_usage_filter.json +#{{ +# +#{ "Dimensions": \{ "Key": "RECORD_TYPE", "Values": [ "SavingsPlanCoveredUsage", "Usage" ] } +#}}} +# costInfo = JSON.parse(jsonOut) +# costText = "" +# if costInfo['ResultsByTime'].respond_to?(:each) +# unblendedCost = 0 +# count = 1 +# costInfo['ResultsByTime'].each do | result | +# unblendedCost += result['Total']['UnblendedCost']['Amount'].to_f +# count += 1 +# end +# avgUnblendedCost = unblendedCost / count +# costText += "Total cost so far: $#{unblendedCost.to_d.truncate(2).to_s}\n" +# costText += "Daily average cost: $#{avgUnblendedCost.to_d.truncate(2).to_s}\n" +# else +# costText += "Unable to get costs for this service." +# end diff --git a/cost-tracker/operator/resource_claim.py b/cost-tracker/operator/resource_claim.py new file mode 100644 index 000000000..475ff71e5 --- /dev/null +++ b/cost-tracker/operator/resource_claim.py @@ -0,0 +1,87 @@ +import re + +from anarchy_subject import AnarchySubject +from aws_sandbox_cost import get_aws_sandbox_cost +from datetime import datetime, timezone + +class ResourceClaim: + def __init__(self, definition): + self.definition = definition + + @property + def annotations(self): + return self.metadata.get('annotations', {}) + + @property + def creation_datetime(self): + return datetime.strptime( + self.creation_timestamp, '%Y-%m-%dT%H:%M:%SZ' + ).replace(tzinfo=timezone.utc) + + @property + def creation_timestamp(self): + return self.metadata['creationTimestamp'] + + @property + def guid(self): + resourceHandleName = self.definition.get('status', {}).get('resourceHandle', {}).get('name') + return re.sub(r'^guid-', '', resourceHandleName) if resourceHandleName else None + + @property + def metadata(self): + return self.definition['metadata'] + + @property + def name(self): + return self.metadata['name'] + + @property + def namespace(self): + return self.metadata['namespace'] + + @property + def status(self): + return self.definition.get('status') + + @property + def status_resources(self): + return self.status.get('resources', []) + + @property + def supports_cost_tracking(self): + if not self.status: + return False + + # If any resource in the claim is an AnarchySubject with an AWS sandbox then + # cost tracking is supported. + for status_resource in self.status_resources: + status_resource_state = status_resource.get('state') + if not status_resource_state: + continue + if status_resource_state['kind'] == 'AnarchySubject': + anarchy_subject = AnarchySubject(definition=status_resource_state) + if anarchy_subject.aws_sandbox_account: + return True + + return False + + @property + def uid(self): + return self.metadata['uid'] + + def update_cost_tracker_state(self, cost_tracker_state): + total_estimated_cost = 0 + + for status_resource in self.status_resources: + status_resource_state = status_resource.get('state') + if not status_resource_state: + continue + if status_resource_state['kind'] == 'AnarchySubject': + anarchy_subject = AnarchySubject(definition=status_resource_state) + if anarchy_subject.aws_sandbox_account: + total_estimated_cost += get_aws_sandbox_cost( + creation_datetime = self.creation_datetime, + sandbox_account = anarchy_subject.aws_sandbox_account + ) + + cost_tracker_state.set_estimated_cost(total_estimated_cost) diff --git a/cost-tracker/requirements.txt b/cost-tracker/requirements.txt new file mode 100644 index 000000000..2ead57e88 --- /dev/null +++ b/cost-tracker/requirements.txt @@ -0,0 +1,2 @@ +boto3==1.24.19 +botocore==1.27.19 diff --git a/openshift/config/common/helm/babylon-cost-tracker b/openshift/config/common/helm/babylon-cost-tracker new file mode 120000 index 000000000..a25ed5751 --- /dev/null +++ b/openshift/config/common/helm/babylon-cost-tracker @@ -0,0 +1 @@ +../../../../cost-tracker/helm \ No newline at end of file diff --git a/openshift/config/common/vars.yaml b/openshift/config/common/vars.yaml index 66653b04b..8700448c1 100644 --- a/openshift/config/common/vars.yaml +++ b/openshift/config/common/vars.yaml @@ -80,6 +80,14 @@ babylon_resources: "replik8sVersion": replik8s_version, } }} + - name: Babylon cost-tracker Helm Template + when: >- + babylon_cost_tracker_deploy | default(false) | bool + helm_template: + dir: babylon-cost-tracker + values: + awsSandboxManagerCredentials: "{{ aws_sandbox_manager_credentials | default({}) }}" + - namespace: babylon-cross-cluster-backup resources: - name: Babylon cross-cluster-backup replik8s