V3.2.0 release (#57)

V3.2.0 release.

.gitignore

@@ -58,8 +58,9 @@ terraform.rc
 logan/util/*
 
 # helm-chart
-charts/oci-onm/Chart.lock
-charts/oci-onm/charts/
+charts/tests/
+charts/**/Chart.lock
+charts/**/charts/
 
 # zip artifacts
-releases/
+releases/

CHANGELOG.md

@@ -1,5 +1,16 @@
 # Change Log
 
+## 2023-11-30
+### Added
+- Added resources information that got created through RM Stack to Stack output.
+- Added new auto created policy for Kubernetes Objects discovery (for future release(s) use). 
+- Added new helm variable for cluster's EntityId (ociLAClusterEntityID) (for future release(s) use). 
+### Changed
+- RM Stack is changed to use remote [helm repo](https://oracle-quickstart.github.io/oci-kubernetes-monitoring), instead of a local copy of helm chart source. 
+- RM Stack is modified to skip recreation of Management Agent Key if the Key already created by Stack. 
+### Breaking Changes
+-  Removed ociLAEntityID input variable of Logan chart. This was an optional and its use-case was not defined so far. Hence, it shouldn't  be a breaking change in general but still documenting for the reference. 
+
 ## 2023-11-07
 ### Added
 - Control Plane Logs Collection for OCNE and Standalone Kubernetes Clusters.

charts/logan/Chart.yaml

@@ -5,7 +5,7 @@ apiVersion: v2
 name: oci-onm-logan
 description: Charts for sending Kubernetes platform logs, compute logs, and Kubernetes Objects information to OCI Logging Analytics.
 type: application
-version: 3.1.1
+version: 3.2.0
 appVersion: "3.0.0"
 
 dependencies:

charts/logan/README.md

@@ -15,7 +15,7 @@ Charts for sending Kubernetes platform logs, compute logs, and Kubernetes Object
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | authtype | string | `"InstancePrincipal"` | Allowed values: InstancePrincipal, config |
-| extraEnv | list | `[]` | Logging Analytics OCID for OKE Cluster ociLAEntityID: Logging Analytics additional metadata. Use this to tag all the collected logs with one or more key:value pairs. Key must be a valid field in Logging Analytics metadata: "Client Host Region": "PCT" "Environment": "Production" "Third key": "Third Value" @param extra environment variables. Example   name: ENV_VARIABLE_NAME   value: ENV_VARIABLE_VALUE |
+| extraEnv | list | `[]` | Use this to tag all the collected logs with one or more key:value pairs. Key must be a valid field in Logging Analytics metadata: "Client Host Region": "PCT" "Environment": "Production" "Third key": "Third Value" @param extra environment variables. Example   name: ENV_VARIABLE_NAME   value: ENV_VARIABLE_VALUE |
 | extraVolumeMounts | list | `[]` | @param extraVolumeMounts Mount extra volume(s). Example:   - name: tmpDir     mountPath: /tmp |
 | extraVolumes | list | `[]` | @param extraVolumes Extra volumes. Example:   - name: tmpDir     hostPath:         path: /tmp log |
 | fluentd.baseDir | string | `"/var/log"` | Base directory on the node (with read write permission) for storing fluentd plugins related data. |
@@ -60,7 +60,7 @@ Charts for sending Kubernetes platform logs, compute logs, and Kubernetes Object
 | global.resourceNamePrefix | string | `"oci-onm"` | Resource names prefix used, where allowed. |
 | image.imagePullPolicy | string | `"Always"` | Image pull policy |
 | image.imagePullSecrets | string | `nil` |  |
-| image.url | string | `"container-registry.oracle.com/oci_observability_management/oci-la-fluentd-collector:1.0.0"` | Replace this value with actual docker image url |
+| image.url | string | `"container-registry.oracle.com/oci_observability_management/oci-la-fluentd-collector:1.1.0"` | Replace this value with actual docker image url |
 | kubernetesClusterID | string | `nil` | OKE Cluster OCID e.g. ocid1.cluster.oc1.phx.aaaaaaaahhbadf3rxa62faaeixanvr7vftmkg6hupycbf4qszctf2wbmqqxq |
 | kubernetesClusterName | string | `nil` | Kubernetes Cluster name. Need not be the OKE Cluster display name. e.g. production-cluster |
 | namespace | string | `"{{ .Values.global.namespace }}"` | Kubernetes Namespace for deploying monitoring resources deployed by this chart. |

charts/logan/templates/logs-configmap.yaml

@@ -102,11 +102,6 @@ data:
        {{- end }}
        oci_la_log_path "${record['tailed_path']}"
        oci_la_log_source_name "{{ $logDefinition.ociLALogSourceName | required (printf "fluentd.kubernetesSystem.logs.%s.ociLALogSourceName is required" $name) }}"
-       {{- if $logDefinition.ociLAEntityID }}
-       oci_la_entity_id "{{ $logDefinition.ociLAEntityID }}"
-       {{- else }}
-       oci_la_entity_id "{{ $.Values.fluentd.kubernetesSystem.ociLAEntityID | default $.Values.ociLAEntityID }}"
-       {{- end }}
        {{- if $logDefinition.ociLALogSet }}
        oci_la_log_set "{{ $logDefinition.ociLALogSet }}"
        {{- else }}
@@ -207,11 +202,6 @@ data:
        {{- end }}
        oci_la_log_path "${record['tailed_path']}"
        oci_la_log_source_name "{{ $logDefinition.ociLALogSourceName | required (printf "fluentd.linuxSystem.logs.%s.ociLALogSourceName is required" $name) }}"
-       {{- if $logDefinition.ociLAEntityID }}
-       oci_la_entity_id "{{ $logDefinition.ociLAEntityID }}"
-       {{- else }}
-       oci_la_entity_id "{{ $.Values.fluentd.linuxSystem.ociLAEntityID | default $.Values.ociLAEntityID }}"
-       {{- end }}
        {{- if $logDefinition.ociLALogSet }}
        oci_la_log_set "{{ $logDefinition.ociLALogSet }}"
        {{- else }}
@@ -296,11 +286,6 @@ data:
        {{- end }}
        oci_la_log_path "${record['tailed_path']}"
        oci_la_log_source_name "{{ required "fluentd.linuxSystem.logs.kubeletlog.ociLALogSourceName is required" $.Values.fluentd.linuxSystem.logs.kubeletlog.ociLALogSourceName }}"
-       {{- if $.Values.fluentd.linuxSystem.logs.kubeletlog.ociLAEntityID }}
-       oci_la_entity_id "{{ $.Values.fluentd.linuxSystem.logs.kubeletlog.ociLAEntityID }}"
-       {{- else }}
-       oci_la_entity_id "{{ $.Values.fluentd.linuxSystem.ociLAEntityID | default $.Values.ociLAEntityID }}"
-       {{- end }}
        {{- if $.Values.fluentd.linuxSystem.logs.kubeletlog.ociLALogSet }}
        oci_la_log_set "{{ $.Values.fluentd.linuxSystem.logs.kubeletlog.ociLALogSet }}"
        {{- else }}
@@ -331,11 +316,6 @@ data:
        {{- end }}
        oci_la_log_path "${record['tailed_path']}"
        oci_la_log_source_name "{{ required "fluentd.linuxSystem.logs.syslog.ociLALogSourceName is required" $.Values.fluentd.linuxSystem.logs.syslog.ociLALogSourceName }}"
-       {{- if $.Values.fluentd.linuxSystem.logs.syslog.ociLAEntityID }}
-       oci_la_entity_id "{{ $.Values.fluentd.linuxSystem.logs.syslog.ociLAEntityID }}"
-       {{- else }}
-       oci_la_entity_id "{{ $.Values.fluentd.linuxSystem.ociLAEntityID | default $.Values.ociLAEntityID }}"
-       {{- end }}
        {{- if $.Values.fluentd.linuxSystem.logs.syslog.ociLALogSet }}
        oci_la_log_set "{{ $.Values.fluentd.linuxSystem.logs.syslog.ociLALogSet }}"
        {{- else }}
@@ -410,7 +390,6 @@ data:
        {{- end }}
        oci_la_log_path "${record['tailed_path']}"
        oci_la_log_source_name "{{ $logDefinition.ociLALogSourceName | required (printf "fluentd.customLogs.%s.ociLALogSourceName is required" $name) }}"
-       oci_la_entity_id "{{ $logDefinition.ociLAEntityID | default $.Values.ociLAEntityID }}"
        oci_la_log_set "{{ $logDefinition.ociLALogSet | default $.Values.ociLALogSet }}"
        {{- if and (ne "false" ($logDefinition.isContainerLog | toString)) (eq $runtime "docker") }}
        message "${record['log']}"
@@ -517,7 +496,6 @@ data:
        oci_la_log_group_id ${record.dig("kubernetes", "annotations", "oracle.com/oci_la_log_group_id") ? record.dig("kubernetes", "annotations", "oracle.com/oci_la_log_group_id") : "{{ $.Values.fluentd.genericContainerLogs.ociLALogGroupID | default $.Values.ociLALogGroupID }}"}
        oci_la_log_path "${record['tailed_path']}"
        oci_la_log_source_name ${record.dig("kubernetes", "annotations", "oracle.com/oci_la_log_source_name") ? record.dig("kubernetes", "annotations", "oracle.com/oci_la_log_source_name") : "{{ $.Values.fluentd.genericContainerLogs.ociLALogSourceName | default "Kubernetes Container Generic Logs" }}"}
-       oci_la_entity_id ${record.dig("kubernetes", "annotations", "oracle.com/oci_la_entity_id") ? record.dig("kubernetes", "annotations", "oracle.com/oci_la_entity_id") : "{{ $.Values.fluentd.genericContainerLogs.ociLAEntityID | default $.Values.ociLAEntityID }}"}
        oci_la_log_set ${record.dig("kubernetes", "annotations", "oracle.com/oci_la_log_set") ? record.dig("kubernetes", "annotations", "oracle.com/oci_la_log_set") : "{{ $.Values.fluentd.genericContainerLogs.ociLALogSet | default $.Values.ociLALogSet }}"}
        {{- if eq $runtime "docker" }}
        message "${record['log']}"

charts/logan/templates/objects-configmap.yaml

@@ -100,7 +100,6 @@ data:
         oci_la_metadata ${{"{{"}}"Kubernetes Cluster Name":"{{ $kubernetesClusterName }}", "Kubernetes Cluster ID": "{{ $kubernetesClusterId }}" {{- range $k, $v := .Values.metadata }},{{ $k | quote }}: {{ $v | quote -}} {{- end }}{{"}}"}}
         {{- end }}
         oci_la_log_group_id "{{ .Values.fluentd.kubernetesObjects.ociLALogGroupID | default .Values.ociLALogGroupID }}"
-        oci_la_entity_id "{{ .Values.fluentd.kubernetesObjects.ociLAEntityID | default .Values.ociLAEntityID }}"
         oci_la_log_set "{{ .Values.fluentd.kubernetesObjects.ociLALogSet | default .Values.ociLALogSet }}"
         oci_la_log_path ${tag}
         oci_la_log_source_name "Kubernetes Object Logs"

charts/logan/values.yaml

@@ -67,14 +67,15 @@ kubernetesClusterID:
 # e.g. production-cluster
 kubernetesClusterName:
 
+# -- Kubernetes Cluster Entity OCID.
+# e.g. ocid1.loganalyticsentity.oc1.phx.amaaaaaabulluiqabqeq4delvhdlmd7aqcjrdla57n2szsxyz7pfdvnhwuua
+ociLAClusterEntityID:
+
 # -- Kubernetes Security Context privileged flag
 # Default: 'false'. This is not a required for OKE clusters.
 # In Kubernetes environments where SELinux mode is enforced, set this flag to 'true' to allow fluentd pods to access log files.
 privileged: false
 
-# -- Logging Analytics OCID for OKE Cluster
-#ociLAEntityID:
-
 # Logging Analytics additional metadata. Use this to tag all the collected logs with one or more key:value pairs.
 # Key must be a valid field in Logging Analytics
 #metadata:
@@ -236,7 +237,6 @@ fluentd:
          #"Client Host Region": "America"
          #"Environment": "Production"
          #"Third Key": "Third Value"
-      #ociLAEntityID:
       #encoding:
       # Worker number in case of multi process workers enabled. If not set when multi process workers enabled, then it defaults to 0.
       #worker:
@@ -254,7 +254,6 @@ fluentd:
                #"Client Host Region": "America"
                #"Environment": "Production"
                #"Third Key": "Third Value"
-            #ociLAEntityID:
             #ociLALogGroupID:
             #encoding:
             # Worker number in case of multi process workers enabled. If not set when multi process workers enabled, then it defaults to 0.
@@ -358,7 +357,6 @@ fluentd:
          #"Client Host Region": "America"
          #"Environment": "Production"
          #"Third Key": "Third Value"
-      #ociLAEntityID:
       #encoding:
       # Worker number in case of multi process workers enabled. If not set when multi process workers enabled, then it defaults to 0.
       #worker:
@@ -468,7 +466,6 @@ fluentd:
             #"Client Host Region": "America"
             #"Environment": "Production"
             #"Third Key": "Third Value"
-         #ociLAEntityID:
          #encoding:
          # Worker number in case of multi process workers enabled. If not set when multi process workers enabled, then it defaults to 0.
          #worker:
@@ -496,7 +493,6 @@ fluentd:
          #"Client Host Region": "America"
          #"Environment": "Production"
          #"Third Key": "Third Value"
-      #ociLAEntityID:
       #ociLALogGroupID:
       objectsList:
          nodes:

charts/oci-onm/Chart.yaml

@@ -18,7 +18,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 3.1.1
+version: 3.2.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
@@ -32,7 +32,7 @@ dependencies:
   repository: "file://../common"
   condition: oci-onm-common.enabled
 - name: oci-onm-logan
-  version: "3.1.1"
+  version: "3.2.0"
   repository: "file://../logan"
   condition: oci-onm-logan.enabled
 - name: oci-onm-mgmt-agent

charts/oci-onm/README.md

@@ -25,7 +25,7 @@ Helm chart for collecting Kubernetes logs & objects and metrics using Fluentd an
 | oci-onm-common.namespace | string | `"{{ .Values.global.namespace }}"` | Kubernetes Namespace in which the serviceaccount to be created. |
 | oci-onm-common.resourceNamePrefix | string | `"{{ .Values.global.resourceNamePrefix }}"` | Prefix to be attached to resources created through this chart. Not all resources may have this prefix. |
 | oci-onm-common.serviceAccount | string | `"{{ .Values.global.resourceNamePrefix }}"` | Name of the Kubernetes ServiceAccount |
-| oci-onm-logan.image.url | string | `"container-registry.oracle.com/oci_observability_management/oci-la-fluentd-collector:1.0.0"` |  |
+| oci-onm-logan.image.url | string | `"container-registry.oracle.com/oci_observability_management/oci-la-fluentd-collector:1.1.0"` |  |
 | oci-onm-logan.kubernetesClusterID | string | `"{{ .Values.global.kubernetesClusterID }}"` |  |
 | oci-onm-logan.kubernetesClusterName | string | `"{{ .Values.global.kubernetesClusterName }}"` |  |
 | oci-onm-logan.namespace | string | `"{{ .Values.global.namespace }}"` |  |

charts/oci-onm/values.yaml

@@ -52,4 +52,4 @@ oci-onm-mgmt-agent:
       # Replace this value with actual docker image URL for Management Agent
       url: container-registry.oracle.com/oci_observability_management/oci-management-agent:1.0.0
       # Image secrets to use for pulling container image (base64 encoded content of ~/.docker/config.json file)
-      secret:
+      secret:

terraform/modules/helm/helm.tf

@@ -1,29 +1,23 @@
 # Copyright (c) 2023, Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v1.0 as shown at https://oss.oracle.com/licenses/upl.
 
-data "oci_containerengine_clusters" "oke_clusters_list" {
-  compartment_id = var.oke_compartment_ocid
-}
-
 locals {
-  oke_clusters_list = data.oci_containerengine_clusters.oke_clusters_list.clusters
-  oke_cluster_name  = [for c in local.oke_clusters_list : c.name if c.id == var.oke_cluster_ocid][0]
+  helm_repo_url   = "https://oracle-quickstart.github.io/oci-kubernetes-monitoring"
+  helm_repo_chart = "oci-onm"
 
   helm_inputs = {
     # global
     "global.namespace"             = var.deploy_mushop_config ? "livelab-test" : var.kubernetes_namespace
     "global.kubernetesClusterID"   = var.oke_cluster_ocid
-    "global.kubernetesClusterName" = local.oke_cluster_name
+    "global.kubernetesClusterName" = var.oke_cluster_name
 
     # oci-onm-logan
     "oci-onm-logan.ociLANamespace"  = var.oci_la_namespace
     "oci-onm-logan.ociLALogGroupID" = var.oci_la_logGroup_id
-    "oci-onm-logan.image.url"       = var.logan_container_image_url
     "oci-onm-logan.fluentd.baseDir" = var.fluentd_baseDir_path
 
     #oci-onm-mgmt-agent
     "oci-onm-mgmt-agent.mgmtagent.installKeyFileContent" = var.mgmt_agent_install_key_content
-    "oci-onm-mgmt-agent.mgmtagent.image.url"             = var.mgmt_agent_container_image_url
     "oci-onm-mgmt-agent.deployMetricServer"              = var.opt_deploy_metric_server
   }
 
@@ -32,12 +26,13 @@ locals {
     "createServiceAccount" = false
     "serviceAccount"       = var.livelab_service_account
   }
-
 }
 
+# Create helm release
 resource "helm_release" "oci-kubernetes-monitoring" {
   name              = "oci-kubernetes-monitoring"
-  chart             = var.helm_abs_path
+  repository        = var.use_local_helm_chart ? null : local.helm_repo_url
+  chart             = var.use_local_helm_chart ? var.helm_abs_path : local.helm_repo_chart
   wait              = true
   dependency_update = true
   atomic            = true
@@ -52,6 +47,14 @@ resource "helm_release" "oci-kubernetes-monitoring" {
     }
   }
 
+  dynamic "set" {
+    for_each = var.oke_cluster_entity_ocid == "DEFAULT" ? [] : ["run_once"]
+    content {
+      name  = "oci-onm-logan.ociLAClusterEntityID"
+      value = var.oke_cluster_entity_ocid
+    }
+  }
+
   dynamic "set" {
     for_each = var.deploy_mushop_config ? local.mushop_helm_inputs : {}
     content {
@@ -60,12 +63,16 @@ resource "helm_release" "oci-kubernetes-monitoring" {
     }
   }
 
-  count = var.generate_helm_template ? 0 : 1
+  count = var.install_helm ? 1 : 0
 }
 
+# Create helm template
 data "helm_template" "oci-kubernetes-monitoring" {
-  name              = "oci-kubernetes-monitoring"
-  chart             = var.helm_abs_path
+  name = "oci-kubernetes-monitoring"
+  # default behaviour is to use remote helm repo | var.use_local_helm_chart = false
+  # the option to use local helm chart is for development purpose only
+  repository        = var.use_local_helm_chart ? null : local.helm_repo_url
+  chart             = var.use_local_helm_chart ? var.helm_abs_path : local.helm_repo_chart
   dependency_update = true
 
   values = var.deploy_mushop_config ? ["${file("${path.module}/mushop_values.yaml")}"] : null
@@ -78,6 +85,14 @@ data "helm_template" "oci-kubernetes-monitoring" {
     }
   }
 
+  dynamic "set" {
+    for_each = var.oke_cluster_entity_ocid == "DEFAULT" ? [] : ["run_once"]
+    content {
+      name  = "oci-onm-logan.ociLAClusterEntityID"
+      value = var.oke_cluster_entity_ocid
+    }
+  }
+
   dynamic "set" {
     for_each = var.deploy_mushop_config ? local.mushop_helm_inputs : {}
     content {
@@ -87,11 +102,4 @@ data "helm_template" "oci-kubernetes-monitoring" {
   }
 
   count = var.generate_helm_template ? 1 : 0
-}
-
-# Helm release artifacts for local testing and validation. Not used by helm resource.
-resource "local_file" "helm_release" {
-  content  = tostring(data.helm_template.oci-kubernetes-monitoring[0].manifest)
-  filename = "${path.module}/local/helmrelease.yaml"
-  count    = var.generate_helm_template ? 1 : 0
 }