diff --git a/canary-checker/docs/reference/1-gcs-database-backup.mdx b/canary-checker/docs/reference/3-gcs-database-backup.mdx similarity index 93% rename from canary-checker/docs/reference/1-gcs-database-backup.mdx rename to canary-checker/docs/reference/3-gcs-database-backup.mdx index 83900cb2..b0afa3bb 100644 --- a/canary-checker/docs/reference/1-gcs-database-backup.mdx +++ b/canary-checker/docs/reference/3-gcs-database-backup.mdx @@ -1,5 +1,6 @@ --- title: GCS Database Backup +sidebar_class_name: beta --- # Google Cloud SQL Backups @@ -42,10 +43,10 @@ Durations are strings with an optional fraction and unit e.g. `300ms`, `1.5h` or | Field | Description | Scheme | Required | | -------------- | ----------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------ | -------- | -| `project` | GCP project\ name | _string_ | Yes | +| `project` | GCP project name | _string_ | Yes | | `instance` | Google CloudSQL instance name | _string_ | Yes | | **Connection** | | | | -| `connection` | Path of an existing connection e.g. `connection://aws/instance`/. Mutually exclusive with `credentials`
| _Connections_ | | +| `connection` | Path of an existing connection e.g. `connection://gcp`/. Mutually exclusive with `credentials`
| _Connections_ | | | `credentials` | GCP Access Token File. Mutually exclusive with `connection` | _EnvVar_ | Yes | ### Connecting to GCP @@ -72,7 +73,7 @@ spec: 3. `accessKey` and `secretKey` _EnvVar_ with the credentials stored in a secret. -```yaml title="aws.yaml" +```yaml title="gcs-database-backup.yaml" apiVersion: canaries.flanksource.com/v1 kind: Canary metadata: diff --git a/canary-checker/docs/scripting/gotemplate.md b/canary-checker/docs/scripting/gotemplate.md index 62498010..901df649 100644 --- a/canary-checker/docs/scripting/gotemplate.md +++ b/canary-checker/docs/scripting/gotemplate.md @@ -31,7 +31,7 @@ To send `{{ .secret }}` as it is do this {{`{{ .secret }}`}} ``` -## Encoding +## base64 ### Encode @@ -46,8 +46,6 @@ Encode data as a Base64 string. Specifically, this uses the standard Base64 enco Decode a Base64 string. This supports both standard ([RFC4648 §4](https://tools.ietf.org/html/rfc4648#section-4)) and URL-safe ([RFC4648 §5](https://tools.ietf.org/html/rfc4648#section-5)) encodings. -This function outputs the data as a string, so it may not be appropriate for decoding binary data. Use [`base64.DecodeBytes`](#base64.DecodeBytes) -for binary data. ```go {{ base64.Decode "aGVsbG8gd29ybGQ=" }} // hello world @@ -56,7 +54,6 @@ for binary data. :::note - For the functions that return an array, a Go `[]interface{}` is returned, regardless of whether or not the input was a different type. ::: @@ -417,7 +414,7 @@ The value in decimal is {{ $val }} ### ParseFloat -_**Note:**_ See [`conv.ToFloat`](#tofloat) instead for a simpler and more flexible variant of this function. +_**Note:**_ See [`conv.ToFloat`](#tofloat64) instead for a simpler and more flexible variant of this function. Parses a string as an float64 for later use. Equivalent to [strconv.ParseFloat](https://golang.org/pkg/strconv/#ParseFloat) @@ -1803,7 +1800,7 @@ Parses a timestamp defined by the given layout. This wraps [`time.Parse`](https: A number of pre-defined layouts are provided as constants, defined [here](https://golang.org/pkg/time/#pkg-constants). -Just like [`time.Now`](#time-now), this is usually used in conjunction with other functions. +Just like [`time.Now`](#now), this is usually used in conjunction with other functions. _Note: In the absence of a time zone indicator, `time.Parse` returns a time in UTC._ diff --git a/common/src/components/Fields.jsx b/common/src/components/Fields.jsx index 02a8a5ce..e8b5c7b6 100644 --- a/common/src/components/Fields.jsx +++ b/common/src/components/Fields.jsx @@ -9,7 +9,12 @@ const schemes = { "EnvVar": "[EnvVar](/reference/env-var)", "[]EnvVar": "[[]EnvVar](/reference/env-var)", "CEL": "[CEL](/reference/scripting/cel)", + "Javascript": "[Javascript](/reference/scripting/javascript)", + "Gotemplate": "[Go Template](/reference/scripting/gotemplate)", "Duration": "[Duration](/reference/types#duration)", + "JSONPathOrString": "`string` or [JSONPath](https://jsonpath.com/)", + "[]JSONPathOrString": "`[]string` or [[]JSONPath](https://jsonpath.com/)", + "JSONPath": "[JSONPath](https://jsonpath.com/)", "Size": "[Size](/reference/types#size)", "Agent": "[Agent](/reference/types#agent)", "ResourceSelector": "[ResourceSelector](/reference/resource-selector)", @@ -17,9 +22,9 @@ const schemes = { "string": "`string`", "Icon": "[Icon](/reference/types#icon)", "bool": "`boolean`", - "NotificationURL": "[url](/reference/notifications/channels)", - "NotificationConnection": "[Connection](/reference/connection)", - "NotificationProperties": "[map[string]string](/reference/notifications/channels#properties)", + "NotificationURL": "[Notification](/reference/notifications)", + "NotificationConnection": "[Connection](/reference/connections)", + "NotificationProperties": "[map[string]string](/reference/notifications#properties)", } export default function Fields({ common = [], rows = [], oneOf, anyOf, connection }) { @@ -52,7 +57,7 @@ export default function Fields({ common = [], rows = [], oneOf, anyOf, connectio rows = rows.concat([ { field: oss ? null : "connection", - description: "The connection to use, mutually exclusive with `username` and `password`", + description: "The connection url to use, mutually exclusive with `username` and `password`", scheme: "Connection", }, { @@ -73,11 +78,38 @@ export default function Fields({ common = [], rows = [], oneOf, anyOf, connectio scheme: "EnvVar", } ]) + } + + if (connection == "sql") { + rows = rows.concat([ + { + field: oss ? null : "connection", + description: "The connection url or name to use`", + scheme: "Connection", + }, + { + field: !oss && rows.filter(i => i.field == "url").length == 0 ? "url" : null, + description: "If `connection` is specified and it also includes a `url`, this field will take precedence", + scheme: "string", + }, + { + field: oss && rows.filter(i => i.field == "url").length == 0 ? "url" : null, + scheme: "string", + }, + { + field: "auth.username", + scheme: "EnvVar", + }, + { + field: "auth.password", + scheme: "EnvVar", + } + ]) } else if (connection == "git") { rows = rows.concat([ { field: oss ? null : "connection", - description: "The connection to use, mutually exclusive with `username` and `password`", + description: "The connection url to use, mutually exclusive with `username` and `password`", scheme: "Connection", }, { @@ -106,7 +138,7 @@ export default function Fields({ common = [], rows = [], oneOf, anyOf, connectio rows = rows.concat([ { field: oss ? null : "connection", - description: "The connection to use, mutually exclusive with `accessKey` and `secretKey`", + description: "The connection url to use, mutually exclusive with `accessKey` and `secretKey`", scheme: "Connection", }, { @@ -136,7 +168,7 @@ export default function Fields({ common = [], rows = [], oneOf, anyOf, connectio rows = rows.concat([ { field: oss ? null : "connection", - description: "The connection to use, mutually exclusive with `credentials`", + description: "The connection url to use, mutually exclusive with `credentials`", scheme: "Connection", }, { @@ -149,7 +181,7 @@ export default function Fields({ common = [], rows = [], oneOf, anyOf, connectio rows = rows.concat([ { field: oss ? null : "connection", - description: "The connection to use, mutually exclusive with `host` and `port`", + description: "The connection url to use, mutually exclusive with `host` and `port`", scheme: "Connection", }, { @@ -175,7 +207,7 @@ export default function Fields({ common = [], rows = [], oneOf, anyOf, connectio rows = rows.concat([ { field: oss ? null : "connection", - description: "The connection to use, mutually exclusive with `host`, `share`, and `credentials`", + description: "The connection url to use, mutually exclusive with `host`, `share`, and `credentials`", scheme: "Connection", }, { @@ -212,7 +244,7 @@ export default function Fields({ common = [], rows = [], oneOf, anyOf, connectio rows = rows.concat([ { field: oss ? null : "connection", - description: "The connection to use, mutually exclusive with `tenantId`, `subscriptionId`, `clientId`, and `clientSecret`", + description: "The connection url to use, mutually exclusive with `tenantId`, `subscriptionId`, `clientId`, and `clientSecret`", scheme: "Connection", }, { @@ -273,11 +305,17 @@ export default function Fields({ common = [], rows = [], oneOf, anyOf, connectio } { - oneOf && + oneOf && oneOf.length == 2 &&

You must specify {oneOf[0]} or {oneOf[1]} but not both

} + { + oneOf && oneOf.length == 3 && +
+

You must specify one of {oneOf[0]}, {oneOf[1]} or {oneOf[2]}

+
+ } ) } diff --git a/common/src/components/Scraper.jsx b/common/src/components/Scraper.jsx new file mode 100644 index 00000000..52f89370 --- /dev/null +++ b/common/src/components/Scraper.jsx @@ -0,0 +1,93 @@ +import React from 'react' +import Admonition from '@theme/Admonition' +import useDocusaurusContext from '@docusaurus/useDocusaurusContext' +import Link from '@docusaurus/Link' +import ReactMarkdown from 'react-markdown' +import clsx from 'clsx' +import Fields from './Fields' + +const customScraperFields = [ + { + "field": "id", + "description": "A deterministic or natural id for the resource", + "scheme": "JSONPathOrString", + "required": true + }, + { + "field": "name", + default: "`id`", + "scheme": "JSONPathOrString" + }, + { + "field": "items", + "description": "Extract multiple config items from this array", + "scheme": "JSONPath" + }, + { + "field": "type", + "description": "e.g. `File::Host`, `File::Tomcat`, `File::Pom`", + "scheme": "JSONPathOrString", + "required": "true" + }, + { + "field": "class", + "scheme": "JSONPathOrString" + }, + { + "field": "format", + "description": "Format of config item e.g. `xml`, `properties`", + "default": "`JSON`", + "scheme": "string" + }, + { + "field": "timestampFormat", + "description": "Format to parse timestamps in `createFields` and `deletedFields`", + "default": "RFC3339", + "scheme": "Go time format" + }, + { + "field": "createFields", + "description": "Identify the created time for a resource (if different to scrape time). If multiple fields are specified, the first non-empty value will be used", + "scheme": "[]JSONPathOrString" + }, + { + "field": "deleteFields", + "description": "Identify when a config item was deleted. If multiple fields are specified, the first non-empty value will be used", + "scheme": "[]JSONPathOrString" + }, +] + +const commonsRows = [ + + { + "field": "ignore", + "description": "Fields to ignore and strip out of the config", + "scheme": "[]JSONPath" + }, + { + "field": "properties", + "description": "Custom templatable properties for the scraped config items.", + "scheme": "[`[]ConfigProperty`](/reference/config-db/properties)" + }, + { + "field": "tags", + "description": "Additional tags ", + "scheme": "map[string]string" + }, +] + +export function Scraper({ name, edition, rows, ...props }) { + + const { siteConfig } = useDocusaurusContext() + + + return + +} + +export function CustomScraper({ name, edition, rows, ...props }) { + + const { siteConfig } = useDocusaurusContext() + return + +} diff --git a/common/src/theme/MDXComponents/index.js b/common/src/theme/MDXComponents/index.js index 6fad3d78..7382ca8e 100644 --- a/common/src/theme/MDXComponents/index.js +++ b/common/src/theme/MDXComponents/index.js @@ -29,6 +29,9 @@ import { FullImage } from '@site/src/components/Badges' import { CommonLink } from '@site/src/components/Link' import { ConfigTransform } from '@site/src/components/ConfigTransform' import { CustomField } from '@site/src/components/Badges' +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import { Scraper, CustomScraper } from '@site/src/components/Scraper' const MDXComponents = { Icon: Icon, @@ -48,6 +51,10 @@ const MDXComponents = { head: MDXHead, code: MDXCode, a: MDXA, + Tabs: Tabs, + Scraper: Scraper, + CustomScraper: CustomScraper, + TabItem: TabItem, pre: MDXPre, details: Details, Screenshot: Screenshot, diff --git a/mission-control/docs/config-db/concepts/transform.md b/mission-control/docs/config-db/concepts/transform.md index 209da806..fb02d63a 100644 --- a/mission-control/docs/config-db/concepts/transform.md +++ b/mission-control/docs/config-db/concepts/transform.md @@ -2,7 +2,7 @@ title: Transform --- -Transformations allows you to transform the scraped config items before they are saved, common use cases include: +Transformations allows you to modify scraped config items before they are saved, common use cases include: - Linking configuration items - Removing extraneous or overly verbose fields @@ -49,7 +49,8 @@ spec: ### Masking -Masking allows replacing sensitive fields with a hash or static string. +Masking replaces sensitive fields with a hash or static string. A hash can be used to determine if a field changed without revealing original values. + ```yaml title="file-mask-scraper.yaml" apiVersion: configs.flanksource.com/v1 @@ -88,10 +89,10 @@ Masks are applied in the order they are specified in the configuration file. ## Changes ### Exclusions +Some configurations can change frequently and may not be relevant. For example, a `Kubernetes::Node` configuration changes often as pods launched and stopped. From the node's perspective, these image changes are irrelevant. -Some configs can have changes in high volume that may not be relevant. Example: A kubernetes Node config changes frequently as the pods in the cluster update their images. From the node's perspective the image changes are irrelevant. +This is where exclusions become useful. Here's an example that ignores all image changes in a `Kubernetes::Node` configuration: -This is where exclusions can become handy. Here's an example that ignore all image changes in a kubernetes node config: ```yaml title="kubernetes-scraper.yaml" apiVersion: configs.flanksource.com/v1 @@ -111,7 +112,7 @@ spec: ### Mapping -When you encounter a diff change, unlike an event based change, it can sometimes appear cryptic. The summary of the change may not immediately indicate what the change is about. For example, the change 'status.images' might not be self-explanatory. To address this issue, we can assign types to these diff changes using mapping. +When you encounter a diff change, unlike an event-based change, it can sometimes appear unclear. The summary of the change may not immediately indicate its purpose. For example, the change 'status.images' might not be self-explanatory. To clarify this, you can assign types to these diff changes using mapping. ```yaml title="kubernetes-scraper.yaml" apiVersion: configs.flanksource.com/v1 @@ -142,11 +143,11 @@ spec: | `filter` | Selects changes to apply the mapping | CEL with [Change Context](/reference/config-db/changes) | | `action` | What action to take on the change, if `delete` then the corresponding config item is marked as deleted | `delete` or `ignore` | | `type` | New change type | `string` | -| `summary` | New summary of the change | [Go Template](/reference/scripting/template) | +| `summary` | New summary of the change | Go Template with [Change Context](/reference/config-db/changes) | ## Scripting -Scripting allows you to modify the scraped configuration using CEL before saving it to the database. This is useful for data normalization, default value population, sensitive field masking etc. +Scripting modifies the scraped configuration using CEL before saving it to the database. This process is beneficial for data normalization, default value population, and sensitive field masking. | Field | Description | Scheme | Context | | ------ | ----------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ | diff --git a/mission-control/docs/config-db/index.mdx b/mission-control/docs/config-db/index.mdx index 8f84c620..48e8ad00 100644 --- a/mission-control/docs/config-db/index.mdx +++ b/mission-control/docs/config-db/index.mdx @@ -1,6 +1,7 @@ --- slug: /config-db hide_title: true +title: Overview hide_table_of_contents: true pagination_next: playbooks/index pagination_prev: @@ -34,7 +35,7 @@ Under the hood the catalog is powered by [github.com/flanksource/config-db](http The catalog is comprised of: * **Config Items** are individual resources e.g. `Pod`, `EBS`, `IAM Role`, `postgres.conf` scraped from external sources like [AWS](/config-db/scrapers/aws), [Kubernetes](/config-db/scrapers/kubernetes), [Azure](/config-db/scrapers/azure), [Azure Devops](/config-db/scrapers/azure-devops) , [Github](/config-db/scrapers/github) -* **Changes** recorded against config items either through automatic change detection (diffs) or from sources like [Cloud Trail](/config-db/scrapers/aws#cloud-trail) or [Kubernetes Events](/config-db/scrapers/kubernetes#events) +* **Changes** recorded against config items either through automatic change detection (diffs) or from sources like [Cloud Trail](/config-db/scrapers/aws#cloudtrail) or [Kubernetes Events](/config-db/scrapers/kubernetes#events) * **Insights** recorded against config items from external sources like [AWS Trusted Advisor](/config-db/scrapers/aws#trusted-advisor) or [Trivy](/config-db/scrapers/trivy) * **Relationships** between configuration items diff --git a/mission-control/docs/config-db/scrapers/azure-devops.mdx b/mission-control/docs/config-db/scrapers/azure-devops.mdx index 787727a2..5d3e7dcb 100644 --- a/mission-control/docs/config-db/scrapers/azure-devops.mdx +++ b/mission-control/docs/config-db/scrapers/azure-devops.mdx @@ -21,7 +21,7 @@ Each time the pipeline is run it will create a change for that configuration ite | `logLevel` | Specify the level of logging. | `string` | | | `schedule` | Specify the interval to scrape in cron format. Defaults to every 60 minutes. | `string` | | | `retention` | Settings for retaining changes, analysis and scraped items | [`Retention`](/config-db/concepts/retention) | | -| `azureDevops` | Specifies the list of Azure DevOps configurations to scrape. | [`[]AzureDevops`](#azuredevops-1) | | +| `azureDevops` | Specifies the list of Azure DevOps configurations to scrape. | [`[]AzureDevops`](#azuredevops) | | ### AzureDevops diff --git a/mission-control/docs/config-db/scrapers/custom/file.md b/mission-control/docs/config-db/scrapers/custom/file.md deleted file mode 100644 index 7d52c210..00000000 --- a/mission-control/docs/config-db/scrapers/custom/file.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -title: File ---- - -The `file` scraper is used to create config items from files in a local folder (or git). This can be used to track changes in files like `/etc/hosts` or `/etc/passwd`, or for service metadata stored in git. - -See [Kubernetes Files](./kubernetes-file) for scraping files inside running kubernetes pods. - -```yaml title='file-scraper.yaml' file=../../../../modules/config-db/fixtures/file-git.yaml -apiVersion: configs.flanksource.com/v1 -kind: ScrapeConfig -metadata: - name: file-scraper -spec: - file: - - type: $.Config.InstanceType - id: $.Config.InstanceId - path: - - config*.json - - test*.json -``` - - -## Scraper - -| Field | Description | Scheme | Required | -| ----------- | ---------------------------------------------------------------------------------- | -------------------------------------------- | -------- | -| `schedule` | Specify the interval to scrape in cron format. Defaults to every 60 minutes. | `string` | | -| `full` | Set to `true` to extract changes from scraped configurations. Defaults to `false`. | `bool` | | -| `retention` | Settings for retaining changes, analysis and scraped items | [`Retention`](/config-db/concepts/retention) | | -| `file` | Specifies the list of File configurations to scrape. | [`[]File`](#file) | | - -### File - -| Field | Description | Scheme | Required | -| ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------- | -------- | -| `id` | A static value or JSONPath expression to use as the ID for the resource. | `string` | `true` | -| `name` | A static value or JSONPath expression to use as the Name for the resource. Default value is the `id`. | `string` | | -| `items` | A JSONPath expression to use to extract individual items from the resource | `string` | | -| `type` | A static value or JSONPath expression to use as the type for the resource. | `string` | `true` | -| `class` | A static value or JSONPath expression to use as the config class. _(Defaults to `type`)_ | `string` | | -| `format` | Format of config item, defaults to JSON, available options are JSON | `string` | | -| `timestampFormat` | TimestampFormat is a Go time format string used to parse timestamps in createFields and DeletedFields. If not specified, the default is `RFC3339`. | `string` | | -| `createFields` | CreateFields is a list of JSONPath expression used to identify the created time of the config. If multiple fields are specified, the first non-empty value will be used | `[]string` | | -| `deleteFields` | DeleteFields is a JSONPath expression used to identify the deleted time of the config. If multiple fields are specified, the first non-empty value will be used | `[]string` | | -| `url` | Specify URL e.g github repository containing the configuration(s) | `string` | | -| `paths` | Specify paths to configuration(s) for scraping | `[]string` | `true` | -| `ignore` | Set configurations to ignore | `[]string` | | -| `properties` | Custom templatable properties for the scraped config items. | [`[]ConfigProperty`](/reference/config-db/properties) | | -| `tags` | set custom tags on the scraped config items | `map[string]string` | | -| `transform` | Field to transform result | [`Transform`](/config-db/concepts/transform) | | | | - - diff --git a/mission-control/docs/config-db/scrapers/custom/file.mdx b/mission-control/docs/config-db/scrapers/custom/file.mdx new file mode 100644 index 00000000..5d29e9b7 --- /dev/null +++ b/mission-control/docs/config-db/scrapers/custom/file.mdx @@ -0,0 +1,51 @@ +--- +title: File +--- + +The `file` scraper is used to create config items from files in a local folder (or git). This can be used to track changes in files like `/etc/hosts` or `/etc/passwd`, or for service metadata stored in git. + +See [Kubernetes Files](./kubernetes-file) for scraping files inside running kubernetes pods. + +```yaml title='file-scraper.yaml' file=../../../../modules/config-db/fixtures/file-git.yaml +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: file-scraper +spec: + file: + - type: $.Config.InstanceType + id: $.Config.InstanceId + path: + - config*.json + - test*.json +``` + + +## Scraper + +| Field | Description | Scheme | Required | +| ----------- | ---------------------------------------------------------------------------------- | -------------------------------------------- | -------- | +| `schedule` | Specify the interval to scrape in cron format. Defaults to every 60 minutes. | `string` | | +| `full` | Set to `true` to extract changes from scraped configurations. Defaults to `false`. | `bool` | | +| `retention` | Settings for retaining changes, analysis and scraped items | [`Retention`](/config-db/concepts/retention) | | +| `file` | Specifies the list of File configurations to scrape. | [`[]File`](#file) | | + +### File + + + + diff --git a/mission-control/docs/config-db/scrapers/custom/index.md b/mission-control/docs/config-db/scrapers/custom/index.md index 1bf5b2f9..4fcd82fa 100644 --- a/mission-control/docs/config-db/scrapers/custom/index.md +++ b/mission-control/docs/config-db/scrapers/custom/index.md @@ -63,15 +63,17 @@ When displaying the config, the UI will automatically convert the JSON data to Y ### XML / Properties -[**Custom**](./concepts/custom-scraper) scrapers can ingest non-JSON config which is represented as: +Non JSON files are stored as JSON using: ```yaml { "format": "xml", - "content": ".." + "content": ".." } ``` +Non JSON content can still be accessed in scripts using `config.content` + The UI will format and render XML appropriately. diff --git a/mission-control/docs/config-db/scrapers/custom/kubernetes-file.md b/mission-control/docs/config-db/scrapers/custom/kubernetes-file.md deleted file mode 100644 index da89d90d..00000000 --- a/mission-control/docs/config-db/scrapers/custom/kubernetes-file.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -title: Kubernetes File ---- - -# Kubernetes File - -The KubernetesFile config type is used to scrape configurations contained in your specified resource e.g Pod. - -```yaml title='k8s-file-scraper.yaml' -apiVersion: configs.flanksource.com/v1 -kind: ScrapeConfig -metadata: - name: k8s-file-scraper -spec: - kubernetesFile: - - selector: - namespace: demo - kind: Statefulset - name: postgresql - files: - - path: - - /var/lib/postgresql/data/pgdata/postgresql.conf - format: properties -``` - -## Scraper - -| Field | Description | Scheme | Required | -| ---------------- | ---------------------------------------------------------------------------------- | -------------------------------------------- | -------- | -| `logLevel` | Specify the level of logging. | `string` | | -| `schedule` | Specify the interval to scrape in cron format. Defaults to every 60 minutes. | `string` | | -| `full` | Set to `true` to extract changes from scraped configurations. Defaults to `false`. | `bool` | | -| `retention` | Settings for retaining changes, analysis and scraped items | [`Retention`](/config-db/concepts/retention) | | -| `kubernetesFile` | Specifies the list of Kubernetes File configurations to scrape. | [`[]KubernetesFile`](#kubernetesfile) | | - -### KubernetesFile - -| Field | Description | Scheme | Required | -| ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------- | -------- | -| `id` | A static value or JSONPath expression to use as the ID for the resource. | `string` | `true` | -| `name` | A static value or JSONPath expression to use as the Name for the resource. Default value is the `id`. | `string` | | -| `type` | A static value or JSONPath expression to use as the type for the resource. | `string` | `true` | -| `items` | A JSONPath expression to use to extract individual items from the resource | `string` | | -| `container` | Set container name | `string` | | -| `createFields` | CreateFields is a list of JSONPath expression used to identify the created time of the config. If multiple fields are specified, the first non-empty value will be used | `[]string` | | -| `deleteFields` | DeleteFields is a JSONPath expression used to identify the deleted time of the config. If multiple fields are specified, the first non-empty value will be used | `[]string` | | -| `files` | Specify path to file contained in Pod | [`[]File`](#file) | | -| `format` | Format of config item, defaults to JSON, available options are JSON | `string` | | -| `selector` | Specify Kubernetes resource for configuration based on `namespace`, `kind`, `name` and more. | [`ResourceSelector`](/reference/resource-selector) | `true` | -| `timestampFormat` | TimestampFormat is a Go time format string used to parse timestamps in createFields and DeletedFields. If not specified, the default is `RFC3339`. | `string` | | -| `transform` | Field to transform result | [`Transform`](/config-db/concepts/transform) | | -| `tags` | set custom tags on the scraped config items | `map[string]string` | | -| `properties` | Custom templatable properties for the scraped config items. | [`[]ConfigProperty`](/reference/config-db/properties) | | - -#### File - -| Field | Description | Scheme | Required | -| -------- | --------------------------------- | ---------- | -------- | -| `path` | Set path to file contained in Pod | `[]string` | | -| `format` | Specify format of file | `string` | | - diff --git a/mission-control/docs/config-db/scrapers/custom/kubernetes-file.mdx b/mission-control/docs/config-db/scrapers/custom/kubernetes-file.mdx new file mode 100644 index 00000000..7be66996 --- /dev/null +++ b/mission-control/docs/config-db/scrapers/custom/kubernetes-file.mdx @@ -0,0 +1,69 @@ +--- +title: Kubernetes File +--- + +# Kubernetes File + +The KubernetesFile config type is used to scrape configurations contained in your specified resource e.g Pod. + +```yaml title='k8s-file-scraper.yaml' +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: k8s-file-scraper +spec: + kubernetesFile: + - selector: + namespace: demo + kind: Statefulset + name: postgresql + files: + - path: + - /var/lib/postgresql/data/pgdata/postgresql.conf + format: properties +``` + +## Scraper + +| Field | Description | Scheme | Required | +| ---------------- | ---------------------------------------------------------------------------------- | -------------------------------------------- | -------- | +| `logLevel` | Specify the level of logging. | `string` | | +| `schedule` | Specify the interval to scrape in cron format. Defaults to every 60 minutes. | `string` | | +| `full` | Set to `true` to extract changes from scraped configurations. Defaults to `false`. | `bool` | | +| `retention` | Settings for retaining changes, analysis and scraped items | [`Retention`](/config-db/concepts/retention) | | +| `kubernetesFile` | Specifies the list of Kubernetes File configurations to scrape. | [`[]KubernetesFile`](#kubernetesfile) | | + + + +### KubernetesFile + + + + +### File + +| Field | Description | Scheme | Required | +| -------- | --------------------------------- | ---------- | -------- | +| `path` | Set path to file contained in Pod | `[]string` | | +| `format` | Specify format of file | `string` | | + diff --git a/mission-control/docs/config-db/scrapers/custom/sql.md b/mission-control/docs/config-db/scrapers/custom/sql.md deleted file mode 100644 index 0fb29aba..00000000 --- a/mission-control/docs/config-db/scrapers/custom/sql.md +++ /dev/null @@ -1,103 +0,0 @@ ---- -title: SQL ---- - -The SQL configuration scraper will execute a SQL query and then create a configuration item for each returned row. - -The example below creates a new `MSSQL::Database` configuration for each database on the sql server, and then creates a roles object container the SQL Server login to database role mapping. With change detection this will highlight when new users are added / modified / removed on an individual databases. - -```yaml title='sql-scraper.yaml' -apiVersion: configs.flanksource.com/v1 -kind: ScrapeConfig -metadata: - name: sql-scraper -spec: - sql: - - connection: 'sqlserver://localhost:1433?database=master' - auth: - username: - value: sa - password: - value: password - type: MSSQL::Database - id: $.name - - transform: - full: true # transform the entire configuration item, and not just the configuration data (row) - script: - javascript: |+ - var dbs = {} - for (var i = 0; i < config.rows.length; i++) { - var db = config.rows[i] - var name = db.DB - if (dbs[db.DB] == null) { - { - config: dbs[db.DB] = { - name: name, - roles: {} - }, - changes: { - - }, - analysis: { - - } - - } - } - dbs[name].roles[db.role] = db.name - } - JSON.stringify(_.values(dbs)) - - query: | - declare @mytable table ( - [DB] [nvarchar](128) NULL, - [name] [nvarchar](255) NOT NULL, - [role] [nvarchar](255) NOT NULL - ) - - - DECLARE @command varchar(1000) - SELECT @command = - 'USE ?; SELECT DB_NAME() as DB, DP1.name AS [user], - isnull (DP2.name, ''No members'') AS [role] - FROM sys.database_role_members AS DRM - RIGHT OUTER JOIN sys.database_principals AS DP1 - ON DRM.role_principal_id = DP1.principal_id - LEFT OUTER JOIN sys.database_principals AS DP2 - ON DRM.member_principal_id = DP2.principal_id - WHERE DP1.type = ''R'' and DP2.name is not null' - - insert into @mytable EXEC sp_MSforeachdb @command - - select * from @mytable -``` - -## Scraper - -| Field | Description | Scheme | Required | -| ----------- | ---------------------------------------------------------------------------------- | -------------------------------------------- | -------- | -| `logLevel` | Specify the level of logging. | `string` | | -| `schedule` | Specify the interval to scrape in cron format. Defaults to every 60 minutes. | `string` | | -| `full` | Set to `true` to extract changes from scraped configurations. Defaults to `false`. | `bool` | | -| `retention` | Settings for retaining changes, analysis and scraped items | [`Retention`](/config-db/concepts/retention) | | -| `sql` | Specifies the list of SQL configurations to scrape. | [`[]SQL`](#sql) | | - -### SQL - -| Field | Description | Scheme | Required | -| ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------- | -------- | -| `id` | A static value or JSONPath expression to use as the ID for the resource. | `string` | `true` | -| `name` | A static value or JSONPath expression to use as the Name for the resource. Default value is the `id`. | `string` | | -| `items` | A JSONPath expression to use to extract individual items from the resource | `string` | | -| `type` | A static value or JSONPath expression to use as the type for the resource. | `string` | `true` | -| `format` | Format of config item, defaults to JSON, available options are JSON | `string` | | -| `timestampFormat` | TimestampFormat is a Go time format string used to parse timestamps in createFields and DeletedFields. If not specified, the default is `RFC3339`. | `string` | | -| `createFields` | CreateFields is a list of JSONPath expression used to identify the created time of the config. If multiple fields are specified, the first non-empty value will be used | `[]string` | | -| `deleteFields` | DeleteFields is a JSONPath expression used to identify the deleted time of the config. If multiple fields are specified, the first non-empty value will be used | `[]string` | | -| - | Specify connection details to the database | [Connection](#connection) | | -| `driver` | Specify the name of the driver to use for connecting to the database | `string` | | -| `query` | Specify the SQL query to execute | `string` | `true` | -| `properties` | Custom templatable properties for the scraped config items. | [`[]ConfigProperty`](/reference/config-db/properties) | | -| `transform` | Specify field to transform result | [`Transform`](/config-db/concepts/transform) | | -| `tags` | set custom tags on the scraped config items | `map[string]string` | | diff --git a/mission-control/docs/config-db/scrapers/custom/sql.mdx b/mission-control/docs/config-db/scrapers/custom/sql.mdx new file mode 100644 index 00000000..b1eaa21f --- /dev/null +++ b/mission-control/docs/config-db/scrapers/custom/sql.mdx @@ -0,0 +1,99 @@ +--- +title: SQL +--- + +The SQL configuration scraper will execute a SQL query and then create a configuration item for each returned row. + +The example below creates a new `MSSQL::Database` configuration for each database on the sql server, and then creates a roles object container the SQL Server login to database role mapping. With change detection this will highlight when new users are added / modified / removed on an individual databases. + +```yaml title='sql-scraper.yaml' +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: sql-scraper +spec: + sql: + - connection: 'sqlserver://localhost:1433?database=master' + auth: + username: + value: sa + password: + value: password + type: MSSQL::Database + id: $.name + + transform: + full: true # transform the entire configuration item, and not just the configuration data (row) + script: + javascript: |+ + var dbs = {} + for (var i = 0; i < config.rows.length; i++) { + var db = config.rows[i] + var name = db.DB + if (dbs[db.DB] == null) { + { + config: dbs[db.DB] = { + name: name, + roles: {} + }, + changes: { + + }, + analysis: { + + } + + } + } + dbs[name].roles[db.role] = db.name + } + JSON.stringify(_.values(dbs)) + + query: | + declare @mytable table ( + [DB] [nvarchar](128) NULL, + [name] [nvarchar](255) NOT NULL, + [role] [nvarchar](255) NOT NULL + ) + + + DECLARE @command varchar(1000) + SELECT @command = + 'USE ?; SELECT DB_NAME() as DB, DP1.name AS [user], + isnull (DP2.name, ''No members'') AS [role] + FROM sys.database_role_members AS DRM + RIGHT OUTER JOIN sys.database_principals AS DP1 + ON DRM.role_principal_id = DP1.principal_id + LEFT OUTER JOIN sys.database_principals AS DP2 + ON DRM.member_principal_id = DP2.principal_id + WHERE DP1.type = ''R'' and DP2.name is not null' + + insert into @mytable EXEC sp_MSforeachdb @command + + select * from @mytable +``` + +## Scraper + +| Field | Description | Scheme | Required | +| ----------- | ---------------------------------------------------------------------------------- | -------------------------------------------- | -------- | +| `logLevel` | Specify the level of logging. | `string` | | +| `schedule` | Specify the interval to scrape in cron format. Defaults to every 60 minutes. | `string` | | +| `full` | Set to `true` to extract changes from scraped configurations. Defaults to `false`. | `bool` | | +| `retention` | Settings for retaining changes, analysis and scraped items | [`Retention`](/config-db/concepts/retention) | | +| `sql` | Specifies the list of SQL configurations to scrape. | [`[]SQL`](#sql) | | + +### SQL + + + diff --git a/mission-control/docs/config-db/scrapers/kubernetes.md b/mission-control/docs/config-db/scrapers/kubernetes.md index 8bf515a2..b48ab5e8 100644 --- a/mission-control/docs/config-db/scrapers/kubernetes.md +++ b/mission-control/docs/config-db/scrapers/kubernetes.md @@ -7,17 +7,15 @@ sidebar_position: 4 :::tip Registry -The registry has an [Kubernetes](/registry/kubernetes) Helm chart that provides a pre-configured Scraper and Topology with some common defaults +The registry has a [Kubernetes](/registry/kubernetes) Helm chart that provides a pre-configured Scraper and Topology with some common defaults ::: -The `kubernetes` config type scrapes the configurations of your Kubernetes resources as specified with the fields; `namespace`, `selector`, `fieldSelector` and more. +The `kubernetes` scraper collects all of the resources and events in a Kubernetes cluster, and then watches for changes. ```yaml title='kubernetes-scraper.yaml' file=../../../modules/config-db/fixtures/kubernetes.yaml ``` -## Scraper - | Field | Description | Scheme | Required | | ------------ | ---------------------------------------------------------------------------- | -------------------------------------------- | -------- | | `logLevel` | Specify the level of logging. | `string` | | @@ -31,24 +29,23 @@ The `kubernetes` config type scrapes the configurations of your Kubernetes resou | ----------------- | ------------------------------------------------------------------------------------------------ | ------------------------------------------------ | -------- | | `clusterName` | Specify cluster name | `string` | | | `event` | Specify configuration to handle Kubernetes events. | [`Event`](#events) | | -| `exclusions` | Specify Kubernetes resources to be excluded from scraping | `[]string` | | -| `fieldSelector` | Specify Kubernetes resource based on value of resource fields. e.g `status.Phase=Running` | `string` | | -| **`kubeconfig`** | Specify kubeconfig for access to your Kubernetes Cluster | []_EnvVar_ | | -| `namespace` | Specify namespace for scraping of Kubernetes resources | `string` | | -| `relationships` | Create relationships between kubernetes objects. | [`[]Relationship`](#kubernetes-relationship) | | +| `exclusions` | Resources to be excluded from scraping | `[]string` | | +| `fieldSelector` | Resources to be included e.g `status.Phase=Running` | `string` | | +| **`kubeconfig`** | Kubeconfig to connect to the cluster | []_EnvVar_ | | +| `namespace` | Include resources only from this namespace | `string` | | +| `relationships` | Create relationships between kubernetes objects. | [`[]Relationship`](#relationships) | | | `scope` | Specify scope for scrape. e.g `cluster` for scraping at Cluster level | `string` | | -| `selector` | Specify Kubernetes resource to scrape based on selector. e.g `matchLabels` | `string` | | +| `selector` | Include resources matching this selector only e.g `matchLabels` | `string` | | | `since` | Set time constraint for scraping resources within the set period | `string` | | -| `properties` | Custom templatable properties for the scraped config items. | [`[]ConfigProperty`](/reference/config-db/properties) | | -| `transform` | Field to transform result | [`Transform`](/config-db/concepts/transform) | | -| `tags` | set custom tags on the scraped config items | `map[string]string` | | +| `properties` | Custom properties to be added for each item | [`[]ConfigProperty`](/reference/config-db/properties) | | +| `transform` | Custom transformations to apply | [`Transform`](/config-db/concepts/transform) | | +| `tags` | Tags to set on each config item. `cluster` and `namespace` are set by default | `map[string]string` | | ## Events -`Config DB` maps Kubernetes Event objects to config changes unlike other objects that are mapped to config items. This configuration allows you to exclude or include the Kubernetes Event objects based on the reason. +`Kubernetes::Event` resources are mapped to config changes. Events can be very verbose so they can be excluded or their severity level changed: -In addition, you can also specify keywords used to identify the severity of the Kubernetes Event based on the reason. | Field | Description | Scheme | Required | | ------------------ | ------------------------------------------------------------------------------------------ | --------------------------------------- | -------- | @@ -59,13 +56,16 @@ In addition, you can also specify keywords used to identify the severity of the | Field | Description | Scheme | Required | | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | -------- | -| `warn` | A list of keywords used to identify a warning severity from the reason. It could also be a match pattern: example "\*" to match all or "!badword" to exclude "badword" | `[]string` | | +| `warn` | A list of keywords used to identify a warning severity from the reason. It could also be a match pattern: e.g. `*` to match all or `!badword` to exclude `badword` | `[]string` | | | `error` | Same as `warn` but used to map to error severity. | `[]string` | | -#### Kubernetes Relationship +## Relationships + +You can create relationships between kubernetes objects on the basis of -You can create relationships between kubernetes objects on the basis of `kind`, `name` and `namespace`. -Kubernetes scraper offers a more tailored relationship selector in addition to the [general relationship selector](../concepts/transform.md#relationshipconfig). +:::info +[Relationships](../concepts/relationships) can also be defined under `transform.relationships`, however by defining them under `kubernetes.relationships` is simpler with specific support for `kind`, `name` and `namespace` fields. +::: ```yaml title="kubernetes-relationship.yaml" kubernetes: @@ -98,13 +98,13 @@ kubernetes: | Field | Description | Scheme | Required | | ----------- | -------------------------------- | -------------------------------------------- | -------- | -| `kind` | `kind` of Kubernetes Object | [`RelationshipLookup`](#relationship-lookup) | `true` | -| `name` | `name` of Kubernetes Object | [`RelationshipLookup`](#relationship-lookup) | `true` | -| `namespace` | `namespace` of Kubernetes Object | [`RelationshipLookup`](#relationship-lookup) | `true` | +| `kind` | `kind` of Kubernetes Object | [`Lookup`](#lookup) | `true` | +| `name` | `name` of Kubernetes Object | [`Lookup`](#lookup) | `true` | +| `namespace` | `namespace` of Kubernetes Object | [`Lookup`](#lookup) | `true` | -##### Relationship Lookup +##### Lookup -RelationshipLookup offers different ways to specify a lookup value +There are 3 different ways to specify which value to use when finding related configs: | Field | Description | Scheme | Required | | ------- | ---------------------------------- | -------- | -------- | diff --git a/mission-control/docs/config-db/scrapers/trivy.mdx b/mission-control/docs/config-db/scrapers/trivy.mdx index 529c7240..fefcb0b9 100644 --- a/mission-control/docs/config-db/scrapers/trivy.mdx +++ b/mission-control/docs/config-db/scrapers/trivy.mdx @@ -26,7 +26,7 @@ _Fig: A detailed view of the analysis on the postgres container_ | ---------- | ---------------------------------------------------------------------------- | --------------------- | -------- | | `logLevel` | Specify the level of logging. | `string` | | | `schedule` | Specify the interval to scrape in cron format. Defaults to every 60 minutes. | `string` | | -| `trivy` | Specifies the list of Trivy configurations to scrape. | [`[]Trivy`](#trivy-1) | | +| `trivy` | Specifies the list of Trivy configurations to scrape. | [`[]Trivy`](#trivy) | | ### Trivy diff --git a/mission-control/docs/index.mdx b/mission-control/docs/index.mdx index 1482408a..6fbc66ba 100644 --- a/mission-control/docs/index.mdx +++ b/mission-control/docs/index.mdx @@ -18,10 +18,10 @@ With Mission Control you can: * [Empower Developers](./playbooks) with self-service playbooks for Day 0-2 operations. * [Run Health Checks](./canary-checker) across both cloud-native and legacy infrastructure and applications. * [Incrementally Adopt GitOps](./playbooks/actions/gitops) with playbooks that perform Git commits in the background. -* [Aggregate Alerts](./canary-checker/probes) from Prometheus, Cloudwatch, Datadog, etc. -* [Visualize Complex Systems](./topology/overview) with a multi-dimensional system topology. +* Aggregate Alerts from [Prometheus](/canary-checker/reference/alert-manager), [Cloudwatch](/canary-checker/reference/aws-cloudwatch), etc. +* [Visualize Complex Systems](./topology) with a multi-dimensional system topology. * [Build Event Driven Control Planes](./playbooks/actions/gitops) with a combination of webhooks, events and GitOps. -* [Notify People and Systems](./notifications/overview) about changes in health and configuration. +* [Notify People and Systems](./notifications) about changes in health and configuration. ### When not to use Mission Control diff --git a/mission-control/docs/installation/_aws_iam.mdx b/mission-control/docs/installation/_aws_iam.mdx new file mode 100644 index 00000000..7ab94861 --- /dev/null +++ b/mission-control/docs/installation/_aws_iam.mdx @@ -0,0 +1,175 @@ + + + +Create a role to allow mission-control to scrape your AWS resources. Attach the following AWS managed policies to the role: + +- `ReadOnlyAccess` +- `AWSConfigUserAccess` +- `AWSQuicksightAthenaAccess` + +
+
+```json title="iam-policy.json" +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "mission-control-config-role", + "Effect": "Allow", + "Action": [ + "acm:Describe*", + "acm:Get*", + "acm:List*", + "cloudtrail:LookupEvents", + "config:BatchGetAggregateResourceConfig", + "config:BatchGetResourceConfig", + "config:Describe*", + "config:Get*", + "config:List*", + "config:SelectAggregateResourceConfig", + "config:SelectResourceConfig", + "ec2:Describe*", + "ecr:Describe*", + "eks:Describe*", + "eks:ListClusters", + "elasticfilesystem:Describe*", + "elasticloadbalancing:Describe*", + "guardduty:Describe*", + "guardduty:Get*", + "guardduty:List*", + "iam:GetAccountName", + "iam:GetAccountSummary", + "iam:GetGroup", + "iam:GetGroupPolicy", + "iam:GetInstanceProfile", + "iam:GetLoginProfile", + "iam:GetPolicy", + "iam:GetRole", + "iam:GetRolePolicy", + "iam:GetUser", + "iam:List*", + "lambda:List*", + "rds:Describe*", + "sts:GetCallerIdentity" + "trustedadvisor:Describe*", + "trustedadvisor:DownloadRisk", + "trustedadvisor:Get*", + "trustedadvisor:List*", + ], + "Resource": "*" + } + ] +} +``` +
+
+ + + + + + + + +a) Get the `OIDC ID` from your EKS Cluster in the **Overview** page: + +![](/img/eks-oidc.png) + +b) Update the trust policy of the IAM role by changing the OIDC arn, OIDC endpoint and the namespace below. +```json +{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": { + "Federated": + "arn:aws:iam:::oidc-provider/oidc.eks.us-east-1.amazonaws.com/id/" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "oidc.eks.:amazonaws.com/id/:sub": "system:serviceaccount:mission-control:config-db-sa", + "oidc.eks..amazonaws.com/id/:sub": "system:serviceaccount:mission-control:mission-control-sa", + "oidc.eks..amazonaws.com/id/:sub": "system:serviceaccount:mission-control:canary-checker-sa", + "oidc.eks..amazonaws.com/id/:sub": "sts.amazonaws.com", + } + } + }] +} +``` + + + + + + + +:::info Prerequisites +- [Pod Identity Agent](https://docs.aws.amazon.com/eks/latest/userguide/pod-id-agent-setup.html) is installed and configured +::: + + + + a) Modify the trust policy of the IAM role +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowEksAuthToAssumeRoleForPodIdentity", + "Effect": "Allow", + "Principal": { + "Service": "pods.eks.amazonaws.com" + }, + "Action": [ + "sts:AssumeRole", + "sts:TagSession" + ] + } + ] +} +``` + +b) Create the Pod Identity Association + +```bash +for role in mission-control-sa canary-checker-sa config-db-sa; do + echo aws eks create-pod-identity-association \ + --cluster-name \ + --namespace mission-control \ + --service-account $role \ + --role-arn +done +``` + + + + + + + + + + + +Update the `values.yaml` to annotate the service accounts: + +```yaml title="values.yaml" +# service account used by for scraping +config-db: + serviceAccount: + annotations: + eks.amazonaws.com/role-arn: + +# service account used by for health checks +canary-checker: + serviceAccount: + annotations: + eks.amazonaws.com/role-arn: + +# service account used for notifications and playbooks +serviceAccount: + annotations: + eks.amazonaws.com/role-arn: +``` + diff --git a/mission-control/docs/installation/saas/eks.mdx b/mission-control/docs/installation/saas/eks.mdx index 2b7d9c7c..3cac0576 100644 --- a/mission-control/docs/installation/saas/eks.mdx +++ b/mission-control/docs/installation/saas/eks.mdx @@ -6,6 +6,7 @@ slug: installation/eks import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; +import AwsIam from "../_aws_iam.mdx" # Install Mission Control on AWS EKS cluster @@ -18,184 +19,10 @@ To install and run Mission Control you need to have the following prerequisites: - (Optional) SMTP Server (For sending notifications and invites) ::: + - - -Create a role to allow mission-control to scrape your AWS resources. Attach the following AWS managed policies to the role: - -- `ReadOnlyAccess` -- `AWSConfigUserAccess` -- `AWSQuicksightAthenaAccess` - -
-
-```json title="iam-policy.json" -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "mission-control-config-role", - "Effect": "Allow", - "Action": [ - "acm:Describe*", - "acm:Get*", - "acm:List*", - "cloudtrail:LookupEvents", - "config:BatchGetAggregateResourceConfig", - "config:BatchGetResourceConfig", - "config:Describe*", - "config:Get*", - "config:List*", - "config:SelectAggregateResourceConfig", - "config:SelectResourceConfig", - "ec2:Describe*", - "ecr:Describe*", - "eks:Describe*", - "eks:ListClusters", - "elasticfilesystem:Describe*", - "elasticloadbalancing:Describe*", - "guardduty:Describe*", - "guardduty:Get*", - "guardduty:List*", - "iam:GetAccountName", - "iam:GetAccountSummary", - "iam:GetGroup", - "iam:GetGroupPolicy", - "iam:GetInstanceProfile", - "iam:GetLoginProfile", - "iam:GetPolicy", - "iam:GetRole", - "iam:GetRolePolicy", - "iam:GetUser", - "iam:List*", - "lambda:List*", - "rds:Describe*", - "sts:GetCallerIdentity" - "trustedadvisor:Describe*", - "trustedadvisor:DownloadRisk", - "trustedadvisor:Get*", - "trustedadvisor:List*", - ], - "Resource": "*" - } - ] -} -``` -
-
- - - - - - - - -a) Get the `OIDC ID` from your EKS Cluster in the **Overview** page: - -![](/img/eks-oidc.png) - -b) Update the trust policy of the IAM role by changing the OIDC arn, OIDC endpoint and the namespace below. -```json -{ - "Version": "2012-10-17", - "Statement": [{ - "Effect": "Allow", - "Principal": { - "Federated": - "arn:aws:iam:::oidc-provider/oidc.eks.us-east-1.amazonaws.com/id/" - }, - "Action": "sts:AssumeRoleWithWebIdentity", - "Condition": { - "StringEquals": { - "oidc.eks.:amazonaws.com/id/:sub": "system:serviceaccount:mission-control:config-db-sa", - "oidc.eks..amazonaws.com/id/:sub": "system:serviceaccount:mission-control:mission-control-sa", - "oidc.eks..amazonaws.com/id/:sub": "system:serviceaccount:mission-control:canary-checker-sa", - "oidc.eks..amazonaws.com/id/:sub": "sts.amazonaws.com", - } - } - }] -} -``` - - - - - - - -:::info Prerequisites -- [Pod Identity Agent](https://docs.aws.amazon.com/eks/latest/userguide/pod-id-agent-setup.html) is installed and configured -::: - - - - a) Modify the trust policy of the IAM role -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "AllowEksAuthToAssumeRoleForPodIdentity", - "Effect": "Allow", - "Principal": { - "Service": "pods.eks.amazonaws.com" - }, - "Action": [ - "sts:AssumeRole", - "sts:TagSession" - ] - } - ] -} -``` - -b) Create the Pod Identity Association - -```bash -for role in mission-control-sa canary-checker-sa config-db-sa; do - echo aws eks create-pod-identity-association \ - --cluster-name \ - --namespace mission-control \ - --service-account $role \ - --role-arn -done -``` - - - - - - - - - - - -Update the `values.yaml` to annotate the service accounts: - -```yaml title="values.yaml" -# service account used by for scraping -config-db: - serviceAccount: - annotations: - eks.amazonaws.com/role-arn: - -# service account used by for health checks -canary-checker: - serviceAccount: - annotations: - eks.amazonaws.com/role-arn: - -# service account used for notifications and playbooks -serviceAccount: - annotations: - eks.amazonaws.com/role-arn: -``` - - ```bash diff --git a/mission-control/docs/installation/saas/fully-hosted.mdx b/mission-control/docs/installation/saas/fully-hosted.mdx new file mode 100644 index 00000000..e69de29b diff --git a/mission-control/docs/installation/saas/getting-started.mdx b/mission-control/docs/installation/saas/getting-started.mdx index ac561a28..ebce5ca8 100644 --- a/mission-control/docs/installation/saas/getting-started.mdx +++ b/mission-control/docs/installation/saas/getting-started.mdx @@ -13,7 +13,7 @@ export const toc = [{ }, { value: "Agent Installation", - id: "agent", + id: "agent-installation", level: 2, }, @@ -36,7 +36,7 @@ The agent based approach has the following benefits: 3. **Local Playbook Execution** on agents, reusing existing secrets or IAM roles. -See [Deployment Models](/installation/deployment-models) for [Self Hosted](/installation/self-hosted/) and [Fully Hosted](/installation/saas/fully-hosted) options +See [Deployment Models](/installation/deployment-models) for [Self Hosted](/installation/self-hosted/getting-started) and [Fully Hosted](/installation/saas/fully-hosted) options ### Signup @@ -54,3 +54,5 @@ c. Invite your team members +### Registry + diff --git a/mission-control/docs/installation/self-hosted/eks.mdx b/mission-control/docs/installation/self-hosted/eks.mdx index e1deef2c..b0fbbb73 100644 --- a/mission-control/docs/installation/self-hosted/eks.mdx +++ b/mission-control/docs/installation/self-hosted/eks.mdx @@ -3,6 +3,8 @@ title: AWS EKS slug: installation/eks --- +import AwsIam from "../_aws_iam.mdx" + # Self Hosted Installation on AWS EKS :::info Prerequisites @@ -56,120 +58,8 @@ helm install mission-control \ - - - -(a) Create a role to allow mission-control to scrape your AWS resources. Attach the following AWS managed policies to the role: - -1. ReadOnlyAccess -2. AWSConfigUserAccess -3. AWSQuicksightAthenaAccess - -
-
-```json title="iam-policy.json" -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "mission-control-config-role", - "Effect": "Allow", - "Action": [ - "acm:Describe*", - "acm:Get*", - "acm:List*", - "cloudtrail:LookupEvents", - "ec2:Describe*", - "ecr:Describe*", - "eks:ListClusters", - "eks:Describe*", - "rds:Describe*", - "elasticfilesystem:Describe*", - "elasticloadbalancing:Describe*", - "sts:GetCallerIdentity" - "config:BatchGetAggregateResourceConfig", - "config:BatchGetResourceConfig", - "config:Describe*", - "config:Get*", - "config:List*", - "config:SelectAggregateResourceConfig", - "config:SelectResourceConfig", - "guardduty:Describe*", - "guardduty:Get*", - "guardduty:List*", - "iam:GetAccountName", - "iam:GetAccountSummary", - "iam:GetGroup", - "iam:GetGroupPolicy", - "iam:GetInstanceProfile", - "iam:GetLoginProfile", - "iam:GetPolicy", - "iam:GetRole", - "iam:GetRolePolicy", - "iam:GetUser", - "iam:List*", - "lambda:List*", - "trustedadvisor:Describe*", - "trustedadvisor:DownloadRisk", - "trustedadvisor:Get*", - "trustedadvisor:List*", - ], - "Resource": "*" - } - ] -} -``` -
-
- - - -(b) Modify the trust policy of the IAM role by changing the OIDC arn, OIDC endpoint and the namespace below. -```json -{ - "Version": "2012-10-17", - "Statement": [{ - "Effect": "Allow", - "Principal": { - "Federated": "arn:aws:iam::7458xxxxxxxx:oidc-provider/oidc.eks.us-east-1.amazonaws.com/id/4D3C9C8xxxx" - }, - "Action": "sts:AssumeRoleWithWebIdentity", - "Condition": { - "StringEquals": { - "oidc.eks.us-east-1.amazonaws.com/id/4D3C9C8xxxx:sub": "system:serviceaccount:namespace:config-db-sa", - "oidc.eks.us-east-1.amazonaws.com/id/4D3C9C8xxxx:sub": "system:serviceaccount:namespace:mission-control-sa", - "oidc.eks.us-east-1.amazonaws.com/id/4D3C9C8xxxx:sub": "system:serviceaccount:namespace:canary-checker-sa", - "oidc.eks.us-east-1.amazonaws.com/id/4D3C9C8xxxx:aud": "sts.amazonaws.com" - } - } - }] -} -``` - - -(c) Annotate the service account - -```yaml title='values.yaml' -# service account used by for scraping -config-db: - serviceAccount: - annotations: - eks.amazonaws.com/role-arn: IAM Role ARN - -# service account used by for health checks -canary-checker: - serviceAccount: - annotations: - eks.amazonaws.com/role-arn: IAM Role ARN - -# service account used for notifications and playbooks -serviceAccount: - annotations: - eks.amazonaws.com/role-arn: IAM Role ARN + -``` - -
diff --git a/mission-control/docs/notifications/concepts/filtering.md b/mission-control/docs/notifications/concepts/filtering.md deleted file mode 100644 index b6c87d0f..00000000 --- a/mission-control/docs/notifications/concepts/filtering.md +++ /dev/null @@ -1,49 +0,0 @@ -# Filtering - -Customizing notifications is made possible through the use of filters, which are driven by CEL expressions _(Check Execution Language)_. - -:::info -Learn more about CEL expressions [here](../../reference/scripting/cel). -::: - -The cel expression receives different input variables based on the event type. To determine which variables are available, refer to the corresponding event details in the [events](../events) section. - -## Examples - -```yaml title="check-alerts.yaml" ---- -# Filter HTTP checks that have failed for more than 5 times in the last 1 hour -apiVersion: mission-control.flanksource.com/v1 -kind: Notification -metadata: - name: check-alerts - namespace: default -spec: - events: - - check.passed - - check.failed - filter: check.type == 'http' && check.uptime.failed > 5 - to: - connection: connection://slack/flanksource - title: Check {{.check.name}} is {{.check.status}} -``` - -```yaml title="kubernetes-node-component-alerts.yaml" ---- -# Send all component status alerts for Kubernetes Node component -apiVersion: mission-control.flanksource.com/v1 -kind: Notification -metadata: - name: kubernetes-node-component-alerts - namespace: default -spec: - events: - - component.status.healthy - - component.status.unhealthy - - component.status.warning - - component.status.error - - component.status.info - filter: component.type == "Kubernetes::Node" - to: - connection: connection://slack/flanksource -``` diff --git a/mission-control/docs/notifications/concepts/templating.md b/mission-control/docs/notifications/concepts/templating.md deleted file mode 100644 index b2d11388..00000000 --- a/mission-control/docs/notifications/concepts/templating.md +++ /dev/null @@ -1,56 +0,0 @@ ---- -title: Templating ---- - -**Templating is:** - -> Simply a way to represent data in different forms. - -Templating is essential for crafting customized notifications. It involves using templates for both the notification title and body. In Mission Control, these templates are rendered using Go templates. To determine which variables you can use in a specific event's template, refer to the corresponding event details in the [events](../events) section. - -## Go Template - -Go templates are a feature of the Go programming language that allow you to define and execute templates. Templates are text files that contain placeholders for data, which are filled in at runtime. The view data for the template is the [search param](./api.md#search-params). - -### Examples - -```yaml title="check-alerts.yaml" -apiVersion: mission-control.flanksource.com/v1 -kind: Notification -metadata: - name: check-alerts - namespace: default -spec: - events: - - check.passed - - check.failed - to: - connection: connection://slack/flanksource - title: Check {{.check.name}} is {{.check.status}} - template: | - Canary: {{.canary.id}} ({{.canary.name}}) - Description: {{.check.description}} - Type: {{.check.type}} -``` - -```yaml title="component-alerts.yaml" ---- -apiVersion: mission-control.flanksource.com/v1 -kind: Notification -metadata: - name: component-alerts - namespace: default -spec: - events: - - component.status.healthy - - component.status.unhealthy - - component.status.warning - - component.status.error - - component.status.info - to: - connection: connection://slack/flanksource - title: Component {{.component.name}} status changed to {{.component.status}} - template: | - Description: {{.component.description}} - Type: {{.component.type}} -``` diff --git a/mission-control/docs/notifications/events/components.md b/mission-control/docs/notifications/events/components.md index 616b8ec6..d6000dde 100644 --- a/mission-control/docs/notifications/events/components.md +++ b/mission-control/docs/notifications/events/components.md @@ -4,13 +4,13 @@ title: Components Component status updates emit the following events -- `component.status.healthy` -- `component.status.unhealthy` -- `component.status.warning` -- `component.status.error` -- `component.status.info` +- `component.healthy` +- `component.unhealthy` +- `component.warning` +- `component.error` +- `component.info` -## Environment variables +## Variables | Field | Description | Schema | Optional | | ----------- | ----------------------------- | ------------------------- | -------- | diff --git a/mission-control/docs/notifications/events/configs.md b/mission-control/docs/notifications/events/configs.md new file mode 100644 index 00000000..41199a81 --- /dev/null +++ b/mission-control/docs/notifications/events/configs.md @@ -0,0 +1,64 @@ +--- +title: Components +--- + +Component status updates emit the following events + +- `config.healthy` +- `config.unhealthy` +- `config.warning` + +## Variables + +| Field | Description | Schema | Optional | +| ----------- | ----------------------------- | ------------------------- | -------- | +| `config` | The component object | [`Component`](#component) | | +| `agent` | Agent details (if applicable) | [`Agent`](#agent) | `true` | +| `permalink` | A link to the health check | `string` | | + +### Component + +| Field | Description | Schema | Optional | +| ----------------- | -------------------------------------------- | ------------------- | -------- | +| `id` | The id of the component | `uuid` | | +| `description` | The description of the component | `string` | | +| `external_id` | The external id of the component | `string` | | +| `hidden` | Whether the component is hidden | `bool` | | +| `labels` | The labels of the component | `map[string]string` | `true` | +| `name` | The name of the component | `string` | | +| `namespace` | The namespace of the component | `string` | | +| `parent_id` | The id of the parent component | `uuid` | `true` | +| `properties` | The properties of the component | `map[string]string` | `true` | +| `silenced` | Whether the component is silenced | `bool` | | +| `status_reason` | The status reason of the component | `string` | | +| `status` | The status of the component | `string` | | +| `summary` | The summary of the component | `map[string]string` | `true` | +| `text` | The text of the component | `string` | | +| `topology_type` | The type of the topology | `string` | | +| `type` | The type of the component | `string` | | +| `cost_per_minute` | The cost per minute of the component | `float64` | | +| `cost_total_1d` | The cost total 1d of the component | `float64` | | +| `cost_total_7d` | The cost total 7d of the component | `float64` | | +| `cost_total_30d` | The cost total 30d of the component | `float64` | | +| `created_by` | Id of the person that created this component | `uuid` | | +| `created_at` | Created timestamp | `time.Time` | | +| `updated_at` | Updated timestamp | `time.Time` | | +| `deleted_at` | Deleted timestamp | `time.Time` | `true` | + +### Agent + +| Field | Description | Schema | Optional | +| ------------- | ------------------------------ | -------- | -------- | +| `id` | The id of the agent | `uuid` | | +| `name` | The name of the agent | `string` | | +| `description` | Short description of the agent | `string` | | + +## Notification Defaults + +``` +# Title +Config {{.config.name}} status updated to {{.config.status}} + +# Body +[Reference]({{.permalink}}) +``` diff --git a/mission-control/docs/notifications/events/health-checks.md b/mission-control/docs/notifications/events/health-checks.md index 29a08e23..5e11d083 100644 --- a/mission-control/docs/notifications/events/health-checks.md +++ b/mission-control/docs/notifications/events/health-checks.md @@ -5,7 +5,7 @@ Health checks emit 2 events 1. `check.passed` 2. `check.failed` -## Environment variables +## Variables | Field | Description | Schema | Optional | | ----------- | ----------------------------- | ------------------------------ | -------- | diff --git a/mission-control/docs/notifications/examples/slack.md b/mission-control/docs/notifications/examples/slack.md index b2ab325b..dbb5e4ba 100644 --- a/mission-control/docs/notifications/examples/slack.md +++ b/mission-control/docs/notifications/examples/slack.md @@ -82,10 +82,9 @@ metadata: namespace: default spec: events: - - check.passed - check.failed to: # use the slack connection as the recipient for this notification - connection: connection://slack/flanksource + connection: connection://flanksource filter: check.type == 'http' ``` diff --git a/mission-control/docs/notifications/index.md b/mission-control/docs/notifications/index.md deleted file mode 100644 index 4c196a11..00000000 --- a/mission-control/docs/notifications/index.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -pagination_next: registry/index -pagination_prev: canary-checker/health-checks -title: Notifications ---- - -Notification allows you to receive alerts on a preferred channel when a particular set of events occur. - -```yaml title="http-check.yaml" ---- -apiVersion: mission-control.flanksource.com/v1 -kind: Notification -metadata: - name: http-check - namespace: default -spec: - events: - - check.passed - - check.failed - filter: check.type == 'http' - to: - connection: connection://slack/flanksource -``` - -## Spec - -| Field | Description | Scheme | Required | Templatable | -| ---------- | ------------------------------------------------------------------ | ------------------------- | -------- | ----------- | -| `events` | [List of events](./events/) that trigger this notification. | `[]string` | `true` | -| `to` | Specify the recipients. | [`Recipient`](#recipient) | `true` | -| `title` | Title of the notification. [See templating](./concepts/templating) | `string` | | `true` | -| `template` | Body of the notification. [See templating](./concepts/templating) | `string` | | `true` | -| `filter` | Filter of notification. | `string` | | | - -:::info -The title and body _(template)_ have default templates based on the event. Eg: [Component](./events/components.md#notification-defaults) -::: - -### Recipient - -| Field | Description | Schema | -| ------------ | ---------------------------------------------------------------- | ------------------- | -| `person` | ID or email of the person | `string` | -| `team` | ID or name of the team | `string` | -| `email` | Email of the recipient | `string` | -| `connection` | connection string for an external service | `string` | -| `url` | Specify shoutrrr URL directly | `string` | -| `properties` | Properties for Shoutrrr (applicable if shoutrrr url is provided) | `map[string]string` | - -:::note -You can only specify one recipient -::: diff --git a/mission-control/docs/notifications/index.mdx b/mission-control/docs/notifications/index.mdx new file mode 100644 index 00000000..34521136 --- /dev/null +++ b/mission-control/docs/notifications/index.mdx @@ -0,0 +1,61 @@ +--- +pagination_next: registry/index +pagination_prev: canary-checker/health-checks +title: Notifications +--- +import Notification from "../reference/notifications/_notification.mdx" + +Notifications can be sent based on different type of events occuring including: + +- Health check failing (`check.failed`) +- Aggregated health of a topology failing (`component.unhealthy`) +- A scraped config item health (`config.unhealthy`) + + +```yaml title="http-check.yaml" +apiVersion: mission-control.flanksource.com/v1 +kind: Notification +metadata: + name: http-check + namespace: default +spec: + events: + - check.passed + filter: check.type == 'http' + to: + email: alerts@acme.com +``` + + + +## Templating + +Notification title and bodies can be templated using Go Templates +```yaml title="template-title.yaml" +apiVersion: mission-control.flanksource.com/v1 +kind: Notification +spec: + // highlight-next-line + title: Check {{.check.name}} is {{.check.health}} +``` + +## Filtering + +Notifications can be filtered using CEL expressions. The filters have access to same [variables](./events) that can be used in the templates. + +```yaml title="filter-http.yaml" +--- +# Only send a notification for HTTP checks +apiVersion: mission-control.flanksource.com/v1 +kind: Notification +metadata: + name: http-alerts +spec: + events: + - check.failed + // highlight-next-line + filter: check.type == 'http' + to: + connection: connection://slack/flanksource + title: Check {{.check.name}} is {{.check.health}} +``` diff --git a/mission-control/docs/playbooks/Actions/index.md b/mission-control/docs/playbooks/Actions/index.md index ef3aba73..c07d6cc6 100644 --- a/mission-control/docs/playbooks/Actions/index.md +++ b/mission-control/docs/playbooks/Actions/index.md @@ -8,7 +8,7 @@ Actions are the fundamental tasks executed by a playbook. A playbook can compris | Field | Description | Scheme | Required | | --------------------- | ------------------------------------------------------------ | ----------------------------------------------------- | -------- | | `name` | Name of action. | `string` | `true` | -| `runsOn` | Specify the [runners](./runners) that can run this action. One will be chosen on random. When empty, the playbook will run on the main instance itself | `[]string` | | +| `runsOn` | Specify the [runners](/playbooks/concepts/runners) that can run this action. One will be chosen on random. When empty, the playbook will run on the main instance itself | `[]string` | | | `templatesOn` | Specify where the templating of the action spec should occur | `host` or `agent` | | | `delay` | A delay before running the action e.g. `8h` | `Duration` or [`Expression`](../concepts/expression) | | | `filter` | Whether to run the step or not | [`Expression`](../concepts/expression) | | diff --git a/mission-control/docs/playbooks/Actions/notification.mdx b/mission-control/docs/playbooks/Actions/notification.mdx index 068224c6..6467d8b5 100644 --- a/mission-control/docs/playbooks/Actions/notification.mdx +++ b/mission-control/docs/playbooks/Actions/notification.mdx @@ -10,17 +10,8 @@ Notification action sends notification. ```yaml title="notify-deployment-restarts.yaml" file=../../../modules/mission-control/fixtures/playbooks/notification-playbook.yaml ``` -| Field | Description | Scheme | Required | Templatable | -| ------------ | --------------------------------------------------------- | ------------------- | -------- | ----------- | -| `url` | Shoutrrr URL [Read more](../../notifications/overview) | `string` | | | -| `connection` | Connection to use to send the notification | `string` | | | -| `title` | Title of the notification | `string` | `true` | `true` | -| `message` | Message is the body of the notification | `string` | `true` | `true` | -| `properties` | Properties for shoutrrr | `map[string]string` | | | - - ## Types @@ -62,7 +75,7 @@ Component events relate to activities on Topology components. | `warning` | when a component has warning | | `error` | when a component has error | -#### Example + ```yaml title="notify-unhealthy-database-component.yaml" apiVersion: mission-control.flanksource.com/v1 @@ -95,7 +108,6 @@ Config events relate to activities on config items. | `updated` | when a config item is updated | | `deleted` | when a config item is deleted | -#### Example ```yaml title="notify-newly-scraped-pod.yaml" apiVersion: mission-control.flanksource.com/v1 diff --git a/mission-control/docs/playbooks/index.mdx b/mission-control/docs/playbooks/index.mdx index f4333d10..e9635503 100644 --- a/mission-control/docs/playbooks/index.mdx +++ b/mission-control/docs/playbooks/index.mdx @@ -1,6 +1,6 @@ --- slug: /playbooks -title: Playbooks +title: Overview sidebar_position: 1 hide_title: true # hide_table_of_contents: true diff --git a/mission-control/docs/playbooks/quick-start.mdx b/mission-control/docs/playbooks/quick-start.mdx index bdc346ad..2703d4f6 100644 --- a/mission-control/docs/playbooks/quick-start.mdx +++ b/mission-control/docs/playbooks/quick-start.mdx @@ -31,7 +31,7 @@ spec: script: kubectl rollout restart deployment {{.config.name}} -n {{.config.tags.namespace}} ``` -See [Playbook](/reference/playbooks/playbook) for the full specification. +See [Playbook](/reference/playbooks) for the full specification. diff --git a/mission-control/docs/playbooks/self-service.mdx b/mission-control/docs/playbooks/self-service.mdx index f72ead37..e6008053 100644 --- a/mission-control/docs/playbooks/self-service.mdx +++ b/mission-control/docs/playbooks/self-service.mdx @@ -5,7 +5,8 @@ sidebar_position: 4 import Parameters from '../reference/playbooks/parameters.md' -Playbooks can be manually triggered on a component, config item or a check. If any of the [playbook's resource filters](../concepts/playbook.md#resourcefilter) matches the type and tags of the component, config or a check then a playbook run dropdown can be seen on that item. +Playbooks can be manually triggered on a component, config item or check. + ```yaml title="restart-deployment.yaml" apiVersion: mission-control.flanksource.com/v1 kind: Playbook @@ -13,6 +14,7 @@ metadata: name: restart-deployment spec: description: Restart deployment + # only allow the playbook to run on Kubernetes::Deployment config items configs: - types: - Kubernetes::Deployment @@ -22,20 +24,38 @@ spec: script: kubectl rollout restart deployment {{.config.name}} -n {{.config.tags.namespace}} ``` +To run the playbook: + +a. Navigate to a Deployment in the **Catalog**
+b. Select **Restart Deployment** from the Playbooks menu
+c. Click **Run**
+ + - +## Resource Filters - +Playbooks can be restricted to run on specific `config`, `check` and `components` using [resource selectors](/reference/resource-selector) +```yaml title="restart-deployment.yaml" +apiVersion: mission-control.flanksource.com/v1 +kind: Playbook +spec: + //highlight-start + configs: # Only applicable to config items + - types: + - Kubernetes::Deployment # of this type + //highlight-end +``` ## Parameters -![Playbook on a config item](/img/playbook-on-a-eks-cluster-config-item.png) -**Fig: Playbook on a config item** + + + ![Playbook parameters](/img/playbook-on-a-eks-cluster-config-item-popup.png) **Fig: Playbook parameters** diff --git a/mission-control/docs/playbooks/webhook.md b/mission-control/docs/playbooks/webhooks.md similarity index 100% rename from mission-control/docs/playbooks/webhook.md rename to mission-control/docs/playbooks/webhooks.md diff --git a/mission-control/docs/reference/canary-checker/index.mdx b/mission-control/docs/reference/canary-checker/index.mdx index 3126fd50..dca906ae 100644 --- a/mission-control/docs/reference/canary-checker/index.mdx +++ b/mission-control/docs/reference/canary-checker/index.mdx @@ -29,7 +29,7 @@ sidebar_position: 3 | `jmeter` | List of JMeter load test checks to run. | [`[]Jmeter`](/canary-checker/reference/jmeter) | | | `junit` | List of JUnit test result checks to run. | [`[]Junit`](/canary-checker/reference/junit) | | | `kubernetes` | List of Kubernetes checks to run. | [`[]Kubernetes`](/canary-checker/reference/kubernetes) | | -| `kubernetes` | Kubernetes Resource Checks | [`[]KubernetesResource`](/canary-checker/reference/kubernetesResource) | | +| `kubernetesResource` | Kubernetes Resource Checks | [`[]KubernetesResource`](/canary-checker/reference/kubernetes-resource) | | | `ldap` | List of LDAP checks to run. | [`[]LDAP`](/canary-checker/reference/./ldap) | |s | `mongodb` | List of MongoDB checks to run. | [`[]MongoDB`](/canary-checker/reference/./mongo) | | | `mssql` | List of MSSQL checks to run. | [`[]Mssql`](/canary-checker/reference/./sql) | | diff --git a/mission-control/docs/reference/connections/index.md b/mission-control/docs/reference/connections/index.md index 6ffd7099..e49961dd 100644 --- a/mission-control/docs/reference/connections/index.md +++ b/mission-control/docs/reference/connections/index.md @@ -28,14 +28,15 @@ spec: key: POSTGRES_PASSWORD ``` +## Referencing Connections + Eventually, the URL that gets templated is used for establishing connections. This can be used for any datasource that authenticates via URL (PostgreSQL, MySQL, MSSQL, Redis, Opensearch, Elasticsearch etc.) -A connection string can be represented in the form of `type/connection_name` or `type/namespace/connection_name` +A connection string can be represented in the form of `namespace/connection_name` -It can then be used in Health Checks via `connection` attribute or during Topology creation in `component.lookup` +It can then be used in Health Checks, Playbooks, Notifications etc via the `connection` field. ```yaml - apiVersion: canaries.flanksource.com/v1 kind: Canary metadata: @@ -43,16 +44,14 @@ metadata: spec: postgres: - name: postgres schemas check - connection: connection://postgres/payments-database + connection: connection://default/payments-database query: SELECT COUNT(*) FROM payments where state = 'pending' ``` This allows us a safe and reusable way to handle authentication :::tip - If the entire URL is in the secrets and cannot be constructed like `scheme://$(username):$(password)@:` you can fetch that directly like - ```yaml kind: Connection metadata: diff --git a/mission-control/docs/reference/notifications/_notification.mdx b/mission-control/docs/reference/notifications/_notification.mdx new file mode 100644 index 00000000..e2afbe53 --- /dev/null +++ b/mission-control/docs/reference/notifications/_notification.mdx @@ -0,0 +1,22 @@ + +| Field | Description | Scheme | Required | Templatable | +| ---------- | ------------------------------------------------------------------ | ------------------------- | -------- | ----------- | +| `events` | | [][Event](./events) | `true` || +| `to` | Notification recipient | [`Recipient`](#recipient) | `true` || +| `title` | Channel dependant e.g. subject for email | Go Template | | `true` | +| `template` |Channel dependant e.g. email body for email | Go Template | | `true` | +| `filter` | Filter out unwanted notifications | CEL | | | + + +### Recipient + +| Field | Description | Schema | +| ------------ | ---------------------------------------------------------------- | ------------------- | +| `person` | | `UUID` or `email` of a person | +| `team` | Every member of the team with an email will receive a notification | `UUID` or `name` of team | +| `email` | Send an email to an external user | `email` | +| `connection` | Connection to external service | `connection` | +| `url` | Custom notification URL | [Channel](./channels) | +| `properties` | *Only applicable if `url` is specified* | `map[string]string` | + +> Only one recipient can be specified diff --git a/mission-control/docs/reference/topology/components.mdx b/mission-control/docs/reference/topology/components.mdx index ca43f72b..d79fef61 100644 --- a/mission-control/docs/reference/topology/components.mdx +++ b/mission-control/docs/reference/topology/components.mdx @@ -33,12 +33,28 @@ Components are the building blocks of a Topology. The component specification pr ### ID -| Field | Description | Scheme | Required | -| ------------ | ------------------------------------------ | -------- | -------- | -| `expr` | Specify expression for ID | `string` | | -| `javascript` | Specify javascript syntax to generate ID | `string` | | -| `jsonPath` | Specify path to JSON element for use in ID | `string` | | -| `template` | Specify Go template for use in ID | `string` | | + + ### RelationshipSpec diff --git a/mission-control/docs/reference/topology/properties.md b/mission-control/docs/reference/topology/properties.md index 38edcf6d..39bd0979 100644 --- a/mission-control/docs/reference/topology/properties.md +++ b/mission-control/docs/reference/topology/properties.md @@ -21,9 +21,6 @@ title: Property | `tooltip` | Set tooltip outlining information pertaining to the component. | `string` | | | `configLookup` | Specify lookup for component config. | [`ConfigLookup`](#config-lookup) | `optional` | - - - ## Link | Field | Description | Schema | Required | @@ -44,5 +41,5 @@ title: Property | `config.type` | The type of config item. | `string` | | | `config.labels` | Match labels of the config item, all labels must match | `map[string]string` | | | `field` | A JSONPath expression to lookup the value in the config. | `string` | `true` | -| `display` | Apply transformations to the value. | [`Display`](../concepts/templating) | | +| `display` | Apply transformations to the value. | [`Display`](./concepts/templating) | | | `id` | The UUID of config item, rarely used | `string` | | diff --git a/mission-control/docs/registry/aws.md b/mission-control/docs/registry/aws.mdx similarity index 65% rename from mission-control/docs/registry/aws.md rename to mission-control/docs/registry/aws.mdx index 6b74aa0f..4c5e9648 100644 --- a/mission-control/docs/registry/aws.md +++ b/mission-control/docs/registry/aws.mdx @@ -2,13 +2,22 @@ title: AWS --- -The AWS bundle installs a [catalog scraper](/config-db/scrapers/aws) that: +import AwsIam from "../installation/_aws_iam.mdx" + + +The AWS bundle installs a catalog [scraper](/config-db/scrapers/aws) that: - Scrapes AWS Resources and detects changes in the resource definition - Ingests changes from CloudTrail - Ingests cost data from AWS Cost & Usage Reporting - Links AWS EKS resources to the corresponding Kubernetes resources + +:::info Prerequisites +- IAM Role for scraping AWS API's +- Mission Control configured to use Pod Identity/IRSA or an AWS access key +::: + ## Setup ```sh @@ -20,22 +29,12 @@ helm install mission-control-aws flanksource/mission-control-aws When you go to the catalog now, you can now see all the AWS Resources -![AWS Catalog](/img/aws-registry-catalog-scraper.png) - - -## Prerequisites - -1. Connection to AWS must be authenticated via IAM Roles for Service Accounts or AWS Access and Secret Key. [Click Here to see how to configure AWS](/installation/aws-eks) + -2. The role should have permissions to fetch the AWS Resources -3. Cost Reporting should also be setup if you want to associate costs to each resource. [Check the guide here](/installation/aws-eks#cost-reporting) + -### Values - -This document provides an overview of configurable values for deploying Mission Control Registry on AWS using Helm. - -## Configuration +## values.yaml The following table lists the configurable parameters and their default values: @@ -44,7 +43,6 @@ The following table lists the configurable parameters and their default values: | `labels` | Additional labels to apply to resources. | {} | | `scraper.name` | Name of the AWS scraper. | "aws" | -### Connection Details | Parameter | Description | Schema | Default | | --- | --- | --- | --- | @@ -58,41 +56,14 @@ The following table lists the configurable parameters and their default values: | `cloudtrail.maxAge` | Maximum age for CloudTrail. | "" | | | `cloudtrail.exclude` | List of excluded items for CloudTrail. | [] | | | `compliance` | Enable or disable compliance. | true | | -| | | | | | `costReporting.enabled` | Enable or disable cost reporting. | false | | | `costReporting.database` | Cost reporting database. | "" | | | `costReporting.region` | Cost reporting region. | "" | | | `costReporting.s3BucketPath` | S3 bucket path for cost reporting. | "" | | | `costReporting.table` | Table for cost reporting. | "" | | -| | | | | -| | | | | -| | | | | | regions | | | | | `includeResources` | List of resources to include. | [] | | | `excludeResources` | List of resources to exclude. | [] | | -| | | | | - -:::info -If you have setup IAM Roles for Service Account, you do not have to do anything else. If you do not have that setup, you can use AWS Access and Secret Keys as well - -Example: - -```yaml title="values.yaml" -connectionDetails: - accessKey: - valueFrom: - secretKeyRef: - name: aws-credentials - key: AWS_ACCESS_KEY - secretKey: - valueFrom: - secretKeyRef: - name: aws-credentials - key: AWS_SECRET_KEY -``` - -::: - @@ -105,6 +76,3 @@ connectionDetails: - - -### diff --git a/mission-control/docs/registry/kubernetes.md b/mission-control/docs/registry/kubernetes.md index df913d83..c8ca2e10 100644 --- a/mission-control/docs/registry/kubernetes.md +++ b/mission-control/docs/registry/kubernetes.md @@ -72,7 +72,7 @@ The following table lists the configurable parameters and their default values: | `scraper.severityKeywords.error` | Keywords indicating error severity. | `["failed", "error"]` | | `scraper.severityKeywords.warn` | Keywords indicating warning severity. | `["backoff", "nodeoutofmemory"]` | | `scraper.retention.changes` | Retention changes. | `[{"name": "ReconciliationSucceeded", "count": 10}]` | -| `scraper.relationships` | Kubernetes Relationships to create via name, namespace and kind. [Kubernetes Relationships](/config-db/scrapers/kubernetes#kubernetesrelationships) | Helm and Argo | +| `scraper.relationships` | Kubernetes Relationships to create via name, namespace and kind. [Kubernetes Relationships](/config-db/scrapers/kubernetes#relationships) | Helm and Argo | ### Playbooks diff --git a/mission-control/docs/topology/concepts/templating.mdx b/mission-control/docs/topology/concepts/templating.mdx index 37e25bf7..1f58315b 100644 --- a/mission-control/docs/topology/concepts/templating.mdx +++ b/mission-control/docs/topology/concepts/templating.mdx @@ -2,6 +2,8 @@ title: Templating --- +When building a topology, templating enables you to transform data from the catalog, external data sources into a components and properties of a topology. + **Templating is:** > Simply a way to represent data in different forms. diff --git a/mission-control/docs/topology/lookups/index.mdx b/mission-control/docs/topology/lookups/index.mdx index 3febcb5a..636dccb3 100644 --- a/mission-control/docs/topology/lookups/index.mdx +++ b/mission-control/docs/topology/lookups/index.mdx @@ -86,8 +86,8 @@ spec: | ------------ | -------------------------------------------------------------- | ------------------------------------------------------------ | -------- | | `components` | Create sub-components for each component | [`[]Component`](/reference/topology/components) | | | `properties` | Create or lookup properties for each component | [`[]Property`](/reference/topology/properties) | | -| `configs` | Link configuration items for each component | [`[]ConfigSelector`](./catalog.md#config-selector) | | -| `checks` | Create or link health checks for each component | [`[]CheckSelector`](./health-checks.md#check) | | +| `configs` | Link configuration items for each component | [`[]ConfigSelector`](/reference/topology/components#config-selector) | | +| `checks` | Create or link health checks for each component | [`[]CheckSelector`](/reference/topology/components#check) | | | `selectors` | Select existing components to be used as the child components. | [`[]ResourceSelector`](/reference/resource-selector) | | ### Templating diff --git a/mission-control/docusaurus.config.js b/mission-control/docusaurus.config.js index f129ed21..efbc6f06 100644 --- a/mission-control/docusaurus.config.js +++ b/mission-control/docusaurus.config.js @@ -164,7 +164,6 @@ export default async function createConfigAsync() { showReadingTime: true }, theme: { - customCss: './src/css/out.css' } }) diff --git a/mission-control/sidebars.js b/mission-control/sidebars.js index 84bd628e..8fd84da2 100644 --- a/mission-control/sidebars.js +++ b/mission-control/sidebars.js @@ -103,22 +103,7 @@ const sidebars = { id: 'notifications/index', label: 'Overview' }, - { - type: 'category', - label: 'Concepts', - items: [ - { - type: 'doc', - id: 'notifications/concepts/filtering', - label: 'Filtering' - }, - { - type: 'doc', - id: 'notifications/concepts/templating', - label: 'Templating' - } - ] - }, + { type: 'category', label: 'Events',