diff --git a/config.toml b/config.toml index d15d6106e..1a6799df1 100644 --- a/config.toml +++ b/config.toml @@ -29,7 +29,7 @@ alpine_js_version = "2.2.1" favicon = "favicon.png" [params.versions] -docs = ["2.15", "2.14", "2.13", "2.12", "2.11", "2.10", "2.9", "2.8", "2.7", "2.6", "2.5", "2.4", "2.3", "2.2", "2.1", "2.0", "1.5", "1.4"] +docs = ["2.16", "2.15", "2.14", "2.13", "2.12", "2.11", "2.10", "2.9", "2.8", "2.7", "2.6", "2.5", "2.4", "2.3", "2.2", "2.1", "2.0", "1.5", "1.4"] # Site fonts. For more options see https://fonts.google.com. [[params.fonts]] diff --git a/content/docs/2.17/_index.md b/content/docs/2.17/_index.md new file mode 100644 index 000000000..3dce1ff48 --- /dev/null +++ b/content/docs/2.17/_index.md @@ -0,0 +1,19 @@ ++++ +title = "Getting Started" +weight = 1 ++++ + +Welcome to the documentation for **KEDA**, the Kubernetes Event-driven Autoscaler. + +Use the navigation bar on the left to learn more about KEDA's architecture and how to deploy and use KEDA. + +## Where to go + +What is your involvement with KEDA? + +| Role | Documentation | +|---------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------| +| User | This documentation is for users who want to deploy KEDA to scale Kubernetes. | +| Core Contributor | To contribute to the core KEDA project see the [KEDA GitHub repo](https://github.com/kedacore/keda). | +| Documentation Contributor | To add or contribute to these docs, or to build and serve the documentation locally, see the [keda-docs GitHub repo](https://github.com/kedacore/keda-docs). | +| Other Contributor | See the [KEDA project on GitHub](https://github.com/kedacore/) for other KEDA repos, including project governance, testing, and external scalers. | diff --git a/content/docs/2.17/authentication-providers/_index.md b/content/docs/2.17/authentication-providers/_index.md new file mode 100644 index 000000000..49a2d9cf6 --- /dev/null +++ b/content/docs/2.17/authentication-providers/_index.md @@ -0,0 +1,8 @@ ++++ +title = "Authentication Providers" +weight = 5 ++++ + +Available authentication providers for KEDA: + +{{< authentication-providers >}} diff --git a/content/docs/2.17/authentication-providers/aws-eks.md b/content/docs/2.17/authentication-providers/aws-eks.md new file mode 100644 index 000000000..51ae83327 --- /dev/null +++ b/content/docs/2.17/authentication-providers/aws-eks.md @@ -0,0 +1,14 @@ ++++ +title = "AWS EKS Pod Identity Webhook" ++++ + +[**EKS Pod Identity Webhook**](https://github.com/aws/amazon-eks-pod-identity-webhook), which is described more in depth [here](https://aws.amazon.com/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/), allows you to provide the role name using an annotation on a service account associated with your pod. + +> ⚠️ **WARNING:** [`aws-eks` auth has been deprecated](https://github.com/kedacore/keda/discussions/5343) and support for it will be removed from KEDA on v3. We strongly encourage the migration to [`aws` auth](./aws.md). + +You can tell KEDA to use EKS Pod Identity Webhook via `podIdentity.provider`. + +```yaml +podIdentity: + provider: aws-eks # Optional. Default: none +``` diff --git a/content/docs/2.17/authentication-providers/aws-secret-manager.md b/content/docs/2.17/authentication-providers/aws-secret-manager.md new file mode 100644 index 000000000..fcd9beaf1 --- /dev/null +++ b/content/docs/2.17/authentication-providers/aws-secret-manager.md @@ -0,0 +1,40 @@ ++++ +title = "AWS Secret Manager" ++++ + +You can integrate AWS Secret Manager secrets into your trigger by configuring the `awsSecretManager` key in your KEDA scaling specification. + +The `podIdentity` section configures the usage of AWS pod identity with the provider set to AWS. + +The `credentials` section specifies AWS credentials, including the `accessKey` and `secretAccessKey`. + +- **accessKey:** Configuration for the AWS access key. +- **secretAccessKey:** Configuration for the AWS secret access key. + +The `region` parameter is optional and represents the AWS region where the secret resides, defaulting to the default region if not specified. + +The `secrets` list within `awsSecretManager` defines the mapping between the AWS Secret Manager secret and the authentication parameter used in your application, including the parameter name, AWS Secret Manager secret name, and an optional version parameter, defaulting to the latest version if unspecified. + +### Configuration + +```yaml +awsSecretManager: + podIdentity: # Optional. + provider: aws # Required. + credentials: # Optional. + accessKey: # Required. + valueFrom: # Required. + secretKeyRef: # Required. + name: {k8s-secret-with-aws-credentials} # Required. + key: {key-in-k8s-secret} # Required. + accessSecretKey: # Required. + valueFrom: # Required. + secretKeyRef: # Required. + name: {k8s-secret-with-aws-credentials} # Required. + key: {key-in-k8s-secret} # Required. + region: {aws-region} # Optional. + secrets: # Required. + - parameter: {param-name-used-for-auth} # Required. + name: {aws-secret-name} # Required. + version: {aws-secret-version} # Optional. +``` diff --git a/content/docs/2.17/authentication-providers/aws.md b/content/docs/2.17/authentication-providers/aws.md new file mode 100644 index 000000000..f47cc38c2 --- /dev/null +++ b/content/docs/2.17/authentication-providers/aws.md @@ -0,0 +1,140 @@ ++++ +title = "AWS (IRSA) Pod Identity Webhook" ++++ + +[**AWS IAM Roles for Service Accounts (IRSA) Pod Identity Webhook**](https://github.com/aws/amazon-eks-pod-identity-webhook) ([documentation](https://aws.amazon.com/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/)) allows you to provide the role name using an annotation on a service account associated with your pod. + +You can tell KEDA to use AWS Pod Identity Webhook via `podIdentity.provider`. + +```yaml +podIdentity: + provider: aws + roleArn: # Optional. + identityOwner: keda|workload # Optional. If not set, 'keda' is default value. Mutually exclusive with 'roleArn' (if set) +``` + +**Parameter list:** + +- `roleArn` - Role ARN to be used by KEDA. If not set the IAM role which the KEDA operator uses will be used. Mutually exclusive with `identityOwner: workload` +- `identityOwner` - Owner of the identity to be used. (Values: `keda`, `workload`, Default: `keda`, Optional) + +> ⚠️ **NOTE:** `podIdentity.roleArn` and `podIdentity.identityOwner` are mutually exclusive, setting both is not supported. + +## How to use + +AWS IRSA will give access to pods with service accounts having appropriate annotations. ([official docs](https://aws.amazon.com/es/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/)) You can set these annotations on the KEDA Operator service account. + +This can be done for you during deployment with Helm with the following flags: + +1. `--set podIdentity.aws.irsa.enabled=true` +2. `--set podIdentity.aws.irsa.roleArn={aws-arn-role}` + +You can override the default KEDA operator IAM role by specifying an `roleArn` parameter under the `podIdentity` field. This allows end-users to use different roles to access various resources which allows for more granular access than having a single IAM role that has access to multiple resources. + +If you would like to use the same IAM credentials as your workload is currently using, `podIdentity.identityOwner` can be set with the value `workload` and KEDA will inspect the workload service account to check if IRSA annotation is there and KEDA will assume that role. + +## AssumeRole or AssumeRoleWithWebIdentity? + +This authentication automatically uses both, falling back from [AssumeRoleWithWebIdentity](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html) to [AssumeRole](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html) if the first one fails. This extends the capabilities because KEDA doesn't need `sts:AssumeRole` permission if you are already working with [WebIdentities](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_oidc.html); in this case, you can add a KEDA service account to the trusted relations of the role. + +## Setting up KEDA role and policy + +The [official AWS docs](https://aws.amazon.com/es/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/) explain how to set up a basic configuration for an IRSA role. The policy changes depending on if you are using the KEDA role (`podIdentity.roleArn` is not set) or workload role (`podIdentity.roleArn` sets a RoleArn or `podIdentity.identityOwner` sets to `workload`). + +### Using KEDA role to access infrastructure + +Attach the desired policies to KEDA's role, granting the access permissions that you want to provide. For example, this could be a policy to use with SQS: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "sqs:GetQueueAttributes", + "Resource": "arn:aws:sqs:*:YOUR_ACCOUNT:YOUR_QUEUE" + } + ] +} +``` + +### Using KEDA role to assume workload role using AssumeRoleWithWebIdentity +In this case, KEDA will use its own (k8s) service account to assume workload role (and to use workload's role attached policies). This scenario requires that KEDA service account is trusted for requesting the role using AssumeRoleWithWebIdentity. + +This is an example of how role policy could look like: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + ... YOUR WORKLOAD TRUSTED RELATION ... + }, + { + "Effect": "Allow", + "Principal": { + "Federated": "YOUR_OIDC_ARN" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "YOUR_OIDC:sub": "system:serviceaccount:keda:keda-operator", + "YOUR_OIDC:aud": "sts.amazonaws.com" + } + } + } + ] +} +``` + +### Using KEDA role to assume workload role using AssumeRole + +In this case, KEDA will use its own role to assume the workload role (and to use workload's role attached policies). This scenario is a bit more complex because we need to establish a trusted relationship between both roles and we need to grant to KEDA's role the permission to assume other roles. + +This is an example of how KEDA's role policy could look like: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "sts:AssumeRole", + "Resource": [ + "arn:aws:iam::ACCOUNT_1:role/ROLE_NAME" + ] + }, + { + "Effect": "Allow", + "Action": "sts:AssumeRole", + "Resource": [ + "arn:aws:iam::ACCOUNT_2:role/*" + ] + } + ] +} +``` +This can be extended so that KEDA can assume multiple workload roles, either as an explicit array of role ARNs, or with a wildcard. +This policy attached to KEDA's role will allow KEDA to assume other roles, now you have to allow the workload roles you want to use all allow to being assumed by the KEDA operator role. To achieve this, you have to add a trusted relation to the workload role: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + // Your already existing relations + "Sid": "", + "Effect": "Allow", + // ... + }, + { + "Sid": "", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::ACCOUNT:role/KEDA_ROLE_NAME" + }, + "Action": "sts:AssumeRole" + } + ] +} +``` diff --git a/content/docs/2.17/authentication-providers/azure-ad-workload-identity.md b/content/docs/2.17/authentication-providers/azure-ad-workload-identity.md new file mode 100644 index 000000000..25d033723 --- /dev/null +++ b/content/docs/2.17/authentication-providers/azure-ad-workload-identity.md @@ -0,0 +1,69 @@ ++++ +title = "Azure AD Workload Identity" ++++ + +[**Azure AD Workload Identity**](https://github.com/Azure/azure-workload-identity) is the newer version of [**Azure AD Pod Identity**](https://github.com/Azure/aad-pod-identity). It lets your Kubernetes workloads access Azure resources using an +[**Azure AD Application**](https://docs.microsoft.com/en-us/azure/active-directory/develop/app-objects-and-service-principals) or [**Azure Managed Identity**](https://docs.microsoft.com/azure/active-directory/managed-identities-azure-resources/overview) without having to specify secrets, using [federated identity credentials](https://azure.github.io/azure-workload-identity/docs/topics/federated-identity-credential.html) - *Don't manage secrets, let Azure AD do the hard work*. + +You can tell KEDA to use Azure AD Workload Identity via `podIdentity.provider`. + +```yaml +podIdentity: + provider: azure-workload # Optional. Default: none + identityId: # Optional. Default: ClientId from annotation on service-account. + identityTenantId: # Optional. Default: TenantId from annotation on service-account. + identityAuthorityHost: # Optional. Default: AZURE_AUTHORITY_HOST environment variable which is injected by azure-wi-webhook-controller-manager. +``` + +Azure AD Workload Identity will give access to pods with service accounts having appropriate labels and annotations. Refer +to these [docs](https://azure.github.io/azure-workload-identity/docs/topics/service-account-labels-and-annotations.html) for more information. You can set these labels and annotations on the KEDA Operator service account. This can be done for you during deployment with Helm with the +following flags - + +1. `--set podIdentity.azureWorkload.enabled=true` +2. `--set podIdentity.azureWorkload.clientId={azure-ad-client-id}` +3. `--set podIdentity.azureWorkload.tenantId={azure-ad-tenant-id}` + +You can override the identity that was assigned to KEDA during installation, by specifying an `identityId` parameter under the `podIdentity` field. This allows end-users to use different identities to access various resources which is more secure than using a single identity that has access to multiple resources. + +Additionally, there might be a need for Azure Workload Identity to authenticate across tenants and/or clouds (e.g., AzureCloud, AzureChinaCloud, AzureUSGovernment, AzureGermanCloud). To authenticate against a different tenant within the same cloud, you can specify the `identityTenantId` parameter under the `podIdentity` field. To authenticate against a tenant within a different cloud, you must specify both the `identityTenantId` and `identityAuthorityHost` parameters under the `podIdentity` field. This is useful when you have resources in different tenants or clouds that is different from the tenant and cloud where the KEDA Operator is running. + +## Considerations about Federations and Overrides + +The concept of "overrides" can be somewhat confusing in certain scenarios, as it may not always be clear which service account needs to be federated with a specific Azure identity to ensure proper functionality. + +Let's clarify this with two examples: + +### Case 1 + +Imagine you have an identity for KEDA that has access to ServiceBus A, ServiceBus B, and ServiceBus C. Additionally, you have separate identities for various workloads, resulting in the following setup: + +- KEDA's identity with access to ServiceBus A, ServiceBus B, and ServiceBus C (the identity set during installation and not overridden). +- Workload A's identity with access to Service Bus A. +- Workload B's identity with access to Service Bus B. +- Workload C's identity with access to Service Bus C. + +In this case, KEDA's Managed Service Identity should only be federated with KEDA's service account. + +### Case 2 + +To avoid granting too many permissions to KEDA's identity, you have a KEDA identity without access to any Service Bus (perhaps unrelated, such as Key Vault). Similar to the previous scenario, you also have separate identities for your workloads: + +- KEDA's identity without access to any Service Bus. +- Workload A's identity with access to Service Bus A. +- Workload B's identity with access to Service Bus B. +- Workload C's identity with access to Service Bus C. + +In this case, you are overriding the default identity set during installation through the "TriggerAuthentication" option (`.spec.podIdentity.identityId`). Each "ScaledObject" now uses its own "TriggerAuthentication," with each specifying an override (Workload A's TriggerAuthentication sets the identityId for Workload A, Workload B's for Workload B, and so on). Consequently, you don't need to stack excessive permissions on KEDA's identity. However, in this scenario, KEDA's service account must be federated with all the identities it may attempt to assume: + +- TriggerAuthentications without overrides will use KEDA's identity (for tasks such as accessing the Key Vault). +- TriggerAuthentications with overrides will use the identity specified in the TriggerAuthentication (requiring KEDA's service account to be federated with them). + +### Case 3 + +Similar to the previous scenario, you also have separate identities for your workloads but in different tenants: + +- KEDA's identity without access to any Service Bus. +- Workload A's identity with access to Service Bus A in Tenant A. +- Workload B's identity with access to Service Bus B in Tenant B. + +In this case, you are overriding the default identity and tenant set during installation through the "TriggerAuthentication" option (`.spec.podIdentity.identityId` and `.spec.podIdentity.identityTenantId`). Each "ScaledObject" now uses its own "TriggerAuthentication," with each specifying an override (Workload A's TriggerAuthentication sets the identityId and identityTenantId for Workload A and Workload B's for Workload B). It is important to note that within this scenario, KEDA's service account must be federated with all the identities in each tenant. diff --git a/content/docs/2.17/authentication-providers/azure-key-vault.md b/content/docs/2.17/authentication-providers/azure-key-vault.md new file mode 100644 index 000000000..674229956 --- /dev/null +++ b/content/docs/2.17/authentication-providers/azure-key-vault.md @@ -0,0 +1,41 @@ ++++ +title = "Azure Key Vault secret" ++++ + + +You can pull secrets from Azure Key Vault into the trigger by using the `azureKeyVault` key. + +The `secrets` list defines the mapping between the key vault secret and the authentication parameter. + +Currently, `azure` and `azure-workload` pod identity providers are supported for Azure Key Vault using `podIdentity` inside `azureKeyVault`. + +Service principal authentication is also supported, needing to register an [application](https://docs.microsoft.com/en-us/azure/active-directory/develop/app-objects-and-service-principals) with Azure Active Directory and specifying its credentials. The `clientId` and `tenantId` for the application are to be provided as part of the spec. The `clientSecret` for the application is expected to be within a kubernetes secret in the same namespace as the authentication resource. + +Ensure that "read secret" permissions have been granted to the Azure AD application on the Azure Key Vault. Learn more in the Azure Key Vault [documentation](https://docs.microsoft.com/en-us/azure/key-vault/general/assign-access-policy?tabs=azure-portal). + +The `cloud` parameter can be used to specify cloud environments besides `Azure Public Cloud`, such as known Azure clouds like +`Azure China Cloud`, etc. and even Azure Stack Hub or Air Gapped clouds. + +```yaml +azureKeyVault: # Optional. + vaultUri: {key-vault-address} # Required. + podIdentity: # Optional. + provider: azure-workload # Required. + identityId: # Optional + credentials: # Optional. + clientId: {azure-ad-client-id} # Required. + clientSecret: # Required. + valueFrom: # Required. + secretKeyRef: # Required. + name: {k8s-secret-with-azure-ad-secret} # Required. + key: {key-within-the-secret} # Required. + tenantId: {azure-ad-tenant-id} # Required. + cloud: # Optional. + type: AzurePublicCloud | AzureUSGovernmentCloud | AzureChinaCloud | AzureGermanCloud | Private # Required. + keyVaultResourceURL: {key-vault-resource-url-for-cloud} # Required when type = Private. + activeDirectoryEndpoint: {active-directory-endpoint-for-cloud} # Required when type = Private. + secrets: # Required. + - parameter: {param-name-used-for-auth} # Required. + name: {key-vault-secret-name} # Required. + version: {key-vault-secret-version} # Optional. +``` diff --git a/content/docs/2.17/authentication-providers/configmap.md b/content/docs/2.17/authentication-providers/configmap.md new file mode 100644 index 000000000..48196fad1 --- /dev/null +++ b/content/docs/2.17/authentication-providers/configmap.md @@ -0,0 +1,14 @@ ++++ +title = "Config Map" ++++ + +You can pull information into the trigger by defining the `name` of the Kubernetes `ConfigMap` and the `key` to use. + +```yaml +configMapTargetRef: # Optional. + - parameter: connectionString # Required - Defined by the scale trigger + name: my-keda-configmap-resource-name # Required. + key: azure-storage-connectionstring # Required. +``` + +**Assumptions:** `namespace` is in the same resource as referenced by `scaleTargetRef.name` in the `ScaledObject`, unless specified otherwise. diff --git a/content/docs/2.17/authentication-providers/environment-variable.md b/content/docs/2.17/authentication-providers/environment-variable.md new file mode 100644 index 000000000..787f143ae --- /dev/null +++ b/content/docs/2.17/authentication-providers/environment-variable.md @@ -0,0 +1,14 @@ ++++ +title = "Environment variable" ++++ + +You can pull information via one or more environment variables by providing the `name` of the variable for a given `containerName`. + +```yaml +env: # Optional. + - parameter: region # Required - Defined by the scale trigger + name: my-env-var # Required. + containerName: my-container # Optional. Default: scaleTargetRef.envSourceContainerName of ScaledObject +``` + +**Assumptions:** `containerName` is in the same resource as referenced by `scaleTargetRef.name` in the ScaledObject, unless specified otherwise. diff --git a/content/docs/2.17/authentication-providers/gcp-secret-manager.md b/content/docs/2.17/authentication-providers/gcp-secret-manager.md new file mode 100644 index 000000000..5bffcc834 --- /dev/null +++ b/content/docs/2.17/authentication-providers/gcp-secret-manager.md @@ -0,0 +1,81 @@ ++++ +title = "GCP Secret Manager" ++++ + +You can pull secrets from GCP Secret Manager into the trigger by using the `gcpSecretManager` key. + +The `secrets` list defines the mapping between the secret and the authentication parameter. + +GCP IAM Service Account credentials can be used for authenticating with the Secret Manager service, which can be provided using a Kubernetes secret. Alternatively, `gcp` pod identity provider is also supported for GCP Secret Manager using `podIdentity` inside `gcpSecretManager`. + +```yaml +gcpSecretManager: # Optional. + secrets: # Required. + - parameter: {param-name-used-for-auth} # Required. + id: {secret-manager-secret-name} # Required. + version: {secret-manager-secret-name} # Optional. + podIdentity: # Optional. + provider: gcp # Required. + credentials: # Optional. + clientSecret: # Required. + valueFrom: # Required. + secretKeyRef: # Required. + name: {k8s-secret-with-gcp-iam-sa-secret} # Required. + key: {key-within-the-secret} # Required. +``` + +### Steps to create the IAM Service Account Kubernetes secret +- Create a new GCP IAM service account. In case you would like to use an existing service account, you can skip this step. + + ```shell + gcloud iam service-accounts create GSA_NAME \ + --project=GSA_PROJECT + ``` + + Replace the following: + + GSA_NAME: the name of the new IAM service account.\ + GSA_PROJECT: the project ID of the Google Cloud project for your IAM service account. + +- Ensure that your IAM service account has [roles](https://cloud.google.com/iam/docs/understanding-roles) which provide sufficient [permissions](https://cloud.google.com/iam/docs/permissions-reference) needed to retrieve the secrets, such as the [Secret Manager Secret Accessor](https://cloud.google.com/secret-manager/docs/access-control#secretmanager.secretAccessor). You can grant additional roles using the following command: + + ```shell + gcloud projects add-iam-policy-binding PROJECT_ID \ + --member "serviceAccount:GSA_NAME@GSA_PROJECT.iam.gserviceaccount.com" \ + --role "ROLE_NAME" + ``` + + Replace the following: + + PROJECT_ID: your Google Cloud project ID. \ + GSA_NAME: the name of your IAM service account. \ + GSA_PROJECT: the project ID of the Google Cloud project of your IAM service account. \ + ROLE_NAME: the IAM role to assign to your service account, like roles/secretmanager.secretaccessor. + +- Either setup [GCP workload identity](./gcp-workload-identity) or create a JSON key credential for authenticating with the service account: + + ```shell + gcloud iam service-accounts keys create KEY_FILE \ + --iam-account=GSA_NAME@PROJECT_ID.iam.gserviceaccount.com + ``` + + Replace the following: + + KEY_FILE: the file path to a new output file for the private key in your local machine. \ + GSA_NAME: the name of your IAM service account. \ + PROJECT_ID: your Google Cloud project ID. + +- Create a Kubernetes secret for storing the SA key file in the same namespace where you will create the `TriggerAuthentication` resource: + + ```shell + kubectl create secret generic NAME --from-file=KEY=KEY_FILE -n NAMESPACE + ``` + + Replace the following: + + NAME: name of the Kubernetes secret resource. \ + KEY: Kubernetes secret key for the SA. \ + KEY_FILE: the file path to the SA in your local machine. \ + NAMESPACE: the namespace in which the `TriggerAuthentication` resource will be created. + +Now you can create the `TriggerAuthentication` resource which references the secret-name and key for the SA. diff --git a/content/docs/2.17/authentication-providers/gcp-workload-identity.md b/content/docs/2.17/authentication-providers/gcp-workload-identity.md new file mode 100644 index 000000000..8f36705f1 --- /dev/null +++ b/content/docs/2.17/authentication-providers/gcp-workload-identity.md @@ -0,0 +1,69 @@ ++++ +title = "GCP Workload Identity" ++++ + +[**GCP Workload Identity**](https://cloud.google.com/kubernetes-engine/docs/concepts/workload-identity) allows workloads in your GKE clusters to impersonate Identity and Access Management (IAM) service accounts to access Google Cloud services. + +You can tell KEDA to use GCP Workload Identity via `podIdentity.provider`. + +```yaml +podIdentity: + provider: gcp # Optional. Default: none +``` +### Steps to set up Workload Identity +If you are using podIdentity provider as `gcp`, you need to set up workload identity as below and your GKE cluster must have Workload Identity enabled. + +* You need to create a GCP IAM service account with proper permissions to retrive metrics for particular scalers. + + ```shell + gcloud iam service-accounts create GSA_NAME \ + --project=GSA_PROJECT + ``` + + Replace the following: \ + GSA_NAME: the name of the new IAM service account.\ + GSA_PROJECT: the project ID of the Google Cloud project for your IAM service account. + + +* Ensure that your IAM service account has the [roles](https://cloud.google.com/iam/docs/understanding-roles) you need. You can grant additional roles using the following command: + + ```shell + gcloud projects add-iam-policy-binding PROJECT_ID \ + --member "serviceAccount:GSA_NAME@GSA_PROJECT.iam.gserviceaccount.com" \ + --role "ROLE_NAME" + ``` + + Replace the following: + + PROJECT_ID: your Google Cloud project ID. \ + GSA_NAME: the name of your IAM service account. \ + GSA_PROJECT: the project ID of the Google Cloud project of your IAM service account. \ + ROLE_NAME: the IAM role to assign to your service account, like roles/monitoring.viewer. + +* Allow the Kubernetes service account to impersonate the IAM service account by adding an IAM policy binding between the two service accounts. This binding allows the Kubernetes service account to act as the IAM service account. + ```shell + gcloud iam service-accounts add-iam-policy-binding GSA_NAME@GSA_PROJECT.iam.gserviceaccount.com \ + --role roles/iam.workloadIdentityUser \ + --member "serviceAccount:PROJECT_ID.svc.id.goog[NAMESPACE/KSA_NAME]" + ``` + Replace the following: + + PROJECT_ID: your Google Cloud project ID. \ + GSA_NAME: the name of your IAM service account. \ + GSA_PROJECT: the project ID of the Google Cloud project of your IAM service account. \ + NAMESPACE: Namespace where keda operator is installed; defaults to `keda` . \ + KSA_NAME: Kubernetes service account name of the keda; defaults to `keda-operator` . +* Then you need to annotate the Kubernetes service account with the email address of the IAM service account. + + ```shell + kubectl annotate serviceaccount keda-operator \ + --namespace keda \ + iam.gke.io/gcp-service-account=GSA_NAME@GSA_PROJECT.iam.gserviceaccount.com + ``` + Replace the following: \ + + GSA_NAME: the name of your IAM service account. \ + GSA_PROJECT: the project ID of the Google Cloud project of your IAM service account. + + + Refer to GCP official [documentation](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity#authenticating_to) for more. diff --git a/content/docs/2.17/authentication-providers/hashicorp-vault.md b/content/docs/2.17/authentication-providers/hashicorp-vault.md new file mode 100644 index 000000000..a09b01a4e --- /dev/null +++ b/content/docs/2.17/authentication-providers/hashicorp-vault.md @@ -0,0 +1,89 @@ ++++ +title = "Hashicorp Vault secret" ++++ + + +You can pull one or more Hashicorp Vault secrets into the trigger by defining the authentication metadata such as Vault `address` and the `authentication` method (token | kubernetes). If you choose kubernetes auth method you should provide `role` and `mount` as well. +`credential` defines the Hashicorp Vault credentials depending on the authentication method, for kubernetes you should provide path to service account token (/var/run/secrets/kubernetes.io/serviceaccount/token) and for token auth method provide the token. +`secrets` list defines the mapping between the path and the key of the secret in Vault to the parameter. +`namespace` may be used to target a given Vault Enterprise namespace. + +> Since version `1.5.0` Vault secrets backend **version 2** is supported. +> The support for Vault secrets backend **version 1** was added on version `2.10`. + +```yaml +hashiCorpVault: # Optional. + address: {hashicorp-vault-address} # Required. + namespace: {hashicorp-vault-namespace} # Optional. Default is root namespace. Useful for Vault Enterprise + authentication: token | kubernetes # Required. + role: {hashicorp-vault-role} # Optional. + mount: {hashicorp-vault-mount} # Optional. + credential: # Optional. + token: {hashicorp-vault-token} # Optional. + serviceAccount: {path-to-service-account-file} # Optional. + secrets: # Required. + - parameter: {scaledObject-parameter-name} # Required. + key: {hashicorp-vault-secret-key-name} # Required. + path: {hashicorp-vault-secret-path} # Required. + type: {hashicorp-vault-secret-type} # Optional. Default to `""`. Allowed values: `secret`, `secretV2`, `pki` + pkidata: {hashicorp-vault-secret-pkidata} # Optional. Data to be send with the secret if `hashicorp-vault-secret-type` is pki request + commonName: {hashicorp-vault-secret-pkidata-commonName} # Optional. + altNames: {hashicorp-vault-secret-pkidata-altNames} # Optional. + ipSans: {hashicorp-vault-secret-pkidata-ipSans} # Optional. + uriSans: {hashicorp-vault-secret-pkidata-uriSans} # Optional. + otherSans: {hashicorp-vault-secret-pkidata-otherSans} # Optional. + ttl: {hashicorp-vault-secret-pkidata-ttl} # Optional. + format: {hashicorp-vault-secret-pkidata-format} # Optional. +``` + +### Example +Vault Secret can be used to provide authentication for a Scaler. If using the [Prometheus scaler](https://keda.sh/docs/2.3/scalers/prometheus/), mTls can be used by the `ScaledObject` to authenticate to the Prometheus server. The following example would request a certificate to Vault dynamically. +```yaml +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: {trigger-authentication-mame} + namespace: default +spec: + hashiCorpVault: + address: {hashicorp-vault-address} + authentication: token + credential: + token: {hashicorp-vault-token} + secrets: + - key: "ca_chain" + parameter: "ca" + path: {hashicorp-vault-secret-path} + type: pki + pki_data: + common_name: {hashicorp-vault-secret-pkidata-commonName} + - key: "private_key" + parameter: "key" + path: {hashicorp-vault-secret-path} + type: pki + pki_data: + common_name: {hashicorp-vault-secret-pkidata-commonName} + - key: "certificate" + parameter: "cert" + path: {hashicorp-vault-secret-path} + type: pki + pki_data: + common_name: {hashicorp-vault-secret-pkidata-commonName} +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {scaled-object-name} + namespace: default +spec: + scaleTargetRef: + name: {deployment-name} + triggers: + - type: prometheus + metadata: + serverAddress: http://:9090 + query: sum(rate(http_requests_total{deployment="my-deployment"}[2m])) + authModes: "tls" + authenticationRef: + name: { trigger-authentication-mame } +``` diff --git a/content/docs/2.17/authentication-providers/secret.md b/content/docs/2.17/authentication-providers/secret.md new file mode 100644 index 000000000..afd535811 --- /dev/null +++ b/content/docs/2.17/authentication-providers/secret.md @@ -0,0 +1,14 @@ ++++ +title = "Secret" ++++ + +You can pull one or more secrets into the trigger by defining the `name` of the Kubernetes Secret and the `key` to use. + +```yaml +secretTargetRef: # Optional. + - parameter: connectionString # Required - Defined by the scale trigger + name: my-keda-secret-entity # Required. + key: azure-storage-connectionstring # Required. +``` + +**Assumptions:** `namespace` is in the same resource as referenced by `scaleTargetRef.name` in the ScaledObject, unless specified otherwise. diff --git a/content/docs/2.17/concepts/_index.md b/content/docs/2.17/concepts/_index.md new file mode 100644 index 000000000..b2860c704 --- /dev/null +++ b/content/docs/2.17/concepts/_index.md @@ -0,0 +1,49 @@ ++++ +title = "KEDA Concepts" +description = "What KEDA is and how it works" +weight = 1 ++++ + +## What is KEDA? + +**KEDA** is a [Kubernetes](https://kubernetes.io)-based Event Driven Autoscaler. With KEDA, you can drive the scaling of any container in Kubernetes based on the number of events needing to be processed. + +**KEDA** is a single-purpose and lightweight component that can be added into any Kubernetes cluster. KEDA works alongside standard Kubernetes components like the [Horizontal Pod Autoscaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) and can extend functionality without overwriting or duplication. With KEDA you can explicitly map the apps you want to use event-driven scale, with other apps continuing to function. This makes KEDA a flexible and safe option to run alongside any number of any other Kubernetes applications or frameworks. + +## How KEDA works + +KEDA performs three key roles within Kubernetes: + +1. **Agent** — KEDA activates and deactivates Kubernetes [Deployments](https://kubernetes.io/docs/concepts/workloads/controllers/deployment) to scale to and from zero on no events. This is one of the primary roles of the `keda-operator` container that runs when you install KEDA. +2. **Metrics** — KEDA acts as a [Kubernetes metrics server](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics) that exposes rich event data like queue length or stream lag to the Horizontal Pod Autoscaler to drive scale out. It is up to the Deployment to consume the events directly from the source. This preserves rich event integration and enables gestures like completing or abandoning queue messages to work out of the box. The metric serving is the primary role of the `keda-operator-metrics-apiserver` container that runs when you install KEDA. +3. **Admission Webhooks** - Automatically validate resource changes to prevent misconfiguration and enforce best practices by using an [admission controller](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/). As an example, it will prevent multiple ScaledObjects to target the same scale target. + +## Architecture + +The diagram below shows how KEDA works in conjunction with the Kubernetes Horizontal Pod Autoscaler, external event sources, and Kubernetes' [etcd](https://etcd.io) data store: + +![KEDA architecture](/img/keda-arch.png) + +### Event sources and scalers + +KEDA has a wide range of [**scalers**](/scalers) that can both detect if a deployment should be activated or deactivated, and feed custom metrics for a specific event source. The following scalers are available: + +{{< scalers-compact >}} + +### Custom Resources (CRD) + +When you install KEDA, it creates four custom resources: + +1. `scaledobjects.keda.sh` +1. `scaledjobs.keda.sh` +1. `triggerauthentications.keda.sh` +1. `clustertriggerauthentications.keda.sh` + +These custom resources enable you to map an event source (and the authentication to that event source) to a Deployment, StatefulSet, Custom Resource or Job for scaling. +- `ScaledObjects` represent the desired mapping between an event source (e.g. Rabbit MQ) and the Kubernetes Deployment, StatefulSet or any Custom Resource that defines `/scale` subresource. +- `ScaledJobs` represent the mapping between event source and Kubernetes Job. +- `ScaledObject`/`ScaledJob` may also reference a `TriggerAuthentication` or `ClusterTriggerAuthentication` which contains the authentication configuration or secrets to monitor the event source. + +## Deploy KEDA + +See the [Deployment](../deploy) documentation for instructions on how to deploy KEDA into any cluster using tools like [Helm](../deploy/#helm). diff --git a/content/docs/2.17/concepts/admission-webhooks.md b/content/docs/2.17/concepts/admission-webhooks.md new file mode 100644 index 000000000..b3e5e0e05 --- /dev/null +++ b/content/docs/2.17/concepts/admission-webhooks.md @@ -0,0 +1,24 @@ ++++ +title = "Admission Webhooks" +description = "Automatically validate resource changes to prevent misconfiguration and enforce best practices" +weight = 600 ++++ + +There are some several misconfiguration scenarios that can produce scaling problems in productive workloads, for example: in Kubernetes a single workload should never be scaled by 2 or more HPA because that will produce conflicts and unintended behaviors. + +Some errors with data format can be detected during the model validation, but these misconfigurations can't be detected in that step because the model is correct indeed. For trying to avoid those misconfigurations at data plane detecting them early, admission webhooks validate all the incoming (KEDA) resources (new or updated) and reject any resource that doesn't match the rules below. + +### Prevention Rules + +KEDA will block all incoming changes to `ScaledObject` that don't match these rules: + +- The scaled workload (`scaledobject.spec.scaleTargetRef`) is already autoscaled by another other sources (other ScaledObject or HPA). +- CPU and/or Memory trigger are used and the scaled workload doesn't have the requests defined. **This rule doesn't apply to all the workload types, only to `Deployment` and `StatefulSet`.** +- CPU and/or Memory trigger are **the only used triggers** and the ScaledObject defines `minReplicaCount:0`. **This rule doesn't apply to all the workload types, only to `Deployment` and `StatefulSet`.** +- In the case of multiple triggers where a `name` is **specified**, the name must be **unique** (it is not allowed to have multiple triggers with the same name) +- The fallback feature is configured with a metrics type other than AverageValue. Fallback feature only supports AverageValue metrics type and doesn't support CPU and Memory metrics. + +KEDA will block all incoming changes to `TriggerAuthentication`/`ClusterTriggerAuthentication` that don't match these rules: + +- The specified identity ID for Azure AD Workload Identity and/or Pod Identity is empty. (Default/unset identity ID will be passed through.) + > NOTE: This only applies if the `TriggerAuthentication/ClusterTriggerAuthentication` is overriding the default identityId provided to KEDA during the installation diff --git a/content/docs/2.17/concepts/authentication.md b/content/docs/2.17/concepts/authentication.md new file mode 100644 index 000000000..d751e01ec --- /dev/null +++ b/content/docs/2.17/concepts/authentication.md @@ -0,0 +1,428 @@ ++++ +title = "Authentication" +weight = 500 ++++ + +Often a scaler will require authentication or secrets and config to check for events. + +KEDA provides a few secure patterns to manage authentication flows: + +* Configure authentication per `ScaledObject` +* Re-use per-namespace credentials or delegate authentication with `TriggerAuthentication` +* Re-use global credentials with `ClusterTriggerAuthentication` + +## Defining secrets and config maps on ScaledObject + +Some metadata parameters will not allow resolving from a literal value, and will instead require a reference to a secret, config map, or environment variable defined on the target container. + +> 💡 ***TIP:*** *If creating a deployment yaml that references a secret, be sure the secret is created before the deployment that references it, and the scaledObject after both of them to avoid invalid references.* + +### Example + +If using the [RabbitMQ scaler](https://keda.sh/docs/2.1/scalers/rabbitmq-queue/), the `host` parameter may include passwords so is required to be a reference. You can create a secret with the value of the `host` string, reference that secret in the deployment, and map it to the `ScaledObject` metadata parameter like below: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: {secret-name} +data: + {secret-key-name}: YW1xcDovL3VzZXI6UEFTU1dPUkRAcmFiYml0bXEuZGVmYXVsdC5zdmMuY2x1c3Rlci5sb2NhbDo1Njcy #base64 encoded per secret spec +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {deployment-name} + namespace: default + labels: + app: {deployment-name} +spec: + selector: + matchLabels: + app: {deployment-name} + template: + metadata: + labels: + app: {deployment-name} + spec: + containers: + - name: {deployment-name} + image: {container-image} + envFrom: + - secretRef: + name: {secret-name} +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {scaled-object-name} + namespace: default +spec: + scaleTargetRef: + name: {deployment-name} + triggers: + - type: rabbitmq + metadata: + queueName: hello + host: {secret-key-name} + queueLength : '5' +``` + +If you have multiple containers in a deployment, you will need to include the name of the container that has the references in the `ScaledObject`. If you do not include a `envSourceContainerName` it will default to the first container. KEDA will attempt to resolve references from secrets, config maps, and environment variables of the container. + +### The downsides + +While this method works for many scenarios, there are some downsides: + +* **Difficult to efficiently share auth** config across `ScaledObjects` +* **No support for referencing a secret directly**, only secrets that are referenced by the container +* **No support for other types of authentication flows** such as *pod identity* where access to a source could be acquired with no secrets or connection strings + +For these and other reasons, we also provide a `TriggerAuthentication` resource to define authentication as a separate resource to a `ScaledObject`. This allows you to reference secrets directly, configure to use pod identity or use authentication object managed by a different team. + +## Re-use credentials and delegate auth with TriggerAuthentication + +`TriggerAuthentication` allows you to describe authentication parameters separate from the `ScaledObject` and the deployment containers. It also enables more advanced methods of authentication like "pod identity", authentication re-use or allowing IT to configure the authentication. + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: {trigger-authentication-name} + namespace: default # must be same namespace as the ScaledObject +spec: + podIdentity: + provider: none | azure-workload | aws | aws-eks | gcp # Optional. Default: none + identityId: # Optional. Only used by azure & azure-workload providers. + roleArn: # Optional. Only used by aws provider. + identityOwner: keda|workload # Optional. Only used by aws provider. + secretTargetRef: # Optional. + - parameter: {scaledObject-parameter-name} # Required. + name: {secret-name} # Required. + key: {secret-key-name} # Required. + env: # Optional. + - parameter: {scaledObject-parameter-name} # Required. + name: {env-name} # Required. + containerName: {container-name} # Optional. Default: scaleTargetRef.envSourceContainerName of ScaledObject + hashiCorpVault: # Optional. + address: {hashicorp-vault-address} # Required. + namespace: {hashicorp-vault-namespace} # Optional. Default is root namespace. Useful for Vault Enterprise + authentication: token | kubernetes # Required. + role: {hashicorp-vault-role} # Optional. + mount: {hashicorp-vault-mount} # Optional. + credential: # Optional. + token: {hashicorp-vault-token} # Optional. + serviceAccount: {path-to-service-account-file} # Optional. + secrets: # Required. + - parameter: {scaledObject-parameter-name} # Required. + key: {hashicorp-vault-secret-key-name} # Required. + path: {hashicorp-vault-secret-path} # Required. + azureKeyVault: # Optional. + vaultUri: {key-vault-address} # Required. + podIdentity: # Optional. Required when using pod identity. + provider: azure-workload # Required. + identityId: # Optional + credentials: # Optional. Required when not using pod identity. + clientId: {azure-ad-client-id} # Required. + clientSecret: # Required. + valueFrom: # Required. + secretKeyRef: # Required. + name: {k8s-secret-with-azure-ad-secret} # Required. + key: {key-within-the-secret} # Required. + tenantId: {azure-ad-tenant-id} # Required. + cloud: # Optional. + type: AzurePublicCloud | AzureUSGovernmentCloud | AzureChinaCloud | AzureGermanCloud | Private # Required. + keyVaultResourceURL: {key-vault-resource-url-for-cloud} # Required when type = Private. + activeDirectoryEndpoint: {active-directory-endpoint-for-cloud} # Required when type = Private. + secrets: # Required. + - parameter: {param-name-used-for-auth} # Required. + name: {key-vault-secret-name} # Required. + version: {key-vault-secret-version} # Optional. + awsSecretManager: + podIdentity: # Optional. + provider: aws # Required. + credentials: # Optional. + accessKey: # Required. + valueFrom: # Required. + secretKeyRef: # Required. + name: {k8s-secret-with-aws-credentials} # Required. + key: AWS_ACCESS_KEY_ID # Required. + accessSecretKey: # Required. + valueFrom: # Required. + secretKeyRef: # Required. + name: {k8s-secret-with-aws-credentials} # Required. + key: AWS_SECRET_ACCESS_KEY # Required. + region: {aws-region} # Optional. + secrets: # Required. + - parameter: {param-name-used-for-auth} # Required. + name: {aws-secret-name} # Required. + version: {aws-secret-version} # Optional. + gcpSecretManager: # Optional. + secrets: # Required. + - parameter: {param-name-used-for-auth} # Required. + id: {secret-manager-secret-name} # Required. + version: {secret-manager-secret-name} # Optional. + podIdentity: # Optional. + provider: gcp # Required. + credentials: # Optional. + clientSecret: # Required. + valueFrom: # Required. + secretKeyRef: # Required. + name: {k8s-secret-with-gcp-iam-sa-secret} # Required. + key: {key-within-the-secret} # Required. +``` + +Based on the requirements you can mix and match the reference types providers in order to configure all required parameters. + +Every parameter you define in `TriggerAuthentication` definition does not need to be included in the `metadata` of the trigger for your `ScaledObject` definition. To reference a `TriggerAuthentication` from a `ScaledObject` you add the `authenticationRef` to the trigger. + +```yaml +# some Scaled Object +# ... + triggers: + - type: {scaler-type} + metadata: + param1: {some-value} + authenticationRef: + name: {trigger-authentication-name} # this may define other params not defined in metadata +``` + +## Authentication scopes: Namespace vs. Cluster + +Each `TriggerAuthentication` is defined in one namespace and can only be used by a `ScaledObject` in that same namespace. For cases where you want to share a single set of credentials between scalers in many namespaces, you can instead create a `ClusterTriggerAuthentication`. As a global object, this can be used from any namespace. To set a trigger to use a `ClusterTriggerAuthentication`, add a `kind` field to the authentication reference: + +```yaml + authenticationRef: + name: {cluster-trigger-authentication-name} + kind: ClusterTriggerAuthentication +``` + +By default, Secrets loaded from a `secretTargetRef` must be in the same namespace as KEDA is deployed in (usually `keda`). This can be overridden by setting a `KEDA_CLUSTER_OBJECT_NAMESPACE` environment variable for the `keda-operator` container. + +Defining a `ClusterTriggerAuthentication` works almost identically to a `TriggerAuthentication`, except there is no `metadata.namespace` value: + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ClusterTriggerAuthentication +metadata: + name: {cluster-trigger-authentication-name} +spec: + # As before ... +``` + +## Authentication parameters + +Authentication parameters can be pulled in from many sources. All of these values are merged together to make the authentication data for the scaler. You can find the all the available authentications [here](./../authentication-providers/). + +### Environment variable(s) + +You can pull information via one or more environment variables by providing the `name` of the variable for a given `containerName`. + +```yaml +env: # Optional. + - parameter: region # Required - Defined by the scale trigger + name: my-env-var # Required. + containerName: my-container # Optional. Default: scaleTargetRef.envSourceContainerName of ScaledObject +``` + +**Assumptions:** `containerName` is in the same resource as referenced by `scaleTargetRef.name` in the ScaledObject, unless specified otherwise. + +### Secret(s) + +You can pull one or more secrets into the trigger by defining the `name` of the Kubernetes Secret and the `key` to use. + +```yaml +secretTargetRef: # Optional. + - parameter: connectionString # Required - Defined by the scale trigger + name: my-keda-secret-entity # Required. + key: azure-storage-connectionstring # Required. +``` + +**Assumptions:** `namespace` is in the same resource as referenced by `scaleTargetRef.name` in the ScaledObject, unless specified otherwise. + +### Hashicorp Vault secret(s) + +You can pull one or more Hashicorp Vault secrets into the trigger by defining the authentication metadata such as Vault `address` and the `authentication` method (token | kubernetes). If you choose kubernetes auth method you should provide `role` and `mount` as well. +`credential` defines the Hashicorp Vault credentials depending on the authentication method, for kubernetes you should provide path to service account token (the default value is `/var/run/secrets/kubernetes.io/serviceaccount/token`) and for token auth method provide the token. +`secrets` list defines the mapping between the path and the key of the secret in Vault to the parameter. +`namespace` may be used to target a given Vault Enterprise namespace. + +```yaml +hashiCorpVault: # Optional. + address: {hashicorp-vault-address} # Required. + namespace: {hashicorp-vault-namespace} # Optional. Default is root namespace. Useful for Vault Enterprise + authentication: token | kubernetes # Required. + role: {hashicorp-vault-role} # Optional. + mount: {hashicorp-vault-mount} # Optional. + credential: # Optional. + token: {hashicorp-vault-token} # Optional. + serviceAccount: {path-to-service-account-file} # Optional. Default is /var/run/secrets/kubernetes.io/serviceaccount/token + secrets: # Required. + - parameter: {scaledObject-parameter-name} # Required. + key: {hashicorp-vault-secret-key-name} # Required. + path: {hashicorp-vault-secret-path} # Required. +``` + +### Azure Key Vault secret(s) + +You can pull secrets from Azure Key Vault into the trigger by using the `azureKeyVault` key. + +The `secrets` list defines the mapping between the key vault secret and the authentication parameter. + +You can use pod identity providers `azure` or `azure-workload` to authenticate to the key vault by specifying it in the +`TriggerAuthentication` / `ClusterTriggerAuthentication` definition. Pod Identity binding needs to be applied in the keda namespace. + +If you do not wish to use a pod identity provider, you need to register an [application](https://docs.microsoft.com/en-us/azure/active-directory/develop/app-objects-and-service-principals) with Azure Active Directory and specify its credentials. The `clientId` and `tenantId` for the application are to be provided as part of the spec. The `clientSecret` for the application is expected to be within a kubernetes secret in the same namespace as the authentication resource. + +Ensure that "read secret" permissions have been granted to the managed identity / Azure AD application on the Azure Key Vault. Learn more in the Azure Key Vault [documentation](https://docs.microsoft.com/en-us/azure/key-vault/general/assign-access-policy?tabs=azure-portal). + +The `cloud` parameter can be used to specify cloud environments besides `Azure Public Cloud`, such as known Azure clouds like +`Azure China Cloud`, etc. and even Azure Stack Hub or Air Gapped clouds. + +```yaml +azureKeyVault: # Optional. + vaultUri: {key-vault-address} # Required. + credentials: # Optional. Required when not using pod identity. + clientId: {azure-ad-client-id} # Required. + clientSecret: # Required. + valueFrom: # Required. + secretKeyRef: # Required. + name: {k8s-secret-with-azure-ad-secret} # Required. + key: {key-within-the-secret} # Required. + tenantId: {azure-ad-tenant-id} # Required. + cloud: # Optional. + type: AzurePublicCloud | AzureUSGovernmentCloud | AzureChinaCloud | AzureGermanCloud | Private # Required. + keyVaultResourceURL: {key-vault-resource-url-for-cloud} # Required when type = Private. + activeDirectoryEndpoint: {active-directory-endpoint-for-cloud} # Required when type = Private. + secrets: # Required. + - parameter: {param-name-used-for-auth} # Required. + name: {key-vault-secret-name} # Required. + version: {key-vault-secret-version} # Optional. +``` + +### GCP Secret Manager secret(s) + +You can pull secrets from GCP Secret Manager into the trigger by using the `gcpSecretManager` key. + +The `secrets` list defines the mapping between the secret and the authentication parameter. + +GCP IAM Service Account credentials can be used for authenticating with the Secret Manager service, which can be provided using a Kubernetes secret. Alternatively, `gcp` pod identity provider is also supported for GCP Secret Manager using `podIdentity` inside `gcpSecretManager`. + +```yaml +gcpSecretManager: # Optional. + secrets: # Required. + - parameter: {param-name-used-for-auth} # Required. + id: {secret-manager-secret-name} # Required. + version: {secret-manager-secret-name} # Optional. + podIdentity: # Optional. + provider: gcp # Required. + credentials: # Optional. + clientSecret: # Required. + valueFrom: # Required. + secretKeyRef: # Required. + name: {k8s-secret-with-gcp-iam-sa-secret} # Required. + key: {key-within-the-secret} # Required. +``` + +### Pod Authentication Providers + +Several service providers allow you to assign an identity to a pod. By using that identity, you can defer authentication to the pod & the service provider, rather than configuring secrets. + +Currently we support the following: + +```yaml +podIdentity: + provider: none | azure-workload | aws | aws-eks | gcp # Optional. Default: none + identityId: # Optional. Only used by azure & azure-workload providers. + roleArn: # Optional. Only used by aws provider. + identityOwner: keda|workload # Optional. Only used by aws provider. +``` + +#### Azure Workload Identity + +[**Azure AD Workload Identity**](https://github.com/Azure/azure-workload-identity) is the newer version of [**Azure AD Pod Identity**](https://github.com/Azure/aad-pod-identity). It lets your Kubernetes workloads access Azure resources using an +[**Azure AD Application**](https://docs.microsoft.com/en-us/azure/active-directory/develop/app-objects-and-service-principals) +without having to specify secrets, using [federated identity credentials](https://azure.github.io/azure-workload-identity/docs/topics/federated-identity-credential.html) - *Don't manage secrets, let Azure AD do the hard work*. + +You can tell KEDA to use Azure AD Workload Identity via `podIdentity.provider`. + +```yaml +podIdentity: + provider: azure-workload # Optional. Default: none + identityId: # Optional. Default: ClientId From annotation on service-account. +``` + +Azure AD Workload Identity will give access to pods with service accounts having appropriate labels and annotations. Refer +to these [docs](https://azure.github.io/azure-workload-identity/docs/topics/service-account-labels-and-annotations.html) for more information. You can set these labels and annotations on the KEDA Operator service account. This can be done for you during deployment with Helm with the +following flags - + +1. `--set podIdentity.azureWorkload.enabled=true` +2. `--set podIdentity.azureWorkload.clientId={azure-ad-client-id}` +3. `--set podIdentity.azureWorkload.tenantId={azure-ad-tenant-id}` + +Setting `podIdentity.azureWorkload.enabled` to `true` is required for workload identity authentication to work. For KEDA to get access to the provided client id federated credential has to be configured on the target Managed Identity / Azure AD application. Refer to these [docs](https://azure.github.io/azure-workload-identity/docs/topics/federated-identity-credential.html). Federated credential should use this subject (if KEDA is installed in `keda` namespace): `system:serviceaccount:keda:keda-operator`. + +You can override the identity that was assigned to KEDA during installation, by specifying an `identityId` parameter under the `podIdentity` field. This allows end-users to use different identities to access various resources which is more secure than using a single identity that has access to multiple resources. In the case of override federated credentials should be configured for each of the used identities. + +### Aws Secret Manager(s) + +You can integrate AWS Secret Manager secrets into your trigger by configuring the `awsSecretManager` key in your KEDA scaling specification. + +The `podIdentity` section configures the usage of AWS pod identity with the provider set to AWS. + +The `credentials` section specifies AWS credentials, including the `accessKey` and `secretAccessKey`. + +- **accessKey:** Configuration for the AWS access key. +- **secretAccessKey:** Configuration for the AWS secret access key. + +The `region` parameter is optional and represents the AWS region where the secret resides, defaulting to the default region if not specified. + +The `secrets` list within `awsSecretManager` defines the mapping between the AWS Secret Manager secret and the authentication parameter used in your application, including the parameter name, AWS Secret Manager secret name, and an optional version parameter, defaulting to the latest version if unspecified. + +```yaml +awsSecretManager: + podIdentity: # Optional. + provider: aws # Required. + credentials: # Optional. + accessKey: # Required. + valueFrom: # Required. + secretKeyRef: # Required. + name: {k8s-secret-with-aws-credentials} # Required. + key: AWS_ACCESS_KEY_ID # Required. + accessSecretKey: # Required. + valueFrom: # Required. + secretKeyRef: # Required. + name: {k8s-secret-with-aws-credentials} # Required. + key: AWS_SECRET_ACCESS_KEY # Required. + region: {aws-region} # Optional. + secrets: # Required. + - parameter: {param-name-used-for-auth} # Required. + name: {aws-secret-name} # Required. + version: {aws-secret-version} # Optional. +``` + +#### AWS Pod Identity Webhook for AWS + +[**AWS IAM Roles for Service Accounts (IRSA) Pod Identity Webhook**](https://github.com/aws/amazon-eks-pod-identity-webhook) ([documentation](https://aws.amazon.com/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/)) allows you to provide the role name using an annotation on a service account associated with your pod. + +You can tell KEDA to use EKS Pod Identity Webhook via `podIdentity.provider`. + +```yaml +podIdentity: + provider: aws # Optional. Default: none + roleArn: # Optional. + identityOwner: keda|workload # Optional. +``` + +#### AWS EKS Pod Identity Webhook + +> [DEPRECATED: This will be removed in KEDA v3](https://github.com/kedacore/keda/discussions/5343) + +[**EKS Pod Identity Webhook**](https://github.com/aws/amazon-eks-pod-identity-webhook), which is described more in depth [here](https://aws.amazon.com/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/), allows you to provide the role name using an annotation on a service account associated with your pod. + +You can tell KEDA to use EKS Pod Identity Webhook via `podIdentity.provider`. + +```yaml +podIdentity: + provider: aws-eks # Optional. Default: none +``` \ No newline at end of file diff --git a/content/docs/2.17/concepts/external-scalers.md b/content/docs/2.17/concepts/external-scalers.md new file mode 100644 index 000000000..d58ffe06d --- /dev/null +++ b/content/docs/2.17/concepts/external-scalers.md @@ -0,0 +1,636 @@ ++++ +title = "External Scalers" +weight = 500 ++++ + +While KEDA ships with a set of [built-in scalers](../scalers), users can also extend KEDA through a [GRPC](https://grpc.io) service that implements the same interface as the built-in scalers. + +Built-in scalers run in the KEDA process/pod, while external scalers require an externally managed GRPC server that's accessible from KEDA with optional [TLS authentication](https://grpc.io/docs/guides/auth/). KEDA itself acts as a GRPC client and it exposes similar service interface for the built-in scalers, so external scalers can fully replace built-in ones. + +This document describes the external scaler interfaces and how to implement them in Go, Node, and .NET; however for more details on GRPC refer to [the official GRPC documentation](https://grpc.io/docs/) + +> Want to learn about existing external scalers? Explore our [external scaler community](https://github.com/kedacore/external-scalers). + +## Overview + +### Built-in scalers interface + +Since external scalers mirror the interface of built-in scalers, it's worth becoming familiar with the Go `interface` that the built-in scalers implement: + +```go +// Scaler interface +type Scaler interface { + // GetMetricsAndActivity returns the metric values and activity for a metric Name + GetMetricsAndActivity(ctx context.Context, metricName string) ([]external_metrics.ExternalMetricValue, bool, error) + // GetMetricSpecForScaling returns the metrics based on which this scaler determines that the ScaleTarget scales. This is used to construct the HPA spec that is created for + // this scaled object. The labels used should match the selectors used in GetMetrics + GetMetricSpecForScaling(ctx context.Context) []v2.MetricSpec + // Close any resources that need disposing when scaler is no longer used or destroyed + Close(ctx context.Context) error +} + +// PushScaler interface +type PushScaler interface { + Scaler + + // Run is the only writer to the active channel and must close it once done. + Run(ctx context.Context, active chan<- bool) +} + +``` + +The `Scaler` interface defines 3 methods: + +- `Close` is called to allow the scaler to clean up connections or other resources. +- `GetMetricSpecForScaling` returns the target value for the HPA definition for the scaler. For more details refer to [Implementing `GetMetricSpec`](#5-implementing-getmetricspec). +- `GetMetricsAndActivity` is called on `pollingInterval` and. When activity returns `true`, KEDA will scale to what is returned by the metric limited by `maxReplicaCount` on the ScaledObject/ScaledJob. + When `false` is returned, KEDA will scale to `minReplicaCount` or optionally `idleReplicaCount`. More details around the defaults and how these options work together can be found on the [ScaledObjectSpec](../reference/scaledobject-spec). +> Refer to the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) for how HPA calculates `replicaCount` based on metric value and target value. KEDA supports both `AverageValue` and `Value` metric target types for external metrics. When `AverageValue` (the default metric type) is used, the metric value returned by the external scaler will be divided by the number of replicas. + +The `PushScaler` interface adds a `Run` method. This method receives a push channel (`active`), on which the scaler can send `true` at any time. The purpose of this mechanism is to initiate a scaling operation independently from `pollingInterval`. + +### External Scaler GRPC interface + +KEDA comes with 2 external scalers [`external`](../scalers/external.md) and [`external-push`](../scalers/external-push.md). + +The configuration in the ScaledObject points to a GRPC service endpoint that implements the [`externalscaler.proto`](https://github.com/kedacore/keda/blob/main/pkg/scalers/externalscaler/externalscaler.proto) GRPC contract: + +```proto +service ExternalScaler { + rpc IsActive(ScaledObjectRef) returns (IsActiveResponse) {} + rpc StreamIsActive(ScaledObjectRef) returns (stream IsActiveResponse) {} + rpc GetMetricSpec(ScaledObjectRef) returns (GetMetricSpecResponse) {} + rpc GetMetrics(GetMetricsRequest) returns (GetMetricsResponse) {} +} +``` + +Much of this contract is similar to the built-in scalers: + +- `GetMetricsSpec` mirrors its counterpart in the `Scaler` interface for creating HPA definition. +- `IsActive` and `GetMetrics` map to the `GetMetricsAndActivity` method on the `Scaler` interface. +- `StreamIsActive` maps to the `Run` method on the `PushScaler` interface. + +There are, however, some notable differences: + +- There is no `Close` method. The scaler is expected to be functional throughout its lifetime. +- `IsActive`, `StreamIsActive`, and `GetMetricsSpec` are called with a `ScaledObjectRef` that contains the scaledObject name/namespace as well as the content of `metadata` defined in the trigger. + +### Example + +Given the following `ScaledObject`: + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: scaledobject-name + namespace: scaledobject-namespace +spec: + scaleTargetRef: + name: deployment-name + triggers: + - type: external-push + metadata: + scalerAddress: service-address.svc.local:9090 + key1: value1 + key2: value2 +``` + +KEDA will attempt a GRPC connection to `service-address.svc.local:9090` immediately after reconciling the `ScaledObject`. It will then make the following RPC calls: + +- `IsActive` - KEDA does an initial call to `IsActive` followed by one call on each `pollingInterval` +- `StreamIsActive` - KEDA does an initial call and the scaler is expected to maintain a long-lived connection (called a `stream` in GRPC terminology). The external push scaler can then send an `IsActive` event back to KEDA at any time. KEDA will only attempt another call to `StreamIsActive` if it needs to re-connect +- `GetMetricsSpec` - KEDA will do an initial call with the following data in the incoming `ScaledObjectRef` parameter: +- `GetMetrics` - KEDA will call this method every `pollingInterval` to get the point-in-time metric values for the names returned by `GetMetricsSpec`. + +```json +{ + "name": "scaledobject-name", + "namespace": "scaledobject-namespace", + "scalerMetadata": { + "scalerAddress": "service-address.svc.local:9090", + "key1": "value1", + "key2": "value2" + } +} +``` + +> **Note**: KEDA will issue all of the above RPC calls except `StreamIsActive` if `spec.triggers.type` is `external`. It _must_ be `external-push` for `StreamIsActive` to be called. + +## Implementing KEDA external scaler GRPC interface + +### Implementing an external scaler + +#### 1. Download [`externalscaler.proto`](https://github.com/kedacore/keda/blob/main/pkg/scalers/externalscaler/externalscaler.proto) + +#### 2. Prepare project + +{{< collapsible "Golang" >}} + +2.1. Download [`./protoc`](https://github.com/protocolbuffers/protobuf/releases) for your platform + +2.2. get `protoc-gen-go` + +```bash +go get github.com/golang/protobuf/protoc-gen-go@v1.3.2 +``` + +2.3. Prepare project + +```bash +go mod init example.com/external-scaler/sample +mkdir externalscaler +protoc externalscaler.proto --go_out=plugins=grpc:externalscaler +``` + +{{< /collapsible >}} + +{{< collapsible "C#" >}} +2.1. Create a new project + +```bash +dotnet new console -o ExternalScalerSample +cd ExternalScalerSample + +# add Grpc.AspNetCore +dotnet add package Grpc.AspNetCore +dotnet add package Newtonsoft.Json + +# Create a Protos and Services folders +mkdir Protos +mkdir Services +``` + +2.2. Move `externalscaler.proto` to `Protos` folder + +2.3. Compile `externalscaler.proto` using this in `ExternalScalerSample.csproj` + +```xml + + + +``` + +{{< /collapsible >}} + +{{< collapsible "Javascript" >}} +2.1. Prepare the project + +```bash +npm install --save grpc request +``` + +{{< /collapsible >}} + +#### 3. Implementing `IsActive` + +Just like `IsActive(ctx context.Context) (bool, error)` in the go interface, the `IsActive` method in the GRPC interface is called every `pollingInterval` with a `ScaledObjectRef` object that contains the scaledObject name, namespace, and scaler metadata. + +This section implements an external scaler that queries earthquakes from [earthquake.usgs.gov](https://earthquake.usgs.gov/) and scales the deployment if there has been more than 2 earthquakes with `magnitude > 1.0` around a particular longitude/latitude in the previous day. + +Submit the following `ScaledObject` to tell KEDA to start making RPC calls to your external scaler (modifying appropriate fields as necessary): + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: scaledobject-name + namespace: scaledobject-namespace +spec: + scaleTargetRef: + name: deployment-name + triggers: + - type: external + metadata: + scalerAddress: earthquake-scaler:9090 + longitude: "-122.335167" + latitude: "47.608013" +``` + +{{< collapsible "Golang" >}} + +The full implementation can be found at [github.com/kedacore/external-scaler-samples](https://github.com/kedacore/external-scaler-samples). + +Put the following code into your `main.go` file: + +```golang +func (e *ExternalScaler) IsActive(ctx context.Context, scaledObject *pb.ScaledObjectRef) (*pb.IsActiveResponse, error) { + // request.Scalermetadata contains the `metadata` defined in the ScaledObject + longitude := scaledObject.ScalerMetadata["longitude"] + latitude := scaledObject.ScalerMetadata["latitude"] + + if len(longitude) == 0 || len(latitude) == 0 { + return nil, status.Error(codes.InvalidArgument, "longitude and latitude must be specified") + } + + startTime := time.Now().AddDate(0, 0, -1).Format("2006-01-02") + endTime := time.Now().Format("2006-01-02") + radiusKM := 500 + query := fmt.Sprintf("format=geojson&starttime=%s&endtime=%s&longitude=%s&latitude=%s&maxradiuskm=%d", startTime, endTime, longitude, latitude, radiusKM) + + resp, err := http.Get(fmt.Sprintf("https://earthquake.usgs.gov/fdsnws/event/1/query?%s", query)) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + payload := USGSResponse{} + err = json.Unmarshal(body, &payload) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + // count how many earthquakes with mag > 1.0 + count := 0 + for _, f := range payload.Features { + if f.Properties.Mag > 1.0 { + count++ + } + } + + // return true if there is more than 2 + return &pb.IsActiveResponse{ + Result: count > 2, + }, nil +} +``` + +{{< /collapsible >}} + +{{< collapsible "C#" >}} + +Full implementation can be found at [github.com/kedacore/external-scaler-samples](https://github.com/kedacore/external-scaler-samples). + +Put the following code into your `Services/ExternalScalerService.cs` file: + +```csharp +public class ExternalScalerService : ExternalScaler.ExternalScalerBase +{ + private static readonly HttpClient _client = new HttpClient(); + + public override async Task IsActive(ScaledObjectRef request, ServerCallContext context) + { + // request.Scalermetadata contains the `metadata` defined in the ScaledObject + if (!request.ScalerMetadata.ContainsKey("latitude") || + !request.ScalerMetadata.ContainsKey("longitude")) { + throw new ArgumentException("longitude and latitude must be specified"); + } + + var longitude = request.ScalerMetadata["longitude"]; + var latitude = request.ScalerMetadata["latitude"]; + var startTime = DateTime.UtcNow.AddDays(-1).ToString("yyyy-MM-dd"); + var endTime = DateTime.UtcNow.ToString("yyyy-MM-dd"); + var radiusKm = 500; + var query = $"format=geojson&starttime={startTime}&endtime={endTime}&longitude={longitude}&latitude={latitude}&maxradiuskm={radiusKm}"; + + var resp = await _client.GetAsync($"https://earthquake.usgs.gov/fdsnws/event/1/query?{query}"); + resp.EnsureSuccessStatusCode(); + var payload = JsonConvert.DeserializeObject(await resp.Content.ReadAsStringAsync()); + + return new IsActiveResponse + { + // return true if there is more than 2 Earthquakes with mag > 1.0 + Result = payload.features.Count(f => f.properties.mag > 1.0) > 2 + }; + } +} +``` + +{{< /collapsible >}} + +{{< collapsible "Javascript" >}} + +Put the following code into your `index.js` file: + +```js +const grpc = require("grpc"); +const request = require("request"); +const externalScalerProto = grpc.load("externalscaler.proto"); + +const server = new grpc.Server(); +server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { + isActive: (call, callback) => { + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; + if (!longitude || !latitude) { + callback({ + code: grpc.status.INVALID_ARGUMENT, + details: "longitude and latitude must be specified", + }); + } else { + const now = new Date(); + const yesterday = new Date(new Date().setDate(new Date().getDate() - 1)); + + const startTime = `${yesterday.getUTCFullYear()}-${yesterday.getUTCMonth()}-${yesterday.getUTCDay()}`; + const endTime = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDay()}`; + const radiusKm = 500; + const query = `format=geojson&starttime=${startTime}&endtime=${endTime}&longitude=${longitude}&latitude=${latitude}&maxradiuskm=${radiusKm}`; + + request.get( + { + url: `https://earthquake.usgs.gov/fdsnws/event/1/query?${query}`, + json: true, + }, + (err, resp, data) => { + if (err) { + callback({ + code: grpc.status.INTERNAL, + details: err, + }); + } else if (resp.statusCode !== 200) { + callback({ + code: grpc.status.INTERNAL, + details: `expected status 200, got ${resp.statusCode}`, + }); + } else { + // count how many earthquakes with mag > 1.0 + let count = 0; + data.features.forEach((i) => { + if (i.properties.mag > 1.0) { + count++; + } + }); + callback(null, { + result: count > 2, + }); + } + } + ); + } + }, +}); + +server.bind("127.0.0.1:9090", grpc.ServerCredentials.createInsecure()); +console.log("Server listening on 127.0.0.1:9090"); + +server.start(); +``` + +{{< /collapsible >}} + +#### 4. Implementing `StreamIsActive` + +Unlike `IsActive`, `StreamIsActive` is called once when KEDA reconciles the `ScaledObject`, and expects the external scaler to maintain a long-lived connection and push `IsActiveResponse` objects whenever the scaler needs KEDA to activate the deployment. + +This implementation creates a timer and queries USGS APIs on that timer, effectively ignoring `pollingInterval` set in the scaledObject. Alternatively any other asynchronous event can be used instead of a timer, like an HTTP request, or a network connection. + +{{< collapsible "Golang" >}} + +```golang +func (e *ExternalScaler) StreamIsActive(scaledObject *pb.ScaledObjectRef, epsServer pb.ExternalScaler_StreamIsActiveServer) error { + longitude := scaledObject.ScalerMetadata["longitude"] + latitude := scaledObject.ScalerMetadata["latitude"] + + if len(longitude) == 0 || len(latitude) == 0 { + return status.Error(codes.InvalidArgument, "longitude and latitude must be specified") + } + + for { + select { + case <-epsServer.Context().Done(): + // call cancelled + return nil + case <-time.Tick(time.Hour * 1): + earthquakeCount, err := getEarthQuakeCount(longitude, latitude) + if err != nil { + // log error + } else if earthquakeCount > 2 { + err = epsServer.Send(&pb.IsActiveResponse{ + Result: true, + }) + } + } + } +} +``` + +{{< /collapsible >}} + +{{< collapsible "C#" >}} + +```csharp +public override async Task StreamIsActive(ScaledObjectRef request, IServerStreamWriter responseStream, ServerCallContext context) +{ + if (!request.ScalerMetadata.ContainsKey("latitude") || + !request.ScalerMetadata.ContainsKey("longitude")) + { + throw new ArgumentException("longitude and latitude must be specified"); + } + + var longitude = request.ScalerMetadata["longitude"]; + var latitude = request.ScalerMetadata["latitude"]; + var key = $"{longitude}|{latitude}"; + + while (!context.CancellationToken.IsCancellationRequested) + { + var earthquakeCount = await GetEarthQuakeCount(longitude, latitude); + if (earthquakeCount > 2) { + await responseStream.WriteAsync(new IsActiveResponse + { + Result = true + }); + } + + await Task.Delay(TimeSpan.FromHours(1)); + } +} +``` + +{{< /collapsible >}} + +{{< collapsible "Javascript" >}} + +```js +server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { + // ... + streamIsActive: (call, callback) => { + const longitude = call.request.scalerMetadata.longitude; + const latitude = call.request.scalerMetadata.latitude; + if (!longitude || !latitude) { + callback({ + code: grpc.status.INVALID_ARGUMENT, + details: "longitude and latitude must be specified", + }); + } else { + const interval = setInterval(() => { + getEarthquakeCount((err, count) => { + if (err) { + console.error(err); + } else if (count > 2) { + call.write({ + result: true, + }); + } + }); + }, 1000 * 60 * 60); + + call.on("end", () => { + clearInterval(interval); + }); + } + }, +}); +``` + +{{< /collapsible >}} + +#### 5. Implementing `GetMetricSpec` + +`GetMetricSpec` returns the `target` value for [the HPA definition for the scaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/). This scaler will define a static target of 10, but the threshold value is often specified in the metadata for other scalers. + +{{< collapsible "Golang" >}} + +```golang +func (e *ExternalScaler) GetMetricSpec(context.Context, *pb.ScaledObjectRef) (*pb.GetMetricSpecResponse, error) { + return &pb.GetMetricSpecResponse{ + MetricSpecs: []*pb.MetricSpec{{ + MetricName: "earthquakeThreshold", + TargetSize: 10, + }}, + }, nil +} +``` + +{{< /collapsible >}} + +{{< collapsible "C#" >}} + +```csharp +public override async Task GetMetricSpec(ScaledObjectRef request, ServerCallContext context) +{ + var resp = new GetMetricSpecResponse(); + + resp.MetricSpecs.Add(new MetricSpec + { + MetricName = "earthquakeThreshold", + TargetSize = 10 + }); + + return Task.FromResult(resp); +} +``` + +{{< /collapsible >}} + +{{< collapsible "Javascript" >}} + +```js +server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { + // ... + getMetricSpec: (call, callback) => { + callback(null, { + metricSpecs: [ + { + metricName: "earthquakeThreshold", + targetSize: 10, + }, + ], + }); + }, +}); +``` + +{{< /collapsible >}} + +#### 6. Implementing `GetMetrics` + +`GetMetrics` returns the value of the metric referred to from `GetMetricSpec`, in this example it's `earthquakeThreshold`. + +{{< collapsible "Golang" >}} + +```golang +func (e *ExternalScaler) GetMetrics(_ context.Context, metricRequest *pb.GetMetricsRequest) (*pb.GetMetricsResponse, error) { + longitude := metricRequest.ScaledObjectRef.ScalerMetadata["longitude"] + latitude := metricRequest.ScaledObjectRef.ScalerMetadata["latitude"] + + if len(longitude) == 0 || len(latitude) == 0 { + return nil, status.Error(codes.InvalidArgument, "longitude and latitude must be specified") + } + + earthquakeCount, err := getEarthQuakeCount(longitude, latitude, 1.0) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + return &pb.GetMetricsResponse{ + MetricValues: []*pb.MetricValue{{ + MetricName: "earthquakeThreshold", + MetricValue: int64(earthquakeCount), + }}, + }, nil +} +``` + +{{< /collapsible >}} + +{{< collapsible "C#" >}} + +```csharp +public override async Task GetMetrics(GetMetricsRequest request, ServerCallContext context) +{ + if (!request.ScaledObjectRef.ScalerMetadata.ContainsKey("latitude") || + !request.ScaledObjectRef.ScalerMetadata.ContainsKey("longitude")) + { + throw new ArgumentException("longitude and latitude must be specified"); + } + + var longitude = request.ScaledObjectRef.ScalerMetadata["longitude"]; + var latitude = request.ScaledObjectRef.ScalerMetadata["latitude"]; + + var earthquakeCount = await GetEarthQuakeCount(longitude, latitude); + + var resp = new GetMetricsResponse(); + resp.MetricValues.Add(new MetricValue + { + MetricName = "earthquakeThreshold", + MetricValue_ = earthquakeCount + }); + + return resp; +} +``` + +{{< /collapsible >}} + +{{< collapsible "Javascript" >}} + +```js +server.addService(externalScalerProto.externalscaler.ExternalScaler.service, { + // ... + getMetrics: (call, callback) => { + const longitude = call.request.scaledObjectRef.scalerMetadata.longitude; + const latitude = call.request.scaledObjectRef.scalerMetadata.latitude; + if (!longitude || !latitude) { + callback({ + code: grpc.status.INVALID_ARGUMENT, + details: "longitude and latitude must be specified", + }); + } else { + getEarthquakeCount((err, count) => { + if (err) { + callback({ + code: grpc.status.INTERNAL, + details: err, + }); + } else { + callback(null, { + metricValues: [ + { + metricName: "earthquakeThreshold", + metricValue: count, + }, + ], + }); + } + }); + } + }, +}); +``` + +{{< /collapsible >}} diff --git a/content/docs/2.17/concepts/scaling-deployments.md b/content/docs/2.17/concepts/scaling-deployments.md new file mode 100644 index 000000000..63a5f6661 --- /dev/null +++ b/content/docs/2.17/concepts/scaling-deployments.md @@ -0,0 +1,207 @@ ++++ +title = "Scaling Deployments, StatefulSets & Custom Resources" +weight = 200 ++++ + +This page describes the deployment scaling behavior of KEDA. + +# Specification + +See the [Scaled Object specification](../reference/scaledobject-spec.md) for details on how to set the behaviors described below. + +# Scaling objects + +## Scaling Deployments and StatefulSets + +Deployments and StatefulSets are the most common way to scale workloads with KEDA. + +It allows you to define the Kubernetes Deployment or StatefulSet that you want KEDA to scale based on a scale trigger. KEDA will monitor that service and based on the events that occur it will automatically scale your resource out/in accordingly. + +Behind the scenes, KEDA acts to monitor the event source and feed that data to Kubernetes and the HPA (Horizontal Pod Autoscaler) to drive rapid scale of a resource. Each replica of a resource is actively pulling items from the event source. With KEDA and scaling Deployments/StatefulSet you can scale based on events while also preserving rich connection and processing semantics with the event source (e.g. in-order processing, retries, deadletter, checkpointing). + +For example, if you wanted to use KEDA with an Apache Kafka topic as event source, the flow of information would be: + +* When no messages are pending processing, KEDA can scale the deployment to zero. +* When a message arrives, KEDA detects this event and activates the deployment. +* When the deployment starts running, one of the containers connects to Kafka and starts pulling messages. +* As more messages arrive at the Kafka Topic, KEDA can feed this data to the HPA to drive scale out. +* Each replica of the deployment is actively processing messages. Very likely, each replica is processing a batch of messages in a distributed manner. + +## Scaling Custom Resources + +With KEDA you can scale any workload defined as any `Custom Resource` (for example `ArgoRollout` [resource](https://argoproj.github.io/argo-rollouts/)). The scaling behaves the same way as scaling for arbitrary Kubernetes `Deployment` or `StatefulSet`. + +The only constraint is that the target `Custom Resource` must define `/scale` [subresource](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#scale-subresource). + +# Features + +## Caching Metrics + +This feature enables caching of metric values during polling interval (as specified in `.spec.pollingInterval`). Kubernetes (HPA controller) asks for a metric every few seconds (as defined by `--horizontal-pod-autoscaler-sync-period`, usually 15s), then this request is routed to KEDA Metrics Server, that by default queries the scaler and reads the metric values. Enabling this feature changes this behavior such that KEDA Metrics Server tries to read metric from the cache first. This cache is updated periodically during the polling interval. + +Enabling [`useCachedMetrics`](../reference/scaledobject-spec/#triggers) can significantly reduce the load on the scaler service. + +This feature is not supported for `cpu`, `memory` or `cron` scaler. + +## Pausing autoscaling + +It can be useful to instruct KEDA to pause the autoscaling of objects, to do to cluster maintenance or to avoid resource starvation by removing non-mission-critical workloads. + +This is preferable to deleting the resource because it removes the instances it is running from operation without touching the applications themselves. When ready, you can then reenable scaling. + +You can pause autoscaling by adding this annotation to your `ScaledObject` definition: + + +```yaml +metadata: + annotations: + autoscaling.keda.sh/paused-replicas: "0" + autoscaling.keda.sh/paused: "true" +``` + +The presence of these annotations will pause autoscaling no matter what number of replicas is provided. + +The annotation `autoscaling.keda.sh/paused` will pause scaling immediately and use the current instance count while the annotation `autoscaling.keda.sh/paused-replicas: ""` will scale your current workload to specified amount of replicas and pause autoscaling. You can set the value of replicas for an object to be paused to any arbitrary number. + +Typically, either one or the other is being used given they serve a different purpose/scenario. However, if both `paused` and `paused-replicas` are set, KEDA will scale your current workload to the number specified count in `paused-replicas` and then pause autoscaling. + +To unpause (reenable) autoscaling again, remove all paused annotations from the `ScaledObject` definition. If you paused with `autoscaling.keda.sh/paused`, you can unpause by setting the annotation to `false`. + + +## Scaling Modifiers + +**Example: compose average value** + +```yaml +advanced: + scalingModifiers: + formula: "(trig_one + trig_two)/2" + target: "2" + activationTarget: "2" + metricType: "AverageValue" +... +triggers: + - type: kubernetes-workload + name: trig_one + metadata: + podSelector: 'pod=workload-test' + - type: metrics-api + name: trig_two + metadata: + url: "https://mockbin.org/bin/336a8d99-9e09-4f1f-979d-851a6d1b1423" + valueLocation: "tasks" +``` + +Formula composes 2 given metrics from 2 triggers `kubernetes-workload` named `trig_one` and `metrics-api` named `trig_two` together as an average value and returns one final metric which is used to make autoscaling decisions on. + +**Example: activationTarget** + +```yaml +advanced: + scalingModifiers: + activationTarget: "2" +``` + +If the calculated value is <=2, the ScaledObject is not `Active` and it'll scale to 0 if it's allowed. + +**Example: ternary operator** + +```yaml +advanced: + scalingModifiers: + formula: "trig_one > 2 ? trig_one + trig_two : 1" +``` + +If metric value of trigger `trig_one` is more than 2, then return `trig_one` + `trig_two` otherwise return 1. + +**Example: count function** + +```yaml +advanced: + scalingModifiers: + formula: "count([trig_one,trig_two,trig_three],{#>1}) > 1 ? 5 : 0" +``` + +If at least 2 metrics (from the list `trig_one`,`trig_two`,`trig_three`) have value of more than 1, then return 5, otherwise return 0 + +**Example: nested conditions and operators** + +```yaml +advanced: + scalingModifiers: + formula: "trig_one < 2 ? trig_one+trig_two >= 2 ? 5 : 10 : 0" +``` + +Conditions can be used within another condition as well. +If value of `trig_one` is less than 2 AND `trig_one`+`trig_two` is at least 2 then return 5, if only the first is true return 10, if the first condition is false then return 0. + +Complete language definition of `expr` package can be found [here](https://expr.medv.io/docs/Language-Definition). Formula must return a single value (not boolean). All formulas are internally wrapped with float cast. + +## Activating and Scaling thresholds + +KEDA has 2 different phases during the autoscaling process. + +- **Activation phase:** The activating (or deactivating) phase is the moment when KEDA (operator) has to decide if the workload should be scaled from/to zero. KEDA takes responsibility for this action based on the result of the scaler `IsActive` function and only applies to 0<->1 scaling. There are use-cases where the activating value (0-1 and 1-0) is totally different than 0, such as workloads scaled with the Prometheus scaler where the values go from -X to X. +- **Scaling phase:** The scaling phase is the moment when KEDA has decided to scale out to 1 instance and now it is the HPA controller who takes the scaling decisions based on the configuration defined in the generated HPA (from ScaledObject data) and the metrics exposed by KEDA (metrics server). This phase applies the to 1<->N scaling. + +KEDA allows you to specify different values for each scenario: + +- **Activation:** Defines when the scaler is active or not and scales from/to 0 based on it. +- **Scaling:** Defines the target value to scale the workload from 1 to _n_ instances and vice versa. To achieve this, KEDA passes the target value to the Horizontal Pod Autoscaler (HPA) and the built-in HPA controller will handle all the autoscaling. + +> ⚠️ **NOTE:** If the minimum replicas is >= 1, the scaler is always active and the activation value will be ignored. + +Each scaler defines parameters for their use-cases, but the activation will always be the same as the scaling value, appended by the prefix `activation` (ie: `threshold` for scaling and `activationThreshold` for activation). + +There are some important topics to take into account: + +- Opposite to scaling value, the activation value is always optional and the default value is 0. +- Activation only occurs when this value is greater than the set value; not greater than or equal to. + - ie, in the default case: `activationThreshold: 0` will only activate when the metric value is 1 or more +- The activation value has more priority than the scaling value in case of different decisions for each. ie: `threshold: 10` and `activationThreshold: 50`, in case of 40 messages the scaler is not active and it'll be scaled to zero even the HPA requires 4 instances. + +> ⚠️ **NOTE:** If a scaler doesn't define "activation" parameter (a property that starts with `activation` prefix), then this specific scaler doesn't support configurable activation value and the activation value is always 0. + +## Transferring ownership of an existing HPA + +If your environment already operates using kubernetes HPA, you can transfer the ownership of this resource to a new ScaledObject: + +```yaml +metadata: + annotations: + scaledobject.keda.sh/transfer-hpa-ownership: "true" +spec: + advanced: + horizontalPodAutoscalerConfig: + name: {name-of-hpa-resource} +``` + +> ⚠️ **NOTE:** You need to specify a custom HPA name in your ScaledObject matching the existing HPA name you want it to manage. + +## Disabling validations on an existing HPA + +You are allowed to disable admission webhooks validations with the following snippet. It grants you better flexibility but also brings vulnerabilities. Do it **at your own risk**. + +```yaml +metadata: + annotations: + validations.keda.sh/hpa-ownership: "true" +``` + +### Long-running executions + +One important consideration to make is how this pattern can work with long-running executions. Imagine a deployment triggers on a RabbitMQ queue message. Each message takes 3 hours to process. It's possible that if many queue messages arrive, KEDA will help drive scaling out to many replicas - let's say 4. Now the HPA makes a decision to scale down from 4 replicas to 2. There is no way to control which of the 2 replicas get terminated to scale down. That means the HPA may attempt to terminate a replica that is 2.9 hours into processing a 3 hour queue message. + +There are two main ways to handle this scenario. + +#### Leverage the container lifecycle + +Kubernetes provides a few [lifecycle hooks](https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/) that can be leveraged to delay termination. Imagine a replica is scheduled for termination and is 2.9 hours into processing a 3 hour message. Kubernetes will send a [`SIGTERM`](https://www.gnu.org/software/libc/manual/html_node/Termination-Signals.html) to signal the intent to terminate. Rather than immediately terminating, a deployment can delay termination until processing the current batch of messages has completed. Kubernetes will wait for a `SIGTERM` response or the `terminationGracePeriodSeconds` before killing the replica. + +> 💡 **NOTE:** There are other ways to delay termination, including the [`preStop` Hook](https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks). + +Using this method can preserve a replica and enable long-running executions. However, one downside of this approach is while delaying termination, the pod phase will remain in the `Terminating` state. That means a pod that is delaying termination for a very long duration may show `Terminating` during that entire period of delay. + +#### Run as jobs + +The other alternative to handling long-running executions is by running the event driven code in Kubernetes Jobs instead of Deployments or Custom Resources. This approach is discussed [in the next section](./scaling-jobs). diff --git a/content/docs/2.17/concepts/scaling-jobs.md b/content/docs/2.17/concepts/scaling-jobs.md new file mode 100644 index 000000000..2f9029332 --- /dev/null +++ b/content/docs/2.17/concepts/scaling-jobs.md @@ -0,0 +1,90 @@ ++++ +title = "Scaling Jobs" +weight = 300 ++++ + +This page describes the job scaling behavior of KEDA. See the [Scaled Job specification](../reference/scaledjob-spec.md) for details on how to set the behaviors described below. + + +# Overview + +As an alternate to [scaling event-driven code as deployments](./scaling-deployments) you can also run and scale your code as Kubernetes Jobs. The primary reason to consider this option is to handle processing long-running executions. Rather than processing multiple events within a deployment, for each detected event a single Kubernetes Job is scheduled. That job will initialize, pull a single event from the message source, and process to completion and terminate. + +For example, if you wanted to use KEDA to run a job for each message that lands on a RabbitMQ queue, the flow may be: + +1. When no messages are awaiting processing, no jobs are created. +1. When a message arrives at the queue, KEDA creates a job. +1. When the job starts running, it pulls *a single* message and processes it to completion. +1. As additional messages arrive, additional jobs are created. Each job processes a single message to completion. +1. Periodically remove completed/failed job by the `SuccessfulJobsHistoryLimit` and `FailedJobsHistoryLimit.` + + +# Pausing autoscaling + +It can be useful to instruct KEDA to pause the autoscaling of objects, to do to cluster maintenance or to avoid resource starvation by removing non-mission-critical workloads. + +This is preferable to deleting the resource because it removes the instances it is running from operation without touching the applications themselves. When ready, you can then reenable scaling. + +You can pause autoscaling by adding this annotation to your `ScaledJob` definition: + +```yaml +metadata: + annotations: + autoscaling.keda.sh/paused: true +``` + +To reenable autoscaling, remove the annotation from the `ScaledJob` definition or set the value to `false`. + +```yaml +metadata: + annotations: + autoscaling.keda.sh/paused: false +``` + + +## Example + +An example configuration for autoscaling jobs using a RabbitMQ scaler is given below. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: rabbitmq-consumer +data: + RabbitMqHost: +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledJob +metadata: + name: rabbitmq-consumer + namespace: default +spec: + jobTargetRef: + template: + spec: + containers: + - name: demo-rabbitmq-client + image: demo-rabbitmq-client:1 + imagePullPolicy: Always + command: ["receive", "amqp://user:PASSWORD@rabbitmq.default.svc.cluster.local:5672"] + envFrom: + - secretRef: + name: rabbitmq-consumer-secrets + restartPolicy: Never + backoffLimit: 4 + pollingInterval: 10 # Optional. Default: 30 seconds + maxReplicaCount: 30 # Optional. Default: 100 + successfulJobsHistoryLimit: 3 # Optional. Default: 100. How many completed jobs should be kept. + failedJobsHistoryLimit: 2 # Optional. Default: 100. How many failed jobs should be kept. + scalingStrategy: + strategy: "custom" # Optional. Default: default. Which Scaling Strategy to use. + customScalingQueueLengthDeduction: 1 # Optional. A parameter to optimize custom ScalingStrategy. + customScalingRunningJobPercentage: "0.5" # Optional. A parameter to optimize custom ScalingStrategy. + triggers: + - type: rabbitmq + metadata: + queueName: hello + host: RabbitMqHost + queueLength : '5' +``` diff --git a/content/docs/2.17/concepts/troubleshooting.md b/content/docs/2.17/concepts/troubleshooting.md new file mode 100644 index 000000000..68b19bf15 --- /dev/null +++ b/content/docs/2.17/concepts/troubleshooting.md @@ -0,0 +1,24 @@ ++++ +title = "Troubleshooting" +weight = 600 ++++ + +## KEDA logging and telemetry + +The first place to look if something isn't behaving correctly is the logs generated from KEDA. After deploying you should have a pod with two containers running within the namespace (by default: `keda`). + +You can view the KEDA operator pod via kubectl: + +```sh +kubectl get pods -n keda +``` + +You can view the logs for the keda operator container with the following: + +```sh +kubectl logs -n keda {keda-pod-name} -c keda-operator +``` + +## Reporting issues + +If you are having issues or hitting a potential bug, please file an issue [in the KEDA GitHub repo](https://github.com/kedacore/keda/issues/new/choose) with details, logs, and steps to reproduce the behavior. diff --git a/content/docs/2.17/deploy.md b/content/docs/2.17/deploy.md new file mode 100644 index 000000000..fbfe5cf08 --- /dev/null +++ b/content/docs/2.17/deploy.md @@ -0,0 +1,162 @@ ++++ +title = "Deploying KEDA" ++++ + +We provide a few approaches to deploy KEDA runtime in your Kubernetes clusters: + +- Helm charts +- Operator Hub +- YAML declarations + +> 💡 **NOTE:** KEDA requires Kubernetes cluster version 1.29 and higher + +Don't see what you need? Feel free to [create an issue](https://github.com/kedacore/keda/issues/new) on our GitHub repo. + +## Deploying with Helm {#helm} + +### Install + +To deploy KEDA with Helm: + +1. Add Helm repo + + ```sh + helm repo add kedacore https://kedacore.github.io/charts + ``` + +2. Update Helm repo + + ```sh + helm repo update + ``` + +3. Install `keda` Helm chart + + **Helm 3** + + ```sh + helm install keda kedacore/keda --namespace keda --create-namespace + ``` + +To deploy the CRDs separately from the Helm chart, use the `keda-2.xx.x-crds.yaml` file provided on the [GitHub releases](https://github.com/kedacore/keda/releases) page. + +> 💡 **NOTE:** Are you upgrading to v2.2.1 or above? Make sure to read [our troubleshooting guide](https://keda.sh/docs/latest/troubleshooting/) to fix potential CRD issues. + +### Uninstall + +If you want to remove KEDA from a cluster, you first need to remove any ScaledObjects and ScaledJobs that you have created. Once that is done, the Helm chart can be uninstalled: + +```sh +kubectl delete $(kubectl get scaledobjects.keda.sh,scaledjobs.keda.sh -A \ + -o jsonpath='{"-n "}{.items[*].metadata.namespace}{" "}{.items[*].kind}{"/"}{.items[*].metadata.name}{"\n"}') +helm uninstall keda -n keda +``` + +Note: if you uninstall the Helm chart without first deleting any ScaledObject or ScaledJob resources you have created, they will become orphaned. In this situation, you will need to patch the resources to remove their finalizers. Once this is done, they should automatically be removed: + +```sh +for i in $(kubectl get scaledobjects -A \ + -o jsonpath='{"-n "}{.items[*].metadata.namespace}{" "}{.items[*].kind}{"/"}{.items[*].metadata.name}{"\n"}'); +do kubectl patch $i -p '{"metadata":{"finalizers":null}}' --type=merge +done + +for i in $(kubectl get scaledjobs -A \ + -o jsonpath='{"-n "}{.items[*].metadata.namespace}{" "}{.items[*].kind}{"/"}{.items[*].metadata.name}{"\n"}'); +do kubectl patch $i -p '{"metadata":{"finalizers":null}}' --type=merge +done +``` + +## Deploying with Operator Hub {#operatorhub} + +### Install + +1. On Operator Hub Marketplace locate and install KEDA operator to namespace `keda` +2. Create `KedaController` resource named `keda` in namespace `keda` +![Operator Hub installation](https://raw.githubusercontent.com/kedacore/keda-olm-operator/main/images/keda-olm-install.gif) +> 💡 **NOTE:** Further information on Operator Hub installation method can be found in the following [repository](https://github.com/kedacore/keda-olm-operator). + +### Uninstall + +Locate installed KEDA Operator in `keda` namespace, then remove created `KedaController` resource and uninstall KEDA operator. + +## Deploying using the deployment YAML files {#yaml} + +### Install + +If you want to try KEDA on [Minikube](https://minikube.sigs.k8s.io) or a different Kubernetes deployment without using Helm you can still deploy it with `kubectl`. + +- We provide sample YAML declaration which includes our CRDs and all other resources in a file which is available on the [GitHub releases](https://github.com/kedacore/keda/releases) page. + - We offer two options to deploy KEDA: + - Use `keda-2.xx.x.yaml` that includes all features, including [admission webhooks](./concepts/admission-webhooks.md) (recommended) + - Use `keda-2.xx.x-core.yaml` that installs the minimal required KEDA components, without admission webhooks + +Run the following command (if needed, replace the version, in this case `2.17.0`, with the one you are using): + +```sh +# Including admission webhooks +kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.17.0/keda-2.17.0.yaml +# Without admission webhooks +kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.17.0/keda-2.17.0-core.yaml +``` + +- Alternatively you can download the file and deploy it from the local path: +```sh +# Including admission webhooks +kubectl apply --server-side -f keda-2.17.0.yaml +# Without admission webhooks +kubectl apply --server-side -f keda-2.17.0-core.yaml +``` + +> 💡 **NOTE:** `--server-side` option is needed because the ScaledJob CRD is too long to process, see [this issue](https://github.com/kedacore/keda/issues/4740) for details. + +- You can also find the same YAML declarations in our `/config` directory on our [GitHub repo](https://github.com/kedacore/keda) if you prefer to clone it. + +```sh +git clone https://github.com/kedacore/keda && cd keda + +VERSION=2.17.0 make deploy +``` + +### Uninstall + +- In case of installing from released YAML file just run the following command (if needed, replace the version, in this case `2.17.0`, with the one you are using): + +```sh +# Including admission webhooks +kubectl delete -f https://github.com/kedacore/keda/releases/download/v2.17.0/keda-2.17.0.yaml +# Without admission webhooks +kubectl delete -f https://github.com/kedacore/keda/releases/download/v2.17.0/keda-2.17.0-core.yaml +``` + +- If you have downloaded the file locally, you can run: + +```sh +# Including admission webhooks +kubectl delete -f keda-2.17.0.yaml +# Without admission webhooks +kubectl delete -f keda-2.17.0-core.yaml +``` + +- You would need to run these commands from within the directory of the cloned [GitHub repo](https://github.com/kedacore/keda): + +```sh +VERSION=2.17.0 make undeploy +``` + +## Deploying KEDA on MicroK8s {#microk8s} + +### Install + +If you want to try KEDA v2 on [MicroK8s](https://microk8s.io/) from `1.20` channel, KEDA is included into MicroK8s add-ons. + +```sh +microk8s enable keda +``` + +### Uninstall + +To uninstall KEDA in MicroK8s, disable the add-on as shown below. + +```sh +microk8s disable keda +``` diff --git a/content/docs/2.17/integrations/_index.md b/content/docs/2.17/integrations/_index.md new file mode 100644 index 000000000..745b6065b --- /dev/null +++ b/content/docs/2.17/integrations/_index.md @@ -0,0 +1,8 @@ ++++ +title = "Integrations" +weight = 6 ++++ + +An overview of tools/products integrating with KEDA: + +{{< integrations >}} diff --git a/content/docs/2.17/integrations/istio-integration.md b/content/docs/2.17/integrations/istio-integration.md new file mode 100644 index 000000000..413cf3ae0 --- /dev/null +++ b/content/docs/2.17/integrations/istio-integration.md @@ -0,0 +1,64 @@ ++++ +title = "KEDA Integration with Istio" +description = "Guidance for running KEDA along with Istio in your cluster" +availability = "v2.14+" +project = "Istio" ++++ + +## Overview + +Integrating KEDA with Istio can present challenges, particularly in environments with enforced mTLS. This document provides guidance on how to configure KEDA to work within an Istio service mesh without disabling Istio sidecar injection. This solution allows KEDA components to communicate securely and effectively while maintaining compliance with security requirements. + +This can be considered as workaround, however it's perfectly valid from the security standpoint. +Keda is still using own mTLS certificates for secure communication between it's components and at the same time it's able to communicate with Istio Mesh services (like Prometheus) through Istio sidecar proxies. + +## Background + +In some scenarios, users might face issues with KEDA components failing discovery checks when Istio sidecar injection is enabled. The current [troubleshooting guide](../../../troubleshooting/istio-keda-faileddiscoverycheck) suggests disabling Istio sidecar injection in the KEDA namespace. However, if this is not feasible due to security policies, the following workaround can be applied. + + +### Requirements + +- Istio version >= 1.18.* +- Kubernetes cluster with KEDA installed + +### Example configuration + +`values.yaml` fragment for the [helm chart](https://github.com/kedacore/charts/blob/main/keda/values.yaml) +``` +... + +podAnnotations: + # -- Pod annotations for KEDA operator + keda: + traffic.sidecar.istio.io/excludeInboundPorts: "9666" + traffic.sidecar.istio.io/excludeOutboundPorts: "9443,6443" + # -- Pod annotations for KEDA Metrics Adapter + metricsAdapter: + traffic.sidecar.istio.io/excludeInboundPorts: "6443" + traffic.sidecar.istio.io/excludeOutboundPorts: "9666,9443" + # -- Pod annotations for KEDA Admission webhooks + webhooks: + traffic.sidecar.istio.io/excludeInboundPorts: "9443" + traffic.sidecar.istio.io/excludeOutboundPorts: "9666,6443" + +... + +``` + +*Check your respective ports set correctly for each component.* + +### Applying the Annotations +- Annotate the KEDA Components: Update the deployment manifests for the KEDA operator, Metrics Adapter, and Admission Webhooks to include the specified pod annotations. +- Deploy Updated Manifests: Apply the updated manifests to your Kubernetes cluster. +- Verify Communication: Ensure that KEDA components can communicate internally and with external mesh services without failing discovery checks. + + +### References +For more information on the annotations used, refer to the Istio documentation on traffic management. +Existing troubleshooting guide for KEDA with Istio. + +### Conclusion +By applying these annotations, you can ensure that KEDA integrates seamlessly with Istio while adhering to security requirements. This configuration allows KEDA to maintain internal mTLS communication and interact properly with other mesh services. + +If you encounter any issues or have further questions, please refer to the KEDA and Istio documentation or reach out to the community for support. diff --git a/content/docs/2.17/integrations/opentelemetry.md b/content/docs/2.17/integrations/opentelemetry.md new file mode 100644 index 000000000..1db48fb1e --- /dev/null +++ b/content/docs/2.17/integrations/opentelemetry.md @@ -0,0 +1,58 @@ ++++ +title= "Integrate with OpenTelemetry Collector (Experimental)" +description= "Detail of integrating OpenTelemetry Collector in KEDA" +availability = "v2.12+" +project = "OpenTelemetry" ++++ + +## Push Metrics to OpenTelemetry Collector (Experimental) + +### Operator + +The KEDA Operator supports outputting metrics to the OpenTelemetry collector. The parameter `--enable-opentelemetry-metrics=true` needs to be set. KEDA will push metrics to the OpenTelemetry collector specified by the `OTEL_EXPORTER_OTLP_ENDPOINT` environment variable. `OTEL_EXPORTER_OTLP_PROTOCOL` will also be used to choose HTTP or GRPC client. Other environment variables in OpenTelemetry are also supported (https://opentelemetry.io/docs/concepts/sdk-configuration/otlp-exporter-configuration/). Here is an example configuration of the operator: +``` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: keda-operator + ... + containers: + - name: keda-operator + image: ghcr.io/kedacore/keda:latest + command: + - /keda + args: + --enable-opentelemetry-metrics=true + ... + ... + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://opentelemetry-collector.default.svc.cluster.local:4318" +``` +The following metrics are being gathered: + +| Metric | Description | +| ------ | ----------- | +| `keda.build.info` | Info metric, with static information about KEDA build like: version, git commit and Golang runtime info. | +| `keda.scaler.active` | This metric marks whether the particular scaler is active (value == 1) or in-active (value == 0). | +| `keda.scaled.object.paused` | This metric indicates whether a ScaledObject is paused (value == 1) or un-paused (value == 0). | +| `keda.scaler.metrics.value` | The current value for each scaler's metric that would be used by the HPA in computing the target average. | +| `keda.scaler.metrics.latency.seconds` | The latency of retrieving current metric from each scaler. | +| `keda.scaler.errors` | The number of errors that have occurred for each scaler. | +| `keda.scaler.errors.total` | The total number of errors encountered for all scalers. | +| `keda.scaled.object.errors` | The number of errors that have occurred for each ScaledObject. | +| `keda.scaled.job.errors` | The number of errors that have occurred for each ScaledJob. | +| `keda.resource.registered.count` | Total number of KEDA custom resources per namespace for each custom resource type (CRD). | +| `keda.trigger.registered.count` | Total number of triggers per trigger type. | +| `keda.internal.scale.loop.latency.seconds` | Total deviation between the expected execution time and the actual execution time for the scaling loop. This latency could be produced due to accumulated scalers latencies or high load. This is an internal metric. | +| `keda.cloudeventsource.events.emitted.count` | Measured emitted cloudevents with destination of this emitted event (eventsink) and emitted state. | +| `keda.cloudeventsource.events.queued` | The number of events that are in the emitting queue. | + +#### Deprecated metrics + +The following metrics are exposed as well, but are deprecated and will be removed in KEDA v2.16. + +| `keda.scaler.metrics.latency` | The latency of retrieving current metric from each scaler. | +| `keda.resource.totals` | Total number of KEDA custom resources per namespace for each custom resource type (CRD). | +| `keda.trigger.totals` | Total number of triggers per trigger type. | +| `keda.internal.scale.loop.latency` | Total deviation (in milliseconds) between the expected execution time and the actual execution time for the scaling loop. This latency could be produced due to accumulated scalers latencies or high load. This is an internal metric. | diff --git a/content/docs/2.17/integrations/prometheus.md b/content/docs/2.17/integrations/prometheus.md new file mode 100644 index 000000000..87b8117b1 --- /dev/null +++ b/content/docs/2.17/integrations/prometheus.md @@ -0,0 +1,86 @@ ++++ +title = "Integrate with Prometheus" +description = "Overview of all Prometheus metrics that KEDA provides" +availability = "v2.0+" +project = "Prometheus" ++++ + +## Prometheus Exporter Metrics + +### Operator + +The KEDA Operator exposes Prometheus metrics which can be scraped on port `8080` at `/metrics`. The following metrics are being gathered: + +- `keda_build_info` - Info metric, with static information about KEDA build like: version, git commit and Golang runtime info. +- `keda_scaler_active` - This metric marks whether the particular scaler is active (value == 1) or in-active (value == 0). +- `keda_scaled_object_paused` - This metric indicates whether a ScaledObject is paused (value == 1) or un-paused (value == 0). +- `keda_scaler_metrics_value` - The current value for each scaler's metric that would be used by the HPA in computing the target average. +- `keda_scaler_metrics_latency_seconds` - The latency of retrieving current metric from each scaler. +- `keda_scaler_detail_errors_total` - The number of errors encountered for each scaler. +- `keda_scaled_object_errors_total` - The number of errors that have occurred for each ScaledObject. +- `keda_scaled_job_errors_total` - The number of errors that have occurred for each ScaledJob. +- `keda_resource_registered_total` - Total number of KEDA custom resources per namespace for each custom resource type (CRD) handled by the operator. +- `keda_trigger_registered_total` - Total number of triggers per trigger type handled by the operator. +- `keda_internal_scale_loop_latency_seconds` - Total deviation (in seconds) between the expected execution time and the actual execution time for the scaling loop. This latency could be produced due to accumulated scalers latencies or high load. This is an internal metric. +- `keda_cloudeventsource_events_emitted_total` - Measured emitted cloudevents with destination of this emitted event (eventsink) and emitted state. +- `keda_cloudeventsource_events_queued` - The number of events that are in the emitting queue. +- `keda_internal_metricsservice_grpc_server_started_total` - Total number of RPCs started on the server. +- `keda_internal_metricsservice_grpc_server_handled_total` - Total number of RPCs completed on the server, regardless of success or failure. +- `keda_internal_metricsservice_grpc_server_msg_received_total` - Total number of RPC stream messages received on the server. +- `keda_internal_metricsservice_grpc_server_msg_sent_total` - Total number of gRPC stream messages sent by the server. +- `keda_internal_metricsservice_grpc_server_handling_seconds` - Histogram of response latency (seconds) of gRPC that had been application-level handled by the server. +- Metrics exposed by the `Operator SDK` framework as explained [here](https://sdk.operatorframework.io/docs/building-operators/golang/advanced-topics/#metrics). + +> Note: When you deploy the KEDA Operator without any scalers deployed, the only metric you will see is `keda_build_info`. As you deploy scalers, you will start to see some of the metrics listed above but it is dependant on the types of scalers you have deployed. + +#### Deprecated metrics + +The following metrics are exposed as well, but are deprecated and will be removed in KEDA v2.16. + +- `keda_scaler_metrics_latency` - The latency of retrieving current metric from each scaler. Replaced by `keda_scaler_metrics_latency_seconds`. +- `keda_scaler_errors` - The number of errors that have occurred for each scaler. Replaced by `keda_scaler_detail_errors_total`. +- `keda_scaler_errors_total` - The total number of errors encountered for all scalers. Replaced by `keda_scaler_detail_errors_total`. +- `keda_scaled_object_errors` - The number of errors that have occurred for each ScaledObject. Replaced by `keda_scaled_object_errors_total`. +- `keda_scaled_job_errors` - The number of errors that have occurred for each ScaledJob. Replace by `keda_scaled_job_errors_total`. +- `keda_resource_totals` - Total number of KEDA custom resources per namespace for each custom resource type (CRD). Replaced by `keda_resource_registered_total`. +- `keda_trigger_totals` - Total number of triggers per trigger type. Replaced by `keda_trigger_registered_total`. +- `keda_internal_scale_loop_latency` - Total deviation (in milliseconds) between the expected execution time and the actual execution time for the scaling loop. This latency could be produced due to accumulated scalers latencies or high load. This is an internal metric. Replaced by `keda_internal_scale_loop_latency_seconds`. + +### Admission Webhooks + +The KEDA Webhooks expose Prometheus metrics which can be scraped on port `8080` at `/metrics`. The following metrics are being gathered: + +- `keda_webhook_scaled_object_validation_total`- The current value for scaled object validations. +- `keda_webhook_scaled_object_validation_errors` - The number of validation errors. + +### Metrics Server + +The KEDA Metrics Adapter exposes Prometheus metrics which can be scraped on port `8080` at `/metrics`. The following metrics are being gathered: + +- `keda_internal_metricsservice_grpc_client_started_total` - Total number of RPCs started on the client. +- `keda_internal_metricsservice_grpc_client_handled_total` - Total number of RPCs completed by the client, regardless of success or failure. +- `keda_internal_metricsservice_grpc_client_msg_received_total` - Total number of RPC stream messages received by the client. +- `keda_internal_metricsservice_grpc_client_msg_sent_total` - Total number of gRPC stream messages sent by the client. +- `keda_internal_metricsservice_grpc_client_handling_seconds` - Histogram of response latency (seconds) of the gRPC until it is finished by the application. +- Metrics exposed by the `Operator SDK` framework as explained [here](https://sdk.operatorframework.io/docs/building-operators/golang/advanced-topics/#metrics). +- Metrics exposed (prepended with `apiserver_`) by [Kubernetes API Server](https://kubernetes.io/docs/reference/instrumentation/metrics/) + +## Premade Grafana dashboard + +A premade [Grafana dashboard](https://github.com/kedacore/keda/tree/main/config/grafana/keda-dashboard.json) is available to visualize metrics exposed by the KEDA Metrics Adapter. + +![KEDA Grafana dashboard](/img/grafana-dashboard.png) + +The dashboard has two sections: + +- Visualization of KEDA's metric server +- Visualization of the scale target and its changes in replicas scaled by KEDA + +On top, the dashboard supports the following variables: + +- datasource +- namespace +- scaledObject +- scaledJob +- scaler +- metric diff --git a/content/docs/2.17/migration.md b/content/docs/2.17/migration.md new file mode 100644 index 000000000..3e52fd485 --- /dev/null +++ b/content/docs/2.17/migration.md @@ -0,0 +1,211 @@ ++++ +title = "Migration Guide" ++++ + +## Migrating from KEDA v1 to v2 + +Please note that you **can not** run both KEDA v1 and v2 on the same Kubernetes cluster. You need to [uninstall](../../1.5/deploy) KEDA v1 first, in order to [install](../deploy) and use KEDA v2. + +> 💡 **NOTE:** When uninstalling KEDA v1 make sure v1 CRDs are uninstalled from the cluster as well. + +KEDA v2 is using a new API namespace for its Custom Resources Definitions (CRD): `keda.sh` instead of `keda.k8s.io` and introduces a new Custom Resource for scaling of Jobs. See full details on KEDA Custom Resources [here](../concepts/#custom-resources-crd). + +Here's an overview of what's changed: + +- [Scaling of Deployments](#scaling-of-deployments) +- [Scaling of Jobs](#scaling-of-jobs) +- [Improved flexibility & usability of trigger metadata](#improved-flexibility--usability-of-trigger-metadata) +- [Scalers](#scalers) +- [TriggerAuthentication](#triggerauthentication) + +### Scaling of Deployments + +In order to scale `Deployments` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + +- Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` +- Rename property `spec.scaleTargetRef.deploymentName` to `spec.scaleTargetRef.name` +- Rename property `spec.scaleTargetRef.containerName` to `spec.scaleTargetRef.envSourceContainerName` +- Label `deploymentName` (in `metadata.labels.`) is no longer needed to be specified on v2 ScaledObject (it was mandatory on older versions of v1) + +Please see the examples below or refer to the full [v2 ScaledObject Specification](./reference/scaledobject-spec) + +**Example of v1 ScaledObject** + +```yaml +apiVersion: keda.k8s.io/v1alpha1 +kind: ScaledObject +metadata: + name: { scaled-object-name } + labels: + deploymentName: { deployment-name } +spec: + scaleTargetRef: + deploymentName: { deployment-name } + containerName: { container-name } + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 + triggers: + # {list of triggers to activate the deployment} +``` + +**Example of v2 ScaledObject** + +```yaml +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +kind: ScaledObject +metadata: # <--- labels.deploymentName is not needed + name: { scaled-object-name } +spec: + scaleTargetRef: + name: { deployment-name } # <--- Property name was changed + envSourceContainerName: { container-name } # <--- Property name was changed + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 + triggers: + # {list of triggers to activate the deployment} +``` + +### Scaling of Jobs + +In order to scale `Jobs` with KEDA v2, you need to do only a few modifications to existing v1 `ScaledObjects` definitions, so they comply with v2: + +- Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` +- Change the value of `kind` property from `ScaledObject` to `ScaledJob` +- Remove property `spec.scaleType` +- Remove properties `spec.cooldownPeriod` and `spec.minReplicaCount` + +You can configure `successfulJobsHistoryLimit` and `failedJobsHistoryLimit`. They will remove the old job histories automatically. + +Please see the examples below or refer to the full [v2 ScaledJob Specification](./reference/scaledjob-spec/) + +**Example of v1 ScaledObject for Jobs scaling** + +```yaml +apiVersion: keda.k8s.io/v1alpha1 +kind: ScaledObject +metadata: + name: { scaled-object-name } +spec: + scaleType: job + jobTargetRef: + parallelism: 1 + completions: 1 + activeDeadlineSeconds: 600 + backoffLimit: 6 + template: + # {job template} + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 100 + triggers: + # {list of triggers to create jobs} +``` + +**Example of v2 ScaledJob** + +```yaml +apiVersion: keda.sh/v1alpha1 # <--- Property value was changed +kind: ScaledJob # <--- Property value was changed +metadata: + name: { scaled-job-name } +spec: # <--- spec.scaleType is not needed + jobTargetRef: + parallelism: 1 + completions: 1 + activeDeadlineSeconds: 600 + backoffLimit: 6 + template: + # {job template} + pollingInterval: 30 # <--- spec.cooldownPeriod and spec.minReplicaCount are not needed + successfulJobsHistoryLimit: 5 # <--- property is added + failedJobsHistoryLimit: 5 # <--- Property is added + maxReplicaCount: 100 + triggers: + # {list of triggers to create jobs} +``` + +### Improved flexibility & usability of trigger metadata + +We've introduced more options to configure trigger metadata to give users more flexibility. + +> 💡 **NOTE:** Changes only apply to trigger metadata and don't impact usage of `TriggerAuthentication` + +Here's an overview: + +| Scaler | 1.x | 2.0 | +| -------------------- | --------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | -------- | +| `azure-blob` | `connection` (**Default**: `AzureWebJobsStorage`) | `connectionFromEnv` | +| `azure-monitor` | `activeDirectoryClientId` `activeDirectoryClientPassword` | `activeDirectoryClientId` `activeDirectoryClientIdFromEnv` `activeDirectoryClientPasswordFromEnv` | +| `azure-queue` | `connection` (**Default**: AzureWebJobsStorage) | `connectionFromEnv` | +| `azure-servicebus` | `connection` | `connectionFromEnv` | +| `azure-eventhub` | `storageConnection` (**Default**: `AzureWebJobsStorage`) `connection` (**Default**: `EventHub`) | `storageConnectionFromEnv` `connectionFromEnv` | +| `aws-cloudwatch` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-kinesis-stream` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `aws-sqs-queue` | `awsAccessKeyID` (**Default**: `AWS_ACCESS_KEY_ID`) `awsSecretAccessKey` (**Default**: `AWS_SECRET_ACCESS_KEY`) | `awsAccessKeyID` `awsAccessKeyIDFromEnv` `awsSecretAccessKeyFromEnv` | +| `kafka` | _(none)_ | _(none)_ | +| `rabbitmq` | `apiHost` `host` | ~~`apiHost`~~ `host` `hostFromEnv` | +| `prometheus` | _(none)_ | _(none)_ | +| `cron` | _(none)_ | _(none)_ | +| `redis` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `redis-streams` | `address` `host` `port` `password` | `address` `addressFromEnv` `host` `hostFromEnv` ~~`port`~~ `passwordFromEnv` | +| `gcp-pubsub` | `credentials` | `credentialsFromEnv` | +| `external` | _(any matching value)_ | _(any matching value with `FromEnv` suffix)_ | +| `liiklus` | _(none)_ | _(none)_ | +| `stan` | _(none)_ | _(none)_ | +| `huawei-cloudeye` | | _(none)_ | _(none)_ | +| `postgresql` | `connection` `password` | `connectionFromEnv` `passwordFromEnv` | +| `mysql` | `connectionString` `password` | `connectionStringFromEnv` `passwordFromEnv` | + +### Scalers + +**Azure Service Bus** + +- `queueLength` was renamed to `messageCount` + +**Kafka** + +- `authMode` property was replaced with `sasl` and `tls` properties. Please refer [documentation](../scalers/apache-kafka/#authentication-parameters) for Kafka Authentication Parameters details. + +**RabbitMQ** + +In KEDA 2.0 the RabbitMQ scaler has only `host` parameter, and the protocol for communication can be specified by +`protocol` (http or amqp). The default value is `amqp`. The behavior changes only for scalers that were using HTTP +protocol. + +Example of RabbitMQ trigger before 2.0: + +```yaml +triggers: + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + includeUnacked: "true" + apiHost: "https://guest:password@localhost:443/vhostname" +``` + +The same trigger in 2.0: + +```yaml +triggers: + - type: rabbitmq + metadata: + queueLength: "20" + queueName: testqueue + protocol: "http" + host: "https://guest:password@localhost:443/vhostname" +``` + +### TriggerAuthentication + +In order to use Authentication via `TriggerAuthentication` with KEDA v2, you need to change: + +- Change the value of `apiVersion` property from `keda.k8s.io/v1alpha1` to `keda.sh/v1alpha1` + +For more details please refer to the full +[v2 TriggerAuthentication Specification](../concepts/authentication/#re-use-credentials-and-delegate-auth-with-triggerauthentication) diff --git a/content/docs/2.17/operate/_index.md b/content/docs/2.17/operate/_index.md new file mode 100644 index 000000000..bfacd27cb --- /dev/null +++ b/content/docs/2.17/operate/_index.md @@ -0,0 +1,13 @@ ++++ +title = "Operate" +description = "Guidance and requirements for operating KEDA" +weight = 1 ++++ + +We provide guidance and requirements around various areas to operate KEDA: + +- [Admission Webhooks](./admission-webhooks) +- [Cluster](./cluster) +- [Kubernetes Events](../reference/events) +- [KEDA Metrics Server](./metrics-server) +- [Security](./security) diff --git a/content/docs/2.17/operate/admission-webhooks.md b/content/docs/2.17/operate/admission-webhooks.md new file mode 100644 index 000000000..74545b8c7 --- /dev/null +++ b/content/docs/2.17/operate/admission-webhooks.md @@ -0,0 +1,118 @@ ++++ +title = "Admission Webhooks" +description = "Admission webhooks configurations guidance" +weight = 100 ++++ + +## Validation Enforcement + +By default, the admission webhooks are registered with `failurePolicy: Ignore`, this won't block the resources creation/update when the admission controller is not available. To ensure that the validation is always required and perform validation, setting `failurePolicy` to `Fail` is required. + +In particular, the admission webhooks for HPA ownership validation can be skipped by setting the annotation `validations.keda.sh/hpa-ownership` to `"false"`. Be cautious when doing so as it exposes the system to potential risks. + +### Cache Miss with Fallback to Direct Client for ScaledObject + +When validation enforcement is enabled, it's possible to run into a race condition when `ScaledObject` is part of the same deployment artifact as the `scaleTargetRef` (see also issue: [#5973](https://github.com/kedacore/keda/issues/5973)). For this purpose it's possible to configure additional argument for the webhook `Deployment`: +``` +--cache-miss-to-direct-client=true +``` +This will ensure that if getting the `scaleTargetRef` from the cached client returns `IsNotFound` error, the webhook will attempt to get the object directly from Kubernetes API. + + +## Custom Validations using Kubernetes ValidatingAdmissionPolicy + +> ⚠️ FEATURE STATE: Kubernetes v1.30 [stable] + +Kubernetes `ValidatingAdmissionPolicy` allows you to create custom validation policies using the [Common Expression Language (CEL)](https://cel.dev). This provides a declarative way to define and enforce validation rules directly within the cluster, helping ensure resource configurations meet your specified requirements. For more details, refer to the Kubernetes `ValidatingAdmissionPolicy` [documentation](https://kubernetes.io/docs/reference/access-authn-authz/validating-admission-policy/). + +Here is an example of a `ValidatingAdmissionPolicy` and its corresponding `ValidatingAdmissionPolicyBinding` to fail/deny `ScaledObject`s with more than 10 replicas in the `default` namespace (i.e., limiting the maximum number of pod replicas to 10): + +```yaml +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: "limit-keda-replicas.example.com" +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: ["keda.sh"] + apiVersions: ["v1alpha1"] + operations: ["CREATE", "UPDATE"] + resources: ["scaledobjects"] + validations: + - expression: "object.spec.maxReplicaCount <= 10" + message: "The maximum allowed number of pod replicas is 10." +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: "limit-keda-replicas-binding.example.com" +spec: + policyName: "limit-keda-replicas.example.com" + validationActions: [Deny] + matchResources: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: default +``` + +Since KEDA manages the `HorizontalPodAutoscaler` (HPA) behind the scenes, here is a complementary configuration to deny scaling for `HPA`, `Deployments`, and `ReplicaSets` with more than 10 replicas: + +```yaml +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: "limit-apps-replicas.example.com" +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: ["apps"] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE"] + resources: ["deployments", "replicasets"] + validations: + - expression: "object.spec.replicas <= 10" + message: "The maximum allowed number of pod replicas is 10." +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: "limit-hpa-replicas.example.com" +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: ["autoscaling"] + apiVersions: ["v2"] + operations: ["CREATE", "UPDATE"] + resources: ["horizontalpodautoscalers"] + validations: + - expression: "object.spec.replicas <= 10" + message: "The maximum allowed number of pod replicas is 10." +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: "limit-apps-replicas-binding.example.com" +spec: + policyName: "limit-apps-replicas.example.com" + validationActions: [Deny] + matchResources: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: default +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: "limit-hpa-replicas-binding.example.com" +spec: + policyName: "limit-hpa-replicas.example.com" + validationActions: [Deny] + matchResources: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: default +``` \ No newline at end of file diff --git a/content/docs/2.17/operate/cloud-events.md b/content/docs/2.17/operate/cloud-events.md new file mode 100644 index 000000000..4ae54d5b5 --- /dev/null +++ b/content/docs/2.17/operate/cloud-events.md @@ -0,0 +1,126 @@ ++++ +title = "CloudEvent Support" +description = "Experimental support for cloud events" +weight = 100 ++++ + +## Subscribing to events with `CloudEventSource` +`CloudEventSource` resource can be used in KEDA for subscribing to events that are emitted to the user's defined CloudEvent sink. + +> 📝 Event will be emitted to both Kubernetes Events and CloudEvents Destination if CloudEventSource resource is created. + +Here is a the schema of the `CloudEventSource` CRD: + +```yaml +apiVersion: eventing.keda.sh/v1alpha1 +kind: CloudEventSource +metadata: + name: {cloud-event-name} +spec: + clusterName: {cluster-name} #Optional. Will be used in the source/subject to specify where the event comes from. The default value is 'kubernetes-default' and it can also be set during the installation of KEDA with --k8sClusterName. This one will overwrite others if set. + authenticationRef: + name: {trigger-authentication-name} #Optional. Used to reference a `TriggerAuthentication` for authentication. + kind: TriggerAuthentication # Optional. Used to choose the authentication scopes. https://keda.sh/docs/latest/concepts/authentication/#authentication-scopes-namespace-vs-cluster + destination: + http: + uri: http://foo.bar + azureEventGridTopic: + endpoint: https://my-topic.eastus-1.eventgrid.azure.net/api/events + + eventSubscription: #Optional. Submit included/excluded event types will filter events when emitting events. + includedEventTypes: #Optional. Only events in this section will be emitted. + - keda.scaledobject.failed.v1 + excludedEventTypes: #Optional. Events in this section will not be emitted. + - keda.scaledobject.ready.v1 +``` + +In general, an event emitted by KEDA would fundamentally come down to the following structure: +```json +{ + "specversion" : "1.0", + "type" : "com.cloudeventsource.keda", + "source" : "/{cluster-name}/{keda-namespace}/keda", + "subject" : "/{cluster-name}/{namespace}/{object-type}/{object-name}", + "id" : "", + "time" : "2018-04-05T17:31:00Z", + "datacontenttype" : "application/json", + "data" : { + "reason":"", + "message":"" + } +} +``` + +## Event Sinks + +There will be multiple types of destination to emit KEDA events to. + +Here is an overview of the supported destinations: + +- [HTTP endpoint](#http-endpoint). +- [Azure Event Grid endpoint](#azure-event-grid). + +### HTTP endpoint +```yaml + destination: + http: + uri: http://foo.bar #An http endpoint that can receive cloudevent +``` + +### Azure Event Grid + +```yaml + destination: + azureEventGrid: + endpoint: foo.bar #endpoint from AzureEventGrid Topic +``` + +Authentication information must be provided by using `authenticationRef` which allows you to provide the access key or managed identity for Azure Event Grid authentication by providing a `TriggerAuthentication`. + +Here is an overview of the supported authentication types: + +#### Connection String Authentication + +- `accessKey` - Access key string for the Azure Event Grid connection auth. + +#### Pod identity based authentication +[Azure AD Workload Identity](https://azure.github.io/azure-workload-identity/docs/) provider can be used. + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: nameOfTriggerAuth + namespace: default +spec: + podIdentity: + provider: azure-workload +``` + +## Event Filter + +You can include filter(s) to define what event types you are interested in, or want to ignore. This is done by using `includedEventTypes` or `excludedEventTypes` respectively for a given sink. + +```yaml +eventSubscription: #Optional. Submit included/excluded event types will filter events when emitting events. + includedEventTypes: #Optional. Only events in this section will be emitted. + - keda.scaledobject.failed.v1 + excludedEventTypes: #Optional. Events in this section will not be emitted. + - keda.scaledobject.ready.v1 +``` + +## Supported Event List +| Event Type | Scenario Description | +|-------------------------------|-----------------------------------------------------------------------------------------------------------------------------| +| `keda.scaledobject.ready.v1` | On the first time a ScaledObject is ready, or if the previous ready condition status of the object was `Unknown` or `False` | +| `keda.scaledobject.failed.v1` | If the check validation for a ScaledObject fails | +| `keda.scaledobject.removed.v1`| When a ScaledObject is deleted | +| `keda.scaledjob.ready.v1`| On the first time a ScaledJob is ready, or if the previous ready condition status of the object was `Unknown` or `False` | +| `keda.scaledjob.failed.v1`| If the check validation for a ScaledJob fails +| `keda.scaledjob.removed.v1`| When a ScaledJob is deleted | +| `keda.authentication.triggerauthentication.created.v1` | On the first time a TriggerAuthentication is created | +| `keda.authentication.triggerauthentication.updated.v1` | When a TriggerAuthentication is updated | +| `keda.authentication.triggerauthentication.removed.v1` | When a TriggerAuthentication is deleted | +| `keda.authentication.clustertriggerauthentication.created.v1`| On the first time a ClusterTriggerAuthentication is created | +| `keda.authentication.clustertriggerauthentication.updated.v1`| When a ClusterTriggerAuthentication is updated | +| `keda.authentication.clustertriggerauthentication.removed.v1`| When a ClusterTriggerAuthentication is deleted | \ No newline at end of file diff --git a/content/docs/2.17/operate/cluster.md b/content/docs/2.17/operate/cluster.md new file mode 100644 index 000000000..8df3cf238 --- /dev/null +++ b/content/docs/2.17/operate/cluster.md @@ -0,0 +1,213 @@ ++++ +title = "Cluster" +description = "Guidance & requirements for running KEDA in your cluster" +weight = 100 ++++ + +## Requirements + +### Kubernetes Compatibility + +The supported window of Kubernetes versions with KEDA is known as "N-2" which means that KEDA will provide support for running on N-2 at least. + +However, maintainers can decide to extend this by supporting more minor versions based on the required CRDs being used; but there is no guarantee. + +As a reference, this compatibility matrix shows supported k8s versions per KEDA version: + +| KEDA | Kubernetes | +| ----- | ------------- | +| v2.17 | TBD | +| v2.16 | v1.29 - v1.31 | +| v2.15 | v1.28 - v1.30 | +| v2.14 | v1.27 - v1.29 | +| v2.13 | v1.27 - v1.29 | +| v2.12 | v1.26 - v1.28 | +| v2.11 | v1.25 - v1.27 | +| v2.10 | v1.24 - v1.26 | +| v2.9 | v1.23 - v1.25 | +| v2.8 | v1.17 - v1.25 | +| v2.7 | v1.17 - v1.25 | + +### Cluster Capacity + +The KEDA runtime require the following resources in a production-ready setup: + +| Deployment | CPU | Memory | +| ------------------ | ----------------------- | ----------------------------- | +| Admission Webhooks | Limit: 1, Request: 100m | Limit: 1000Mi, Request: 100Mi | +| Metrics Server | Limit: 1, Request: 100m | Limit: 1000Mi, Request: 100Mi | +| Operator | Limit: 1, Request: 100m | Limit: 1000Mi, Request: 100Mi | + +These are used by default when deploying through YAML. + +> 💡 For more info on CPU and Memory resource units and their meaning, see [this](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-units-in-kubernetes) link. + +### Firewall + +KEDA requires to be accessible inside the cluster to be able to autoscale. + +Here is an overview of the required ports that need to be accessible for KEDA to work: + +| Port | Why? | Remarks | +| ------ | -------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `443` | Used by Kubernetes API server to get metrics | Required for all platforms because it uses Control Plane → port 443 on the Service IP range communication. This is not applicable for Google Cloud. | +| `6443` | Used by Kubernetes API server to get metrics | Only required for Google Cloud because it uses Control Plane → port 6443 on the Pod IP range for communication | + +## High Availability + +KEDA does not provide full support for high-availability due to upstream limitations. + +Here is an overview of all KEDA deployments and the HA notes: + +| Deployment | Support Replicas | Note | +| -------------- | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Metrics Server | 1 | You can run multiple replicas of our metrics sever, and it is recommended to add the `--enable-aggregator-routing=true` CLI flag to the kube-apiserver so that requests sent to our metrics servers are load balanced. However, [you can only run one active metric server in a Kubernetes cluster serving external.metrics.k8s.io](https://github.com/kubernetes-sigs/custom-metrics-apiserver/issues/70) which has to be the KEDA metric server. | +| Operator | 2 | While you can run multiple replicas of our operator, only one operator instance will be active. The rest will be standing by, which may reduce downtime during a failure. Multiple replicas will not improve the performance of KEDA, it could only reduce a downtime during a failover. | + +## HTTP Timeouts + +Some scalers issue HTTP requests to external servers (i.e. cloud services). Each applicable scaler uses its own dedicated HTTP client with its own connection pool, and by default each client is set to time out any HTTP request after 3 seconds. + +You can override this default by setting the `KEDA_HTTP_DEFAULT_TIMEOUT` environment variable on the KEDA operator deployment to your desired timeout in milliseconds. + +> ⚠️ All applicable scalers will use this timeout and setting this on a per-scaler is currently not supported. + +## HTTP Connection: Disable Keep Alive + +Keep alive behaviour is enabled by default for every HTTP connection, this could stack a huge amount of connections (one per scaler) in some scenarios. + +You can disable keep alive for every HTTP connection by adding the relevant environment variable to both the KEDA Operator, and KEDA Metrics Server deployments: + +```yaml +- env: + KEDA_HTTP_DISABLE_KEEP_ALIVE: true +``` + +All applicable scalers will use this keep alive behaviour. Setting a per-scaler keep alive behaviour is currently unsupported. + +## HTTP Proxies + +Some scalers issue HTTP requests to external servers (i.e. cloud services). As certain companies require external servers to be accessed by proxy servers, adding the relevant environment variables to both the KEDA Operator, and KEDA Metrics Server deployments (HTTP_PROXY, HTTPS_PROXY, NO_PROXY, etc.) would allow the scaler to connect via the desired proxy. + +```yaml +- env: + HTTP_PROXY: http://proxy.server:port + HTTPS_PROXY: http://proxy.server:port + NO_PROXY: 10.0.0.0/8 +``` + +## HTTP TLS min version + +Our industry has seen an evolution of TLS versions and some are more secure than another. For example, TLS1.0 and TLS1.1 have known vulnerabilities. + +By default, KEDA uses TLS1.2 as a minimum TLS version given it is the lowest version without vulnerabilities. However, if you need to support another version you can configure it by using the environment variable `KEDA_HTTP_MIN_TLS_VERSION`. + +For example: + +```yaml +- env: + KEDA_HTTP_MIN_TLS_VERSION: TLS13 +``` + +The following values are allowed: `TLS13`, `TLS12`, `TLS11` and `TLS10`. + +## Kubernetes Client Parameters + +The Kubernetes client config used within KEDA Operator and KEDA Metrics Adapter can be adjusted by passing the following command-line flags to the binary: + +| Adapter Flag | Client Config Setting | Default Value | Description | +| ------------------- | ---------------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| kube-api-qps | cfg.QPS | 20.0 | Set the QPS rate for throttling requests sent to the apiserver | +| kube-api-burst | cfg.Burst | 30 | Set the burst for throttling requests sent to the apiserver | +| disable-compression | cfg.DisableCompression | true | Disable compression for response in k8s restAPI in client-go, see [this Kubernetes issue](https://github.com/kubernetes/kubernetes/issues/112296) for details | + +## gRPC Metrics Service Parameters + +The gRPC Metrics Service is part of the KEDA Operator deployment and serves scaling events and metrics from the scalers over gRPC to the Metrics API Service, that in turn serves them to the Kubernetes API Server. The gRPC Metrics Service config used by the KEDA Metrics Adapter to connect to the KEDA Operator can be adjusted by passing the following command-line flags to the Adapter binary: + +| Adapter Flag | Default Value | Description | +| ------------------------------ | ----------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| metrics-service-address | keda-operator.keda.svc.cluster.local:9666 | The address of the gRPC Metrics Service Server | +| metrics-service-grpc-authority | "" | Host Authority override for the Metrics Service if the Host Authority is not the same as the address used for the gRPC Metrics Service Server. This is required for mutual TLS when the identity of the adapter server as presented in its TLS certificate is not the same as the metrics-service-address | + +## Configure `MaxConcurrentReconciles` for Controllers + +To implement internal controllers KEDA uses the [controller-runtime project](https://github.com/kubernetes-sigs/controller-runtime), that enables configuration of [MaxConcurrentReconciles property](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/controller#Options), ie. the maximum number of concurrent reconciles which can be run for a controller. + +KEDA Operator exposes properties for specifying `MaxConcurrentReconciles` for following controllers/reconcilers: + +- `ScaledObjectReconciler` - responsible for watching and managing `ScaledObjects`, ie. validates input trigger specification, starts scaling logic and manages dependent HPA. +- `ScaledJobReconciler` - responsible for watching and managing `ScaledJobs` and dependent Kubernetes Jobs + +KEDA Metrics Server exposes property for specifying `MaxConcurrentReconciles` for `MetricsScaledObjectReconciler`, that manages Metrics Names exposes by KEDA and which are being consumed by Kubernetes server and HPA controller. + +To modify this properties you can set environment variables on both KEDA Operator and Metrics Server Deployments: + +| Environment variable name | Deployment | Default Value | Affected reconciler | +| ------------------------------------- | ---------- | ------------- | ---------------------- | +| KEDA_SCALEDOBJECT_CTRL_MAX_RECONCILES | Operator | 5 | ScaledObjectReconciler | +| KEDA_SCALEDJOB_CTRL_MAX_RECONCILES | Operator | 1 | ScaledJobReconciler | + +## Configure Leader Election + +Like reconciliation, KEDA Operator also uses the [controller-runtime project](https://github.com/kubernetes-sigs/controller-runtime) for electing the leader replica. The following properties can be configured for the Operator Deployment: + +- [`LeaseDuration`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#Options.LeaseDuration) +- [`RenewDeadline`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#Options.RenewDeadline) +- [`RetryPeriod`](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#Options.RetryPeriod) + +To specify values other than their defaults, you can set the following environment variables: + +| Environment variable name | Deployment | Default Value | Manager Property | +| -------------------------------------------- | -------------- | ------------- | ---------------- | +| KEDA_OPERATOR_LEADER_ELECTION_LEASE_DURATION | Operator | 15s | LeaseDuration | +| KEDA_OPERATOR_LEADER_ELECTION_RENEW_DEADLINE | Operator | 10s | RenewDeadline | +| KEDA_OPERATOR_LEADER_ELECTION_RETRY_PERIOD | Operator | 2s | RetryPeriod | + +## Restrict the Namespaces KEDA is Watching + +By default, KEDA controller watches for events in all namespaces in Kubernetes cluster. However, this can be restricted by environment variable `WATCH_NAMESPACE`. +It accepts either a single namespace, list of namespaces separated by comma or an empty string that denotes all namespaces. + +When a certain namespace is configured, and then a `ScaledObject` or `ScaledJob` is created in a different namespaces, it will be ignored by the operator. + +Example: + +```yaml +- env: + WATCH_NAMESPACE: keda,production +``` + +## Certificates used by KEDA Metrics Server + +To learn more please refer to [security section](./security#use-your-own-tls-certificates) + +## Restrict Secret Access + +By default, KEDA requires adding `secrets` to the cluster role as following: + +```yaml +- apiGroups: + - "" + resources: + - external + - pods + - secrets + - services + verbs: + - get + - list + - watch +``` + +However, this might lead to security risk (especially in production environment) since it will grant permission to read `secrets` from all namespaces. + +To restrict `secret` access and limited to KEDA namespace, you could add `KEDA_RESTRICT_SECRET_ACCESS` as environment variable to both KEDA Operator and KEDA Metrics Server: + +```yaml +env: + - name: KEDA_RESTRICT_SECRET_ACCESS + value: "true" +``` + +This allows you to omit `secrets` from the cluster role, which will disallow `TriggerAuthentication` to be used for your triggers if the `TriggerAuthentication` is using secrets. You can, however, still use `ClusterTriggerAuthentication`. diff --git a/content/docs/2.17/operate/metrics-server.md b/content/docs/2.17/operate/metrics-server.md new file mode 100644 index 000000000..45e9699fa --- /dev/null +++ b/content/docs/2.17/operate/metrics-server.md @@ -0,0 +1,73 @@ ++++ +title = "KEDA Metrics Server" +description = "Details on KEDA Metrics Server" +weight = 100 ++++ + +## Querying metrics exposed by KEDA Metrics Server + +The metrics exposed by KEDA Metrics Server can be queried directly using `kubectl`: +```bash +kubectl get --raw "/apis/external.metrics.k8s.io/v1beta1" +``` + +This will return a json with the list of metrics exposed by KEDA (external metrics only): +```json +{ + "kind": "APIResourceList", + "apiVersion": "v1", + "groupVersion": "external.metrics.k8s.io/v1beta1", + "resources": [ + { + "name": "externalmetrics", + "singularName": "", + "namespaced": true, + "kind": "ExternalMetricValueList", + "verbs": [ + "get" + ] + } + ] +} +``` + +In order to query a specific metric value, you also can do it using `kubectl`: +```bash +kubectl get --raw "/apis/external.metrics.k8s.io/v1beta1/namespaces/YOUR_NAMESPACE/YOUR_METRIC_NAME?labelSelector=scaledobject.keda.sh%2Fname%3D{SCALED_OBJECT_NAME}" +``` + +At this point, you should take in consideration that KEDA metrics are namespaced, this means that you have to specify the namespace where the `ScaledObject` is placed inside. + +For example, if you want to get the value of the metric named `s1-rabbitmq-queueName2`, that is used by ScaledObject named `my-scaled-object` in namespace `sample-ns`, the query will be like this: +```bash +kubectl get --raw "/apis/external.metrics.k8s.io/v1beta1/namespaces/sample-ns/s1-rabbitmq-queueName2?labelSelector=scaledobject.keda.sh%2Fname%3Dmy-scaled-object" +``` + +And it will show a json like this: + +```json +{ + "kind": "ExternalMetricValueList", + "apiVersion": "external.metrics.k8s.io/v1beta1", + "metadata": {}, + "items": [ + { + "metricName": "s1-rabbitmq-queueName2", + "metricLabels": null, + "timestamp": "2021-10-20T10:48:17Z", + "value": "0" + } + ] +} +``` + +> **Note:** There are 2 exceptions in querying metrics and those are `cpu` and `memory` scalers. When KEDA creates the HPA object, it uses standard `cpu` and `memory` metrics from the Kubernetes Metrics Server. If you want to query these 2 specific values, you should do it using `/apis/metrics.k8s.io/v1beta1` instead of `/apis/external.metrics.k8s.io/v1beta1`. + +## How to get metric names from ScaledObject + +During its work, KEDA updates each ScaledObject with some relevant information which it needs to work. Part of that information is metric names generated from the triggers inside the own ScaledObject. + +You can recover the metric names from a ScaledObject using `kubectl`: +```bash + kubectl get scaledobject SCALEDOBJECT_NAME -n NAMESPACE -o jsonpath={.status.externalMetricNames} +``` diff --git a/content/docs/2.17/operate/security.md b/content/docs/2.17/operate/security.md new file mode 100644 index 000000000..fa93994b0 --- /dev/null +++ b/content/docs/2.17/operate/security.md @@ -0,0 +1,29 @@ ++++ +title = "Security" +description = "Guidance to configure security options" +weight = 100 ++++ + +## Use your own TLS Certificates + +KEDA uses self-signed certificates for different things. These certificates are generated and rotated by the operator. Certificates are stored in a Kubernetes secret (`kedaorg-certs`) that it's mounted to all KEDA components in the (default) path `/certs`. Generated files are named `tls.crt` and `tls.key` for TLS certificate and `ca.crt` and `ca.key` for CA certificate. KEDA also patches Kubernetes resources to include the `caBundle`, making Kubernetes to trust in the CA. + +The KEDA operator is responsible for generating certificates for all the services, certificates are by default generated for following DNS names: +``` + -> eg. keda-operator +.svc -> eg. keda-operator.svc +.svc. -> eg. keda-operator.svc.cluster.local +``` +To change the default cluster domain (`cluster.local`), parameter `--k8s-cluster-domain="my-domain"` on KEDA operator can be used. Helm Charts set this automatically from `clusterDomain` value. + +While this is a good starting point, some end-users may want to use their own certificates which are generated from their own CA in order to improve security. This can be done by disabling the certificate generation/rotation in the operator and updating default values in other components (if required). + +Certificates generation in the KEDA operator can be disabled by removing the console argument `--enable-cert-rotation=true` or setting it to `false`. Once this setting is disabled, user given certs can be placed in the secret `kedaorg-certs` which is automatically mounted in all the components or they can be patched to use other secret (this can be done throught helm values too). + +All components inspect the folder `/certs` for any certificates inside it. Argument `--cert-dir` can be used to specify another folder to be used as a source for certificates, this argument can be patched in the manifests or using Helm values. Because these certificates are also used for internal communication between KEDA components, the CA is also required to be registered as a trusted CA inside KEDA components. + +## Register your own CA in KEDA Operator Trusted Store + +There are use cases where we need to use self-signed CAs (cases like AWS where their CA isn't registered as trusted etc.). Some scalers allow skipping the cert validation by setting the `unsafeSsl` parameter, but this isn't ideal because it allows any certificate, which is not secure. + +To overcome this problem, KEDA supports registering custom CAs to be used by SDKs where it is possible. To register custom CAs, place the certificates in a directory, then pass the directory to the KEDA operator using the `--ca-dir=` flag. By default, the KEDA operator looks in the `/custom/ca` directory. Multiple directories can be specified by providing the `--ca-dir=` flag multiple times. KEDA will try to register as trusted CAs all certificates inside these directories. If using kustomize or helm, CA certificate directories can be specified via `certificates.operator.caDirs` and certificate volumes can be mounted using `volumes.keda.extraVolumes` and `volumes.keda.extraVolumeMounts`. diff --git a/content/docs/2.17/reference/_index.md b/content/docs/2.17/reference/_index.md new file mode 100644 index 000000000..14b0e6a6b --- /dev/null +++ b/content/docs/2.17/reference/_index.md @@ -0,0 +1,13 @@ ++++ +title = "Reference" +weight = 2 ++++ + +Reference information for the KEDA autoscaler. + +- [ScaledObject specification](./scaledobject-spec) +- [ScaledJob specification](./scaledjob-spec) +- [Kubernetes Events](./events) +- [Firewall requirements](../operate/cluster#firewall) +- [FAQ](./faq.md) +- [Glossary](./glossary.md) diff --git a/content/docs/2.17/reference/events.md b/content/docs/2.17/reference/events.md new file mode 100644 index 000000000..aa264c0d8 --- /dev/null +++ b/content/docs/2.17/reference/events.md @@ -0,0 +1,28 @@ ++++ +title = "Events reference" +description = "Kubernetes Events emitted by KEDA" +weight = 2500 ++++ + +KEDA emits the following [Kubernetes Events](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#event-v1-core): + +| Event | Type | Description | +|---------------------------------------|-----------|-----------------------------------------------------------------------------------------------------------------------------| +| `ScaledObjectReady` | `Normal` | On the first time a ScaledObject is ready, or if the previous ready condition status of the object was `Unknown` or `False` | +| `ScaledJobReady` | `Normal` | On the first time a ScaledJob is ready, or if the previous ready condition status of the object was `Unknown` or `False` | +| `ScaledObjectCheckFailed` | `Warning` | If the check validation for a ScaledObject fails | | +| `ScaledJobCheckFailed` | `Warning` | If the check validation for a ScaledJob fails | | +| `ScaledObjectDeleted` | `Normal` | When a ScaledObject is deleted and removed from KEDA watch | | +| `ScaledJobDeleted` | `Normal` | When a ScaledJob is deleted and removed from KEDA watch | | +| `KEDAScalersStarted` | `Normal` | When Scalers watch loop have started for a ScaledObject or ScaledJob | | +| `KEDAScalersStopped` | `Normal` | When Scalers watch loop have stopped for a ScaledObject or a ScaledJob | | +| `KEDAScalerFailed` | `Warning` | When a Scaler fails to create or check its event source | | +| `KEDAScaleTargetActivated` | `Normal` | When the scale target (Deployment, StatefulSet, etc) of a ScaledObject is scaled to 1, triggered by {scalers1;scalers2;...} | | +| `KEDAScaleTargetDeactivated` | `Normal` | When the scale target (Deployment, StatefulSet, etc) of a ScaledObject is scaled to 0 | | +| `KEDAScaleTargetActivationFailed` | `Warning` | When KEDA fails to scale the scale target of a ScaledObject to 1 | | +| `KEDAScaleTargetDeactivationFailed` | `Warning` | When KEDA fails to scale the scale target of a ScaledObject to 0 | | +| `KEDAJobsCreated` | `Normal` | When KEDA creates jobs for a ScaledJob | | +| `TriggerAuthenticationAdded` | `Normal` | When a new TriggerAuthentication is added | | +| `TriggerAuthenticationDeleted` | `Normal` | When a TriggerAuthentication is deleted | | +| `ClusterTriggerAuthenticationAdded` | `Normal` | When a new ClusterTriggerAuthentication is added | | +| `ClusterTriggerAuthenticationDeleted` | `Normal` | When a ClusterTriggerAuthentication is deleted | | diff --git a/content/docs/2.17/reference/faq.md b/content/docs/2.17/reference/faq.md new file mode 100644 index 000000000..95365417f --- /dev/null +++ b/content/docs/2.17/reference/faq.md @@ -0,0 +1,6 @@ ++++ +title = "FAQ" +weight = 2000 ++++ + +{{< faq20 versionData="faq2_15" >}} diff --git a/content/docs/2.17/reference/glossary.md b/content/docs/2.17/reference/glossary.md new file mode 100644 index 000000000..2451000bf --- /dev/null +++ b/content/docs/2.17/reference/glossary.md @@ -0,0 +1,88 @@ ++++ +title = "Glossary" +weight = 1000 ++++ + +This document defines the various terms needed to understand the documentation and set up and use KEDA. + +## Admission Webhook + +[In Kubernetes](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/), an HTTP callback that handle admission requests. KEDA uses an admission webhook to validate and mutate ScaledObject resources. + +## Agent + +A primary role held by the KEDA operator. The Agent activates and deactivates Kubernetes Deployments to scale to and from zero. + +## Cluster + +[In Kubernetes](https://kubernetes.io/docs/reference/glossary/?fundamental=true#term-cluster), a set of one or more nodes that run containerized applications. + +## CRD + +Custom Resource Definition. [In Kubernetes](https://kubernetes.io/docs/reference/glossary/?fundamental=true#term-CustomResourceDefinition), a custom resource that extends the Kubernetes API with custom resources like ScaledObjects that have custom fields and behavior. + +## Event + +A notable occurrence captured by an event source that KEDA may use as a trigger to scale a container or deployment. + +## Event Source + +An external system like Kafka, RabbitMQ, that generates events that KEDA can monitor using a scaler. + +## Grafana + +An open-source monitoring platform that can visualize metrics collected by KEDA. + +## gRPC Remote Procedure Calls (gRPC) + +gRPC Remote Procedure Calls (gRPC). An open-source remote procedure call framework used by KEDA components to communicate. + +## HPA + +Horizontal Pod Autoscaler. Kubernetes autoscaler. By default, scales based on CPU/memory usage. KEDA uses HPA to scale Kubernetes clusters and deployments. + +## KEDA + +Kubernetes Event-Driven Autoscaling. A single-purpose, lightweight autoscaler that can scale a Kubernetes workload based on event metrics. + +## Metric + +Measurement of an event source such as queue length or response lag that KEDA uses to determine scaling. + +## OpenTelemetry + +An observability framework used by KEDA to instrument applications and collect metrics. + +## Operator + +The core KEDA component that monitors metrics and scales workloads accordingly. + +## Prometheus + +An open-source monitoring system that can scrape and store metrics from KEDA. + +## Scaled Object + +A custom resource that defines how KEDA should scale a workload based on events. + +## Scaled Job + +A custom resource KEDA uses to scale an application. + +## Scaler + +A component that integrates KEDA with a specific event source to collect metrics. + +## Stateful Set + +A Kubernetes workload with persistent data. KEDA can scale stateful sets. + +## TLS + +Transport Layer Security. KEDA uses TLS to encrypt communications between KEDA components. + +## Webhook + +An HTTP callback used to notify KEDA of events from external sources. + +[In Kubernetes](https://kubernetes.io/docs/reference/access-authn-authz/webhook/), an HTTP callback used as an event notification mechanism. diff --git a/content/docs/2.17/reference/scaledjob-spec.md b/content/docs/2.17/reference/scaledjob-spec.md new file mode 100644 index 000000000..3f056efa1 --- /dev/null +++ b/content/docs/2.17/reference/scaledjob-spec.md @@ -0,0 +1,287 @@ ++++ +title = "ScaledJob specification" +weight = 4000 ++++ + +## Overview + +This specification describes the `ScaledJob` custom resource definition that defines the triggers and scaling behaviors use by KEDA to scale jobs. The `.spec.ScaleTargetRef` section holds the reference to the job, defined in [_scaledjob_types.go_](https://github.com/kedacore/keda/blob/main/apis/keda/v1alpha1/scaledjob_types.go). + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledJob +metadata: + name: {scaled-job-name} + labels: + my-label: {my-label-value} # Optional. ScaledJob labels are applied to child Jobs + annotations: + autoscaling.keda.sh/paused: true # Optional. Use to pause autoscaling of Jobs + my-annotation: {my-annotation-value} # Optional. ScaledJob annotations are applied to child Jobs +spec: + jobTargetRef: + parallelism: 1 # [max number of desired pods](https://kubernetes.io/docs/concepts/workloads/controllers/job/#controlling-parallelism) + completions: 1 # [desired number of successfully finished pods](https://kubernetes.io/docs/concepts/workloads/controllers/job/#controlling-parallelism) + activeDeadlineSeconds: 600 # Specifies the duration in seconds relative to the startTime that the job may be active before the system tries to terminate it; value must be positive integer + backoffLimit: 6 # Specifies the number of retries before marking this job failed. Defaults to 6 + template: + # describes the [job template](https://kubernetes.io/docs/concepts/workloads/controllers/job) + pollingInterval: 30 # Optional. Default: 30 seconds + successfulJobsHistoryLimit: 5 # Optional. Default: 100. How many completed jobs should be kept. + failedJobsHistoryLimit: 5 # Optional. Default: 100. How many failed jobs should be kept. + envSourceContainerName: {container-name} # Optional. Default: .spec.JobTargetRef.template.spec.containers[0] + minReplicaCount: 10 # Optional. Default: 0 + maxReplicaCount: 100 # Optional. Default: 100 + rolloutStrategy: gradual # Deprecated: Use rollout.strategy instead (see below). + rollout: + strategy: gradual # Optional. Default: default. Which Rollout Strategy KEDA will use. + propagationPolicy: foreground # Optional. Default: background. Kubernetes propagation policy for cleaning up existing jobs during rollout. + scalingStrategy: + strategy: "custom" # Optional. Default: default. Which Scaling Strategy to use. + customScalingQueueLengthDeduction: 1 # Optional. A parameter to optimize custom ScalingStrategy. + customScalingRunningJobPercentage: "0.5" # Optional. A parameter to optimize custom ScalingStrategy. + pendingPodConditions: # Optional. A parameter to calculate pending job count per the specified pod conditions + - "Ready" + - "PodScheduled" + - "AnyOtherCustomPodCondition" + multipleScalersCalculation : "max" # Optional. Default: max. Specifies how to calculate the target metrics when multiple scalers are defined. + triggers: + # {list of triggers to create jobs} +``` + +You can find all supported triggers [here](../scalers). + +## jobTargetRef + +```yaml + jobTargetRef: + parallelism: 1 # Optional. Max number of desired instances ([docs](https://kubernetes.io/docs/concepts/workloads/controllers/job/#controlling-parallelism)) + completions: 1 # Optional. Desired number of successfully finished instances ([docs](https://kubernetes.io/docs/concepts/workloads/controllers/job/#controlling-parallelism)) + activeDeadlineSeconds: 600 # Optional. Specifies the duration in seconds relative to the startTime that the job may be active before the system tries to terminate it; value must be positive integer + backoffLimit: 6 # Optional. Specifies the number of retries before marking this job failed. Defaults to 6 +``` + +The `jobTargetRef` is a batch/v1 `JobSpec` object; refer to the Kubernetes API for [more details](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/job-v1/#JobSpec) about the fields. The `template` field is required. + + +## pollingInterval + +```yaml + pollingInterval: 30 # Optional. Default: 30 seconds +``` + +This is the interval to check each trigger on. By default, KEDA will check each trigger source on every ScaledJob every 30 seconds. + + +## successfulJobsHistoryLimit, failedJobsHistoryLimit + +```yaml + successfulJobsHistoryLimit: 5 # Optional. Default: 100. How many completed jobs should be kept. + failedJobsHistoryLimit: 5 # Optional. Default: 100. How many failed jobs should be kept. +``` + +The `successfulJobsHistoryLimit` and `failedJobsHistoryLimit` fields are optional. These fields specify how many completed and failed jobs should be kept. By default, they are set to 100. + +This concept is similar to [Jobs History Limits](https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#jobs-history-limits) allowing you to learn what the outcomes of your jobs are. + +The actual number of jobs could exceed the limit in a short time. However, it is going to resolve in the cleanup period. Currently, the cleanup period is the same as the Polling interval. + + +## envSourceContainerName + +```yaml + envSourceContainerName: {container-name} # Optional. Default: .spec.JobTargetRef.template.spec.containers[0] +``` + +This optional property specifies the name of container in the Job, from which KEDA should try to get environment properties holding secrets etc. If it is not defined it, KEDA will try to get environment properties from the first Container, ie. from `.spec.JobTargetRef.template.spec.containers[0]`. + +___ +## minReplicaCount + +```yaml + minReplicaCount: 10 # Optional. Default: 0 +``` + +The min number of jobs that is created by default. This can be useful to avoid bootstrapping time of new jobs. If minReplicaCount is greater than maxReplicaCount, minReplicaCount will be set to maxReplicaCount. + +New messages may create new jobs - within the limits imposed by maxReplicaCount - in order to reach the state where minReplicaCount jobs are always running. For example, if one sets minReplicaCount to 2 then there will be 2 jobs running permanently. Using a targetValue of 1, if 3 new messages are sent, 2 of those messages will be processed on the already running jobs but another 3 jobs will be created in order to fulfill the desired state dictated by the minReplicaCount parameter that is set to 2. + +## maxReplicaCount + +```yaml + maxReplicaCount: 100 # Optional. Default: 100 +``` + +The max number of pods that is created within a single polling period. If there are running jobs, the number of running jobs will be deducted. This table is an example of the scaling logic. + +| Queue Length | Max Replica Count | Target Average Value | Running Job Count | Number of the Scale | +|--------------|-------------------|----------------------|-------------------|---------------------| +| 10 | 3 | 1 | 0 | 3 | +| 10 | 3 | 2 | 0 | 3 | +| 10 | 3 | 1 | 1 | 2 | +| 10 | 100 | 1 | 0 | 10 | +| 4 | 3 | 5 | 0 | 1 | + +* **Queue Length:** The number of items in the queue. +* **Target Average Value:** The number of messages that will be consumed on a job. It is defined on the scaler side. e.g. `queueLength` on `Azure Storage Queue` scaler. +* **Running Job Count:** How many jobs are running. +* **Number of the Scale:** The number of the job that is created. + + +## rollout + +```yaml + rollout: + strategy: gradual # Optional. Default: default. Which Rollout Strategy KEDA will use. + propagationPolicy: foreground # Optional. Default: background. Kubernetes propagation policy for cleaning up existing jobs during +``` + +The optional property rollout.strategy specifies the rollout strategy KEDA will use while updating an existing ScaledJob. +Possible values are `default` or `gradual`. \ +When using the `default` rolloutStrategy, KEDA will terminate existing Jobs whenever a ScaledJob is being updated. Then, it will recreate those Jobs with the latest specs. The order in which this termination happens can be configured via the rollout.propagationPolicy property. By default, the kubernetes background propagation is used. To change this behavior specify set propagationPolicy to `foreground`. For further information see [Kubernetes Documentation](https://kubernetes.io/docs/tasks/administer-cluster/use-cascading-deletion/#use-foreground-cascading-deletion). +On the `gradual` rolloutStartegy, whenever a ScaledJob is being updated, KEDA will not delete existing Jobs. Only new Jobs will be created with the latest specs. + + +## scalingStrategy + +```yaml +scalingStrategy: + strategy: "default" # Optional. Default: default. Which Scaling Strategy to use. +``` + +Select a Scaling Strategy. Possible values are `default`, `custom`, `accurate`, or `eager`. The default value is `default`. + +> 💡 **NOTE:** +> +>`maxScale` is not the running Job count. It is measured as follows: + >```go + >maxScale = min(scaledJob.MaxReplicaCount(), divideWithCeil(queueLength, targetAverageValue)) + >``` + >That means it will use the value of `queueLength` divided by `targetAvarageValue` unless it is exceeding the `MaxReplicaCount`. +> +>`RunningJobCount` represents the number of jobs that are currently running or have not finished yet. +> +>It is measured as follows: +>```go +>if !e.isJobFinished(&job) { +> runningJobs++ +>} +>``` +>`PendingJobCount` provides an indication of the amount of jobs that are in pending state. Pending jobs can be calculated in two ways: +> - Default behavior - Job that have not finished yet **and** the underlying pod is either not running or has not been completed yet +> - Setting `pendingPodConditions` - Job that has not finished yet **and** all specified pod conditions of the underlying pod mark as `true` by kubernetes. +> +>It is measured as follows: +>```go +>if !e.isJobFinished(&job) { +> if len(scaledJob.Spec.ScalingStrategy.PendingPodConditions) > 0 { +> if !e.areAllPendingPodConditionsFulfilled(&job, scaledJob.Spec.ScalingStrategy.PendingPodConditions) { +> pendingJobs++ +> } +> } else { +> if !e.isAnyPodRunningOrCompleted(&job) { +> pendingJobs++ +> } +> } +>} +>``` + +**default** +This logic is the same as Job for V1. The number of the scale will be calculated as follows. + +_The number of the scale_ + +```go +maxScale - runningJobCount +``` + +**custom** +You can customize the default scale logic. You need to configure the following parameters. If you don't configure it, then the strategy will be `default.` + +```yaml +customScalingQueueLengthDeduction: 1 # Optional. A parameter to optimize custom ScalingStrategy. +customScalingRunningJobPercentage: "0.5" # Optional. A parameter to optimize custom ScalingStrategy. +``` + +_The number of the scale_ + +```go +min(maxScale-int64(*s.CustomScalingQueueLengthDeduction)-int64(float64(runningJobCount)*(*s.CustomScalingRunningJobPercentage)), maxReplicaCount) +``` + +**accurate** +If the scaler returns `queueLength` (number of items in the queue) that does not include the number of locked messages, this strategy is recommended. `Azure Storage Queue` is one example. You can use this strategy if you delete a message once your app consumes it. + +```go +if (maxScale + runningJobCount) > maxReplicaCount { + return maxReplicaCount - runningJobCount + } + return maxScale - pendingJobCount +``` +For more details, you can refer to [this PR](https://github.com/kedacore/keda/pull/1227). + +**eager** +When adopting the **default** strategy, you are likely to come into a subtle case where messages need to be consumed by spawning jobs but remain in the queue, even when there are available slots between `runningJobCount` and `maxReplicaCount`. The **eager** strategy comes to the rescue. It addresses this issue by utilizing all available slots up to the maxReplicaCount, ensuring that waiting messages are processed as quickly as possible. + +For example, let's assume we configure a ScaledJob in a cluster as below: +```yaml + ### + # A job that runs for a minimum of 3 hours. + ### + pollingInterval: 10 # Optional. Default: 30 seconds + maxReplicaCount: 10 # Optional. Default: 100 + triggers: + - type: rabbitmq + metadata: + queueName: woker_queue + hostFromEnv: RABBITMQ_URL + mode: QueueLength + value: "1" +``` +We send 3 messages to the Rabbitmq and wait longer enough than the `pollingInterval`, then send another 3. + +With the `default` scaling strategy, we are supposed to see the metrics changes in the following table: + +| | initial | incoming 3 messages | after poll | incoming 3 messages | after poll | +|-------------|---------|---------------------|------------|---------------------|------------| +| queueLength | 0 | 3 | 3 | 6 | 6 | +| runningJobs | 0 | 0 | 3 | 3 | 3 | + + +If we switch to `eager`, the result becomes: + +| | initial | incoming 3 messages | after poll | incoming 3 messages | after poll | +|-------------|---------|---------------------|------------|---------------------|------------| +| queueLength | 0 | 3 | 3 | 6 | 6 | +| runningJobs | 0 | 0 | 3 | 3 | 6 | + +We can identify the difference in their final states. + + +You may also refer to [this original issue](https://github.com/kedacore/keda/issues/5114) for more information. + +--- + +```yaml +scalingStrategy: + multipleScalersCalculation : "max" # Optional. Default: max. Specifies how to calculate the target metrics (`queueLength` and `maxScale`) when multiple scalers are defined. +``` +Select a behavior if you have multiple triggers. Possible values are `max`, `min`, `avg`, or `sum`. The default value is `max`. + +* **max:** - Use metrics from the scaler that has the max number of `queueLength`. (default) +* **min:** - Use metrics from the scaler that has the min number of `queueLength`. +* **avg:** - Sum up all the active scalers metrics and divide by the number of active scalers. +* **sum:** - Sum up all the active scalers metrics. + + +### multipleScalersCalculation + +```yaml +scalingStrategy: + multipleScalersCalculation : "max" # Optional. Default: max. Specifies how to calculate the target metrics (`queueLength` and `maxScale`) when multiple scalers are defined. +``` +Select a behavior if you have multiple triggers. Possible values are `max`, `min`, `avg`, or `sum`. The default value is `max`. + +* **max:** - Use metrics from the scaler that has the max number of `queueLength`. (default) +* **min:** - Use metrics from the scaler that has the min number of `queueLength`. +* **avg:** - Sum up all the active scalers metrics and divide by the number of active scalers. +* **sum:** - Sum up all the active scalers metrics. diff --git a/content/docs/2.17/reference/scaledobject-spec.md b/content/docs/2.17/reference/scaledobject-spec.md new file mode 100644 index 000000000..74a5200d5 --- /dev/null +++ b/content/docs/2.17/reference/scaledobject-spec.md @@ -0,0 +1,259 @@ + ++++ +title = "ScaledObject specification" +weight = 3000 ++++ + +## Overview + +This specification describes the `ScaledObject` Custom Resource definition that defines the triggers and scaling behaviors used by KEDA to scale `Deployment`, `StatefulSet` and `Custom Resource` target resources. The `.spec.ScaleTargetRef` section holds the reference to the target resource, defined in [_scaledobject_types.go_](https://github.com/kedacore/keda/blob/main/apis/keda/v1alpha1/scaledobject_types.go). + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {scaled-object-name} + annotations: + scaledobject.keda.sh/transfer-hpa-ownership: "true" # Optional. Use to transfer an existing HPA ownership to this ScaledObject + validations.keda.sh/hpa-ownership: "true" # Optional. Use to disable HPA ownership validation on this ScaledObject + autoscaling.keda.sh/paused: "true" # Optional. Use to pause autoscaling of objects explicitly +spec: + scaleTargetRef: + apiVersion: {api-version-of-target-resource} # Optional. Default: apps/v1 + kind: {kind-of-target-resource} # Optional. Default: Deployment + name: {name-of-target-resource} # Mandatory. Must be in the same namespace as the ScaledObject + envSourceContainerName: {container-name} # Optional. Default: .spec.template.spec.containers[0] + pollingInterval: 30 # Optional. Default: 30 seconds + cooldownPeriod: 300 # Optional. Default: 300 seconds + initialCooldownPeriod: 0 # Optional. Default: 0 seconds + idleReplicaCount: 0 # Optional. Default: ignored, must be less than minReplicaCount + minReplicaCount: 1 # Optional. Default: 0 + maxReplicaCount: 100 # Optional. Default: 100 + fallback: # Optional. Section to specify fallback options + failureThreshold: 3 # Mandatory if fallback section is included + replicas: 6 # Mandatory if fallback section is included + advanced: # Optional. Section to specify advanced options + restoreToOriginalReplicaCount: true/false # Optional. Default: false + horizontalPodAutoscalerConfig: # Optional. Section to specify HPA related options + name: {name-of-hpa-resource} # Optional. Default: keda-hpa-{scaled-object-name} + behavior: # Optional. Use to modify HPA's scaling behavior + scaleDown: + stabilizationWindowSeconds: 300 + policies: + - type: Percent + value: 100 + periodSeconds: 15 + triggers: + # {list of triggers to activate scaling of the target resource} +``` + +## scaleTargetRef + +```yaml + scaleTargetRef: + apiVersion: {api-version-of-target-resource} # Optional. Default: apps/v1 + kind: {kind-of-target-resource} # Optional. Default: Deployment + name: {name-of-target-resource} # Mandatory. Must be in the same namespace as the ScaledObject + envSourceContainerName: {container-name} # Optional. Default: .spec.template.spec.containers[0] +``` + +The reference to the resource this ScaledObject is configured for. This is the resource KEDA will scale up/down and set up an HPA for, based on the triggers defined in `triggers:`. + +To scale Kubernetes Deployments only `name` need be specified. To scale a different resource such as StatefulSet or Custom Resource (that defines `/scale` subresource), appropriate `apiVersion` (following standard Kubernetes convention, ie. `{api}/{version}`) and `kind` need to be specified. + +`envSourceContainerName` is an optional property that specifies the name of container in the target resource, from which KEDA should try to get environment properties holding secrets etc. If it is not defined, KEDA will try to get environment properties from the first Container, ie. from `.spec.template.spec.containers[0]`. + +**Assumptions:** Resource referenced by `name` (and `apiVersion`, `kind`) is in the same namespace as the ScaledObject + + +## pollingInterval +```yaml + pollingInterval: 30 # Optional. Default: 30 seconds +``` +This is the interval to check each trigger on. By default, KEDA will check each trigger source on every ScaledObject every 30 seconds. + +When scaling from 0 to 1, the polling interval is controlled by KEDA. For example, if this parameter is set to `60`, KEDA will poll for a metric value every 60 seconds while the number of replicas is 0. + +While scaling from 1 to N, on top of KEDA, the HPA will also poll regularly for metrics, based on the [`--horizontal-pod-autoscaler-sync-period`](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/#options) parameter to the `kube-controller-manager`, which by default is 15 seconds. For example, if the `kube-controller-manager` was started with `--horizontal-pod-autoscaler-sync-period=30`, the HPA will poll for a metric value every 30 seconds while the number of replicas is between 1 and N. + +If you want respect the polling interval, the feature [`caching metrics`](../concepts/scaling-deployments/#caching-metrics) enables caching of metric values during polling interval. + +**Example:** in a queue scenario, KEDA will check the queueLength every `pollingInterval` while the number of replicas is 0, and scale the resource up an down accordingly. + +## cooldownPeriod +```yaml + cooldownPeriod: 300 # Optional. Default: 300 seconds +``` + +The period to wait after the last trigger reported active before scaling the resource back to 0, in seconds. By default, it's 300 (5 minutes). + +The `cooldownPeriod` only applies after a trigger occurs; when you first create your `Deployment` (or `StatefulSet`/`CustomResource`), KEDA will immediately scale it to `minReplicaCount`. Additionally, the KEDA `cooldownPeriod` only applies when scaling to 0; scaling from 1 to N replicas is handled by the [Kubernetes Horizontal Pod Autoscaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/). + +**Example:** wait 5 minutes after the last time KEDA checked the queue and it was empty. (this is obviously dependent on `pollingInterval`) + + +## initialCooldownPeriod +```yaml + initialCooldownPeriod: 120 # Optional. Default: 0 seconds +``` +The delay before the `cooldownPeriod` starts after the initial creation of the `ScaledObject`, in seconds. By default, it's 0, meaning the `cooldownPeriod` begins immediately upon creation. If set to a value such as 120 seconds, the `cooldownPeriod` will only start after the `ScaledObject` has been active for that duration. + +This parameter is particularly useful for managing the scale-down behavior during the initial phase of a `ScaledObject`. For instance, if `initialCooldownPeriod` is set to 120 seconds, KEDA will not scale the resource back to 0 until 120 seconds have passed since the `ScaledObject` creation, regardless of the activity triggers. This allows for a grace period in situations where immediate scaling down after creation is not desirable. + +**Example:** Wait 120 seconds after the `ScaledObject` is created before starting the `cooldownPeriod`. For instance, if the `initialCooldownPeriod` is set to 120 seconds, KEDA will not initiate the cooldown process until 120 seconds have passed since the `ScaledObject` was first created, regardless of the triggers' activity. This ensures a buffer period where the resource won’t be scaled down immediately after creation. (Note: This setting is independent of the `pollingInterval`.) + + +## idleReplicaCount + +```yaml + idleReplicaCount: 0 # Optional. Default: ignored, must be less than minReplicaCount +``` + +> 💡 **NOTE:** Due to limitations in HPA controller the only supported value for this property is 0, it will not work correctly otherwise. See this [issue](https://github.com/kedacore/keda/issues/2314) for more details. +> +> In some cases, you always need at least `n` pod running. Thus, you can omit this property and set `minReplicaCount` to `n`. +> +> **Example** You set `minReplicaCount` to 1 and `maxReplicaCount` to 10. If there’s no activity on triggers, the target resource is scaled down to `minReplicaCount` (1). Once there are activities, the target resource will scale base on the HPA rule. If there’s no activity on triggers, the resource is again scaled down to `minReplicaCount` (1). + +If this property is set, KEDA will scale the resource down to this number of replicas. If there's some activity on target triggers KEDA will scale the target resource immediately to `minReplicaCount` and then will be scaling handled by HPA. When there is no activity, the target resource is again scaled down to `idleReplicaCount`. This setting must be less than `minReplicaCount`. + +**Example:** If there's no activity on triggers the target resource is scaled down to `idleReplicaCount` (0), once there is an activity the target resource is immediately scaled to `minReplicaCount` (10) and then up to `maxReplicaCount` (100) as needed. If there's no activity on triggers the resource is again scaled down to `idleReplicaCount` (0). + + +## minReplicaCount + +```yaml + minReplicaCount: 1 # Optional. Default: 0 +``` + +Minimum number of replicas KEDA will scale the resource down to. By default, it's scale to zero, but you can use it with some other value as well. + +## maxReplicaCount + +```yaml + maxReplicaCount: 100 # Optional. Default: 100 +``` +This setting is passed to the HPA definition that KEDA will create for a given resource and holds the maximum number of replicas of the target resource. + + +## fallback +```yaml + fallback: # Optional. Section to specify fallback options + failureThreshold: 3 # Mandatory if fallback section is included + replicas: 6 # Mandatory if fallback section is included +``` + +The `fallback` section is optional. It defines a number of replicas to fall back to if a scaler is in an error state. + +KEDA will keep track of the number of consecutive times each scaler has failed to get metrics from its source. Once that value passes the `failureThreshold`, instead of not propagating a metric to the HPA (the default error behaviour), the scaler will, instead, return a normalised metric using the formula: +``` +target metric value * fallback replicas +``` +Due to the HPA metric being of type `AverageValue` (see below), this will have the effect of the HPA scaling the deployment to the defined number of fallback replicas. + +**Example:** When my instance of prometheus is unavailable 3 consecutive times, KEDA will change the HPA metric such that the deployment will scale to 6 replicas. + +There are a few limitations to using a fallback: + - It only supports scalers whose target is an `AverageValue` metric. Thus, it is **not** supported by the CPU & memory scalers, or by scalers whose metric target type is `Value`. In these cases, it will assume that fallback is disabled. + - It is only supported by `ScaledObjects` **not** `ScaledJobs`. + + +## advanced + +### restoreToOriginalReplicaCount + +```yaml +advanced: + restoreToOriginalReplicaCount: true/false # Optional. Default: false +``` + +This property specifies whether the target resource (`Deployment`, `StatefulSet`,...) should be scaled back to original replicas count, after the `ScaledObject` is deleted. +Default behavior is to keep the replica count at the same number as it is in the moment of `ScaledObject's` deletion. + +For example a `Deployment` with `3 replicas` is created, then `ScaledObject` is created and the `Deployment` is scaled by KEDA to `10 replicas`. Then `ScaledObject` is deleted: + 1. if `restoreToOriginalReplicaCount = false` (default behavior) then `Deployment` replicas count is `10` + 2. if `restoreToOriginalReplicaCount = true` then `Deployment` replicas count is set back to `3` (the original value) + + +### horizontalPodAutoscalerConfig + +```yaml +advanced: + horizontalPodAutoscalerConfig: # Optional. Section to specify HPA related options + name: {name-of-hpa-resource} # Optional. Default: keda-hpa-{scaled-object-name} + behavior: # Optional. Use to modify HPA's scaling behavior + scaleDown: + stabilizationWindowSeconds: 300 + policies: + - type: Percent + value: 100 + periodSeconds: 15 +``` + +#### horizontalPodAutoscalerConfig.name + +The name of the HPA resource KEDA will create. By default, it's `keda-hpa-{scaled-object-name}` + +#### horizontalPodAutoscalerConfig.behavior + +Starting from Kubernetes v1.18 the autoscaling API allows scaling behavior to be configured through the HPA behavior field. This way one can directly affect scaling of 1<->N replicas, which is internally being handled by HPA. KEDA would feed values from this section directly to the HPA's `behavior` field. Please follow [Kubernetes documentation](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#configurable-scaling-behavior) for details. + +**Assumptions:** KEDA must be running on Kubernetes cluster v1.18+, in order to be able to benefit from this setting. + + + +```yaml +advanced: + scalingModifiers: # Optional. Section to specify scaling modifiers + target: {target-value-to-scale-on} # Mandatory. New target if metrics are anyhow composed together + activationTarget: {activation-target-value-to-scale-on} # Optional. New activation target if metrics are anyhow composed together + metricType: {metric-tipe-for-the-modifier} # Optional. Metric type to be used if metrics are anyhow composed together + formula: {formula-for-fetched-metrics} # Mandatory. Formula for calculation +``` + +### scalingModifiers + +If defined, both `target` and `formula` are mandatory. Using this structure creates `composite-metric` for the HPA that will replace all requests for external metrics and handle them internally. With `scalingModifiers` each trigger used in the `formula` **must** have a name defined. + +#### scalingModifiers.target + +`target` defines new target value to scale on for the composed metric. + +#### scalingModifiers.activationTarget + +`activationTarget` defines a new [activation target value](../concepts/scaling-deployments.md#activating-and-scaling-thresholds) to scale on for the composed metric. (Default: `0`, Optional) + +#### scalingModifiers.metricType + +`metricType` defines metric type used for this new `composite-metric`. (Values: `AverageValue`, `Value`, Default: `AverageValue`, Optional) + +#### scalingModifiers.formula + + `formula` composes metrics together and allows them to be modified/manipulated. It accepts mathematical/conditional statements using [this external project](https://github.com/antonmedv/expr). If the `fallback` scaling feature is in effect, the `formula` will NOT modify its metrics (therefore it modifies metrics only when all of their triggers are healthy). Complete language definition of `expr` package can be found [here](https://expr.medv.io/docs/Language-Definition). Formula must return a single value (not boolean). + +For examples of this feature see section [Scaling Modifiers](../concepts/scaling-deployments.md). + + +## triggers + +```yaml + triggers: + # {list of triggers to activate scaling of the target resource} +``` + +> 💡 **NOTE:** You can find all supported triggers [here](../scalers). + +Trigger fields: +- **type**: The type of trigger to use. (Mandatory) +- **metadata**: The configuration parameters that the trigger requires. (Mandatory) +- **name**: Name for this trigger. This value can be used to easily distinguish this specific trigger and its metrics when consuming [Prometheus metrics](../integrations/prometheus.md). By default, the name is generated from the trigger type. (Optional) +- **useCachedMetrics**: Enables caching of metric values during polling interval (as specified in `.spec.pollingInterval`). For more information, see ["Caching Metrics"](../concepts/scaling-deployments.md#caching-metrics). (Values: `false`, `true`, Default: `false`, Optional) +- **authenticationRef**: A reference to the `TriggerAuthentication` or `ClusterTriggerAuthentication` object that is used to authenticate the scaler with the environment. + - More details can be found [here](../concepts/authentication). (Optional) +- **metricType**: The type of metric that should be used. (Values: `AverageValue`, `Value`, `Utilization`, Default: `AverageValue`, Optional) + - Learn more about how the [Horizontal Pod Autoscaler (HPA) calculates `replicaCount`](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/) based on metric type and value. + - To show the differences between the metric types, let's assume we want to scale a deployment with 3 running replicas based on a queue of messages: + - With `AverageValue` metric type, we can control how many messages, on average, each replica will handle. If our metric is the queue size, the threshold is 5 messages, and the current message count in the queue is 20, HPA will scale the deployment to 20 / 5 = 4 replicas, regardless of the current replica count. + - The `Value` metric type, on the other hand, can be used when we don't want to take the average of the given metric across all replicas. For example, with the `Value` type, we can control the average time of messages in the queue. If our metric is average time in the queue, the threshold is 5 milliseconds, and the current average time is 20 milliseconds, HPA will scale the deployment to 3 * 20 / 5 = 12. + +> ⚠️ **NOTE:** All scalers, except CPU and Memory, support metric types `AverageValue` and `Value` while CPU and Memory scalers both support `AverageValue` and `Utilization`. diff --git a/content/docs/2.17/scalers/_index.md b/content/docs/2.17/scalers/_index.md new file mode 100644 index 000000000..6e7428923 --- /dev/null +++ b/content/docs/2.17/scalers/_index.md @@ -0,0 +1,6 @@ ++++ +title = "Scalers" +weight = 2 ++++ + +KEDA **scalers** can both detect if a deployment should be activated or deactivated, and feed custom metrics for a specific event source. diff --git a/content/docs/2.17/scalers/activemq.md b/content/docs/2.17/scalers/activemq.md new file mode 100644 index 000000000..b9c012798 --- /dev/null +++ b/content/docs/2.17/scalers/activemq.md @@ -0,0 +1,90 @@ ++++ +title = "ActiveMQ" +availability = "v2.6+" +maintainer = "Community" +category = "Messaging" +description = "Scale applications based on ActiveMQ Queue." +go_file = "activemq_scaler" ++++ + +### Trigger Specification + +This specification describes the `activemq` trigger that scales based on a ActiveMQ Queue. + +```yaml +triggers: +- type: activemq + metadata: + managementEndpoint: "activemq.activemq-test:8161" + destinationName: "testQueue" + brokerName: "activemq_broker" + targetQueueSize: "100" + activationTargetQueueSize: "10" +``` + +**Parameter list:** + +- `managementEndpoint` - ActiveMQ management endpoint in format: `:`. +- `destinationName` - Name of the queue to check for the message count. +- `brokerName` - Name of the broker as defined in ActiveMQ. +- `targetQueueSize` - Target value for queue length passed to the scaler. The scaler will cause the replicas to increase if the queue message count is greater than the target value per active replica. (Default: `10`, Optional) +- `activationTargetQueueSize` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) +- `restAPITemplate` - Template to build REST API url to get queue size. (Default: `"http://{{.ManagementEndpoint}}/api/jolokia/read/org.apache.activemq:type=Broker,brokerName={{.BrokerName}},destinationType=Queue,destinationName={{.DestinationName}}/QueueSize"`, Optional) +- `corsHeader` - Value to populate the Origin header field for CORS filtering. (Default: `"http://<>"`, Optional) + +**Parameter Requirements:** + +- In case of `restAPITemplate` parameter is not used, parameters resolving the REST API Template are all **required**: `managementEndpoint`, `destinationName`, `brokerName`. +- ActiveMQ Scaler polls the ActiveMQ REST API to monitor message count of target queue. Currently, the scaler supports basic authentication. `username` and `password` are **required**. See [Authentication Parameters](#authentication-parameters) below. + +### Authentication Parameters + +You can authenticate by using username and password via `TriggerAuthentication` configuration. + +**Username and Password based Authentication:** + +- `username` - Username for connect to the management endpoint of ActiveMQ. +- `password` - Password for connect to the management endpoint of ActiveMQ. + +### Example + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: activemq-secret +type: Opaque +data: + activemq-password: ACTIVEMQ_PASSWORD + activemq-username: ACTIVEMQ_USERNAME +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: trigger-auth-activemq +spec: + secretTargetRef: + - parameter: username + name: activemq-secret + key: activemq-username + - parameter: password + name: activemq-secret + key: activemq-password +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: activemq-scaledobject +spec: + scaleTargetRef: + name: nginx-deployment + triggers: + - type: activemq + metadata: + managementEndpoint: "activemq.activemq-test:8161" + destinationName: "testQ" + brokerName: "localhost" + targetQueueSize: "50" + authenticationRef: + name: trigger-auth-activemq +``` diff --git a/content/docs/2.17/scalers/apache-kafka-go.md b/content/docs/2.17/scalers/apache-kafka-go.md new file mode 100644 index 000000000..e2c7b7fd8 --- /dev/null +++ b/content/docs/2.17/scalers/apache-kafka-go.md @@ -0,0 +1,337 @@ ++++ +title = "Apache Kafka (Experimental)" +availability = "v2.12+" +maintainer = "Community" +category = "Messaging" +description = "Experimental scaler based on 'segmentio/kafka-go' library. Scale applications based on an Apache Kafka topic or other services that support Kafka protocol." +go_file = "apache_kafka_scaler" ++++ + +> **Notice:** +> - This is an experimental Kafka scaler based on [kafka-go](https://github.com/segmentio/kafka-go) library. +> - This scaler is not fully compatible with the existing [Kafka scaler](./apache-kafka.md). There are some differences in the configuration and behavior. Please read the documentation carefully before using it. +> - If you are using OAuth authentication, please use the existing Kafka scaler, as this scaler does not yet support OAuth2 authentication. +> - This scaler has support for AWS MSK IAM based authentication. +> - By default, the number of replicas will not exceed: +> - The number of partitions on a topic when a topic is specified; +> - The number of partitions of *all topics* in the consumer group when no topic is specified; +> - `maxReplicaCount` specified in `ScaledObject`/`ScaledJob`. If not specified, then the default value of `maxReplicaCount` is taken into account; +> - The number of partitions with non-zero lag if `limitToPartitionsWithLag` is set to `true` +> +> That is, if `maxReplicaCount` is set more than number of partitions, the scaler won't scale up to target maxReplicaCount. See `allowIdleConsumers` below to disable this default behavior. +> - This is so because if there are more number of consumers than the number of partitions in a topic, then extra consumer will have to sit idle. + +### Trigger Specification + +This specification describes the `apache-kafka` trigger for an Apache Kafka topic. + +```yaml +triggers: +- type: apache-kafka + metadata: + bootstrapServers: kafka.svc:9092 + consumerGroup: my-group + topic: test-topic + lagThreshold: '5' + activationLagThreshold: '3' + offsetResetPolicy: latest + allowIdleConsumers: false + scaleToZeroOnInvalidOffset: false + excludePersistentLag: false + limitToPartitionsWithLag: false + partitionLimitation: '1,2,10-20,31' + tls: enable + sasl: plaintext +``` + +**Parameter list:** + +- `bootstrapServers` - Comma separated list of Kafka brokers "hostname:port" to connect to for bootstrap. +- `consumerGroup` - Name of the consumer group used for checking the offset on the topic and processing the related lag. +- `topic` - Name of the topic on which processing the offset lag. (Optional, see note below) +- `lagThreshold` - Average target value to trigger scaling actions. (Default: `5`, Optional) +- `activationLagThreshold` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) +- `offsetResetPolicy` - The offset reset policy for the consumer. (Values: `latest`, `earliest`, Default: `latest`, Optional) +- `allowIdleConsumers` - When set to `true`, the number of replicas can exceed the number of +partitions on a topic, allowing for idle consumers. (Default: `false`, Optional) +- `scaleToZeroOnInvalidOffset` - This parameter controls what the scaler does when a partition doesn't have a valid offset. +If 'false' (the default), the scaler will keep a single consumer for that partition. Otherwise ('true'), the consumers for that +partition will be scaled to zero. See the [discussion](https://github.com/kedacore/keda/issues/2612) about this parameter. +- `excludePersistentLag` - When set to `true`, the scaler will exclude partition lag for partitions which current offset is the same as the current offset of the previous polling cycle. This parameter is useful to prevent scaling due to partitions which current offset message is unable to be consumed. If `false` (the default), scaler will include all consumer lag in all partitions as per normal. (Default: `false`, Optional) +- `limitToPartitionsWithLag` - When set to `true`, the number of replicas will not exceed the number of partitions having non-zero lag. `topic` must be speicied when this parameter is set to `true`. `allowIdleConsumers` cannot be `true` when this parameter is `true`. (Default: `false`, Optional) +- `partitionLimitation` - Comma separated list of partition ids to scope the scaling on. Allowed patterns are "x,y" and/or ranges "x-y". If set, the calculation of the lag will only take these ids into account. (Default: All partitions, Optional) +- `sasl` - Kafka SASL auth mode. (Values: `plaintext`, `scram_sha256`, `scram_sha512`, `aws_msk_iam` or `none`, Default: `none`, Optional). This parameter could also be specified in `sasl` in TriggerAuthentication +- `tls` - To enable SSL auth for Kafka, set this to `enable`. If not set, TLS for Kafka is not used. (Values: `enable`, `disable`, Default: `disable`, Optional). This parameter could also be specified in `tls` in TriggerAuthentication + +> **Note:** +> +> When `topic` is unspecified, total offset lag will be calculated with all topics within the consumer group. +> - When there are **active** consumer instances, _all topics_ includes: +> - Topics the consumer is *currently* subscribing to; +> - Topics that the consumer group *had prior commit history* (up to retention period for `__consumer_offset`, default to 7 days, see [KIP-186](https://cwiki.apache.org/confluence/display/KAFKA/KIP-186%3A+Increase+offsets+retention+default+to+7+days)); +> - When there are **no active** consumer instances, the feature is still a WIP and as such would highly recommend to specify `topic` to avoid unexpected behavior. Namely the scaler will not be able to determine the topics it had subscribed to in the past and will not be able to calculate the lag for those topics. + +### Authentication Parameters + + You can use `TriggerAuthentication` CRD to configure the authenticate by providing `sasl`. + + If TLS is required you should set tls to enable. If required for your Kafka configuration, you may also provide a ca, cert, key and keyPassword. cert and key must be specified together. Another alternative is to specify tls and sasl in ScaledObject instead of tls and sasl in TriggerAuthentication, respectively. + + In case of SASL based authetication provide the `username` and `password`. + + For AWS MSK IAM authentication provide `aws_msk_iam` as `sasl`. You don't need to set `username` and `password` in this case. However, you need to enable TLS by setting `tls` to `enable`. + +**Credential based authentication:** + +**SASL:** + +- `sasl` - Kafka SASL auth mode. (Values: `plaintext`, `scram_sha256`, `scram_sha512`, `aws_msk_iam` or `none`, Default: `none`, Optional) +- `username` - Username used for sasl authentication. (Required if `sasl` is not `none` or `aws_msk_iam`) +- `password` - Password used for sasl authentication. (Required if `sasl` is not `none` or `aws_msk_iam`) + + +**TLS:** + +- `tls` - To enable SSL auth for Kafka, set this to `enable`. If not set, TLS for Kafka is not used. (Values: `enable`, `disable`, Default: `disable`, Optional) +- `ca` - Certificate authority file for TLS client authentication. (Optional) +- `cert` - Certificate for client authentication. (Optional) +- `key` - Key for client authentication. (Optional) +- `keyPassword` - If set the `keyPassword` is used to decrypt the provided `key`. (Optional) + + +**AWS MSK IAM Specific Configuration:** + +- `awsRegion` - AWS region of your MSK cluster. (Optional, required for AWS MSK IAM authentication) + +For authentication, you can also use `TriggerAuthentication` CRD to configure the authenticate by providing `awsAccessKeyID` and `awsSecretAccessKey` or `awsRoleArn`. + +**Role based authentication:** + +- `awsRoleArn` - Amazon Resource Names (ARNs) uniquely identify AWS resource. + +**Credential based authentication:** + +- `awsAccessKeyID` - Id of the user. +- `awsSecretAccessKey` - Access key for the user to authenticate with. + + +### New Consumers and Offset Reset Policy + +When a new Kafka consumer is created, it must determine its consumer group initial position, i.e. the offset it will start to read from. The position is decided in Kafka consumers via a parameter `auto.offset.reset` and the possible values to set are `latest` (Kafka default), and `earliest`. This parameter in KEDA should be set accordingly. In this initial status, no offset has been committed to Kafka for the consumer group and any request for offset metadata will return an `INVALID_OFFSET`; so KEDA has to manage the consumer pod's autoscaling in relation to the offset reset policy that has been specified in the parameters: + +- If the policy is set to `earliest` (a new consumer wants to replay everything in the topic from its beginning) and no offset is committed, the scaler will return a lag value equal to the last offset in the topic. In the case of a new topic the last offset will be 0, so it will scale the deployment to 0 replicas. If a new message is produced to the topic, KEDA will return the new value of the offset (1), and will scale the deployments to consume the message. +- If the policy is set to `latest` (so the new consumer will only consume new messages) and no offset is committed, the scaler will return a negative lag value, and will also tell the HPA to remain `active`, hence the deployment should have the minimum number of replicas running. This is to allow the consumer to read any new message on the topic, and commit its offset. + +### Example + +#### Your kafka cluster has no SASL/TLS auth: + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: kafka-scaledobject + namespace: default +spec: + scaleTargetRef: + name: azure-functions-deployment + pollingInterval: 30 + triggers: + - type: kafka + metadata: + bootstrapServers: localhost:9092 + consumerGroup: my-group # Make sure that this consumer group name is the same one as the one that is consuming topics + topic: test-topic + # Optional + lagThreshold: "50" + offsetResetPolicy: latest +``` + +#### Your kafka cluster turns on SASL/TLS auth: + +##### Method 1: `tls` and `sasl` are in TriggerAuthentication + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-kafka-secrets + namespace: default +data: + sasl: "plaintext" + username: "admin" + password: "admin" + tls: "enable" + ca: + cert: + key: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-kafka-credential + namespace: default +spec: + secretTargetRef: + - parameter: sasl + name: keda-kafka-secrets + key: sasl + - parameter: username + name: keda-kafka-secrets + key: username + - parameter: password + name: keda-kafka-secrets + key: password + - parameter: tls + name: keda-kafka-secrets + key: tls + - parameter: ca + name: keda-kafka-secrets + key: ca + - parameter: cert + name: keda-kafka-secrets + key: cert + - parameter: key + name: keda-kafka-secrets + key: key +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: kafka-scaledobject + namespace: default +spec: + scaleTargetRef: + name: azure-functions-deployment + pollingInterval: 30 + triggers: + - type: kafka + metadata: + bootstrapServers: localhost:9092 + consumerGroup: my-group # Make sure that this consumer group name is the same one as the one that is consuming topics + topic: test-topic + # Optional + lagThreshold: "50" + offsetResetPolicy: latest + authenticationRef: + name: keda-trigger-auth-kafka-credential +``` + +##### Method 2: `tls` and `sasl` are in ScaledObject + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-kafka-secrets + namespace: default +data: + username: "admin" + password: "admin" + ca: + cert: + key: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-kafka-credential + namespace: default +spec: + secretTargetRef: + - parameter: username + name: keda-kafka-secrets + key: username + - parameter: password + name: keda-kafka-secrets + key: password + - parameter: ca + name: keda-kafka-secrets + key: ca + - parameter: cert + name: keda-kafka-secrets + key: cert + - parameter: key + name: keda-kafka-secrets + key: key +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: kafka-scaledobject + namespace: default +spec: + scaleTargetRef: + name: azure-functions-deployment + pollingInterval: 30 + triggers: + - type: kafka + metadata: + bootstrapServers: localhost:9092 + consumerGroup: my-group # Make sure that this consumer group name is the same one as the one that is consuming topics + topic: test-topic + tls: enable + sasl: plaintext + # Optional + lagThreshold: "50" + offsetResetPolicy: latest + authenticationRef: + name: keda-trigger-auth-kafka-credential +``` + +#### Your AWS MSK has IAM auth: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-kafka-secrets + namespace: default +data: + sasl: "aws_msk_iam" + tls: "enable" + awsAccessKeyID: + awsSecretAccessKey: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-kafka-credential + namespace: default +spec: + secretTargetRef: + - parameter: sasl + name: keda-kafka-secrets + key: sasl + - parameter: tls + name: keda-kafka-secrets + key: tls + - parameter: awsAccessKeyID + name: keda-kafka-secrets + key: awsAccessKeyID + - parameter: awsSecretAccessKey + name: keda-kafka-secrets + key: awsSecretAccessKey +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: kafka-scaledobject + namespace: default +spec: + scaleTargetRef: + name: azure-functions-deployment + pollingInterval: 30 + triggers: + - type: apache-kafka + metadata: + bootstrapServers: localhost:9092 + consumerGroup: my-group # Make sure that this consumer group name is the same one as the one that is consuming topics + topic: test-topic + awsRegion: us-east-1 # AWS region of your MSK cluster + # Optional + lagThreshold: "50" + offsetResetPolicy: latest + authenticationRef: + name: keda-trigger-auth-kafka-credential +``` diff --git a/content/docs/2.17/scalers/apache-kafka.md b/content/docs/2.17/scalers/apache-kafka.md new file mode 100644 index 000000000..8197d1fc0 --- /dev/null +++ b/content/docs/2.17/scalers/apache-kafka.md @@ -0,0 +1,606 @@ ++++ +title = "Apache Kafka" +availability = "v1.0+" +maintainer = "Microsoft" +category = "Messaging" +description = "Scale applications based on an Apache Kafka topic or other services that support Kafka protocol." +go_file = "kafka_scaler" ++++ + +> **Notice:** +> - By default, the number of replicas will not exceed: +> - The number of partitions on a topic when a topic is specified; +> - The number of partitions of *all topics* in the consumer group when no topic is specified; +> - `maxReplicaCount` specified in `ScaledObject`/`ScaledJob`. If not specified, then the default value of `maxReplicaCount` is taken into account; +> - The number of partitions with non-zero lag if `limitToPartitionsWithLag` is set to `true` +> That is, if `maxReplicaCount` is set more than number of partitions, the scaler won't scale up to target maxReplicaCount. See `allowIdleConsumers` below to disable this default behavior. +> - This is so because if there are more number of consumers than the number of partitions in a topic, then extra consumer will have to sit idle. + +### Trigger Specification + +This specification describes the `kafka` trigger for an Apache Kafka topic. + +```yaml +triggers: +- type: kafka + metadata: + bootstrapServers: kafka.svc:9092 + consumerGroup: my-group + topic: test-topic + lagThreshold: '5' + activationLagThreshold: '3' + offsetResetPolicy: latest + allowIdleConsumers: false + scaleToZeroOnInvalidOffset: false + excludePersistentLag: false + limitToPartitionsWithLag: false + version: 1.0.0 + partitionLimitation: '1,2,10-20,31' + sasl: plaintext + tls: enable + unsafeSsl: 'false' +``` + +**Parameter list:** + +- `bootstrapServers` - Comma separated list of Kafka brokers "hostname:port" to connect to for bootstrap. +- `consumerGroup` - Name of the consumer group used for checking the offset on the topic and processing the related lag. +- `topic` - Name of the topic on which processing the offset lag. (Optional, see note below) +- `lagThreshold` - Average target value to trigger scaling actions. (Default: `5`, Optional) +- `activationLagThreshold` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) +- `offsetResetPolicy` - The offset reset policy for the consumer. (Values: `latest`, `earliest`, Default: `latest`, Optional) +- `allowIdleConsumers` - When set to `true`, the number of replicas can exceed the number of +partitions on a topic, allowing for idle consumers. (Default: `false`, Optional) +- `scaleToZeroOnInvalidOffset` - This parameter controls what the scaler does when a partition doesn't have a valid offset. +If 'false' (the default), the scaler will keep a single consumer for that partition. Otherwise ('true'), the consumers for that +partition will be scaled to zero. See the [discussion](https://github.com/kedacore/keda/issues/2612) about this parameter. +- `excludePersistentLag` - When set to `true`, the scaler will exclude partition lag for partitions which current offset is the same as the current offset of the previous polling cycle. This parameter is useful to prevent scaling due to partitions which current offset message is unable to be consumed. If `false` (the default), scaler will include all consumer lag in all partitions as per normal. (Default: `false`, Optional) +- `limitToPartitionsWithLag` - When set to `true`, the number of replicas will not exceed the number of partitions having non-zero lag. `topic` must be speicied when this parameter is set to `true`. `allowIdleConsumers` cannot be `true` when this parameter is `true`. (Default: `false`, Optional) +- `version` - Version of your Kafka brokers. See [samara](https://github.com/Shopify/sarama) version (Default: `1.0.0`, Optional) +- `partitionLimitation` - Comma separated list of partition ids to scope the scaling on. Allowed patterns are "x,y" and/or ranges "x-y". If set, the calculation of the lag will only take these ids into account. (Default: All partitions, Optional) +- `sasl` - Kafka SASL auth mode. (Values: `plaintext`, `scram_sha256`, `scram_sha512`, `gssapi`, `oauthbearer`, or `none`, Default: `none`, Optional). This parameter could also be specified in `sasl` in TriggerAuthentication +- `saslTokenProvider` - Kafka SASL token provider when `sasl` is `oauthbearer`. (Values: `bearer`, `aws_msk_iam`, Default: `bearer`, Optional). This parameter could also be specified in `saslTokenProvider` in TriggerAuthentication +- `tls` - To enable SSL auth for Kafka, set this to `enable`. If not set, TLS for Kafka is not used. (Values: `enable`, `disable`, Default: `disable`, Optional). This parameter could also be specified in `tls` in TriggerAuthentication +- `unsafeSsl` - Skip certificate validation when connecting over HTTPS. (Values: `true`, `false`, Default: `false`, Optional) +- `awsRegion` - AWS region of your MSK cluster. (Optional, required for AWS MSK IAM authentication) + +> **Note:** +> +> When `topic` is unspecified, total offset lag will be calculated with all topics within the consumer group. +> - When there are **active** consumer instances, _all topics_ includes: +> - Topics the consumer is *currently* subscribing to; +> - Topics that the consumer group *had prior commit history* (up to retention period for `__consumer_offset`, default to 7 days, see [KIP-186](https://cwiki.apache.org/confluence/display/KAFKA/KIP-186%3A+Increase+offsets+retention+default+to+7+days)); +> - When there are **no active** consumer instances, _all topics_ only includes topics that the consumer group *had prior commit history*; +> --- +> An edge case exists where scaling could be **effectively disabled**: +> - Consumer never makes a commit (no record in `__consumer_offset`); +> - and `ScaledObject` had `minReplicaCount` as 0; +> +> In such case, KEDA could scale the consumer down to 0 when there is no lag and won't be able scale up due to the topic could not be auto discovered. +> +> Fix for such case: +> - Set `minReplicaCount` > 0; +> - or use multiple triggers where one supplies `topic` to ensure lag for that topic will always be detected; +### Authentication Parameters + + You can use `TriggerAuthentication` CRD to configure the authentication by providing `sasl`, `username` and `password`, in case your Kafka cluster has SASL authentication turned on. If you are using SASL/GSSAPI, you will need to provide Kerberos user, password or keytab, realm and krb5.conf file. If you are using SASL/OAuthbearer you will need to provide `oauthTokenEndpointUri` and `scopes` as required by your OAuth2 provider. You can also add custom SASL extension for OAuthbearer (see [KIP-342](https://cwiki.apache.org/confluence/display/KAFKA/KIP-342%3A+Add+support+for+Custom+SASL+extensions+in+OAuthBearer+authentication)) using `oauthExtensions`. + If TLS is required you should set `tls` to `enable`. If required for your Kafka configuration, you may also provide a `ca`, `cert`, `key` and `keyPassword`. `cert` and `key` must be specified together. + Another alternative is to specify `tls` and `sasl` in ScaledObject instead of `tls` and `sasl` in TriggerAuthentication, respectively. For AWS MSK IAM authentication, you only need to set `awsRegion` in ScaledObject and you also need to enable TLS by setting `tls` to enable. + + +**Credential based authentication:** + +**SASL:** + +- `sasl` - Kafka SASL auth mode. (Values: `plaintext`, `scram_sha256`, `scram_sha512`, `gssapi`, `oauthbearer` or `none`, Default: `none`, Optional) +- `saslTokenProvider` - Kafka SASL token provider. (Values: `bearer`, `aws_msk_iam`, Default: `bearer`, Optional). +- `username` - Username used for sasl authentication. (Optional) +- `password` - Password used for sasl authentication. (Optional) +- `keytab` - Kerberos keytab. Either `password` or `keytab` is required in case of `gssapi`. (Optional) +- `realm` - Kerberos realm. (Optional unless sasl mode is `gssapi`) +- `kerberosConfig` - Kerberos configuration file. (Optional unless sasl mode is `gssapi`) +- `kerberosServiceName` - Kerberos service name. (Optional takes default value of `kafka` if not provided) +- `kerberosDisableFAST` - To disable FAST negotation, set this to `true` (Values: `true`, `false`, Default: `false`, Optional) +- `oauthTokenEndpointUri` - The OAuth Access Token URI used for oauthbearer token requests. (Optional unless sasl mode set to oauthbearer) +- `scopes` - A comma separated lists of OAuth scopes used in the oauthbearer token requests. (Optional) +- `oauthExtensions` - A comma separated lists of key value pairs in the format key=value OAuth extensions used in the oauthbearer token. (Optional) + +**TLS:** + +- `tls` - To enable SSL auth for Kafka, set this to `enable`. If not set, TLS for Kafka is not used. (Values: `enable`, `disable`, Default: `disable`, Optional) +- `ca` - Certificate authority file for TLS client authentication. (Optional) +- `cert` - Certificate for client authentication. (Optional) +- `key` - Key for client authentication. (Optional) +- `keyPassword` - If set the `keyPassword` is used to decrypt the provided `key`. (Optional) + +**AWS MSK IAM Specific Configuration:** + +For authentication, you must use `TriggerAuthentication` CRD to configure the authenticate by providing `awsAccessKeyID` and `awsSecretAccessKey` or `awsRoleArn` or a pod identity configuration. + +**Role based authentication:** + +- `awsRoleArn` - Amazon Resource Names (ARNs) uniquely identify AWS resource. + +**Credential based authentication:** + +- `awsAccessKeyID` - Id of the user. +- `awsSecretAccessKey` - Access key for the user to authenticate with. + +### New Consumers and Offset Reset Policy + +When a new Kafka consumer is created, it must determine its consumer group initial position, i.e. the offset it will start to read from. The position is decided in Kafka consumers via a parameter `auto.offset.reset` and the possible values to set are `latest` (Kafka default), and `earliest`. This parameter in KEDA should be set accordingly. In this initial status, no offset has been committed to Kafka for the consumer group and any request for offset metadata will return an `INVALID_OFFSET`; so KEDA has to manage the consumer pod's autoscaling in relation to the offset reset policy that has been specified in the parameters: + +- If the policy is set to `earliest` (a new consumer wants to replay everything in the topic from its beginning) and no offset is committed, the scaler will return a lag value equal to the last offset in the topic. In the case of a new topic the last offset will be 0, so it will scale the deployment to 0 replicas. If a new message is produced to the topic, KEDA will return the new value of the offset (1), and will scale the deployments to consume the message. +- If the policy is set to `latest` (so the new consumer will only consume new messages) and no offset is committed, the scaler will return a negative lag value, and will also tell the HPA to remain `active`, hence the deployment should have the minimum number of replicas running. This is to allow the consumer to read any new message on the topic, and commit its offset. + +### Example + +#### Your kafka cluster has no SASL/TLS auth: + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: kafka-scaledobject + namespace: default +spec: + scaleTargetRef: + name: azure-functions-deployment + pollingInterval: 30 + triggers: + - type: kafka + metadata: + bootstrapServers: localhost:9092 + consumerGroup: my-group # Make sure that this consumer group name is the same one as the one that is consuming topics + topic: test-topic + # Optional + lagThreshold: "50" + offsetResetPolicy: latest +``` + +#### Your kafka cluster turns on SASL/TLS auth: + +##### Method 1: `tls` and `sasl` are in TriggerAuthentication + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-kafka-secrets + namespace: default +data: + sasl: "plaintext" + username: "admin" + password: "admin" + tls: "enable" + ca: + cert: + key: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-kafka-credential + namespace: default +spec: + secretTargetRef: + - parameter: sasl + name: keda-kafka-secrets + key: sasl + - parameter: username + name: keda-kafka-secrets + key: username + - parameter: password + name: keda-kafka-secrets + key: password + - parameter: tls + name: keda-kafka-secrets + key: tls + - parameter: ca + name: keda-kafka-secrets + key: ca + - parameter: cert + name: keda-kafka-secrets + key: cert + - parameter: key + name: keda-kafka-secrets + key: key +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: kafka-scaledobject + namespace: default +spec: + scaleTargetRef: + name: azure-functions-deployment + pollingInterval: 30 + triggers: + - type: kafka + metadata: + bootstrapServers: localhost:9092 + consumerGroup: my-group # Make sure that this consumer group name is the same one as the one that is consuming topics + topic: test-topic + # Optional + lagThreshold: "50" + offsetResetPolicy: latest + authenticationRef: + name: keda-trigger-auth-kafka-credential +``` + +##### Method 2: `tls` and `sasl` are in ScaledObject + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-kafka-secrets + namespace: default +data: + username: "admin" + password: "admin" + ca: + cert: + key: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-kafka-credential + namespace: default +spec: + secretTargetRef: + - parameter: username + name: keda-kafka-secrets + key: username + - parameter: password + name: keda-kafka-secrets + key: password + - parameter: ca + name: keda-kafka-secrets + key: ca + - parameter: cert + name: keda-kafka-secrets + key: cert + - parameter: key + name: keda-kafka-secrets + key: key +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: kafka-scaledobject + namespace: default +spec: + scaleTargetRef: + name: azure-functions-deployment + pollingInterval: 30 + triggers: + - type: kafka + metadata: + bootstrapServers: localhost:9092 + consumerGroup: my-group # Make sure that this consumer group name is the same one as the one that is consuming topics + topic: test-topic + tls: enable + sasl: plaintext + # Optional + lagThreshold: "50" + offsetResetPolicy: latest + authenticationRef: + name: keda-trigger-auth-kafka-credential +``` + +#### Your kafka cluster turns on SASL OAuthbearer/TLS auth: + +##### Method 1: `tls` and `sasl` are in TriggerAuthentication + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-kafka-secrets + namespace: default +data: + sasl: "oauthbearer" + username: "admin" + password: "admin" + oauthTokenEndpointUri: "https://tokenendpoint.com/token" + scopes: "default" + oauthExtensions: "extension_logicalCluster=1,extension_identityPoolId=2" + tls: "enable" + ca: + cert: + key: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-kafka-credential + namespace: default +spec: + secretTargetRef: + - parameter: sasl + name: keda-kafka-secrets + key: sasl + - parameter: username + name: keda-kafka-secrets + key: username + - parameter: password + name: keda-kafka-secrets + key: password + - parameter: oauthTokenEndpointUri + name: keda-kafka-secrets + key: oauthTokenEndpointUri + - parameter: scopes + name: keda-kafka-secrets + key: scopes + - parameter: oauthExtensions + name: keda-kafka-secrets + key: oauthExtensions + - parameter: tls + name: keda-kafka-secrets + key: tls + - parameter: ca + name: keda-kafka-secrets + key: ca + - parameter: cert + name: keda-kafka-secrets + key: cert + - parameter: key + name: keda-kafka-secrets + key: key +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: kafka-scaledobject + namespace: default +spec: + scaleTargetRef: + name: azure-functions-deployment + pollingInterval: 30 + triggers: + - type: kafka + metadata: + bootstrapServers: localhost:9092 + consumerGroup: my-group # Make sure that this consumer group name is the same one as the one that is consuming topics + topic: test-topic + # Optional + lagThreshold: "50" + offsetResetPolicy: latest + authenticationRef: + name: keda-trigger-auth-kafka-credential +``` + +##### Method 2: `tls` and `sasl` are in ScaledObject + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-kafka-secrets + namespace: default +data: + username: "admin" + password: "admin" + oauthTokenEndpointUri: "https://tokenendpoint.com/token" + scopes: "default" + oauthExtensions: "extension_logicalCluster=1,extension_identityPoolId=2" + ca: + cert: + key: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-kafka-credential + namespace: default +spec: + secretTargetRef: + - parameter: username + name: keda-kafka-secrets + key: username + - parameter: password + name: keda-kafka-secrets + key: password + - parameter: oauthTokenEndpointUri + name: keda-kafka-secrets + key: oauthTokenEndpointUri + - parameter: scopes + name: keda-kafka-secrets + key: scopes + - parameter: oauthExtensions + name: keda-kafka-secrets + key: oauthExtensions + - parameter: ca + name: keda-kafka-secrets + key: ca + - parameter: cert + name: keda-kafka-secrets + key: cert + - parameter: key + name: keda-kafka-secrets + key: key +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: kafka-scaledobject + namespace: default +spec: + scaleTargetRef: + name: azure-functions-deployment + pollingInterval: 30 + triggers: + - type: kafka + metadata: + bootstrapServers: localhost:9092 + consumerGroup: my-group # Make sure that this consumer group name is the same one as the one that is consuming topics + topic: test-topic + tls: enable + sasl: oauthbearer + # Optional + lagThreshold: "50" + offsetResetPolicy: latest + authenticationRef: + name: keda-trigger-auth-kafka-credential +``` + +#### Your kafka cluster turns on SASL/GSSAPI auth without TLS: + +##### `sasl/gssapi` in manager.yaml + +If you use YAML declarations to deploy KEDA, add below volume mount and volume to supply writable location for required GSSAPI configurations for the `keda-operator` container. + +``` + volumeMounts: + - mountPath: /tmp/kerberos + name: temp-kerberos-vol + readOnly: false + + volumes: + - name: temp-kerberos-vol + emptyDir: + medium: Memory +``` + +##### `sasl/gssapi` in keda-charts + +If you use Helm Charts to deploy KEDA, add below volume mount and volume to supply writable location for required gssapi configurations. + +``` +volumes.keda.extraVolumeMounts +- mountPath: /tmp/kerberos + name: temp-kerberos-vol + readOnly: false + +volumes.keda.extraVolumes +- name: temp-kerberos-vol + emptyDir: + medium: Memory +``` + +##### `sasl/gssapi` in TriggerAuthentication + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-kafka-secrets + namespace: default +data: + sasl: "gssapi" + tls: "disable" + username: "admin" + realm: + keytab: + kerberosConfig: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-kafka-credential + namespace: default +spec: + secretTargetRef: + - parameter: sasl + name: keda-kafka-secrets + key: sasl + - parameter: tls + name: keda-kafka-secrets + key: tls + - parameter: username + name: keda-kafka-secrets + key: username + - parameter: realm + name: keda-kafka-secrets + key: realm + - parameter: keytab + name: keda-kafka-secrets + key: keytab + - parameter: kerberosConfig + name: keda-kafka-secrets + key: kerberosConfig +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: kafka-scaledobject + namespace: default +spec: + scaleTargetRef: + name: azure-functions-deployment + pollingInterval: 30 + triggers: + - type: kafka + metadata: + bootstrapServers: localhost:9092 + consumerGroup: my-group # Make sure that this consumer group name is the same one as the one that is consuming topics + topic: test-topic + # Optional + lagThreshold: "50" + offsetResetPolicy: latest + authenticationRef: + name: keda-trigger-auth-kafka-credential +``` + +#### Your AWS MSK has IAM auth: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-kafka-secrets + namespace: default +data: + sasl: "oauthbearer" + saslTokenProvider: "aws_msk_iam" + tls: "enable" + awsAccessKeyID: + awsSecretAccessKey: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-kafka-credential + namespace: default +spec: + secretTargetRef: + - parameter: sasl + name: keda-kafka-secrets + key: sasl + - parameter: saslTokenProvider + name: keda-kafka-secrets + key: saslTokenProvider + - parameter: tls + name: keda-kafka-secrets + key: tls + - parameter: awsAccessKeyID + name: keda-kafka-secrets + key: awsAccessKeyID + - parameter: awsSecretAccessKey + name: keda-kafka-secrets + key: awsSecretAccessKey +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: kafka-scaledobject + namespace: default +spec: + scaleTargetRef: + name: azure-functions-deployment + pollingInterval: 30 + triggers: + - type: apache-kafka + metadata: + bootstrapServers: localhost:9092 + consumerGroup: my-group # Make sure that this consumer group name is the same one as the one that is consuming topics + topic: test-topic + awsRegion: us-east-1 # AWS region of your MSK cluster + # Optional + lagThreshold: "50" + offsetResetPolicy: latest + authenticationRef: + name: keda-trigger-auth-kafka-credential +``` diff --git a/content/docs/2.17/scalers/arangodb.md b/content/docs/2.17/scalers/arangodb.md new file mode 100644 index 000000000..f3c1c6eb4 --- /dev/null +++ b/content/docs/2.17/scalers/arangodb.md @@ -0,0 +1,174 @@ ++++ +title = "ArangoDB" +availability = "v2.10+" +maintainer = "Community" +category = "Datastore" +description = "Scale applications based on ArangoDB query result." +go_file = "arangodb_scaler" ++++ + +### Trigger Specification + +This specification describes the `arangodb` trigger that scales based on a ArangoDB query result. Here is an example of providing values in metadata: + +```yaml +triggers: +- type: arangodb + metadata: + # Required fields: + endpoints: "https://:8529,https://:8529" # Note: add one or more comma separated URL endpoints of all the coordinators + query: FOR students IN class COLLECT WITH COUNT INTO length RETURN {"value":length} # Note: the query should return only a single numeric value in the JSON format {"value":} + queryValue: '3' + dbName: gradesheet + collection: class + # Optional fields: + activationQueryValue: '3' + connectionLimit: 13 + unsafeSsl: "false" # Default is `false`, Used for skipping certificate check when having self-signed certs +``` + +**Parameter list:** + +- `endpoints` - ArangoDB server endpoint URL or comma separated URL endpoints of all the coordinators. It can also be provided as an authentication parameter. +- `query` - ArangoDB query to scale for. Please note that the query should return only a single numeric value, i.e. an integer or a float, in the JSON format `{"value":}`. +- `dbName` - Name of the database. It can also be provided as an authentication parameter. +- `collection` - Name of the collection. +- `threshold` - A threshold that will define when scaling should occur. +- `activationThreshold` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) +- `serverID` - The unique ArangoDB server ID. Only required if bearer JWT is being used. (Optional) +- `unsafeSsl` - Used for skipping certificate check e.g: using self-signed certs. (Values: `true`,`false`, Default: `false`, Optional) +- `connectionLimit` - Specify the max size of the active connection pool. (Optional) +- `authModes` - Authentication mode to be used. (Values: `bearer`,`basic`, Optional) + +### Authentication Parameters + +ArangoDB provides SSL/TLS configured out of the box. For authentication, it can be configured along with a Basic Auth or Bearer Auth. + +You can use `TriggerAuthentication` CRD to configure the authentication. Specify `authModes` and other trigger parameters along with secret credentials in `TriggerAuthentication` as mentioned below: + +**Bearer authentication:** +- `authModes`: It must contain `bearer` in case of Bearer Authentication. Specify this in trigger configuration. +- `bearerToken`: The token needed for authentication. + +**Basic authentication:** +- `authModes`: It must contain `basic` in case of Basic Authentication. Specify this in trigger configuration. +- `username` - Provide the username to be used for basic authentication. +- `password` - Provide the password to be used for authentication. (Optional, For convenience this has been marked optional as many applications implement basic auth with a username as apikey and password as empty.) + +Additionally, the parameters `endpoints` and `dbName` can also be provided as authentication parameters. + +### Example + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: arangodb-scaledobject + namespace: default +spec: + maxReplicaCount: 12 + scaleTargetRef: + name: arangodb-deployment + triggers: + - type: arangodb + metadata: + endpoints: https://:8529 + queryValue: '3' + activationQueryValue: '3' + dbName: gradesheet + collection: class + query: FOR students IN class COLLECT WITH COUNT INTO length RETURN {"value":length} +``` + +Here is an example of a arangodb scaler with Bearer Authentication, where the `Secret` and `TriggerAuthentication` are defined as follows: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-arangodb-secret + namespace: default +data: + bearerToken: "BEARER_TOKEN" +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-arangodb-creds + namespace: default +spec: + secretTargetRef: + - parameter: bearerToken + name: keda-arangodb-secret + key: bearerToken +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: arangodb-scaledobject + namespace: default +spec: + maxReplicaCount: 12 + scaleTargetRef: + name: nginx + triggers: + - type: arangodb + metadata: + endpoints: https://:8529 + queryValue: '3' + dbName: gradesheet + collection: class + query: FOR students IN class COLLECT WITH COUNT INTO length RETURN {"value":length} + serverID: "uDmcE-0Zd" + authModes: "bearer" + authenticationRef: + name: keda-arangodb-creds +``` + +Here is an example of a arangodb scaler with Basic Authentication, where the `Secret` and `TriggerAuthentication` are defined as follows: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-arangodb-secret + namespace: default +data: + username: dXNlcm5hbWU= + password: cGFzc3dvcmQ= +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-arangodb-creds + namespace: default +spec: + secretTargetRef: + - parameter: username + name: keda-arangodb-secret + key: username + - parameter: password + name: keda-arangodb-secret + key: password +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: arangodb-scaledobject + namespace: default +spec: + maxReplicaCount: 12 + scaleTargetRef: + name: nginx + triggers: + - type: arangodb + metadata: + endpoints: https://:8529 + queryValue: '3' + dbName: gradesheet + collection: class + query: FOR students IN class COLLECT WITH COUNT INTO length RETURN {"value":length} + authModes: "basic" + authenticationRef: + name: keda-arangodb-creds +``` diff --git a/content/docs/2.17/scalers/artemis.md b/content/docs/2.17/scalers/artemis.md new file mode 100644 index 000000000..19500e25c --- /dev/null +++ b/content/docs/2.17/scalers/artemis.md @@ -0,0 +1,97 @@ ++++ +title = "ActiveMQ Artemis" +availability = "v1.5+" +maintainer = "Community" +category = "Messaging" +description = "Scale applications based on ActiveMQ Artemis queues" +go_file = "artemis_scaler" ++++ + +### Trigger Specification + +This specification describes the `artemis-queue` trigger for ActiveMQ Artemis queues. + +```yaml +triggers: +- type: artemis-queue + metadata: + managementEndpoint: "artemis-activemq.artemis:8161" + queueName: "test" + brokerName: "artemis-activemq" + brokerAddress: "test" + queueLength: '10' + activationQueueLength: '1' + username: 'ARTEMIS_USERNAME' + password: 'ARTEMIS_PASSWORD' + restApiTemplate: # Optional. Default : "http://<>/console/jolokia/read/org.apache.activemq.artemis:broker=\"<>\",component=addresses,address=\"<>\",subcomponent=queues,routing-type=\"anycast\",queue=\"<>\"/MessageCount" +``` + +**Parameter list:** + +- `managementEndpoint` - ActiveMQ Artemis management endpoint to connect to in `:` format. +- `queueName` - Name of the queue to check for the number of messages available. +- `brokerName` - Name of the broker as defined in Artemis. +- `brokerAddress` - Address of the broker. +- `queueLength` - Target value for queue length passed to the scaler. Example: if one pod can handle 10 messages, set the queue length target to 10. If the actual number of messages in the queue is 30, the scaler scales to 3 pods. (default: 10) +- `activationQueueLength` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) +- `restApiTemplate` - Template to build REST API url to get queue size. (Default: `"http://<>/console/jolokia/read/org.apache.activemq.artemis:broker=\"<>\",component=addresses,address=\"<>\",subcomponent=queues,routing-type=\"anycast\",queue=\"<>\"/MessageCount"`, Optional) +- `corsHeader` - Value to populate the Origin header field for CORS filtering. (Default: `"http://<>"`, Optional) + +### Authentication Parameters + + You can use `TriggerAuthentication` CRD to configure the `username` and `password` to connect to the management endpoint. + +**Username and Password based authentication:** + +- `username` - The username to use to connect to the broker's management endpoint. +- `password` - The password to use to connect to the broker's management endpoint. + +### Example + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: kedartemis + namespace: kedartemis + labels: + app: kedartemis +type: Opaque +data: + artemis-password: "YXJ0ZW1pcw==" + artemis-username: "YXJ0ZW1pcw==" +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: trigger-auth-kedartemis + namespace: kedartemis +spec: + secretTargetRef: + - parameter: username + name: kedartemis + key: artemis-username + - parameter: password + name: kedartemis + key: artemis-password +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: kedartemis-consumer-scaled-object + namespace: kedartemis +spec: + scaleTargetRef: + name: kedartemis-consumer + triggers: + - type: artemis-queue + metadata: + managementEndpoint: "artemis-activemq.artemis:8161" + queueName: "test" + queueLength: "50" + brokerName: "artemis-activemq" + brokerAddress: "test" + restApiTemplate: # Optional. Default: "http://<>/console/jolokia/read/org.apache.activemq.artemis:broker=\"<>\",component=addresses,address=\"<>\",subcomponent=queues,routing-type=\"anycast\",queue=\"<>\"/MessageCount" + authenticationRef: + name: trigger-auth-kedartemis +``` \ No newline at end of file diff --git a/content/docs/2.17/scalers/aws-cloudwatch.md b/content/docs/2.17/scalers/aws-cloudwatch.md new file mode 100644 index 000000000..05f448157 --- /dev/null +++ b/content/docs/2.17/scalers/aws-cloudwatch.md @@ -0,0 +1,170 @@ ++++ +title = "AWS CloudWatch" +availability = "v1.0+" +maintainer = "Community" +category = "Metrics" +description = "Scale applications based on AWS CloudWatch." +go_file = "aws_cloudwatch_scaler" ++++ + +### Trigger Specification + +This specification describes the `aws-cloudwatch` trigger that scales based on a AWS CloudWatch. + +```yaml +triggers: +- type: aws-cloudwatch + metadata: + # Optional: namespace + namespace: AWS/SQS + # Optional: Dimension Name + dimensionName: QueueName + # Optional: Dimension Value + dimensionValue: keda + # Optional: Expression query + expression: SELECT MAX("ApproximateNumberOfMessagesVisible") FROM "AWS/SQS" WHERE QueueName = 'keda' + # Optional: metricName + metricName: ApproximateNumberOfMessagesVisible + targetMetricValue: "2.1" + minMetricValue: "1.5" + # Optional: ignoreNullValues + ignoreNullValues: false + # Required: region + awsRegion: "eu-west-1" + # Optional: AWS endpoint url + awsEndpoint: "" + # Optional: AWS Access Key ID, can use TriggerAuthentication as well + awsAccessKeyIDFromEnv: AWS_ACCESS_KEY_ID # default AWS_ACCESS_KEY_ID + # Optional: AWS Secret Access Key, can use TriggerAuthentication as well + awsSecretAccessKeyFromEnv: AWS_SECRET_ACCESS_KEY # default AWS_SECRET_ACCESS_KEY + # DEPRECATED: This parameter is deprecated as of KEDA v2.13 and will be removed in v3. Optional # Optional. Default: pod + identityOwner: pod | operator + # Optional: Collection Time + metricCollectionTime: "300" # default 300 + # Optional: Metric Statistic + metricStat: "Average" # default "Average" + # Optional: Metric Statistic Period + metricStatPeriod: "300" # default 300 + # Optional: Metric Unit + metricUnit: "Count" # default "" + # Optional: Metric EndTime Offset + metricEndTimeOffset: "60" # default 0 +``` + +**Parameter list:** + +- `awsRegion` - AWS Region for the AWS Cloudwatch. +- `awsEndpoint` - Endpoint URL to override the default AWS endpoint. (Default: `""`, Optional) +- `namespace` - AWS Cloudwatch namespace where the metric is located (Optional, Required when `expression` is not specified) +- `metricName` - AWS Cloudwatch metric name (Optional, Required when `expression` is not specified) +- `dimensionName` - Supports specifying multiple dimension names by using ";" as a separator i.e. dimensionName: QueueName;QueueName (Optional, Required when `expression` is not specified) +- `dimensionValue` - Supports specifying multiple dimension values by using ";" as a separator i.e. dimensionValue: queue1;queue2 (Optional, Required when `expression` is not specified) +- `expression` - Supports query with [expression](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch-metrics-insights-querylanguage.html) (Optional, Required when `dimensionName` & `dimensionValue` are not specified) + +- `identityOwner` - Receive permissions for CloudWatch via Pod Identity or from the KEDA operator itself (see below). (DEPRECATED: This parameter is deprecated as of KEDA v2.13 and will be removed in version `3`, Values: `pod`, `operator`, Default: `pod`, Optional, This field only applies for `aws-eks` authentication) + +> When `identityOwner` set to `operator` - the only requirement is that the KEDA operator has the correct IAM permissions on the CloudWatch. Additional Authentication Parameters are not required. + +- `metricCollectionTime` - How long in the past (seconds) should the scaler check AWS Cloudwatch. Used to define **StartTime** ([official documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_GetMetricData.html)). The value of `metricCollectionTime` must be greater than the `metricStatPeriod`, providing a value which is a multiple of the `metricStatPeriod` can improve performance on fetching data from Cloudwatch. In practice setting `metricCollectionTime` 2-to-3 times more than the `metricStatPeriod` value can make sure the scaler is able to get data points back from Cloudwatch, the scaler will always use the most up-to-date datapoint if more datapoints are returned. (Default: `300`, Optional) +- `metricStat` - Which statistics metric to be used by the query. Used to define **Stat** ([official documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html#Statistic)). (Default: `Average`, Optional) +- `metricStatPeriod` - Which frequency to be used by the related query. Used to define **Period**. The value cannot be an arbitrary number, it must be a value supported by Cloudwatch (1, 5, 10, 30, or a multiple of 60). More details can be found from ([official documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html#CloudWatchPeriods)). (Default: `300`, Optional) +- `metricUnit` - Which unit to be used by the query. Used to define **Unit** ([official documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html#Unit)). (Default: `none`, Optional) +- `metricEndTimeOffset` - How long in seconds to offset the **EndTime** ([official documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_GetMetricData.html)). Due to the eventual consistency model which is used by Cloudwatch, the latest datapoint one can get from Cloudwatch might not be accurate. The `metricEndTimeOffset` config provides a way to skip the most recent datapoint if needed. (Default: `0`, Optional) +- `minMetricValue`- Returned value in case of empty response from cloudwatch. (Default: 0, This value can be a float) +- `ignoreNullValues`- Describes the behaviour when the metric query returns no metric values in the response. If set to `true`, the scaler will scale the workload based on the `minMetricValue` provided. If set to `false`, the scaler will return an error and not adjust the scale of the workload. When set to `false` this will take precedence over `minMetricValue`. (Default: `true`, Optional) + +> Both `minMetricValue` and `ignoreNullValues` are used to handle the case when the metric query returns no metric values in the response from AWS CloudWatch. `minMetricValue` will scale the workload based on the value provided, while `ignoreNullValues`, if false, will return an error and not adjust the scale of the workload. + +- `targetMetricValue`- Target value for the metric. (Default: 0, This value can be a float) +- `activationTargetMetricValue`- Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds).(Default: `0`, Optional, This value can be a float) + +### Authentication Parameters + +You can use `TriggerAuthentication` CRD to configure authentication by providing either a role ARN or a set of IAM credentials. + +**Pod identity based authentication:** + +- `podIdentity.provider` - Needs to be set the `TriggerAuthentication` and the pod/service account must be configured correctly for your pod identity provider. + +**Role based authentication:** + +- `awsRoleArn` - Amazon Resource Names (ARNs) uniquely identify AWS resource. (This field is deprecated and only applies for `aws-eks` authentication, for `aws` is set in the auth) + +**Credential based authentication:** + +- `awsAccessKeyID` - Id of the user. +- `awsSecretAccessKey` - Access key for the user to authenticate with. +- `awsSessionToken` - Session token, only required when using [temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html). + +The user will need access to read data from AWS CloudWatch. + +### IAM Permissions + +The user or role used to authenticate with AWS CloudWatch must have the `cloudwatch:GetMetricData` permissions. The following is an example IAM policy that grants the necessary permissions to read data from CloudWatch: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowCloudWatchGetMetricData", + "Effect": "Allow", + "Action": "cloudwatch:GetMetricData", + "Resource": "*" + } + ] +} +``` + +For more information, see the [AWS CloudWatch IAM documentation](https://docs.aws.amazon.com/service-authorization/latest/reference/list_amazoncloudwatch.html). + +### Example + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: test-secrets +data: + AWS_ACCESS_KEY_ID: # Required. + AWS_SECRET_ACCESS_KEY: # Required. + AWS_SESSION_TOKEN: # Required when using temporary credentials. +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-aws-credentials + namespace: keda-test +spec: + secretTargetRef: + - parameter: awsAccessKeyID # Required. + name: test-secrets # Required. + key: AWS_ACCESS_KEY_ID # Required. + - parameter: awsSecretAccessKey # Required. + name: test-secrets # Required. + key: AWS_SECRET_ACCESS_KEY # Required. + - parameter: awsSessionToken # Required when using temporary credentials. + name: test-secrets # Required when using temporary credentials. + key: AWS_SESSION_TOKEN # Required when using temporary credentials. +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: aws-cloudwatch-queue-scaledobject + namespace: keda-test +spec: + scaleTargetRef: + name: nginx-deployment + triggers: + - type: aws-cloudwatch + metadata: + namespace: AWS/SQS + dimensionName: QueueName + dimensionValue: keda + metricName: ApproximateNumberOfMessagesVisible + targetMetricValue: "2.1" + minMetricValue: "0" + awsRegion: "eu-west-1" + authenticationRef: + name: keda-trigger-auth-aws-credentials +``` diff --git a/content/docs/2.17/scalers/aws-dynamodb-streams.md b/content/docs/2.17/scalers/aws-dynamodb-streams.md new file mode 100644 index 000000000..9a3fcb142 --- /dev/null +++ b/content/docs/2.17/scalers/aws-dynamodb-streams.md @@ -0,0 +1,151 @@ ++++ +title = "AWS DynamoDB Streams" +availability = "v2.8+" +maintainer = "Community" +category = "Data & Storage" +description = "Scale applications based on AWS DynamoDB Streams" +go_file = "aws_dynamodb_streams_scaler" ++++ + +### Trigger Specification + +This specification describes the `aws-dynamodb-streams` trigger that scales based on the shard count of AWS DynamoDB Streams. + +```yaml +triggers: +- type: aws-dynamodb-streams + metadata: + # Required: awsRegion + awsRegion: "ap-northeast-1" + # Optional: awsEndpoint + awsEndpoint: "" + # Required: tableName + tableName: myTableName + # Optional targetValue + shardCount: "2" + # DEPRECATED: This parameter is deprecated as of KEDA v2.13 and will be removed in v3. Optional # Optional. Default: pod + identityOwner: pod | operator +``` + +**Parameter list:** + +- `awsRegion` - AWS Region for the DynamoDB. +- `awsEndpoint` - Endpoint URL to override the default AWS endpoint. (Default: `""`, Optional) +- `tableName` - The target DynamoDB table to which the stream belongs. +- `shardCount` - The target value that a DynamoDB streams consumer can handle. (Default: `2`, Optional) +- `activationShardCount` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) +- `identityOwner` - Receive permissions on the DynamoDB and DynamoDB Streams via Pod Identity or from the KEDA operator itself (see below). (DEPRECATED: This parameter is deprecated as of KEDA v2.13 and will be removed in version `3`, Values: `pod`, `operator`, Default: `pod`, Optional, This field only applies for `aws-eks` authentication) + +> When `identityOwner` set to `operator` - the only requirement is that the KEDA operator has the correct IAM permissions on the DynamoDB and Dynamodb Streams. Additional Authentication Parameters are not required. + +### Authentication Parameters + +You can use `TriggerAuthentication` CRD to configure the authenticate by providing either a role ARN or a set of IAM credentials. + +**Pod identity based authentication:** + +- `podIdentity.provider` - Needs to be set on the `TriggerAuthentication` and the pod/service account must be configured correctly for your pod identity provider. + +**Role based authentication:** + +- `awsRoleArn` - Amazon Resource Names (ARNs) uniquely identify AWS resource. (This field is deprecated and only applies for `aws-eks` authentication, for `aws` is set in the auth) + +**Credential based authentication:** + +- `awsAccessKeyID` - Id of the user. +- `awsSecretAccessKey` - Access key for the user to authenticate with. +- `awsSessionToken` - Session token, only required when using [temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html). + +The user will need access to read properties from the specified AWS DynamoDB and DynamoDB Streams. + +### Example + +#### Scaling a deployment using IAM Role + + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: test-secrets + namespace: keda-test +data: + AWS_ROLE_ARN: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-aws-credentials + namespace: keda-test +spec: + secretTargetRef: + - parameter: awsRoleArn # The property in KEDA. + name: test-secrets # The name of the kubernetes secret. + key: AWS_ROLE_ARN # The key from the kubernetes secret. +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: aws-dynamodb-streams-scaledobject + namespace: keda-test +spec: + scaleTargetRef: + name: nginx-deployment + triggers: + - type: aws-dynamodb-streams + authenticationRef: + name: keda-trigger-auth-aws-credentials + metadata: + awsRegion: ap-northeast-1 + tableName: keda-events + shardCount: "2" +``` + + +#### Scaling a deployment using IAM Users + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: test-secrets + namespace: keda-test +data: + AWS_ACCESS_KEY_ID: # Required. + AWS_SECRET_ACCESS_KEY: # Required. + AWS_SESSION_TOKEN: # Required when using temporary credentials. +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-aws-credentials + namespace: keda-test +spec: + secretTargetRef: + - parameter: awsAccessKeyID # Required. + name: test-secrets # Required. + key: AWS_ACCESS_KEY_ID # Required. + - parameter: awsSecretAccessKey # Required. + name: test-secrets # Required. + key: AWS_SECRET_ACCESS_KEY # Required. + - parameter: awsSessionToken # Required when using temporary credentials. + name: test-secrets # Required when using temporary credentials. + key: AWS_SESSION_TOKEN # Required when using temporary credentials. +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: aws-dynamodb-streams-scaledobject + namespace: keda-test +spec: + scaleTargetRef: + name: nginx-deployment + triggers: + - type: aws-dynamodb-streams + authenticationRef: + name: keda-trigger-auth-aws-credentials + metadata: + awsRegion: ap-northeast-1 + tableName: keda-events + shardCount: "2" +``` diff --git a/content/docs/2.17/scalers/aws-dynamodb.md b/content/docs/2.17/scalers/aws-dynamodb.md new file mode 100644 index 000000000..f01141549 --- /dev/null +++ b/content/docs/2.17/scalers/aws-dynamodb.md @@ -0,0 +1,167 @@ ++++ +title = "AWS DynamoDB" +availability = "v2.7+" +maintainer = "Community" +category = "Data & Storage" +description = "Scale applications based on the records count in AWS DynamoDB" +go_file = "aws_dynamodb_scaler" ++++ + +### Trigger Specification + +This specification describes the AWS DynamoDB scaler. This scaler uses a specified DynamoDB query to determine if and when to scale a given workload. + +```yaml +triggers: +- type: aws-dynamodb + metadata: + # Required: awsRegion + awsRegion: "eu-west-1" + # Optional: awsEndpoint + awsEndpoint: "" + # Required: tableName + tableName: myTableName + # Optional: indexName + indexName: "" + # Required: targetValue + targetValue: "1" + # Required: expressionAttributeNames + expressionAttributeNames: '{ "#k" : "partition_key_name"}' + # Required: keyConditionExpression + keyConditionExpression: "#k = :key" + # Required: expressionAttributeValues + expressionAttributeValues: '{ ":key" : {"S":"partition_key_target_value"}}' + # DEPRECATED: This parameter is deprecated as of KEDA v2.13 and will be removed in v3. Optional # Optional. Default: pod + identityOwner: pod | operator +``` + +**Parameter list:** + +- `awsRegion` - AWS Region for the DynamoDB Table. +- `awsEndpoint` - Endpoint URL to override the default AWS endpoint. (Default: `""`, Optional) +- `tableName` - The target table where the scaler execute the query. +- `indexName` - The index used by the DynamoDB Query. (Optional) +- `targetValue` - The target value for the number of items retrieved by the query. +- `activationTargetValue` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) +- `expressionAttributeNames` - one or more substitution tokens for attribute names in an expression. Defined as JSON. +- `keyConditionExpression` - the condition that specifies the key values for items to be retrieved by the Query action. +- `expressionAttributeValues` - one or more values that can be substituted in an expression. Defined as JSON. +- `identityOwner` - Receive permissions on the DynamoDB Table via Pod Identity or from the KEDA operator itself (see below). (DEPRECATED: This parameter is deprecated as of KEDA v2.13 and will be removed in version `3`, Values: `pod`, `operator`, Default: `pod`, Optional, This field only applies for `aws-eks` authentication) + +> When `identityOwner` set to `operator` - the only requirement is that the KEDA operator has the correct IAM permissions on the DynamoDB Table. Additional Authentication Parameters are not required. + +### Authentication Parameters + +You can use `TriggerAuthentication` CRD to configure the authenticate by providing either a role ARN or a set of IAM credentials. + +**Pod identity based authentication:** + +- `podIdentity.provider` - Needs to be set on the `TriggerAuthentication` and the pod/service account must be configured correctly for your pod identity provider. + +**Role based authentication:** + +- `awsRoleArn` - Amazon Resource Names (ARNs) uniquely identify AWS resource. (This field is deprecated and only applies for `aws-eks` authentication, for `aws` is set in the auth) + +**Credential based authentication:** + +- `awsAccessKeyID` - Id of the user. +- `awsSecretAccessKey` - Access key for the user to authenticate with. +- `awsSessionToken` - Session token, only required when using [temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html). + +The user will need access to read properties from the specified AWS SQS queue. + +### Example + +#### Scaling a deployment using IAM Role + + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: test-secrets +data: + AWS_ROLE_ARN: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-aws-credentials + namespace: default +spec: + secretTargetRef: + - parameter: awsRoleArn # The property in KEDA. + name: test-secrets # The name of the kubernetes secret. + key: AWS_ROLE_ARN # The key from the kubernetes secret. +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: aws-dynamodb-table-scaledobject + namespace: keda-test +spec: + scaleTargetRef: + name: nginx-deployment + triggers: + - type: aws-dynamodb + authenticationRef: + name: keda-trigger-auth-aws-credentials + metadata: + awsRegion: eu-west-2 + tableName: keda-events + expressionAttributeNames: '{ "#k" : "event_type"}' + keyConditionExpression: "#k = :key" + expressionAttributeValues: '{ ":key" : {"S":"scaling_event"}}' + targetValue: "5" +``` + + +#### Scaling a deployment using IAM Users + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: test-secrets +data: + AWS_ACCESS_KEY_ID: # Required. + AWS_SECRET_ACCESS_KEY: # Required. + AWS_SESSION_TOKEN: # Required when using temporary credentials. +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-aws-credentials + namespace: default +spec: + secretTargetRef: + - parameter: awsAccessKeyID # Required. + name: test-secrets # Required. + key: AWS_ACCESS_KEY_ID # Required. + - parameter: awsSecretAccessKey # Required. + name: test-secrets # Required. + key: AWS_SECRET_ACCESS_KEY # Required. + - parameter: awsSessionToken # Required when using temporary credentials. + name: test-secrets # Required when using temporary credentials. + key: AWS_SESSION_TOKEN # Required when using temporary credentials. +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: aws-dynamodb-table-scaledobject + namespace: keda-test +spec: + scaleTargetRef: + name: nginx-deployment + triggers: + - type: aws-dynamodb + authenticationRef: + name: keda-trigger-auth-aws-credentials + metadata: + awsRegion: eu-west-2 + tableName: keda-events + expressionAttributeNames: '{ "#k" : "event_type"}' + keyConditionExpression: "#k = :key" + expressionAttributeValues: '{ ":key" : {"S":"scaling_event"}}' + targetValue: "5" +``` diff --git a/content/docs/2.17/scalers/aws-kinesis.md b/content/docs/2.17/scalers/aws-kinesis.md new file mode 100644 index 000000000..901863983 --- /dev/null +++ b/content/docs/2.17/scalers/aws-kinesis.md @@ -0,0 +1,108 @@ ++++ +title = "AWS Kinesis Stream" +availability = "v1.1+" +maintainer = "Community" +category = "Messaging" +description = "Scale applications based on AWS Kinesis Stream." +go_file = "aws_kinesis_stream_scaler" ++++ + +### Trigger Specification + +This specification describes the `aws-kinesis-stream` trigger that scales based on the shard count of AWS Kinesis Stream. + +```yaml +triggers: +- type: aws-kinesis-stream + metadata: + # Required + streamName: myKinesisStream + # Required + awsRegion: "eu-west-1" + # Optional: awsEndpoint + awsEndpoint: "" + # Optional: Default: 2 + shardCount: "2" + # DEPRECATED: This parameter is deprecated as of KEDA v2.13 and will be removed in v3. Optional # Optional. Default: pod + identityOwner: pod | operator +``` + +**Parameter list:** + +- `streamName` - Name of AWS Kinesis Stream. +- `shardCount` - The target value that a Kinesis data streams consumer can handle. (Default: `2`, Optional) +- `activationShardCount` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) +- `awsRegion` - AWS Region for the Kinesis Stream. +- `awsEndpoint` - Endpoint URL to override the default AWS endpoint. (Default: `""`, Optional) +- `identityOwner` - Receive permissions on the Kinesis Stream via Pod Identity or from the KEDA operator itself (see below). (DEPRECATED: This parameter is deprecated as of KEDA v2.13 and will be removed in version `3`, Values: `pod`, `operator`, Default: `pod`, Optional, This field only applies for `aws-eks` authentication) + +> When `identityOwner` set to `operator` - the only requirement is that the KEDA operator has the correct IAM permissions on the Kinesis Stream. Additional Authentication Parameters are not required. + +### Authentication Parameters + +You can use `TriggerAuthentication` CRD to configure the authenticate by providing either a role ARN or a set of IAM credentials, or use other [KEDA supported authentication methods](https://keda.sh/concepts/authentication). + +#### Delegate auth with TriggerAuthentication + +**Role based authentication:** + +- `awsRoleArn` - Amazon Resource Names (ARNs) uniquely identify AWS resource. (This field is deprecated and only applies for `aws-eks` authentication, for `aws` is set in the auth) + +**Credential based authentication:** + +- `awsAccessKeyID` - Id of the user. +- `awsSecretAccessKey` - Access key for the user to authenticate with. +- `awsSessionToken` - Session token, only required when using [temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html). + +The user will need `DescribeStreamSummary` IAM permission policy to read data from AWS Kinesis Streams. + +### Example + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: test-secrets + namespace: keda-test +data: + AWS_ACCESS_KEY_ID: # Required. + AWS_SECRET_ACCESS_KEY: # Required. + AWS_SESSION_TOKEN: # Required when using temporary credentials. +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-aws-credentials + namespace: keda-test +spec: + secretTargetRef: + - parameter: awsAccessKeyID # Required. + name: test-secrets # Required. + key: AWS_ACCESS_KEY_ID # Required. + - parameter: awsSecretAccessKey # Required. + name: test-secrets # Required. + key: AWS_SECRET_ACCESS_KEY # Required. + - parameter: awsSessionToken # Required when using temporary credentials. + name: test-secrets # Required when using temporary credentials. + key: AWS_SESSION_TOKEN # Required when using temporary credentials. +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: aws-kinesis-stream-scaledobject + namespace: keda-test +spec: + scaleTargetRef: + name: nginx-deployment + triggers: + - type: aws-kinesis-stream + authenticationRef: + name: keda-trigger-auth-aws-credentials + metadata: + # Required + streamName: myKinesisStream + # Required + awsRegion: "eu-west-1" + # Optional: Default: 2 + shardCount: "2" +``` diff --git a/content/docs/2.17/scalers/aws-sqs.md b/content/docs/2.17/scalers/aws-sqs.md new file mode 100644 index 000000000..10f897f7a --- /dev/null +++ b/content/docs/2.17/scalers/aws-sqs.md @@ -0,0 +1,234 @@ ++++ +title = "AWS SQS Queue" +availability = "v1.0+" +maintainer = "Community" +category = "Messaging" +description = "Scale applications based on AWS SQS Queue." +go_file = "aws_sqs_queue_scaler" ++++ + +### Trigger Specification + +This specification describes the `aws-sqs-queue` trigger that scales based on an AWS SQS Queue. + +```yaml +triggers: +- type: aws-sqs-queue + metadata: + # Required: queueURL or queueURLFromEnv. If both provided, uses queueURL + queueURL: https://sqs.eu-west-1.amazonaws.com/account_id/QueueName + queueURLFromEnv: QUEUE_URL # Optional. You can use this instead of `queueURL` parameter + queueLength: "5" # Default: "5" + # Required: awsRegion + awsRegion: "eu-west-1" + # Optional: awsEndpoint + awsEndpoint: "" + # DEPRECATED: This parameter is deprecated as of KEDA v2.13 and will be removed in v3. Optional # Optional. Default: pod + identityOwner: pod | operator + +``` + +**Parameter list:** + +- `queueURL` - Full URL for the SQS Queue. The short name of the queue can be used if there's no ambiguity. (Optional. Only one of `queueURL` and `queueURLFromEnv` is required. If both are provided, `queueURL` is used.) +- `queueURLFromEnv` - Name of the environment variable on the scale target to read the queue URL from. (Optional. Only one of `queueURL` and `queueURLFromEnv` is required.) +- `queueLength` - Target value for queue length passed to the scaler. Example: if one pod can handle 10 messages, set the queue length target to 10. If the actual messages in the SQS Queue is 30, the scaler scales to 3 pods. (default: 5) +- `activationQueueLength` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) + +> For the purposes of scaling, the default formula for "actual messages" is equal to `ApproximateNumberOfMessages` + `ApproximateNumberOfMessagesNotVisible`, since `NotVisible` in SQS terms means the message is still in-flight/processing. If you wish to only scale on `ApproximateNumberOfMessages` set `scaleOnInFlight` to `false`. You can also include the number of delayed messages when calculating "actual messages" by setting `scaleOnDelayed` to `true`. With `scaleOnInFlight` and `scaleOnDelayed` set to `true` the formula for "actual messages" is equal to `ApproximateNumberOfMessages` + `ApproximateNumberOfMessagesNotVisible` + `ApproximateNumberOfMessagesDelayed`. + +- `scaleOnInFlight` - Indication of whether or not to include in-flight messages when calculating the number of SQS messages. (default: true, Optional) +- `scaleOnDelayed` - Indication of whether or not to include delayed messages when calculating the number of SQS messages. (default: false, Optional) +- `awsRegion` - AWS Region for the SQS Queue. +- `awsEndpoint` - Endpoint URL to override the default AWS endpoint. (Default: `""`, Optional) +- `identityOwner` - Receive permissions on the SQS Queue via Pod Identity or from the KEDA operator itself (see below). (DEPRECATED: This parameter is deprecated as of KEDA v2.13 and will be removed in version `3`, Values: `pod`, `operator`, Default: `pod`, Optional, This field only applies for `aws-eks` authentication) + +> When `identityOwner` set to `operator` - the only requirement is that the KEDA operator has the correct IAM permissions on the SQS queue. Additional Authentication Parameters are not required. + +### Authentication Parameters + +You can use `TriggerAuthentication` CRD to configure the authenticate by providing either a role ARN or a set of IAM credentials. + +**Pod identity based authentication:** + +- `podIdentity.provider` - Needs to be set on the `TriggerAuthentication` and the pod/service account must be configured correctly for your pod identity provider. + +**Role based authentication:** + +- `awsRoleArn` - Amazon Resource Names (ARNs) uniquely identify AWS resource. (This field is deprecated and only applies for `aws-eks` authentication, for `aws` is set in the auth) + +**Credential based authentication:** + +- `awsAccessKeyID` - Id of the user. +- `awsSecretAccessKey` - Access key for the user to authenticate with. +- `awsSessionToken` - Session token, only required when using [temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html). + +The user will need access to read properties from the specified AWS SQS queue. + +### Example + +#### Scaling a deployment using podIdentity providers + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-aws-credentials + namespace: keda-test +spec: + podIdentity: + provider: aws +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: aws-sqs-queue-scaledobject + namespace: keda-test +spec: + scaleTargetRef: + name: nginx-deployment + triggers: + - type: aws-sqs-queue + authenticationRef: + name: keda-trigger-auth-aws-credentials + metadata: + queueURL: myQueue + queueLength: "5" + awsRegion: "eu-west-1" +``` + +#### Scaling a deployment using IAM Role + +When you need to specify the IAM Role used to access the sqs queue. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: test-secrets +data: + AWS_ROLE_ARN: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-aws-credentials + namespace: keda-test +spec: + secretTargetRef: + - parameter: awsRoleArn # The property in KEDA. + name: test-secrets # The name of the kubernetes secret. + key: AWS_ROLE_ARN # The key from the kubernetes secret. +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: aws-sqs-queue-scaledobject + namespace: keda-test +spec: + scaleTargetRef: + name: nginx-deployment + triggers: + - type: aws-sqs-queue + authenticationRef: + name: keda-trigger-auth-aws-credentials + metadata: + queueURL: myQueue + queueLength: "5" + awsRegion: "eu-west-1" +``` + + +#### Scaling a deployment using IAM Users + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: test-secrets +data: + AWS_ACCESS_KEY_ID: # Required. + AWS_SECRET_ACCESS_KEY: # Required. + AWS_SESSION_TOKEN: # Required when using temporary credentials. +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-aws-credentials + namespace: keda-test +spec: + secretTargetRef: + - parameter: awsAccessKeyID # Required. + name: test-secrets # Required. + key: AWS_ACCESS_KEY_ID # Required. + - parameter: awsSecretAccessKey # Required. + name: test-secrets # Required. + key: AWS_SECRET_ACCESS_KEY # Required. + - parameter: awsSessionToken # Required when using temporary credentials. + name: test-secrets # Required when using temporary credentials. + key: AWS_SESSION_TOKEN # Required when using temporary credentials. +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: aws-sqs-queue-scaledobject + namespace: keda-test +spec: + scaleTargetRef: + name: nginx-deployment + triggers: + - type: aws-sqs-queue + authenticationRef: + name: keda-trigger-auth-aws-credentials + metadata: + queueURL: myQueue + queueLength: "5" + awsRegion: "eu-west-1" +``` + +#### Scaling on ApproximateNumberOfMessages only +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: test-secrets +data: + AWS_ACCESS_KEY_ID: # Required. + AWS_SECRET_ACCESS_KEY: # Required. + AWS_SESSION_TOKEN: # Required when using temporary credentials. +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-aws-credentials + namespace: keda-test +spec: + secretTargetRef: + - parameter: awsAccessKeyID # Required. + name: test-secrets # Required. + key: AWS_ACCESS_KEY_ID # Required. + - parameter: awsSecretAccessKey # Required. + name: test-secrets # Required. + key: AWS_SECRET_ACCESS_KEY # Required. + - parameter: awsSessionToken # Required when using temporary credentials. + name: test-secrets # Required when using temporary credentials. + key: AWS_SESSION_TOKEN # Required when using temporary credentials. +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: aws-sqs-queue-scaledobject + namespace: keda-test +spec: + scaleTargetRef: + name: nginx-deployment + triggers: + - type: aws-sqs-queue + authenticationRef: + name: keda-trigger-auth-aws-credentials + metadata: + queueURL: myQueue + queueLength: "5" + awsRegion: "eu-west-1" + scaleOnInFlight: false +``` diff --git a/content/docs/2.17/scalers/azure-app-insights.md b/content/docs/2.17/scalers/azure-app-insights.md new file mode 100644 index 000000000..cf53d5e45 --- /dev/null +++ b/content/docs/2.17/scalers/azure-app-insights.md @@ -0,0 +1,213 @@ ++++ +title = "Azure Application Insights" +availability = "v2.6+" +maintainer = "Microsoft" +category = "Metrics" +description = "Scale applications based on Azure Application Insights metrics." +go_file = "azure_app_insights_scaler" ++++ + +### Trigger Specification + +This specification describes the `azure-app-insights` trigger that scales based on an Azure Application Insights metric. + +```yaml +triggers: +- type: azure-app-insights + metadata: + metricAggregationTimespan: "0:1" + metricAggregationType: avg + metricFilter: cloud/roleName eq 'role_name' + metricId: "customMetrics/example-metric" + targetValue: "1.5" + activationTargetValue: "5.5" + activeDirectoryClientIdFromEnv: CLIENT_ID_ENV_NAME # Optional, can use TriggerAuthentication as well + activeDirectoryClientPasswordFromEnv: CLIENT_PASSWORD_ENV_NAME # Optional, can use TriggerAuthentication as well + applicationInsightsIdFromEnv: APP_ID # Optional, can use TriggerAuthentication as well + tenantIdFromEnv: TENANT_ID` # Optional, can use TriggerAuthentication as well + # Optional (Default: AzurePublicCloud) + cloud: Private + # Required when cloud = Private + appInsightsResourceURL: https://api.applicationinsights.airgap.io/ + # Required when cloud = Private. + activeDirectoryEndpoint: https://login.airgap.example/ + ignoreNullValues: true # Default is `false`. Set to `true` to ignore any errors with data extraction from a successful query. Set to `false` the scaler will return error when null values are discovered +``` + +This scaler is backed by the Azure Application Insights REST API. Please see [this](https://docs.microsoft.com/en-us/rest/api/application-insights/metrics/get) page +for further details. + +**Parameter list:** + +- `tenantId` - Id of the tenant that contains the Azure resource. This is used for authentication. +- `metricId` - The name of the Application Insights metric to query. Use the [Azure Command Line Interface](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) to run `az monitor app-insights metrics get-metadata` to see a list of available metrics. +- `targetValue` - Target value to trigger scaling actions. (This value can be a float) +- `activationTargetValue` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds).(Default: `0`, Optional, This value can be a float) +- `metricAggregationType` - Aggregation method of the Azure Application Insights metric. The aggregation methods vary from metric to metric. The `az monitor app-insights metrics get-metadata` command can be used to determine which methods apply to a given metric. (Some common aggregation methods are `avg`, `count`, `sum`, `min`, and `max`) +- `metricAggregationInterval` - Collection time of the metric in format `"hh:mm"`. +- `applicationInsightsId` - Id of the Application Insights instance to query. This is a GUID that can be retrieved from the Application Insight's `API Access` blade in the Azure Portal. +- `activeDirectoryClientId` - Id of the Active Directory client. The client must have `Monitoring Reader` permissions for the Application Insights instance. +- `activeDirectoryClientPassword` - Password of the Active Directory client password. +- `metricFilter` - Further specify the metrics query using a filter. For example `cloud/roleName eq 'example`. (Optional) +- `cloud` - Name of the cloud environment that the Azure Application Insights instance belongs to. (Values: `AzurePublicCloud`, `AzureUSGovernmentCloud`, `AzureChinaCloud`, `Private`, Default: `AzurePublicCloud`, Optional) +- `appInsightsResourceURL` - Application Insights REST API URL of the cloud environment. (Required when `cloud` is set to `Private`, e.g. `https://api.applicationinsights.azure.cn/` for `AzureChinaCloud`). +- `activeDirectoryEndpoint` - Active Directory endpoint of the cloud environment. (Required when `cloud` is set to `Private`, e.g. `https://login.chinacloudapi.cn/` for `AzureChinaCloud`). +- `ignoreNullValues` - Ignore any errors with data extraction from a successful query (Values: `true`,`false`, Default: `false`, Optional). + +Some parameters can be provided using environment variables, instead of setting them directly in metadata. Here is a list of parameters you can use to retrieve values from environment variables: + +- `activeDirectoryClientIdFromEnv` - Name of the environment variable that contains the Id of the Active Directory application. (Optional) +- `activeDirectoryClientPasswordFromEnv` - Name of the environment variable that contains the Active Directory client password. (Optional) +- `applicationInsightsIdFromEnv` - Name of the environment variable that contains the Application Insights Id. (Optional) +- `tenantIdFromEnv` - Name of the environment variable that contains the Id of the tenant that contains the Application Insights instance. (Optional) + +### Authentication Parameters + +You can use the `TriggerAuthentication` CRD to configure the authentication by providing a set of Azure Active Directory credentials or by using pod identity. + +**Credential based authentication:** + +- `activeDirectoryClientId` - Id of the Active Directory application which requires at least `Monitoring Reader` permissions. +- `activeDirectoryClientPassword` - Password of the Active Directory application. +- `applicationInsightsId` - Id of the Application Insights instance to query. +- `tenantId` - Id of the tenant that contains the Azure resource. + +The principal will need `Monitoring Reader` access to query metrics from the Application Insights instance. + +**Pod identity based authentication:** + +[Azure AD Workload Identity](https://azure.github.io/azure-workload-identity/docs/) provider can be used in place of credential based authentication. The following section contains an example of a `TriggerAuthentication` using pod identity. + +### Example + +The following example illustrates the use of a TriggerAuthentication to connect to Application Insights. +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: azure-app-insights-secrets +data: + activeDirectoryClientId: + activeDirectoryClientPassword: + applicationInsightsId: + tenantId: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: azure-app-insights-trigger-auth +spec: + secretTargetRef: + - parameter: activeDirectoryClientId + name: azure-app-insights-secrets + key: activeDirectoryClientId + - parameter: activeDirectoryClientPassword + name: azure-app-insights-secrets + key: activeDirectoryClientPassword + - parameter: applicationInsightsId + name: azure-app-insights-secrets + key: applicationInsightsId + - parameter: tenantId + name: azure-app-insights-secrets + key: tenantId + # or Pod Identity, kind: Secret is not required in case of pod Identity + podIdentity: + provider: azure-workload +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: azure-app-insights-scaler +spec: + scaleTargetRef: + name: azure-app-insights-example + minReplicaCount: 1 + maxReplicaCount: 10 + triggers: + - type: azure-app-insights + metadata: + metricId: "customMetrics/example-metric" + metricAggregationTimespan: "0:5" + metricAggregationType: avg + metricFilter: cloud/roleName eq 'example' + targetValue: "1" + authenticationRef: + name: azure-app-insights-trigger-auth +``` + +The following example illustrates the use of environment variables to connect to Application Insights. +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: azure-app-insights-secrets +type: Opaque +data: + activeDirectoryClientId: + activeDirectoryClientPassword: + applicationInsightsId: + tenantId: +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: azure-app-insights-example +spec: + replicas: 0 + selector: + matchLabels: + app: azure-app-insights-example + template: + metadata: + labels: + app: azure-app-insights-example + spec: + containers: + - name: example + image: nginx:1.16.1 + env: + - name: ACTIVE_DIRECTORY_ID + valueFrom: + secretKeyRef: + name: azure-app-insights-secrets + key: activeDirectoryClientId + - name: ACTIVE_DIRECTORY_PASSWORD + valueFrom: + secretKeyRef: + name: azure-app-insights-secrets + key: activeDirectoryClientPassword + - name: APP_INSIGHTS_APP_ID + valueFrom: + secretKeyRef: + name: azure-app-insights-secrets + key: applicationInsightsId + - name: TENANT_ID + valueFrom: + secretKeyRef: + name: azure-app-insights-secrets + key: tenantId +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: azure-app-insights-scaler +spec: + scaleTargetRef: + name: azure-app-insights-example + pollingInterval: 5 + cooldownPeriod: 5 + minReplicaCount: 0 + maxReplicaCount: 2 + triggers: + - type: azure-app-insights + metadata: + metricId: "customMetrics/example-metric" + metricAggregationTimespan: "0:5" + metricAggregationType: avg + metricFilter: cloud/roleName eq 'example' + targetValue: "1" + activeDirectoryClientIdFromEnv: ACTIVE_DIRECTORY_ID + activeDirectoryClientPasswordFromEnv: ACTIVE_DIRECTORY_PASSWORD + applicationInsightsIdFromEnv: APP_INSIGHTS_APP_ID + tenantIdFromEnv: TENANT_ID +``` diff --git a/content/docs/2.17/scalers/azure-data-explorer.md b/content/docs/2.17/scalers/azure-data-explorer.md new file mode 100644 index 000000000..787c05188 --- /dev/null +++ b/content/docs/2.17/scalers/azure-data-explorer.md @@ -0,0 +1,246 @@ ++++ +title = "Azure Data Explorer" +availability = "v2.7+" +maintainer = "Microsoft" +category = "Data & Storage" +description = "Scale applications based on Azure Data Explorer query result." +go_file = "azure_data_explorer_scaler" ++++ + +### Trigger Specification + +This specification describes the `azure-data-explorer` trigger that scales based on an Azure Data Explorer query result. + +```yaml +triggers: +- type: azure-data-explorer + metadata: + endpoint: https://keda.eastus.kusto.windows.net + databaseName: kedadb + query: | + StormEvents + | summarize StormCount = count() by State + | top 1 by StormCount desc + threshold: "10.5" + activationThreshold: "10.5" + tenantId: 045ef409-6dee-4893-a824-5612eac467b1 # Can use TriggerAuthentication as well + clientId: 4ba039f1-d69c-434e-9268-4a2bb7bba90d # Can use TriggerAuthentication as well + # Alternatively, you can use existing environment variables to read aad app creds from: + clientIdFromEnv: AAD_APP_CLIENT_ID_ENV_VAR_NAME # Optional. You can use this instead of `clientId` parameter. + clientSecretFromEnv: AAD_APP_SECRET_ENV_VAR_NAME # Optional. You can use this instead of `clientSecret` parameter. + tenantIdFromEnv: AAD_APP_TENANT_ID_ENV_VAR_NAME # Optional. You can use this instead of `tenantId` parameter. + # Optional (Default: AzurePublicCloud) + cloud: Private + # Required when cloud = Private. + activeDirectoryEndpoint: https://login.airgap.example/ +``` + +**Parameter list:** + +- `endpoint` - The endpoint to query your Data Explorer Cluster. +- `databaseName` - The name of the Data Explorer Database to query. +- `query` - Data Explorer query. +- `threshold` - Value that is used as a threshold to calculate # of pods for scale target. (This value can be a float) +- `activationThreshold` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds).(Default: `0`, Optional, This value can be a float) +- `tenantId` - Id of the Azure AD tenant. +- `clientId` - Id of the Azure AD application. +- `cloud` - Name of the cloud environment that the Azure Data Explorer cluster belongs to. (Values: `AzurePublicCloud`, `AzureUSGovernmentCloud`, `AzureChinaCloud`, `Private`, Default: `AzurePublicCloud`, Optional) +- `activeDirectoryEndpoint` - Active Directory endpoint of the cloud environment. (Required when `cloud` is set to `Private`, e.g. `https://login.chinacloudapi.cn/` for `AzureChinaCloud`). + +The authentication parameters could be provided using environmental variables, instead of setting them directly in metadata. Here is a list of parameters you can use to retrieve values from environment variables: + +- `tenantIdFromEnv` - An environmental variable name, that stores Azure AD tenant id. (Optional) +- `clientIdFromEnv` - An environmental variable name, that stores application id of your Azure AD Application. (Optional) +- `clientSecretFromEnv` - An environmental variable name, that stores password of the Azure AD application. (Optional) + +### Query Guidance + +It is important to design your query to return 1 row. A good practice is to add `| limit 1` at the end of your query. + +The only supported data types for your query result are `real`, `int` or `long`. + +Be careful with defining `pollingInterval` and using long-running queries. Make sure to test your query before using it. + +### Authentication Parameters + +You can use the `TriggerAuthentication` CRD to configure the authentication by providing a set of Azure Active Directory credentials or by using pod identity. + +The AD identity that will be used requires `DatabaseViewer` role to query metrics from the Data Explorer Cluster. + +💡You can use [this guide ](https://docs.microsoft.com/en-us/cli/azure/kusto/database?view=azure-cli-latest#az-kusto-database-add-principal) to assign your principal with the right access permissions through the Azure CLI. + +**Credential based authentication:** + +- `clientId` - Id of the Azure AD application. Use [this](https://docs.microsoft.com/en-us/cli/azure/ad/sp?view=azure-cli-latest) guide to create your service principal. +- `clientSecret` - Password of the Azure AD application. +- `tenantId` - Id of the Azure AD tenant. Use [this](https://docs.microsoft.com/en-us/cli/azure/account?view=azure-cli-latest#az-account-show) guide to retrieve your tenant id. + +**Pod identity based authentication:** + +[Azure AD Workload Identity](https://azure.github.io/azure-workload-identity/docs/) provider can be used. + +### Examples + +### Use TriggerAuthentication with Azure AD Application + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: azure-data-explorer-secret +data: + clientId: # Base64 encoded + clientSecret: # Base64 encoded + tenantId: # Base64 encoded +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: azure-data-explorer-trigger-auth +spec: + secretTargetRef: + - parameter: clientId + name: azure-data-explorer-secret # Required. Refers to the name of the secret + key: clientId + - parameter: clientSecret + name: azure-data-explorer-secret + key: clientSecret + - parameter: tenantId + name: azure-data-explorer-secret + key: tenantId +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: azure-data-explorer-scaler +spec: + scaleTargetRef: + kind: StatefulSet # Optional: Default: Deployment, Available Options: ReplicaSet Deployment, DaemonSet, StatefulSet + name: azure-data-explorer-example + pollingInterval: 30 + cooldownPeriod: 45 + minReplicaCount: 0 + maxReplicaCount: 10 + triggers: + - type: azure-data-explorer + metadata: + databaseName: Weather + endpoint: https://keda.eastus.kusto.windows.net + query: | + StormEvents + | summarize StormCount = count() by State + | top 1 by StormCount desc + threshold: "1000" + authenticationRef: + name: azure-data-explorer-trigger-auth +``` + +### Use TriggerAuthentication with Azure Pod Identity + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: azure-data-explorer-trigger-auth +spec: + podIdentity: + provider: azure-workload +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: azure-data-explorer-scaler +spec: + scaleTargetRef: + kind: StatefulSet # Optional: Default: Deployment, Available Options: ReplicaSet Deployment, DaemonSet, StatefulSet + name: azure-data-explorer-example + pollingInterval: 30 + cooldownPeriod: 45 + minReplicaCount: 0 + maxReplicaCount: 10 + triggers: + - type: azure-data-explorer + metadata: + databaseName: Weather + endpoint: https://keda.eastus.kusto.windows.net + query: | + StormEvents + | summarize StormCount = count() by State + | top 1 by StormCount desc + threshold: "1000" + authenticationRef: + name: azure-data-explorer-trigger-auth +``` + +### Use TriggerAuthentication with Azure AD Application through environment variables + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: azure-data-explorer-secrets +type: Opaque +data: + clientId: # Base64 encoded + clientSecret: # Base64 encoded + tenantId: # Base64 encoded +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: azure-data-explorer-example +spec: + replicas: 0 + selector: + matchLabels: + app: azure-data-explorer-example + template: + metadata: + labels: + app: azure-data-explorer-example + spec: + containers: + - name: example + image: nginx:1.16.1 + env: + - name: AAD_APP_CLIENT_ID + valueFrom: + secretKeyRef: + name: azure-data-explorer-secret + key: clientId + - name: AAD_APP_SECRET + valueFrom: + secretKeyRef: + name: azure-data-explorer-secret + key: clientSecret + - name: AAD_APP_TENANT_ID + valueFrom: + secretKeyRef: + name: azure-data-explorer-secret + key: tenantId +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: azure-data-explorer-scaler +spec: + scaleTargetRef: + name: azure-data-explorer-example + pollingInterval: 30 + cooldownPeriod: 45 + minReplicaCount: 0 + maxReplicaCount: 10 + triggers: + - type: azure-data-explorer + metadata: + clientIdFromEnv: AAD_APP_CLIENT_ID + clientSecretFromEnv: AAD_APP_SECRET + tenantIdFromEnv: AAD_APP_TENANT_ID + databaseName: Weather + endpoint: https://keda.eastus.kusto.windows.net + query: | + StormEvents + | summarize StormCount = count() by State + | top 1 by StormCount desc + threshold: "1000" +``` diff --git a/content/docs/2.17/scalers/azure-event-hub.md b/content/docs/2.17/scalers/azure-event-hub.md new file mode 100644 index 000000000..cb1601cf3 --- /dev/null +++ b/content/docs/2.17/scalers/azure-event-hub.md @@ -0,0 +1,149 @@ ++++ +title = "Azure Event Hubs" +availability = "v1.0+" +maintainer = "Microsoft" +category = "Messaging" +description = "Scale applications based on Azure Event Hubs." +go_file = "azure_eventhub_scaler" ++++ + +### Trigger Specification + +This specification describes the `azure-eventhub` trigger for Azure Event Hubs. + +```yaml +triggers: +- type: azure-eventhub + metadata: + connectionFromEnv: EVENTHUB_CONNECTIONSTRING_ENV_NAME + storageConnectionFromEnv: STORAGE_CONNECTIONSTRING_ENV_NAME + consumerGroup: $Default + unprocessedEventThreshold: '64' + activationUnprocessedEventThreshold: '10' + blobContainer: 'name_of_container' + # Optional (Default: AzurePublicCloud) + cloud: Private + # Required when cloud = Private + endpointSuffix: servicebus.airgap.example + # Required when cloud = Private + storageEndpointSuffix: airgap.example + # Required when using pod identity authentication with blob storage + storageAccountName: 'name_of_account' +``` + +**Parameter list:** + +- `connectionFromEnv` - Name of the environment variable your deployment uses to get the connection string appended with `EntityPath=`. If the connection string does not end with `EntityPath=`, then the parameters `eventHubName` / `eventHubNameFromEnv` must be used to provide the name of the Event Hub. +- `storageConnectionFromEnv` - Name of the environment variable that provides connection string for Azure Storage Account to store checkpoint. As of now the Event Hub scaler only reads from Azure Blob Storage. (Only required when not using pod identity) +- `consumerGroup` - Consumer group of Azure Event Hub consumer. (default: `$default`, Optional) +- `unprocessedEventThreshold` - Average target value to trigger scaling actions. (Default: `64`, Optional) +- `activationUnprocessedEventThreshold` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) +- `blobContainer` - Container name to store checkpoint. This is needed for every `checkpointStrategy` except of `AzureFunction`. With Azure Functions the `blobContainer` is autogenerated and cannot be overridden. +- `eventHubNamespace` - Name of the Event Hub namespace which has the Event Hub. (Optional) +- `eventHubNamespaceFromEnv` - Name of the environment variable that provides the name of the Event Hub namespace, which has the Event Hub. (Optional) +- `eventHubName` - Name of the Event Hub containing the messages. (Optional) +- `eventHubNameFromEnv` - Name of the environment variable that provides the name of the Event Hub, containing the messages. (Optional) +- `storageAccountName` - Account name for blob storage used for checkpoints. (Required when `storageConnectionFromEnv` is not specified. The storage account name is the first part in the hostname of a Blob Storage endpoint. E.g.: for `examplename.blob.core.windows.net` the account name is `examplename`.) +- `checkpointStrategy` - configure the checkpoint behaviour of different Event Hub SDKs. (Values: `azureFunction`, `blobMetadata`, `goSdk`, default: `""`, Optional) + - `azureFunction` - Suitable for Azure Functions & Azure WebJobs SDK. This is the default setting, when `blobcontainer` is not specified. + - `blobMetadata` - For all implementations that store checkpoint information on blob metadata such as current C#, Python, Java and JavaScript Event Hub SDKs. + - `goSdk` - For all implementations using the [Golang SDK](https://github.com/Azure/azure-event-hubs-go)'s checkpointing. + - `dapr` - Suitable for Dapr pubsub and bindings, depending on the used Dapr version: + - pubsub components: >= Dapr 1.6 (older versions need the GoSdk checkpointer) + - binding components: >= Dapr 1.9 (older versions need the GoSdk checkpointer) + - When no checkpoint strategy is specified, the Event Hub scaler will use backwards compatibility and able to scale older implementations of C#, Python or Java Event Hub SDKs. (see "Legacy checkpointing"). If this behaviour should be used, `blobContainer` is also required. +- `cloud` - Name of the cloud environment that the Event Hub belongs to. (Values: `AzurePublicCloud`, `AzureUSGovernmentCloud`, `AzureChinaCloud`, `AzureGermanCloud`, `Private`, Default: `AzurePublicCloud`, Optional) +- `endpointSuffix` - Service Bus endpoint suffix of the cloud environment. (Required when `cloud` is set to `Private`, e.g. `servicebus.cloudapi.de` for `AzureGermanCloud`). +- `storageEndpointSuffix` - Blob Storage endpoint of the cloud environment. (Required when `cloud` is set to `Private`, e.g. `airgap.example`. Do not include the `blob` part of the endpoint.) + +> When using [pod identity](../authentication-providers/azure-ad-workload-identity.md), Microsoft Entra ID endpoint is recovered via `AZURE_AUTHORITY_HOST` env var provided by https://azure.github.io/azure-workload-identity/docs/installation/mutating-admission-webhook.html + +> 💡 Learn more about the checkpointing behaviour in this [section](#checkpointing-behaviour). + +> 💡 The Azure Storage connection string is not compatible with connection string created from a Shared Access Signature. + +### Authentication Parameters + +You can authenticate by using pod identity or connection string authentication. + +**Connection String Authentication:** + +- `connection` - Connection string for the Azure Event Hubs Namespace. + + The following formats are supported. + + - With **SharedAccessKey** - `Endpoint=sb://.servicebus.windows.net/;SharedAccessKeyName=;SharedAccessKey=;EntityPath=`. + +- `storageConnection` - Connection string for the Azure Storage Account used to store checkpoint information. + +> 💡 When providing `connection`, `EntityPath` is optional. If it is not provided, then `eventHubName` must be used to provide the name of the Azure Event Hub instance to use inside the namespace. + +**Pod identity based authentication:** + +[Azure AD Workload Identity](https://azure.github.io/azure-workload-identity/docs/) provider can be used. + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: nameOfTriggerAuth + namespace: default +spec: + podIdentity: + provider: azure-workload +``` + +When you do so, the Event Hub scaler will depend on the existence of two configurations you have to provide: `eventHubNamespace` and `eventHubName`. You can also configure `storageAccountName` if you wish to use Azure AD Pod / Workload Identity to authenticate to Azure Blob Storage instead of a connection string. + +> 💡 When using Azure AD Pod Identity to authenticate the identity must have appropriate [RBAC role-assignments](https://docs.microsoft.com/azure/role-based-access-control/role-assignments-steps) for both Event Hub and Storage Account. Permissions covered by `Azure Event Hubs Data Receiver` and `Storage Blob Data Reader` are required. + +### Checkpointing Behaviour + +The list of available checkpointing strategies can be found in the trigger specification [section](#trigger-specification). The way checkpoints are stored +has changed with updates to the EventHub SDKs. + +* **Legacy behaviour:** The older implementations are based on the `EventProcessorHost` client, which stores the checkpoint information as contents of +the storage blob. This is the default behaviour when no `checkpointStrategy` is specified. +This is applicable for the following scenarios: + - .NET applications using `Microsoft.Azure.EventHubs` NuGet package. + - Java applications using `azure-eventhubs-eph` package. + - Python applications using `azure-eventhub` package below v5. + +* **Current behaviour:** The newer implementations are based on the `EventProcessorClient`, which stores the checkpoint information as metadata on +the storage blob. This is the behaviour when `checkpointStrategy` is set to `blobMetadata`. +This is applicable for the following scenarios: + - .NET applications using `Azure.Messaging.EventHubs` NuGet package. + - Python applications using `azure-eventhub` v5. + - .NET Azure Functions using `Microsoft.Azure.WebJobs.Extensions.EventHubs` v5. + - Azure Functions in other languages using `Microsoft.Azure.Functions.ExtensionBundle` v3. + +> 💡 `blobContainer` name is required for applications following legacy behaviour. + +> 💡 Users should set `blobContainer` to `azure-webjobs-eventhub` for Azure Functions using `blobMetadata` as `checkpointStrategy`. + +### Example + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: azure-eventhub-scaledobject + namespace: default +spec: + scaleTargetRef: + name: azureeventhub-function + triggers: + - type: azure-eventhub + metadata: + # Required + storageConnectionFromEnv: AzureWebJobsStorage + # Required if not using Pod Identity + connectionFromEnv: EventHub + # Required if using Pod Identity + eventHubNamespace: AzureEventHubNameSpace + eventHubName: NameOfTheEventHub +# Optional + consumerGroup: $Default # default: $Default + unprocessedEventThreshold: '64' # default 64 events. + blobContainer: ehcontainer +``` diff --git a/content/docs/2.17/scalers/azure-log-analytics.md b/content/docs/2.17/scalers/azure-log-analytics.md new file mode 100644 index 000000000..26a75a522 --- /dev/null +++ b/content/docs/2.17/scalers/azure-log-analytics.md @@ -0,0 +1,306 @@ ++++ +title = "Azure Log Analytics" +availability = "v2.0+" +maintainer = "Microsoft" +category = "Data & Storage" +description = "Scale applications based on Azure Log Analytics query result" +go_file = "azure_log_analytics_scaler" ++++ + +### Trigger Specification + +This specification describes the `azure-log-analytics` trigger for Azure Log Analytics query result. Here is an example of providing values in metadata: + +```yaml +triggers: + - type: azure-log-analytics + metadata: + tenantId: "AZURE_AD_TENANT_ID" + clientId: "SERVICE_PRINCIPAL_CLIENT_ID" + clientSecret: "SERVICE_PRINCIPAL_PASSWORD" + workspaceId: "LOG_ANALYTICS_WORKSPACE_ID" + query: | + let AppName = "web"; + let ClusterName = "demo-cluster"; + let AvgDuration = ago(10m); + let ThresholdCoefficient = 0.8; + Perf + | where InstanceName contains AppName + | where InstanceName contains ClusterName + | where CounterName == "cpuUsageNanoCores" + | where TimeGenerated > AvgDuration + | extend AppName = substring(InstanceName, indexof((InstanceName), "/", 0, -1, 10) + 1) + | summarize MetricValue=round(avg(CounterValue)) by CounterName, AppName + | join (Perf + | where InstanceName contains AppName + | where InstanceName contains ClusterName + | where CounterName == "cpuLimitNanoCores" + | where TimeGenerated > AvgDuration + | extend AppName = substring(InstanceName, indexof((InstanceName), "/", 0, -1, 10) + 1) + | summarize arg_max(TimeGenerated, *) by AppName, CounterName + | project Limit = CounterValue, TimeGenerated, CounterPath, AppName) + on AppName + | project MetricValue, Threshold = Limit * ThresholdCoefficient + threshold: "10.7" + activationThreshold: "1.7" + # Alternatively, you can use existing environment variables to read configuration from: + # See details in "Parameter list" section + workspaceIdFromEnv: LOG_ANALYTICS_WORKSPACE_ID_ENV_NAME # Optional. You can use this instead of `workspaceId` parameter. + clientIdFromEnv: SERVICE_PRINCIPAL_CLIENT_ID_ENV_NAME # Optional. You can use this instead of `clientId` parameter. + tenantIdFromEnv: AZURE_AD_TENANT_ID_ENV_NAME # Optional. You can use this instead of `tenantId` parameter. + clientSecretFromEnv: SERVICE_PRINCIPAL_PASSWORD_ENV_NAME # Optional. You can use this instead of `clientSecret` parameter. + # Optional (Default: AzurePublicCloud) + cloud: Private + # Required when cloud = Private + logAnalyticsResourceURL: https://api.loganalytics.airgap.io/ + # Required when cloud = Private. + activeDirectoryEndpoint: https://login.airgap.example/ + # Optional (Default: false) + unsafeSsl: "false" +``` + +**Parameter list:** + +- `tenantId` - Id of the Azure Active Directory tenant. Follow [this](https://docs.microsoft.com/en-us/cli/azure/account?view=azure-cli-latest#az-account-show) link to retrieve your tenant id. +- `clientId` - Id of the application from your Azure AD Application/service principal. Follow [this](https://docs.microsoft.com/en-us/cli/azure/ad/sp?view=azure-cli-latest) link to create your service principal. +- `clientSecret` - Password from your Azure AD Application/service principal. +- `workspaceId` - Id of Log Analytics workspace. Follow [this](https://docs.microsoft.com/en-us/cli/azure/monitor/log-analytics/workspace?view=azure-cli-latest#az-monitor-log-analytics-workspace-list) link to get your Log Analytics workspace id. +- `query` - Log Analytics [kusto](https://docs.microsoft.com/en-us/azure/azure-monitor/log-query/get-started-queries) query, JSON escaped. You can use [this](https://www.freeformatter.com/json-escape.html) tool to convert your query from Log Analytics query editor to JSON escaped string, and then review YAML specific escapes. +- `threshold` - Value that is used as a threshold to calculate # of pods for scale target. (This value can be a float) +- `activationThreshold` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds).(Default: `0`, Optional, This value can be a float) +- `cloud` - Name of the cloud environment that the Azure Log Analytics workspace belongs to. (Values: `AzurePublicCloud`, `AzureUSGovernmentCloud`, `AzureChinaCloud`, `Private`, Default: `AzurePublicCloud`, Optional) +- `logAnalyticsResourceURL` - Log Analytics REST API URL of the cloud environment. (Required when `cloud` is set to `Private`, e.g. `https://api.loganalytics.azure.cn/` for `AzureChinaCloud`). +- `activeDirectoryEndpoint` - Active Directory endpoint of the cloud environment. (Required when `cloud` is set to `Private`, e.g. `https://login.chinacloudapi.cn/` for `AzureChinaCloud`). +- `unsafeSsl` - Determines whether or not KEDA will verify the server certificate's chain and host name. (Default: `false`, Optional, This value can be a bool) + + +The authentication parameters could be provided using environmental variables, instead of setting them directly in metadata. Here is a list of parameters you can use to retrieve values from environment variables: + +- `tenantIdFromEnv` - An environmental variable name, that stores Azure Active Directory tenant id. Follow [this](https://docs.microsoft.com/en-us/cli/azure/account?view=azure-cli-latest#az-account-show) link to retrieve your tenant id. (Optional) +- `clientIdFromEnv` - An environmental variable name, that stores Application id from your Azure AD Application/service principal. Follow [this](https://docs.microsoft.com/en-us/cli/azure/ad/sp?view=azure-cli-latest) link to create your service principal. (Optional) +- `clientSecretFromEnv` - An environmental variable name, that stores password from your Azure AD Application/service principal. (Optional) +- `workspaceIdFromEnv` - An environmental variable name, that stores your Log Analytics workspace id. Follow [this](https://docs.microsoft.com/en-us/cli/azure/monitor/log-analytics/workspace?view=azure-cli-latest#az-monitor-log-analytics-workspace-list) link to get your Log Analytics workspace id. (Optional) + +> 💡 **NOTE:** The workspaceID for Log Analytics is called the `customerId`; +it's not the full `id`! the example `az` command below can be used. + +```sh +az monitor log-analytics workspace list --query '[]. {ResourceGroup:resourceGroup,WorkspaceName:name,"workspaceID (customerId)":customerId}' -o table +``` + +### Query Guidance + +It is important to design your query to return 1 table with 1 row. A good practice is to add "| limit 1" at the end of your query. + +Scaler will take value from: + +- 1st cell as Metrics Value. +- 2nd cell as Threshold (optional). + +You can define threshold in trigger metadata, it will be used if your query results only 1 cell, that will be interpreted as metric value. Be aware, even if you have defined threshold in metadata, it can be overwritten by your query. + +Data types of your query result should be: real, int or long. Other data types are not supported. Later, during runtime, your data will be converted to int64. + +Be careful with setting up "pollingInterval" and long-running queries. Test your query before. + +Example query to get `MetricValue` and `Threshold` based on CPU usage and limits, defined for the pod. + +```kusto +let AppName = "web"; +let ClusterName = "demo-cluster"; +let AvgDuration = ago(10m); +let ThresholdCoefficient = 0.8; +Perf +| where InstanceName contains AppName +| where InstanceName contains ClusterName +| where CounterName == "cpuUsageNanoCores" +| where TimeGenerated > AvgDuration +| extend AppName = substring(InstanceName, indexof((InstanceName), "/", 0, -1, 10) + 1) +| summarize MetricValue=round(avg(CounterValue)) by CounterName, AppName +| join (Perf + | where InstanceName contains AppName + | where InstanceName contains ClusterName + | where CounterName == "cpuLimitNanoCores" + | where TimeGenerated > AvgDuration + | extend AppName = substring(InstanceName, indexof((InstanceName), "/", 0, -1, 10) + 1) + | summarize arg_max(TimeGenerated, *) by AppName, CounterName + | project Limit = CounterValue, TimeGenerated, CounterPath, AppName) + on AppName +| project MetricValue, Threshold = Limit * ThresholdCoefficient +``` + +Example result: + +![Azure Log Analytics query example](/img/azure-log-analytics-scaler-query-example.png) + +### Scaler Limitations + +- As it was mentioned before, you can define a threshold using query (2nd cell of query result will be interpret as threshold). Be aware! Threshold from query result will be set only once, during scaler creation. So, if your query will return different threshold values during runtime, they will not be propagated to Horizontal Pod Autoscaler target. + +### Authentication Parameters + +You can use `TriggerAuthentication` CRD to configure the authentication by providing a set of Azure Active Directory credentials and resource identifiers. + +**Service Principal based authentication:** + +- `tenantId` - Azure Active Directory tenant id. Follow [this](https://docs.microsoft.com/en-us/cli/azure/account?view=azure-cli-latest#az-account-show) link to retrieve your tenant id. +- `clientId` - Application id from your Azure AD Application/service principal. Follow [this](https://docs.microsoft.com/en-us/cli/azure/ad/sp?view=azure-cli-latest) link to create your service principal. +- `clientSecret` - Password from your Azure AD Application/service principal. +- `workspaceId` - Your Log Analytics workspace id. Follow [this](https://docs.microsoft.com/en-us/cli/azure/monitor/log-analytics/workspace?view=azure-cli-latest#az-monitor-log-analytics-workspace-list) link to get your Log Analytics workspace id. + +**Managed identity based authentication:** + +You can use managed identity to request access token for Log Analytics API. The advantage of this approach is that there is no need to store secrets in Kubernetes. Read [more](https://docs.microsoft.com/en-us/azure/aks/use-managed-identity) about managed identities in Azure Kubernetes Service. + +[Azure AD Workload Identity](https://azure.github.io/azure-workload-identity/docs/) provider can be used. + +### Example + +#### Service Principal based authentication + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: kedaloganalytics + namespace: kedaloganalytics + labels: + app: kedaloganalytics +type: Opaque +data: + tenantId: "QVpVUkVfQURfVEVOQU5UX0lE" #Base64 encoded Azure Active Directory tenant id + clientId: "U0VSVklDRV9QUklOQ0lQQUxfQ0xJRU5UX0lE" #Base64 encoded Application id from your Azure AD Application/service principal + clientSecret: "U0VSVklDRV9QUklOQ0lQQUxfUEFTU1dPUkQ=" #Base64 encoded Password from your Azure AD Application/service principal + workspaceId: "TE9HX0FOQUxZVElDU19XT1JLU1BBQ0VfSUQ=" #Base64 encoded Log Analytics workspace id +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: trigger-auth-kedaloganalytics + namespace: kedaloganalytics +spec: + secretTargetRef: + - parameter: tenantId + name: kedaloganalytics + key: tenantId + - parameter: clientId + name: kedaloganalytics + key: clientId + - parameter: clientSecret + name: kedaloganalytics + key: clientSecret + - parameter: workspaceId + name: kedaloganalytics + key: workspaceId +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: kedaloganalytics-consumer-scaled-object + namespace: kedaloganalytics + labels: + deploymentName: kedaloganalytics-consumer +spec: + scaleTargetRef: + kind: #Optional: Default: Deployment, Available Options: ReplicaSet, Deployment, DaemonSet, StatefulSet + name: kedaloganalytics-consumer + pollingInterval: 30 + cooldownPeriod: 30 + minReplicaCount: 1 + maxReplicaCount: 10 + triggers: + - type: azure-log-analytics + metadata: + query: | + let AppName = "web"; + let ClusterName = "demo-cluster"; + let AvgDuration = ago(10m); + let ThresholdCoefficient = 0.8; + Perf + | where InstanceName contains AppName + | where InstanceName contains ClusterName + | where CounterName == "cpuUsageNanoCores" + | where TimeGenerated > AvgDuration + | extend AppName = substring(InstanceName, indexof((InstanceName), "/", 0, -1, 10) + 1) + | summarize MetricValue=round(avg(CounterValue)) by CounterName, AppName + | join (Perf + | where InstanceName contains AppName + | where InstanceName contains ClusterName + | where CounterName == "cpuLimitNanoCores" + | where TimeGenerated > AvgDuration + | extend AppName = substring(InstanceName, indexof((InstanceName), "/", 0, -1, 10) + 1) + | summarize arg_max(TimeGenerated, *) by AppName, CounterName + | project Limit = CounterValue, TimeGenerated, CounterPath, AppName) + on AppName + | project MetricValue, Threshold = Limit * ThresholdCoefficient + threshold: "1900000000" + authenticationRef: + name: trigger-auth-kedaloganalytics +``` + +#### Managed identity based authentication + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: trigger-auth-kedaloganalytics + namespace: kedaloganalytics +spec: + podIdentity: + provider: azure-workload +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: kedaloganalytics-consumer-scaled-object + namespace: kedaloganalytics + labels: + deploymentName: kedaloganalytics-consumer +spec: + scaleTargetRef: + kind: #Optional: Default: Deployment, Available Options: ReplicaSet, Deployment, DaemonSet, StatefulSet + name: kedaloganalytics-consumer + pollingInterval: 30 + cooldownPeriod: 30 + minReplicaCount: 1 + maxReplicaCount: 10 + triggers: + - type: azure-log-analytics + metadata: + workspaceId: "81963c40-af2e-47cd-8e72-3002e08aa2af" + query: | + let AppName = "web"; + let ClusterName = "demo-cluster"; + let AvgDuration = ago(10m); + let ThresholdCoefficient = 0.8; + Perf + | where InstanceName contains AppName + | where InstanceName contains ClusterName + | where CounterName == "cpuUsageNanoCores" + | where TimeGenerated > AvgDuration + | extend AppName = substring(InstanceName, indexof((InstanceName), "/", 0, -1, 10) + 1) + | summarize MetricValue=round(avg(CounterValue)) by CounterName, AppName + | join (Perf + | where InstanceName contains AppName + | where InstanceName contains ClusterName + | where CounterName == "cpuLimitNanoCores" + | where TimeGenerated > AvgDuration + | extend AppName = substring(InstanceName, indexof((InstanceName), "/", 0, -1, 10) + 1) + | summarize arg_max(TimeGenerated, *) by AppName, CounterName + | project Limit = CounterValue, TimeGenerated, CounterPath, AppName) + on AppName + | project MetricValue, Threshold = Limit * ThresholdCoefficient + threshold: "1900000000" + authenticationRef: + name: trigger-auth-kedaloganalytics +``` + +### Guides + +### Links + +- [Use managed identities in Azure Kubernetes Service](https://docs.microsoft.com/en-us/azure/aks/use-managed-identity) +- [Azure Pod Identity on keda.sh](https://keda.sh/docs/2.0/concepts/authentication/#azure-pod-identity) +- [Best practices for authentication and authorization in Azure Kubernetes Service (AKS)](https://docs.microsoft.com/en-us/azure/aks/operator-best-practices-identity) + diff --git a/content/docs/2.17/scalers/azure-monitor.md b/content/docs/2.17/scalers/azure-monitor.md new file mode 100644 index 000000000..330c583de --- /dev/null +++ b/content/docs/2.17/scalers/azure-monitor.md @@ -0,0 +1,129 @@ ++++ +title = "Azure Monitor" +availability = "v1.3+" +maintainer = "Microsoft" +category = "Metrics" +description = "Scale applications based on Azure Monitor metrics." +go_file = "azure_monitor_scaler" ++++ + +### Trigger Specification + +This specification describes the `azure-monitor` trigger that scales based on an Azure Monitor metric. + +```yaml +triggers: +- type: azure-monitor + metadata: + resourceURI: Microsoft.ContainerService/managedClusters/azureMonitorCluster + tenantId: xxx-xxx-xxx-xxx-xxx + subscriptionId: yyy-yyy-yyy-yyy-yyy + resourceGroupName: azureMonitor + metricName: kube_pod_status_ready + metricFilter: namespace eq 'default' + metricAggregationInterval: "0:1:0" + targetValue: "0.5" + activationTargetValue: "3.5" + activeDirectoryClientId: # Optional, can use TriggerAuthentication as well + activeDirectoryClientIdFromEnv: CLIENT_ID_ENV_NAME # Optional, can use TriggerAuthentication as well + activeDirectoryClientPasswordFromEnv: CLIENT_PASSWORD_ENV_NAME # Optional, can use TriggerAuthentication as well + # Optional (Default: AzurePublicCloud) + cloud: Private + # Required when cloud = Private + azureResourceManagerEndpoint: https://management.azure.airgap.com/ + # Required when cloud = Private. + activeDirectoryEndpoint: https://login.airgap.example/ +``` + +**Parameter list:** + +- `resourceURI` - Shortened URI to the Azure resource with format `"//"`. +- `tenantId` - Id of the tenant that contains the Azure resource. This is used for authentication. +- `subscriptionId` - Id of Azure subscription that contains the Azure resource. This is used for determining the full resource URI. +- `resourceGroupName` - Name of the resource group for the Azure resource. +- `metricName` - Name of the metric to query. + - Azure metrics are available as a list in the [official documentation](https://docs.microsoft.com/en-us/azure/azure-monitor/platform/metrics-supported). + - Custom metric name when querying Azure Monitor [custom metrics](https://docs.microsoft.com/en-us/azure/azure-monitor/essentials/metrics-custom-overview). +- `metricNamespace` - Name of the metric namespace. Required when `metricName` is a custom metric. +- `targetValue` - Target value to trigger scaling actions. (This value can be a float) +- `activationTargetValue` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds).(Default: `0`, Optional, This value can be a float) +- `metricAggregationType` - Aggregation method of the Azure Monitor metric. Options include `Average`, `Total`, `Maximum` with a full list in the [official documentation](https://docs.microsoft.com/en-us/azure/azure-monitor/platform/metrics-supported). +- `metricFilter` - Name of the filter to be more specific by using dimensions listed in the [official documentation](https://docs.microsoft.com/en-us/azure/azure-monitor/platform/metrics-supported). (Optional) +- `metricAggregationInterval` - Collection time of the metric in format `"hh:mm:ss"` (Default: `"0:5:0"`, Optional) +- `activeDirectoryClientId` - Id of the Active Directory application which requires at least `Monitoring Reader` permissions. (Optional) +- `activeDirectoryClientPasswordFromEnv` - Name of the environment variable that contains the active directory client password. (Optional) +- `cloud` - Name of the cloud environment that the Azure resource belongs to. (Values: `AzurePublicCloud`, `AzureUSGovernmentCloud`, `AzureGermanCloud`, `AzureChinaCloud`, `Private`, Default: `AzurePublicCloud`, Optional) +- `azureResourceManagerEndpoint` - Azure Resource Manager endpoint of the cloud environment. (Required when `cloud` is set to `Private`, e.g. `https://management.chinacloudapi.cn/` for `AzureChinaCloud`). +- `activeDirectoryEndpoint` - Active Directory endpoint of the cloud environment. (Required when `cloud` is set to `Private`, e.g. `https://login.chinacloudapi.cn/` for `AzureChinaCloud`). + +Some parameters could be provided using environmental variables, instead of setting them directly in metadata. Here is a list of parameters you can use to retrieve values from environment variables: + +- `activeDirectoryClientIdFromEnv` - Id of the Active Directory application which requires at least `Monitoring Reader` permissions, similar to `activeDirectoryClientId`, but reads it from an environment variable on the scale target. (Optional) + +### Authentication Parameters + +You can use `TriggerAuthentication` CRD to configure the authentication by providing a set of Azure Active Directory credentials or by using pod identity. + +**Credential based authentication:** + +- `activeDirectoryClientId` - Id of the Active Directory application which requires at least `Monitoring Reader` permissions. +- `activeDirectoryClientPassword` - Password of the Active Directory application. + +The user will need access to read data from the Azure resource. + +**Pod identity based authentication:** + +[Azure AD Workload Identity](https://azure.github.io/azure-workload-identity/docs/) provider can be used. + +### Example + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: azure-monitor-secrets +data: + activeDirectoryClientId: + activeDirectoryClientPassword: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: azure-monitor-trigger-auth +spec: + secretTargetRef: + - parameter: activeDirectoryClientId + name: azure-monitor-secrets + key: activeDirectoryClientId + - parameter: activeDirectoryClientPassword + name: azure-monitor-secrets + key: activeDirectoryClientPassword + # or Pod Identity, kind: Secret is not required in case of pod Identity + podIdentity: + provider: azure-workload +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: azure-monitor-scaler +spec: + scaleTargetRef: + name: azure-monitor-example + minReplicaCount: 1 + maxReplicaCount: 10 + triggers: + - type: azure-monitor + metadata: + resourceURI: Microsoft.ContainerService/managedClusters/azureMonitorCluster + tenantId: xxx-xxx-xxx-xxx-xxx + subscriptionId: yyy-yyy-yyy-yyy-yyy + resourceGroupName: azureMonitor + metricName: pod_custom_metric + metricNamespace: pod_custom_metrics_namespace # required when monitoring custom metrics + metricFilter: namespace eq 'default' + metricAggregationInterval: "0:1:0" + metricAggregationType: Average + targetValue: "1" + authenticationRef: + name: azure-monitor-trigger-auth +``` diff --git a/content/docs/2.17/scalers/azure-pipelines.md b/content/docs/2.17/scalers/azure-pipelines.md new file mode 100644 index 000000000..cbb532b34 --- /dev/null +++ b/content/docs/2.17/scalers/azure-pipelines.md @@ -0,0 +1,266 @@ ++++ +title = "Azure Pipelines" +availability = "v2.3+" +maintainer = "Microsoft" +category = "CI/CD" +description = "Scale applications based on agent pool queues for Azure Pipelines." +go_file = "azure_pipelines_scaler" ++++ + +### Trigger Specification + +This specification describes the `azure-pipelines` trigger for Azure Pipelines. It scales based on the amount of pipeline runs pending in a given agent pool. + +```yaml +triggers: + - type: azure-pipelines + metadata: + # Optional: Name of the pool in Azure DevOps + poolName: "{agentPoolName}" + # Optional: Learn more in 'How to determine your pool ID' + poolID: "{agentPoolId}" + # Optional: Azure DevOps organization URL, can use TriggerAuthentication as well + organizationURLFromEnv: "AZP_URL" + # Optional: Azure DevOps Personal Access Token, can use TriggerAuthentication as well + personalAccessTokenFromEnv: "AZP_TOKEN" + # Optional: Target queue length + targetPipelinesQueueLength: "1" # Default 1 + activationTargetPipelinesQueueLength: "5" # Default 0 + # Optional: Parent template to read demands from + parent: "{parent ADO agent name}" + # Optional: Demands string to read demands from ScaledObject + demands: "{demands}" + # Optional: Demands of jobs must exactly match the capabilities the trigger defines + requireAllDemands: false + # Optional: How many jobs to fetch for the pool in the API (default: 250) + jobsToFetch: "{jobsToFetch}" + authenticationRef: + name: pipeline-trigger-auth +``` + +**Parameter list:** + +- `poolName` - Name of the pool. (Optional, either `poolID` or `poolName` must be configured) +- `poolID` - Id of the pool. (Optional, either `poolID` or `poolName` must be configured) +- `organizationURLFromEnv` - Name of the environment variable your deployment uses to get the URL for your Azure DevOps organization. +- `personalAccessTokenFromEnv` - Name of the environment variable that provides the personal access token (PAT) for Azure DevOps. Learn more about how to create one [in the official docs](https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=preview-page). +- `targetPipelinesQueueLength` - Target value for the amount of pending jobs in the queue to scale on. (Default: `1`, Optional) + - Example - If one pod can handle 10 jobs, set the queue length target to 10. If the actual number of jobs in the queue is 30, the scaler scales to 3 pods. +- `activationTargetPipelinesQueueLength` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) +- `parent` - Put the name of the ADO agent that matched the ScaledObject. e.g. mavenagent-scaledobject may have an initial deployment called "mavenagent-keda-template"; this is the deployment that is made offline. This name is provided to the initial deployment as the environment variable "AZP_NAME" +- `demands` - Put the demands string that was provided to the ScaledObject. This MUST be a subset of the actual capability list the agent has. e.g. `maven,docker` +- `jobsToFetch` - The number of the jobs that KEDA will fetch for the pool from Azure Pipeline API (Default: `250`, Optional) + +> 💡 **NOTE:** You can either use `poolID` or `poolName`. If both are specified, then `poolName` will be used. + +### Authentication Parameters + +As an alternative to using environment variables, you can authenticate with Azure Devops using a Personal Access Token or Managed identity via `TriggerAuthentication` configuration. If `personalAccessTokenFromEnv` or `personalAccessTokenFrom` is empty `TriggerAuthentication` must be configured using podIdentity. + +**Personal Access Token Authentication:** + +- `organizationURL` - The URL of the Azure DevOps organization. +- `personalAccessToken` - The Personal Access Token (PAT) for Azure DevOps. + +**Pod Identity Authentication** + +[Azure AD Workload Identity](https://azure.github.io/azure-workload-identity/docs/) provider can be used. + +### How to determine your pool ID + +There are several ways to get the `poolID`. The easiest could be using `az cli` to get it using the command `az pipelines pool list --pool-name {agentPoolName} --organization {organizationURL} --query [0].id`. + +It is also possible to get the pool ID using the UI by browsing to the agent pool from the organization (Organization settings -> Agent pools -> `{agentPoolName}`) and getting it from the URL. +The URL should be similar to `https://dev.azure.com/{organization}/_settings/agentpools?poolId={poolID}&view=jobs` + +> Careful - You should determine this on an organization-level, not project-level. Otherwise, you might get an incorrect id. + +Finally, it is also possible get the pool ID from the response of a HTTP request by calling the `https://dev.azure.com/{organizationName}/_apis/distributedtask/pools?poolname={agentPoolName}` endpoint in the key `value[0].id`. + +### Supporting demands in agents + +By default, if you do not use demands in your agent scaler then it scales based on the pool's queue length. + +Demands (Capabilities) are useful when you have multiple agents with different capabilities existing within the same pool, +for instance in a kube cluster you may have an agent supporting dotnet5, dotnet6, java or maven; +particularly these would be exclusive agents where jobs would fail if run on the wrong agent. This is Microsoft's demands feature. + +- **Using Parent:** Azure DevOps is able to determine which agents can match any job it is waiting for. If you specify a parent template then KEDA will further interrogate the job request to determine if the parent is able to fulfill the job. If the parent is able to complete the job it scales the workload fulfill the request. The parent template that is generally offline must stay in the Pool's Agent list. + +- **Using demands:** KEDA will determine which agents can fulfill the job based on the demands provided. The demands are provided as a comma-separated list and must be a subset of the actual capabilities of the agent. (For example `maven,java,make`. Note: `Agent.Version` is ignored). If `requireAllDemands` is set to `true` it is checked if a jobs demands are fulfilled exactly by a trigger and only scales if this is true. This means a job with demands `maven` will not match an agent with capabilities `maven,java`. + +Microsoft's documentation: [https://learn.microsoft.com/en-us/azure/devops/pipelines/process/demands?view=azure-devops&tabs=yaml](https://learn.microsoft.com/en-us/azure/devops/pipelines/process/demands?view=azure-devops&tabs=yaml) + +Please note that the parent template feature is exclusive to KEDA and not Microsoft and is another way of supporting demands. + +If you wish to use demands in your agent scaler then you can do so by adding the following to your pipeline: + +```yaml + pool: + - name: "{agentPoolName}" + demands: + - example-demands + - another-demand -equals /bin/executable +``` + +Then, you can use the `demands` parameter to specify the demands that your agent supports or the `parent` parameter to link a template that matches you scaled object. + +KEDA will use the following evaluation order: +1) If neither parent nor demands are defined in the scaling definition, it will scale the workload to fulfill the job. +2) If `parent` is set, KEDA will interrogate the job request to determine if the parent is able to fulfill the job. If the parent is able to complete the job it scales the workload to fulfill the request. +3) Finally, if the demands are set in the scaling definition then KEDA will determine which agents can fulfill the job based on the demands provided. + +> Note: If more than one scaling definition is able to fulfill the demands of the job then they will both spin up an agent. + +#### How it works under the hood + +Azure DevOps has a Job Request API with returns a list of all jobs, and the agent that they are assigned to, or could potentially be assigned to. This is an undocumented Microsoft API which is available on `https://dev.azure.com//_apis/distributedtask/pools//jobrequests`. + +KEDA will interpret this request to find any matching template from the defined parent in the scaling definition, or any agent that can satisfy the demands specified in the scaling definition. + +Once it finds it, it will scale the workload that matched the definition and Azure DevOps will assign it to that agent. + +### Configuring the agent container + +Microsoft self-hosted docker agent documentation: https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/docker?view=azure-devops#linux + +Please use the script in Step 5 as the entrypoint for your agent container. + +You will need to change this section of the shell script so that the agent will terminate and cleanup itself when the job is complete by using the `--once` switch. +The if statement for cleanup is only required if you are using the auto-deployment parent template method. + +``` +print_header "4. Running Azure Pipelines agent..." + +trap 'cleanup; exit 0' EXIT +trap 'cleanup; exit 130' INT +trap 'cleanup; exit 143' TERM + +chmod +x ./run-docker.sh + +# To be aware of TERM and INT signals call run.sh +# Running it with the --once flag at the end will shut down the agent after the build is executed +./run-docker.sh "$@" & wait $! +``` + +to + + +``` +print_header "4. Running Azure Pipelines agent..." + +if ! grep -q "template" <<< "$AZP_AGENT_NAME"; then + echo "Cleanup Traps Enabled" + + trap 'cleanup; exit 0' EXIT + trap 'cleanup; exit 130' INT + trap 'cleanup; exit 143' TERM + +fi + +chmod +x ./run-docker.sh + +# To be aware of TERM and INT signals call run.sh +# Running it with the --once flag at the end will shut down the agent after the build is executed +./run-docker.sh "$@" --once & wait $! +``` + +### Example for ScaledObject + +```yaml +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: pipeline-auth +data: + personalAccessToken: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: pipeline-trigger-auth + namespace: default +spec: + secretTargetRef: + - parameter: personalAccessToken + name: pipeline-auth + key: personalAccessToken +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: azure-pipelines-scaledobject + namespace: default +spec: + scaleTargetRef: + name: azdevops-deployment + minReplicaCount: 1 + maxReplicaCount: 5 + triggers: + - type: azure-pipelines + metadata: + poolID: "1" + organizationURLFromEnv: "AZP_URL" + parent: "example-keda-template" + demands: "maven,docker" + authenticationRef: + name: pipeline-trigger-auth +``` + +### Example for Parent Deployment or StatefulSet + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: agent +spec: + replicas: 1 + selector: + matchLabels: + app: agent + spec: + containers: + - name: agent + image: [SAME AS SCALED JOB] + envFrom: + - secretRef: + name: ado-pat-tokens + env: + - name: AZP_AGENT_NAME + value: example-keda-template # Matches Scaled Job Parent + +``` + +### Example for using pod identity authentication + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: pipeline-trigger-auth +spec: + podIdentity: + provider: azure-workload +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: azure-pipelines-scaledobject + namespace: default +spec: + scaleTargetRef: + name: azdevops-deployment + minReplicaCount: 1 + maxReplicaCount: 5 + triggers: + - type: azure-pipelines + metadata: + poolID: "1" + organizationURLFromEnv: "AZP_URL" + parent: "example-keda-template" + demands: "maven,docker" + authenticationRef: + name: pipeline-trigger-auth +``` diff --git a/content/docs/2.17/scalers/azure-service-bus.md b/content/docs/2.17/scalers/azure-service-bus.md new file mode 100644 index 000000000..80873efbb --- /dev/null +++ b/content/docs/2.17/scalers/azure-service-bus.md @@ -0,0 +1,121 @@ ++++ +title = "Azure Service Bus" +maintainer = "Microsoft" +category = "Messaging" +description = "Scale applications based on Azure Service Bus Queues or Topics." +availability = "v1.0+" +go_file = "azure_servicebus_scaler" ++++ + +### Trigger Specification + +This specification describes the `azure-servicebus` trigger for Azure Service Bus Queue or Topic. + +> ⚠️ **WARNING:** KEDA is not in charge of managing entities. If the queue, topic or subscription does not exist, it will not create them automatically. + +```yaml +triggers: +- type: azure-servicebus + metadata: + # Required: queueName OR topicName and subscriptionName + queueName: functions-sbqueue + # or + topicName: functions-sbtopic + subscriptionName: sbtopic-sub1 + # Optional, required when pod identity is used + namespace: service-bus-namespace + # Optional, can use TriggerAuthentication as well + connectionFromEnv: SERVICEBUS_CONNECTIONSTRING_ENV_NAME + # Optional + messageCount: "5" # Optional. Count of messages to trigger scaling on. Default: 5 messages + activationMessageCount: "2" + cloud: Private # Optional. Default: AzurePublicCloud + endpointSuffix: servicebus.airgap.example # Required when cloud=Private +``` + +**Parameter list:** + +- `messageCount` - Amount of active messages in your Azure Service Bus queue or topic to scale on. +- `activationMessageCount` - Target value for activating the scaler. Learn more about activation [here](../concepts/scaling-deployments#activating-and-scaling-thresholds). (Default: `0`, Optional) +- `queueName` - Name of the Azure Service Bus queue to scale on. (Optional) +- `topicName` - Name of the Azure Service Bus topic to scale on. (Optional) +- `subscriptionName` - Name of the Azure Service Bus queue to scale on. (Optional*, Required when `topicName` is specified) +- `namespace` - Name of the Azure Service Bus namespace that contains your queue or topic. (Optional*, Required when pod identity is used) +- `connectionFromEnv` - Name of the environment variable your deployment uses to get the connection string of the Azure Service Bus namespace. (Optional) +- `useRegex` - Provides indication whether or not a regex is used in the `queueName` or `subscriptionName` parameters. (Values: `true`, `false`, Default: `false`, Optional) +- `operation` - Defines how to compute the number of messages when `useRegex` is set to `true`. (Values: `sum`, `max`, or `avg`, Default: `sum`, Optional). +- `cloud` - Name of the cloud environment that the service bus belongs to. Must be a known Azure cloud environment, or `Private` for Azure Stack Hub or Air Gapped clouds. (valid values: `AzurePublicCloud`, `AzureUSGovernmentCloud`, `AzureChinaCloud`, `AzureGermanCloud`, `Private`; default: `AzurePublicCloud`) + +When `cloud` is set to `Private`, the `endpointSuffix` parameter is required. Otherwise, it is automatically generated based on the cloud environment. `endpointSuffix` represents the service bus endpoint suffix of the cloud environment that the service bus belongs to, e.g. `servicebus.usgovcloudapi.net` for `AzureUSGovernmentCloud`. + +> 💡 **NOTE:** Service Bus Shared Access Policy needs to be of type `Manage`. Manage access is required for KEDA to be able to get metrics from Service Bus. + +### Authentication Parameters + +You can authenticate by using pod identity or connection string authentication. + +**Connection String Authentication:** + +- `connection` - Connection string for the Azure Service Bus Namespace. + + The following formats are supported. + + - With **SharedAccessKey** - + `Endpoint=sb://.servicebus.windows.net/;SharedAccessKeyName=;SharedAccessKey=` + - With **SharedAccessSignature** - + `Endpoint=sb://.servicebus.windows.net/;SharedAccessSignature=SharedAccessSignature sig=&se=&skn=&sr=` + Refer to this [page](https://docs.microsoft.com/en-us/azure/service-bus-messaging/service-bus-sas) for more information + on using Shared Access Signatures. + +**Pod identity based authentication:** + +[Azure AD Workload Identity](https://azure.github.io/azure-workload-identity/docs/) provider can be used. + +### Example + +Here is an example of how to use managed identity: + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: azure-servicebus-auth +spec: + podIdentity: + provider: azure-workload +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: azure-servicebus-queue-scaledobject + namespace: default +spec: + scaleTargetRef: + name: azure-servicebus-queue-function + triggers: + - type: azure-servicebus + metadata: + # Required: queueName OR topicName and subscriptionName + queueName: functions-sbqueue + # or + topicName: functions-sbtopic + subscriptionName: sbtopic-sub1 + # Required: Define what Azure Service Bus to authenticate to with Managed Identity + namespace: service-bus-namespace + # Optional + messageCount: "5" # default 5 + cloud: AzureGermanCloud # Optional. Default: AzurePublicCloud + authenticationRef: + name: azure-servicebus-auth # authenticationRef would need either podIdentity or define a connection parameter +``` + +### Troubleshooting + +#### KEDA is unable to get queue information due to "invalid queue runtime properties: no CountDetails element" + +When KEDA logs show errors similar to `invalid queue runtime properties: no CountDetails element` it usually is caused because of throttling by Azure Service Bus. + +Consider applying one of the following mitigations: +- Scaling the Azure Service Bus namespace to a higher SKU, or use premium +- Increase the polling interval of the ScaledObject/ScaledJob +- Use [caching of metrics](../concepts/scaling-deployments/#caching-metrics) diff --git a/content/docs/2.17/scalers/azure-storage-blob.md b/content/docs/2.17/scalers/azure-storage-blob.md new file mode 100644 index 000000000..1f895ad3b --- /dev/null +++ b/content/docs/2.17/scalers/azure-storage-blob.md @@ -0,0 +1,101 @@ ++++ +title = "Azure Blob Storage" +availability = "v1.1+" +maintainer = "Microsoft" +category = "Data & Storage" +description = "Scale applications based on the count of blobs in a given Azure Blob Storage container." +notice = "As of now, this Azure Blob Storage scaler scales based on the count of the blobs in a container as opposed to the Azure Functions behavior where code is only triggered on new blobs." +go_file = "azure_blob_scaler" ++++ + +### Trigger Specification + +This specification describes the `azure-blob` trigger for Azure Blob Storage. It scales based on the count of blobs in a given blob storage container and assumes the worker is responsible for clearing the container by deleting/moving the blobs once the blob processing completed. + +```yaml +triggers: +- type: azure-blob + metadata: + blobContainerName: functions-blob + blobCount: '5' + activationBlobCount: '50' + connectionFromEnv: STORAGE_CONNECTIONSTRING_ENV_NAME + accountName: storage-account-name + blobPrefix: myprefix + blobDelimiter: /example + cloud: Private + endpointSuffix: blob.core.airgap.example # Required when cloud=Private + recursive: false + globPattern: glob-pattern +``` + +**Parameter list:** + +- `blobContainerName` - Name of container in an Azure Storage account. +- `blobCount` - Average target value to trigger scaling actions. (Default: `5`, Optional) +- `activationBlobCount` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) +- `connectionFromEnv` - Name of the environment variable your deployment uses to get the connection string. +- `accountName` - Name of the storage account that the container belongs to. +- `blobPrefix` - Prefix for the Blob. Use this to specify sub path for the blobs if required. (Default: `""`, Optional) +- `blobDelimiter` - Delimiter for identifying the blob prefix. (Default: `/`, Optional) +- `cloud` - Name of the cloud environment that the blob belongs to. Must be a known Azure cloud environment, or `Private` for Azure Stack Hub or Air Gapped clouds. (valid values: `AzurePublicCloud`, `AzureUSGovernmentCloud`, `AzureChinaCloud`, `AzureGermanCloud`, `Private`; default: `AzurePublicCloud`) +- `recursive` - Indicates whether or not blobs should be counted recursively. (Values: `true`, `false`, Default: `false`, Optional, Equivalent to setting `blobDelimiter` as `""`) +- `globPattern` - Glob pattern for blob names. The scaler will count all blobs whose names match the pattern +as metric value. (Default: `""`, Optional, The `blobPrefix`, `blobDelimiter` and `recursive` parameters are ignored when this is specified.) + +When `cloud` is set to `Private`, the `endpointSuffix` parameter is required. Otherwise, it is automatically generated based on the cloud environment. `endpointSuffix` represents the storage blob endpoint suffix of the cloud environment that the blob belongs to, e.g. `blob.core.cloudapi.de` for `AzureGermanCloud`. + + +### Authentication Parameters + +You can authenticate by using pod identity or connection string authentication. + +**Connection String Authentication:** + +- `connection` - Connection string for Azure Storage Account. + +**Pod Identity Authentication** + +[Azure AD Workload Identity](https://azure.github.io/azure-workload-identity/docs/) provider can be used. + +- `accountName` - Name of the Azure Storage Account. + +### Example + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: azure-blob-auth +spec: + podIdentity: + provider: azure-workload +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: azure-blob-scaledobject + namespace: default +spec: + scaleTargetRef: + name: azureblob-function + triggers: + - type: azure-blob + metadata: + # Required + blobContainerName: functionsblob + # Optional, required when pod identity is used + accountName: storage-account-name + # Optional, connection OR authenticationRef that defines the connection + connectionFromEnv: STORAGE_CONNECTIONSTRING_ENV_NAME # Reference to a connection string in deployment + # or authenticationRef as defined below + # + # Optional + blobCount: "5" # default 5 + blobPrefix: blobsubpath # Default : "" + blobDelimiter: "/" # Default: "/" + # Optional, default: AzurePublicCloud + cloud: AzureChinaCloud + authenticationRef: + name: azure-blob-auth # authenticationRef would need either podIdentity or define a connection parameter +``` diff --git a/content/docs/2.17/scalers/azure-storage-queue.md b/content/docs/2.17/scalers/azure-storage-queue.md new file mode 100644 index 000000000..ef55c5983 --- /dev/null +++ b/content/docs/2.17/scalers/azure-storage-queue.md @@ -0,0 +1,89 @@ ++++ +title = "Azure Storage Queue" +availability = "v1.0+" +maintainer = "Microsoft" +category = "Messaging" +description = "Scale applications based on Azure Storage Queues." +go_file = "azure_queue_scaler" ++++ + +### Trigger Specification + +This specification describes the `azure-queue` trigger for Azure Storage Queue. + +```yaml +triggers: +- type: azure-queue + metadata: + queueName: orders + queueLength: '5' + queueLengthStrategy: all|visibleonly + activationQueueLength: '50' + connectionFromEnv: STORAGE_CONNECTIONSTRING_ENV_NAME + accountName: storage-account-name + cloud: AzureUSGovernmentCloud +``` + +**Parameter list:** + +- `queueName` - Name of the queue. +- `queueLength` - Target value for queue length passed to the scaler. Example: if one pod can handle 10 messages, set the queue length target to 10. If the actual number of messages in the queue is 30, the scaler scales to 3 pods. (Default: `5`, Optional) +- `queueLengthStrategy` - `all` considers both visible and invisible messages, while `visibleonly` uses Peek to count only visible messages. In `visibleonly`, if the count of messages is 32 or higher, it falls back to the default `all` strategy, counting both visible and invisible messages. (Default: `all`, Optional) +- `activationQueueLength` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) +- `connectionFromEnv` - Name of the environment variable your deployment uses to get the connection string. +- `accountName` - Name of the storage account that the queue belongs to. +- `cloud` - Name of the cloud environment that the queue belongs to. Must be a known Azure cloud environment, or `Private` for Azure Stack Hub or Air Gapped clouds. (valid values: `AzurePublicCloud`, `AzureUSGovernmentCloud`, `AzureChinaCloud`, `AzureGermanCloud`, `Private`; default: `AzurePublicCloud`) + +When `cloud` is set to `Private`, the `endpointSuffix` parameter is required. Otherwise, it is automatically generated based on the cloud environment. `endpointSuffix` represents the storage queue endpoint suffix of the cloud environment that the queue belongs to, e.g. `queue.core.windows.net` for `AzurePublicCloud`. + +### Authentication Parameters + +You can authenticate by using pod identity or connection string authentication. + +**Connection String Authentication:** + +- `connection` - Connection string for Azure Storage Account. + +**Pod identity based authentication:** + +[Azure AD Workload Identity](https://azure.github.io/azure-workload-identity/docs/) provider can be used. + +### Example + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: azure-queue-auth +spec: + podIdentity: + provider: azure-workload +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: azure-queue-scaledobject + namespace: default +spec: + scaleTargetRef: + name: azurequeue-function + triggers: + - type: azure-queue + metadata: + # Required + queueName: functionsqueue + # Optional, required when pod identity is used + accountName: storage-account-name + # Optional: connection OR authenticationRef that defines connection + connectionFromEnv: STORAGE_CONNECTIONSTRING_ENV_NAME # Default: AzureWebJobsStorage. Reference to a connection string in deployment + # or authenticationRef as defined below + # + # Optional + queueLength: "5" # default 5 + # Optional + queueLengthStrategy: "all" # or visibleonly. Default: all + cloud: Private + endpointSuffix: queue.local.azurestack.external # Required when cloud=Private + authenticationRef: + name: azure-queue-auth # authenticationRef would need either podIdentity or define a connection parameter +``` diff --git a/content/docs/2.17/scalers/beanstalkd.md b/content/docs/2.17/scalers/beanstalkd.md new file mode 100644 index 000000000..6998b9d0a --- /dev/null +++ b/content/docs/2.17/scalers/beanstalkd.md @@ -0,0 +1,83 @@ ++++ +title = "Beanstalkd" +availability = "v2.16+" +maintainer = "Community" +description = "Scale applications based on beanstalkd queues" +go_file = "beanstalkd_scaler" ++++ + +### Trigger Specification + +This specification describes the `beanstalkd` trigger that scales based on the number of jobs in a Beanstalkd queue. + +```yaml +triggers: + - type: beanstalkd + metadata: + server: beanstalkd.internal-namespace:11300 + includeDelayed: "false" + tube: "default" + activationValue: "10" + value: "15" + timeout: "30" +``` + +**Parameter list:** + +- `server` - Address of beanstalkd server `:`. If no port is specified then the scaler will default to `11300`. +- `includeDelayed` - Whether to include delayed jobs in the count used for scaling. Defaults to false so only `ready` and `reserved` jobs are counted. +- `tube` - Name of the tube to scale on. +- `activationValue` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional, This value can be a float) +- `value` - Number of jobs in the queue to trigger on. Defaults to `ready`+`reserved` jobs if `includeDelayed` isn't set. +- `timeout` - Timeout in seconds for establishing a connection to the beanstalkd server. (Default: `30`, Optional) + +### Authentication Parameters + +No authentication should be needed to connect to the beanstalkd server. + +### Example + +#### Default tube with activation value + +Here the scaler will not be active until there are at least 10 jobs in the tube. Delayed jobs are not included as `includeDelayed` will be set to a default of `false`. + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: beanstalkd-scaledobject +spec: + scaleTargetRef: + name: beanstalkd-consumer + maxReplicaCount: 20 + triggers: + - type: beanstalkd + metadata: + server: beanstalkd.internal-namespace:11300 + tube: 'default' + activationValue: "10" + value: "20" +``` + +#### User-created tube with minimum replicas + +A minimum number of replicas is specified here so the scaler will always be active. This means scaling will occur as soon as the job count goes above 5. Delayed jobs are included here. + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: beanstalkd-scaledobject +spec: + scaleTargetRef: + name: beanstalkd-consumer + minReplicaCount: 1 + maxReplicaCount: 20 + triggers: + - type: beanstalkd + metadata: + server: beanstalkd.internal-namespace:11300 + tube: 'scaling-tube' + value: "5" + includeDelayed: "true" +``` diff --git a/content/docs/2.17/scalers/cassandra.md b/content/docs/2.17/scalers/cassandra.md new file mode 100644 index 000000000..04c151c3f --- /dev/null +++ b/content/docs/2.17/scalers/cassandra.md @@ -0,0 +1,188 @@ ++++ +title = "Cassandra" +availability = "v2.5+" +maintainer = "Community" +category = "Data & Storage" +description = "Scale applications based on Cassandra query results." +go_file = "cassandra_scaler" ++++ + +### Trigger Specification + +This specification describes the `cassandra` trigger that scales based on the outputs of a Cassandra query. + +```yaml +triggers: + - type: cassandra + metadata: + username: "cassandra" + port: "9042" + clusterIPAddress: "cassandra.default" + consistency: "Quorum" + protocolVersion: "4" + keyspace: "test_keyspace" + query: "SELECT COUNT(*) FROM test_keyspace.test_table;" + targetQueryValue: "1" + activationTargetQueryValue: "10" +``` + +**Parameter list:** + +- `username` - The username credential for connecting to the Cassandra instance. +- `port` - The port number of the Cassandra instance. (Optional, Can be set either here or in `clusterIPAddress`) +- `clusterIPAddress` - The IP address or the host name of the Cassandra instance. +- `consistency` - Configuration for a session or per individual read operation. (Values: `LOCAL_ONE`, `LOCAL_QUORUM`, `EACH_QUORUM`, `LOCAL_SERIAL`, `ONE`, `TWO`, `THREE`, `QUORUM`, `SERIAL`, `ALL`, Default: `ONE`, Optional) +- `protocolVersion` - CQL Binary Protocol. (Default: `4`, Optional) +- `keyspace` - The name of the keyspace used in Cassandra. +- `query` - A Cassandra query that should return single numeric value. +- `targetQueryValue` - The threshold value that is provided by the user and used as `targetValue` or `targetAverageValue` (depending on the trigger metric type) in the Horizontal Pod Autoscaler (HPA). +- `activationTargetQueryValue` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) + +### Authentication Parameters + +You can authenticate by using a password via `TriggerAuthentication` configuration. + +**Password Authentication:** + +- `password` - Password for configured user to log in to the Cassandra instance. +- `tls` - To enable SSL auth for Cassandra session, set this to enable. If not set, TLS for Cassandra is not used. (Values: enable, disable, Default: disable, Optional). +- `cert` - Certificate path for client authentication. Mandatory if tls enabled. (Optional) +- `key` - Key path for client authentication. Mandatory if tls enabled. (Optional) + +### Example with no TLS auth + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: cassandra-secrets +type: Opaque +data: + cassandra_password: CASSANDRA_PASSWORD +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-cassandra-secret +spec: + secretTargetRef: + - parameter: password + name: cassandra-secrets + key: cassandra_password +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: cassandra-scaledobject +spec: + scaleTargetRef: + name: nginx-deployment + triggers: + - type: cassandra + metadata: + username: "cassandra" + port: "9042" + clusterIPAddress: "cassandra.default" + consistency: "Quorum" + protocolVersion: "4" + query: "SELECT COUNT(*) FROM test_keyspace.test_table;" + targetQueryValue: "1" + authenticationRef: + name: keda-trigger-auth-cassandra-secret +``` + +### Example with TLS auth + +Since we are passing Cert and Key content as inputs to the scaler, we have to supply writable location for required GSSAPI configurations for the `keda-operator` container. + +##### `sasl/gssapi` in manager.yaml + +If you use YAML declarations to deploy KEDA, add below volume mount and volume to supply writable location for required GSSAPI configurations for the `keda-operator` container. + +``` + volumeMounts: + - mountPath: /tmp/cassandra + name: temp-cassandra-vol + readOnly: false + + volumes: + - name: temp-cassandra-vol + emptyDir: + medium: Memory +``` + +##### `sasl/gssapi` in keda-charts + +If you use Helm Charts to deploy KEDA, add below volume mount and volume to supply writable location for required gssapi configurations. + +``` +volumes.keda.extraVolumeMounts +- mountPath: /tmp/cassandra + name: temp-cassandra-vol + readOnly: false + +volumes.keda.extraVolumes +- name: temp-cassandra-vol + emptyDir: + medium: Memory +``` + +Once we have the writable mount path set up for the certificates and keys. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: cassandra-secrets +type: Opaque +data: + cassandra_password: CASSANDRA_PASSWORD + tls: enable + cert: + key: + ## Optional parameter ca ## + ca: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-cassandra-secret +spec: + secretTargetRef: + - parameter: password + name: cassandra-secrets + key: cassandra_password + - parameter: tls + name: cassandra-secrets + key: tls + - parameter: cert + name: cassandra-secrets + key: cert + - parameter: key + name: cassandra-secrets + key: key + ## Optional parameter ca ## + - parameter: ca + name: cassandra-secrets + key: ca +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: cassandra-scaledobject +spec: + scaleTargetRef: + name: nginx-deployment + triggers: + - type: cassandra + metadata: + username: "cassandra" + port: "9042" + clusterIPAddress: "cassandra.default" + consistency: "Quorum" + protocolVersion: "4" + query: "SELECT COUNT(*) FROM test_keyspace.test_table;" + targetQueryValue: "1" + authenticationRef: + name: keda-trigger-auth-cassandra-secret +``` diff --git a/content/docs/2.17/scalers/couchdb.md b/content/docs/2.17/scalers/couchdb.md new file mode 100644 index 000000000..444126c19 --- /dev/null +++ b/content/docs/2.17/scalers/couchdb.md @@ -0,0 +1,94 @@ ++++ +title = "CouchDB" +availability = "v2.9+" +maintainer = "Community" +category = "Data & Storage" +description = "Scale applications based on CouchDB query results." +go_file = "couchdb_scaler" ++++ + +### Trigger Specification + +This specification describes the `couchdb` trigger that scales based on the outputs of a CouchDB query. + +```yaml +triggers: +- type: couchdb + metadata: + connectionStringFromEnv: "CONNECTION_STRING" + host: "test-release-svc-couchdb.couchdb-test-ns.svc.cluster.local" + port: "5984" + dbName: "animals" + queryValue: "1" + query: '{ "selector": { "feet": { "$gt": 0 } }, "fields": ["_id", "feet", "greeting"] }' + activationQueryValue: "1" +``` + +**Parameter list:** + +- `host` - The hostname for connecting to the CouchDB service. (Optional, Required if `connectionString` and `connectionStringFromEnv` is not set) +- `dbName` - Name of the database. (Optional, Required if `connectionString` and `connectionStringFromEnv` is not set) +- `queryValue` - A threshold that will define when scaling should occur. (Optional, Required if `connectionString` and `connectionStringFromEnv` is not set) +- `port` - The port number of the CouchDB service. (Optional, Required if `connectionString` and `connectionStringFromEnv` is not set) +- `query` - A CouchDB query that should return single numeric value. (Optional) +- `activationQueryValue` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) +- `connectionStringFromEnv` - Environment variable from workload with the connection string. (Optional, Required if `connectionString` and connection parameters aren't set) + +### Authentication Parameters + +You can authenticate by using a username and password via `TriggerAuthentication` configuration. + +**ConnectionString:** + +- `connectionString` - Connection string for CouchDB database. (Optional, Required if `connectionStringFromEnv` and connection parameters aren't set) + +**Password Authentication:** + +- `password` - Password for configured user to login to the Couchdb instance. +- `username` - Username for configured user to login to the Couchdb instance. + +### Example + +Here is an example of a couchdb scaler with Basic Authentication, where the `Secret` and `TriggerAuthentication` are defined as follows: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: couchdb-secrets +data: + username: COUCHDB_USERNAME + password: COUCHDB_PASSWORD +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-couchdb-secret +spec: + secretTargetRef: + - parameter: password + name: couchdb-secrets + key: password + - parameter: username + name: couchdb-secrets + key: username +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: couchdb-scaledobject +spec: + scaleTargetRef: + name: nginx-deployment + triggers: + - type: couchdb + metadata: + hostname: "test-release-svc-couchdb.couchdb-test-ns.svc.cluster.local" + port: "5984" + dbName: "animals" + queryValue: "1" + query: '{ "selector": { "feet": { "$gt": 0 } }, "fields": ["_id", "feet", "greeting"] }' + activationQueryValue: "1" + authenticationRef: + name: keda-trigger-auth-couchdb-secret +``` diff --git a/content/docs/2.17/scalers/cpu.md b/content/docs/2.17/scalers/cpu.md new file mode 100644 index 000000000..7fc63a0bc --- /dev/null +++ b/content/docs/2.17/scalers/cpu.md @@ -0,0 +1,97 @@ ++++ +title = "CPU" +availability = "v2.0+" +maintainer = "Community" +category = "Apps" +description = "Scale applications based on cpu metrics." +go_file = "cpu_memory_scaler" ++++ + +> **Notice:** +> - This scaler **requires prerequisites**. See the 'Prerequisites' section. +> - This scaler can scale to 0 only when user defines at least one additional scaler which is not CPU or Memory (eg. Kafka + CPU, or Prometheus + CPU) and `minReplicaCount` is 0. +> - This scaler only applies to ScaledObject, not to Scaling Jobs. + +### Prerequisites + +KEDA uses standard `cpu` and `memory` metrics from the Kubernetes Metrics Server, which is not installed by default on certain Kubernetes deployments such as EKS on AWS. Additionally, the `resources` section of the relevant Kubernetes Pods must include at least one of `requests` or `limits`. + +- The Kubernetes Metrics Server must be installed. Installation instructions vary based on your Kubernetes provider. +- The configuration for your Kubernetes Pods must include a `resources` section with specified `requests` (or `limits`). See [Resource Management for Pods and Containers](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/). If the resources section is empty (`resources: {}` or similar), KEDA checks if the `defaultRequest` (or `default` for limits) is set in `LimitRange` for the `Container` type in the same namespace. If `defaultRequest` (or `default` for limits) is missing too, the error `missing request for {cpu/memory}` occurs. + +```yaml +# a working example of resources with specified requests +spec: + containers: + - name: app + image: images.my-company.example/app:v4 + resources: + requests: + memory: "128Mi" + cpu: "500m" +``` + +### Trigger Specification + +This specification describes the `cpu` trigger that scales based on cpu metrics. + +```yaml +triggers: +- type: cpu + metricType: Utilization # Allowed types are 'Utilization' or 'AverageValue' + metadata: + type: Utilization # DEPRECATED: This parameter is deprecated in favor of trigger.metricType and will be removed in version v2.18; allowed types are 'Utilization' or 'AverageValue' + value: "60" + containerName: "" # Optional. You can use this to target a specific container in a pod +``` + +**Parameter list:** + +- `type` - Type of metric to use. Options are `Utilization`, or `AverageValue`. +- `value` - Value to trigger scaling actions for: + - When using `Utilization`, the target value is the average of the resource metric across all relevant pods, represented as a percentage of the requested value of the resource for the pods. + - When using `AverageValue`, the target value is the target value of the average of the metric across all relevant pods (quantity). +- `containerName` - Name of the specific container to scale based on its CPU, rather than the entire pod. Defaults to empty if not specified. + +> 💡 **NOTE:** `containerName` parameter requires Kubernetes cluster version 1.20 or higher with `HPAContainerMetrics` feature enabled. Please see [container resource metrics](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#container-resource-metrics) for more information. + +> 💡 **NOTE:** The `type` parameter is deprecated in favor of the global `metricType` and will be removed in a future release. Users are advised to use `metricType` instead. + +### Example + +The following example targets CPU utilization of entire pod. If the pod has multiple containers, it will be sum of all the containers in it. + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: cpu-scaledobject + namespace: default +spec: + scaleTargetRef: + name: my-deployment + triggers: + - type: cpu + metricType: Utilization # Allowed types are 'Utilization' or 'AverageValue' + metadata: + value: "50" +``` + +The following example targets CPU utilization of a specific container (`foo`) in a pod. + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: cpu-scaledobject + namespace: default +spec: + scaleTargetRef: + name: my-deployment + triggers: + - type: cpu + metricType: Utilization # Allowed types are 'Utilization' or 'AverageValue' + metadata: + value: "50" + containerName: "foo" +``` diff --git a/content/docs/2.17/scalers/cron.md b/content/docs/2.17/scalers/cron.md new file mode 100644 index 000000000..3a526742d --- /dev/null +++ b/content/docs/2.17/scalers/cron.md @@ -0,0 +1,84 @@ ++++ +title = "Cron" +availability = "v1.5+" +maintainer = "Community" +category = "Scheduling" +description = "Scale applications based on a cron schedule." +go_file = "cron_scaler" ++++ + +### Trigger Specification + +This specification describes the `cron` trigger that scales workloads in/out based on a cron Schedule. + +```yaml +triggers: +- type: cron + metadata: + # Required + timezone: Asia/Kolkata # The acceptable values would be a value from the IANA Time Zone Database. + start: 0 6 * * * # At 6:00 AM + end: 0 20 * * * # At 8:00 PM + desiredReplicas: "10" +``` + +**Parameter list:** + +- `timezone` - One of the acceptable values from the IANA Time Zone Database. The list of timezones can be found [here](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones). +- `start` - Cron expression indicating the start of the cron schedule. +- `end` - Cron expression indicating the end of the cron schedule. +- `desiredReplicas` - Number of replicas to which the resource has to be scaled **between the start and end** of the cron schedule. + +> 💡 **Note:** `start`/`end` support ["Linux format cron"](https://en.wikipedia.org/wiki/Cron) (Minute Hour Dom Month Dow). + +> **Notice:** +> **Start and end should not be same.** +> +> For example, the following schedule is not valid: +> ```yaml +> start: 0 6 * * * # At 6:00 AM +> end: 0 6 * * * # also at 6:00 AM +>``` + +### How does it work? + +The CRON scaler allows you to define a time range in which you want to scale your workloads out/in. + +When the time window starts, it will scale from the minimum number of replicas to the desired number of replicas based on your configuration. + +![](/img/scalers/cron/how-it-works.png) + +What the CRON scaler does **not** do, is scale your workloads based on a recurring schedule. + +### Scale to 0 during off hours + +If you want to scale you deployment to 0 outside office hours / working hours, +you need to set `minReplicaCount: 0` in the ScaledObject, and increase the +replicas during work hours. That way the Deployment will be scaled to 0 ouside +that time window. By default the ScaledObject `cooldownPeriod` is 5 minutes, so the actual +scaling down will happen 5 minutes after the cron schedule `end` parameter. + +It's almost always an error to try to do the other way +around, i.e. set `desiredReplicas: 0` in the cron trigger. + +#### Example: scale up to 10 replicas from 6AM to 8PM and scale down to 0 replicas otherwise + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: cron-scaledobject + namespace: default +spec: + scaleTargetRef: + name: my-deployment + minReplicaCount: 0 + cooldownPeriod: 300 + triggers: + - type: cron + metadata: + timezone: Asia/Kolkata + start: 0 6 * * * + end: 0 20 * * * + desiredReplicas: "10" +``` diff --git a/content/docs/2.17/scalers/datadog.md b/content/docs/2.17/scalers/datadog.md new file mode 100644 index 000000000..222e70563 --- /dev/null +++ b/content/docs/2.17/scalers/datadog.md @@ -0,0 +1,454 @@ ++++ +title = "Datadog" +availability = "v2.6+" +maintainer = "Datadog" +category = "Metrics" +description = "Scale applications based on Datadog." +go_file = "datadog_scaler" ++++ + +> 💡 **NOTE:** Take into account [API Datadog endpoints rate +limits](https://docs.datadoghq.com/api/latest/rate-limits/) when defining +polling interval. For more detailed information about polling intervals check +[the Polling intervals and Datadog rate limiting +section](#polling-intervals-and-datadog-rate-limiting). + +There are two ways to poll Datadog for a query value using the Datadog scaler: using the REST API endpoints, or using the [Datadog Cluster Agent](https://docs.datadoghq.com/containers/cluster_agent/) as proxy. Using the Datadog Cluster Agent as proxy reduces the chance of reaching rate limits. As both types are different in terms of usage and authentication, this documentation handles them separately. + +## Using the Datadog Cluster Agent (Experimental) + +With this method, the Datadog scaler will be connecting to the Datadog Cluster Agent to retrieve the query values that will be used to drive the KEDA scaling events. This reduces the risk of reaching rate limits for the Datadog API, as the Cluster Agent retrieves metric values in batches. + +### Deploy the Datadog Cluster Agent with enabled external metrics + +First, deploy the Datadog Cluster Agent enabling the external metrics provider, but without registering it as an `APIService` (to avoid clashing with KEDA). + +If you are using Helm to deploy the Cluster Agent, set: + +* `clusterAgent.metricsProvider.enabled` to `true` +* `clusterAgent.metricsProvider.registerAPIService` to `false` +* `clusterAgent.metricsProvider.useDatadogMetrics` to `true` +* `clusterAgent.env` to `[{name: DD_EXTERNAL_METRICS_PROVIDER_ENABLE_DATADOGMETRIC_AUTOGEN, value: "false"}]` + +If you are using the Datadog Operator, add the following options to your `DatadogAgent` object: + +``` +apiVersion: datadoghq.com/v2alpha1 +kind: DatadogAgent +metadata: + name: datadog +spec: + features: + externalMetricsServer: + enabled: true + useDatadogMetrics: true + registerAPIService: false + override: + clusterAgent: + env: [{name: DD_EXTERNAL_METRICS_PROVIDER_ENABLE_DATADOGMETRIC_AUTOGEN, value: "false"}] +[...] +``` + +NOTE: Using the Datadog Operator for this purpose requires version 1.8.0 of the operator or later. + +### Create a DatadogMetric object for each scaling query + +To use the Datadog Cluster Agent to retrieve the query values from Datadog, first, create a [`DatadogMetric`](https://docs.datadoghq.com/containers/guide/cluster_agent_autoscaling_metrics/?tab=helm#create-the-datadogmetric-object) object with the query to drive your scaling events. You will need to add the `external-metrics.datadoghq.com/always-active: "true"` annotation, to ensure the Cluster Agent retrieves the query value. Example: + +```yaml +apiVersion: datadoghq.com/v1alpha1 +kind: DatadogMetric +metadata: + annotations: + external-metrics.datadoghq.com/always-active: "true" + name: nginx-hits +spec: + query: sum:nginx.net.request_per_s{kube_deployment:nginx} +``` + +### Trigger Specification + +This specification describes the `datadog` trigger that scales based on a Datadog query, using the Datadog Cluster Agent as proxy. + +```yaml +triggers: +- type: datadog + metricType: Value + metadata: + useClusterAgentProxy: "true" + datadogMetricName: "nginx-hits" + datadogMetricNamespace: "default" + targetValue: "7.75" + activationQueryValue: "1.1" + type: "global" # Deprecated in favor of trigger.metricType + metricUnavailableValue: "1.5" +``` + +**Parameter list:** + +- `useClusterAgentProxy` - Whether to use the Cluster Agent as proxy to get the query values. (Values: true, false, Default: false, Optional) +- `datadogMetricName` - The name of the `DatadogMetric` object to drive the scaling events. +- `datadogMetricNamespace` - The namespace of the `DatadogMetric` object to drive the scaling events. +- `targetValue` - Value to reach to start scaling (This value can be a float). +- `activationQueryValue` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds).(Default: `0`, Optional, This value can be a float) +- `type` - Whether to start scaling based on the value or the average between pods. (Values: `average`, `global`, Default:`average`, Optional) +- `age`: The time window (in seconds) to retrieve metrics from Datadog. (Default: `90`, Optional) +- `lastAvailablePointOffset`: The offset to retrieve the X to last data point. The value of last data point of some queries might be inaccurate [because of the implicit rollup function](https://docs.datadoghq.com/dashboards/functions/rollup/#rollup-interval-enforced-vs-custom), try to adjust to `1` if you encounter this issue. (Default: `0`, Optional) +- `metricUnavailableValue`: The value of the metric to return to the HPA if Datadog doesn't find a metric value for the specified time window. If not set, an error will be returned to the HPA, which will log a warning. (Optional, This value can be a float) + +> 💡 **NOTE:** The `type` parameter is deprecated in favor of the global `metricType` and will be removed in a future release. Users are advised to use `metricType` instead. + +### Authentication + +The Datadog scaler with Cluster Agent supports one type of authentication - Bearer authentication. + +You can use `TriggerAuthentication` CRD to configure the authentication. Specify `authMode` and other trigger parameters along with secret credentials in `TriggerAuthentication` as mentioned below: + +**Common to all authentication types** +- `authMode` - The authentication mode to connect to the Cluster Agent. (Values: bearer, Default: bearer, Optional) +- `datadogNamespace` - The namespace where the Datadog Cluster Agent is deployed. +- `datadogMetricsService` - The service name for the Cluster Agent metrics server. To find the name of the service, check the available services in the Datadog namespace and look for the `*-cluster-agent-metrics*` name pattern. +- `datadogMetricsServicePort` - The port of the service for the Cluster Agent Metrics API. (Default: 8443, Optional) +- `unsafeSsl` - Skip certificate validation when connecting over HTTPS. (Values: true, false, Default: false, Optional) + +**Bearer authentication:** +- `token` - The ServiceAccount token to connect to the Datadog Cluster Agent. The service account needs to have permissions to `get`, `watch`, and `list` all `external.metrics.k8s.io` resources. + +### Example + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: datadog-config + namespace: my-project +type: Opaque +data: + datadogNamespace: # Required: base64 encoded value of the namespace where the Datadog Cluster Agent is deployed + datadogMetricsService: # Required: base64 encoded value of the Cluster Agent metrics server service + unsafeSsl: # Optional: base64 encoded value of `true` or `false` + authMode: # Required: base64 encoded value of the authentication mode (in this case, bearer) +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: datadog-cluster-agent-creds + namespace: my-project +spec: + secretTargetRef: + - parameter: token + name: dd-cluster-agent-token + key: token + - parameter: datadogNamespace + name: datadog-config + key: datadogNamespace + - parameter: unsafeSsl + name: datadog-config + key: unsafeSsl + - parameter: authMode + name: datadog-config + key: authMode + - parameter: datadogMetricsService + name: datadog-config + key: datadogMetricsService +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: datadog-scaledobject + namespace: my-project +spec: + scaleTargetRef: + name: nginx + maxReplicaCount: 3 + minReplicaCount: 1 + pollingInterval: 60 + triggers: + - type: datadog + metadata: + useClusterAgentProxy: "true" + datadogMetricName: "nginx-hits" + datadogMetricNamespace: "default" + targetValue: "2" + type: "global" + authenticationRef: + name: datadog-cluster-agent-creds +``` + +## Using the Datadog REST API + +### Trigger Specification + +This specification describes the `datadog` trigger that scales based on a Datadog query, using the Datadog REST API. + +```yaml +triggers: +- type: datadog + metricType: Value + metadata: + useClusterAgentProxy: "false" + query: "sum:trace.redis.command.hits{env:none,service:redis}.as_count()" + queryValue: "7.75" + activationQueryValue: "1.1" + queryAggregator: "max" + type: "global" # Deprecated in favor of trigger.metricType + age: "120" + timeWindowOffset: "30" + lastAvailablePointOffset: "1" + metricUnavailableValue: "1.5" +``` + +**Parameter list:** + +- `useClusterAgentProxy` - Whether to use the Cluster Agent as proxy to get the query values. (Default: false) +- `query` - The Datadog query to run. +- `queryValue` - Value to reach to start scaling (This value can be a float). +- `activationQueryValue` - Target value for activating the scaler. Learn more about activation [here](../concepts/scaling-deployments#activating-and-scaling-thresholds).(Default: `0`, Optional, This value can be a float) +- `queryAggregator` - When `query` is multiple queries, comma-seperated, this sets how to aggregate the multiple results. (Values: `max`, `average`, Required only when `query` contains multiple queries) +- `type` - Whether to start scaling based on the value or the average between pods. (Values: `average`, `global`, Default:`average`, Optional) +- `age`: The time window (in seconds) to retrieve metrics from Datadog. (Default: `90`, Optional) +- `timeWindowOffset`: The delayed time window offset (in seconds) to wait for the metric to be available. The values of some queries might be not available at now and need a small delay to become available, try to adjust `timeWindowOffset` if you encounter this issue. (Default: `0`, Optional) +- `lastAvailablePointOffset`: The offset to retrieve the X to last data point. The value of last data point of some queries might be inaccurate [because of the implicit rollup function](https://docs.datadoghq.com/dashboards/functions/rollup/#rollup-interval-enforced-vs-custom), try to adjust to `1` if you encounter this issue. (Default: `0`, Optional) +- `metricUnavailableValue`: The value of the metric to return to the HPA if Datadog doesn't find a metric value for the specified time window. If not set, an error will be returned to the HPA, which will log a warning. (Optional, This value can be a float) + +> 💡 **NOTE:** The `type` parameter is deprecated in favor of the global `metricType` and will be removed in a future release. Users are advised to use `metricType` instead. + +### Authentication + +Datadog requires both an API key and an APP key to retrieve metrics from your account. + +You should use `TriggerAuthentication` CRD to configure the authentication: + +**Parameter list:** + +- `apiKey` - Datadog API key. +- `appKey` - Datadog APP key. +- `datadogSite` - Datadog site where to get the metrics from. This is commonly referred as DD_SITE in Datadog documentation. (Default: `datadoghq.com`, Optional) + +### Example + +The example below uses the default KEDA polling interval (30 seconds). Take into +account that [API Datadog endpoints are rate +limited](https://docs.datadoghq.com/api/latest/rate-limits/) and reducing the +polling interval can accelerate reaching it. If your account has reached its +rate limit, a relevant error will be logged in KEDA. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: datadog-secrets + namespace: my-project +type: Opaque +data: + apiKey: # Required: base64 encoded value of Datadog apiKey + appKey: # Required: base64 encoded value of Datadog appKey + datadogSite: # Optional: base64 encoded value of Datadog site +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-datadog-secret + namespace: my-project +spec: + secretTargetRef: + # Required: API key for your Datadog account + - parameter: apiKey + name: datadog-secrets + key: apiKey + # Required: APP key for your Datadog account + - parameter: appKey + name: datadog-secrets + key: appKey + # Optional: Datadog site. Default: "datadoghq.com" + - parameter: datadogSite + name: datadog-secrets + key: datadogSite +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: datadog-scaledobject + namespace: my-project +spec: + scaleTargetRef: + name: worker + triggers: + - type: datadog + # Optional: (Value or AverageValue). Whether the target value is global or average per pod. Default: AverageValue + metricType: "Value" + metadata: + # Required: datadog metric query + query: "sum:trace.redis.command.hits{env:none,service:redis}.as_count()" + # Required: according to the number of query result, to scale the TargetRef + queryValue: "7" + # Optional: The time window (in seconds) to retrieve metrics from Datadog. Default: 90 + age: "120" + # Optional: The metric value to return to the HPA if a metric value wasn't found for the specified time window + metricUnavailableValue: "0" + authenticationRef: + name: keda-trigger-auth-datadog-secret +``` + +### Polling intervals and Datadog rate limiting + +[API Datadog endpoints are rate +limited](https://docs.datadoghq.com/api/latest/rate-limits/). Depending on the +state of the `ScaledObject` there are two different parameters to control how +often (per `ScaledObject`) we query Datadog for a metric. + +When scaling from 0 to 1, the polling interval is controlled by KEDA, using [the +`spec.pollingInterval` parameter in the `ScaledObject` +definition](../reference/scaledobject-spec/#pollinginterval). For example, if +this parameter is set to `60`, KEDA will poll Datadog for a metric value every +60 seconds while the number of replicas is 0. + +While scaling from 1 to N, on top of KEDA, the HPA will also poll regularly +Datadog for metrics, based on [the `--horizontal-pod-autoscaler-sync-period` +parameter to the +`kube-controller-manager`](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/#options), +which by default is 15 seconds. For example, if the `kube-controller-manager` +was started with `--horizontal-pod-autoscaler-sync-period=30`, the HPA will poll +Datadog for a metric value every 30 seconds while the number of replicas is +between 1 and N. + +### Multi-Query Support + +To reduce issues with API rate limiting from Datadog, it is possible to send a single query, which contains multiple queries, comma-seperated. +When doing this, the results from each query are aggregated based on the `queryAggregator` value (eg: `max` or `average`). + +> 💡 **NOTE:** Because the average/max aggregation operation happens at the scaler level, there won't be any validation or errors if the queries don't make sense to aggregate. Be sure to read and understand the two patterns below before using Multi-Query. + +#### Example 1 - Aggregating Similar Metrics + +Simple aggregation works well, when wanting to scale on more than one metric with similar return values/scale (ie. where multiple metrics can use a single `queryValue` and still make sense). + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: datadog-scaledobject + namespace: my-project +spec: + scaleTargetRef: + name: worker + triggers: + - type: datadog + metricType: "AverageValue" + metadata: + # Comma-seperated querys count as a single API call: + query: "per_second(sum:http.requests{service:myservice1}).rollup(max, 300)),per_second(sum:http.requests{service:myservice1}).rollup(avg, 600)" + # According to aggregated results, how to scale the TargetRef + queryValue: "100" + # How to aggregate results from multi-query queries. Default: 'max' + queryAggregator: "average" + # Optional: The time window (in seconds) to retrieve metrics from Datadog. Default: 90 + age: "600" + # Optional: The metric value to return to the HPA if a metric value wasn't found for the specified time window + metricUnavailableValue: "0" + authenticationRef: + name: keda-trigger-auth-datadog-secret +``` + +The example above looks at the `http.requests` value for a service; taking two views of the same metric (max vs avg, and different time windows), and then uses a scale value which is the average of them both. + +This works particularly well when scaling against the same metric, but with slightly different parameters, or methods like ```week_before()``` for example. + +#### Example 2 - Driving scale directly + +When wanting to scale on non-similar metrics, whilst still benefiting from reduced API calls with multi-query support, the easiest way to do this is to make each query directly return the desired scale (eg: number of pods), and then `max` or `average` the results to get the desired target scale. + +This can be done by adding arthmetic to the queries, which makes them directly return the number of pods that should be running. + +Following this pattern, and then setting `queryValue: 1` and `metricType: AverageValue` results in the desired number of pods being spawned directly from the results of the metric queries. + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: datadog-scaledobject + namespace: my-project +spec: + scaleTargetRef: + name: worker + triggers: + - type: datadog + # `AverageValue` tracks the query results divided by the number of running containers + metricType: "AverageValue" + metadata: + # Comma-seperated queries count as a single API call: + ## This example returns "http.requests" @ 180 requests-per-second per-pod, + ## and "http.backlog" size of 30 per-pod + query: "per_second(sum:http.requests{service:myservice1}).rollup(max, 300))/180,per_second(sum:http.backlog{service:myservice1}).rollup(max, 300)/30" + # Setting query value to 1 and metricType to "AverageValue" allows the metric to dictate the number of pods from it's own arthimetic. + queryValue: "1" + # How to aggregate results from multi-query queries. Default: 'max' + queryAggregator: "max" + authenticationRef: + name: keda-trigger-auth-datadog-secret +``` + +Using the example above, if we assume that `http.requests` is currently returning `360`, dividing that by `180` in the query, results in a value of `2`; if `http.backlog` returns `90`, dividing that by `30` in the query, results in a value of `3`. With the `max` Aggregator set, the scaler will set the target scale to `3` as that is the higher value from all returned queries. + +### Cases of unexpected metrics value in DataDog API response + +#### Latest data point is unavailable + +By default, Datadog scaler retrieves the metrics with time window from `now - metadata.age (in seconds)` to `now`, however, some kinds of queries need a small delay (usually 30 secs - 2 mins) before data is available when querying from the API. In this case, adjust `timeWindowOffset` to ensure that the latest point of your query is always available. + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: datadog-scaledobject + namespace: my-project +spec: + scaleTargetRef: + name: worker + triggers: + - type: datadog + metricType: "AverageValue" + metadata: + query: "sum:trace.express.request.hits{*}.as_rate()" + queryValue: "100" + age: "90" + metricUnavailableValue: "0" + # Optional: The delayed time window offset (in seconds) to wait for the metric to be available. The values of some queries might be not available at now and need a small delay to become available, try to adjust it if you encounter this issue. Default: 0 + timeWindowOffset: "30" + # Optional: The offset to retrieve the X to last data point. The value of last data point of some queries might be inaccurate, try to adjust to 1 if you encounter this issue. Default: 0 + lastAvailablePointOffset: "1" + authenticationRef: + name: keda-trigger-auth-datadog-secret +``` +Check [here](https://github.com/kedacore/keda/pull/3954#discussion_r1042820206) for the details of this issue + +#### The value of last data point is inaccurate + +Datadog implicitly rolls up data points automatically with the `avg` method, effectively displaying the average of all data points within a time interval for a given metric. Essentially, there is a rollup for each point. The values at the end attempt to have the rollup applied. When this occurs, it looks at a X second bucket according to your time window, and will default average those values together. Since this is the last point in the query, there are no other values to average with in that X second bucket. This leads to the value of last data point that was not rolled up in the same fashion as the others, and leads to an inaccurate number. In these cases, adjust `lastAvailablePointOffset` to 1 to use the second to last points of an API response would be the most accurate. + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: datadog-scaledobject + namespace: my-project +spec: + scaleTargetRef: + name: worker + triggers: + - type: datadog + metricType: "AverageValue" + metadata: + query: "sum:trace.express.request.hits{*}.as_rate()" + queryValue: "100" + age: "90" + metricUnavailableValue: "0" + # Optional: The delayed time window offset (in seconds) to wait for the metric to be available. The values of some queries might be not available at now and need a small delay to become available, try to adjust it if you encounter this issue. Default: 0 + timeWindowOffset: "30" + # Optional: The offset to retrieve the X to last data point. The value of last data point of some queries might be inaccurate, try to adjust to 1 if you encounter this issue. Default: 0 + lastAvailablePointOffset: "1" + authenticationRef: + name: keda-trigger-auth-datadog-secret +``` + +Check [here](https://github.com/kedacore/keda/pull/3954#discussion_r1042820206) for the details of this issue. diff --git a/content/docs/2.17/scalers/dynatrace.md b/content/docs/2.17/scalers/dynatrace.md new file mode 100644 index 000000000..848ee3253 --- /dev/null +++ b/content/docs/2.17/scalers/dynatrace.md @@ -0,0 +1,93 @@ ++++ +title = "Dynatrace" +availability = "2.15+" +maintainer = "Community" +category = "Metrics" +description = "Scale applications based on Dynatrace metric data points" +go_file = "dynatrace_scaler" ++++ + +### Trigger Specification + +This specification describes the `dynatrace` trigger that scales based on Dynatrace metric data points. + +```yaml +triggers: + - type: dynatrace + metadata: + host: https://dummy-instance.live.dynatrace.com/ + metricSelector: 'MyCustomEvent:filter(eq("someProperty","someValue")):count:splitBy("dt.entity.process_group"):fold' + # Optional + from: now-2d + threshold: "10" + # Optional + activationThreshold: "5" +``` + +**Parameter list:** + +- `host` - The Dynatrace instance to query against +- `metricSelector` - The metric selector query and any transformations that should be applied to it ([transformations docs](https://docs.dynatrace.com/docs/dynatrace-api/environment-api/metric-v2/metric-selector)). + + Some relevant aspects: + - The selector should focus on a **single metric and dimension**: if multiple are found, a warning is issued and only the first one is considered + - The metric data points should be aggregated to produce a single output value (e.g., using the [fold transformation](https://docs.dynatrace.com/docs/shortlink/api-metrics-v2-selector#fold)): if multiple values are found, only the first one is considered + - If you need to use the entity selector, do it through the `:filter` transformation in the metric selector +- `from` - How far back the metric selector should consider when fetching data points. [syntax supported](https://docs.dynatrace.com/docs/dynatrace-api/environment-api/metric-v2/get-data-points#parameters). (Optional, default `now-2h`, i.e., the last 2 hours) +- `threshold` - A threshold that is used as the `targetValue` or `targetAverageValue` (depending on the trigger metric type) in the HPA configuration. (This value can be a float) +- `activationThreshold` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Optional, default `0`, can be a float) + +### Authentication Parameters + +You can use `TriggerAuthentication` CRD to configure authentication the `host` and `token` parameters. + +**Authentication:** + +- `host` - The Dynatrace instance to query against +- `token` - The API key that will be leveraged to connect to Dynatrace and make requests ([official documentation](https://docs.dynatrace.com/docs/dynatrace-api/basics/dynatrace-api-authentication)). Requires the `metrics.read` scope + +### Example + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: dynatrace-secret + namespace: my-project +type: Opaque +data: + host: aHR0cHM6Ly9kdW1teS1pbnN0YW5jZS5saXZlLmR5bmF0cmFjZS5jb20vCg== # base64 encoding of https://dummy-instance.live.dynatrace.com/ + token: ZHQwczAxLlNUMkVZNzJLUUlOTUg1NzRXTU5WSTdZTi5HM0RGUEJFSllNT0RJREFFWDQ1NE03WVdCVVZFRk9XS1BSVk1XRkFTUzY0TkZINTJQWDZCTkRWRkZNNTcyUlpNCg== # base64 encoding of the dynatrace example api key dt0s01.ST2EY72KQINMH574WMNVI7YN.G3DFPBEJYMODIDAEX454M7YWBUVEFOWKPRVMWFASS64NFH52PX6BNDVFFM572RZM +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-dynatrace + namespace: my-project +spec: + secretTargetRef: + - parameter: token + name: dynatrace-secret + key: token + - parameter: host + name: dynatrace-secret + key: host +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: dynatrace-scaledobject + namespace: keda +spec: + maxReplicaCount: 12 + scaleTargetRef: + name: dummy + triggers: + - type: dynatrace + metadata: + metricSelector: 'MyCustomEvent:filter(eq("someProperty","someValue")):count:splitBy("dt.entity.process_group"):fold' + from: 'now-30m' + threshold: '1000' + authenticationRef: + name: keda-trigger-auth-dynatrace +``` diff --git a/content/docs/2.17/scalers/elasticsearch.md b/content/docs/2.17/scalers/elasticsearch.md new file mode 100644 index 000000000..c53b22f35 --- /dev/null +++ b/content/docs/2.17/scalers/elasticsearch.md @@ -0,0 +1,175 @@ ++++ +title = "Elasticsearch" +availability = "v2.5+" +maintainer = "Community" +category = "Data & Storage" +description = "Scale applications based on 'elasticsearch search template query' or 'elasticsearch query' result." +go_file = "elasticsearch_scaler" ++++ + +### Trigger Specification + +This specification describes the `elasticsearch` trigger that scales based on result of an [elasticsearch search template](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-template.html) query or [elasticsearch query](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html). + +The trigger always requires the following information, but requires either only searchTemplateName **or** only query: + +```yaml +triggers: + - type: elasticsearch + metadata: + addresses: "http://localhost:9200" + username: "elastic" + passwordFromEnv: "ELASTIC_PASSWORD" + index: "my-index" + searchTemplateName: "my-search-template-name" + query: "my-query" + parameters: "param1:value1;param2:value2" + valueLocation: "hits.total.value" + targetValue: "1.1" + activationTargetValue: "5.5" +``` + +**Parameter list:** + +- `addresses` - Comma separated list of hosts and ports of the Elasticsearch cluster client nodes. +- `username` - Username to authenticate with to Elasticsearch cluster. +- `passwordFromEnv` - Environment variable to read the authentication password from to authenticate with the Elasticsearch cluster. +- `index` - Index to run the search template query on. It supports multiple indexes separated by a semicolon character ( `;` ). +- `searchTemplateName` - The search template name to run. +- `query` - The query to run. +- `targetValue` - Target value to scale on. When the metric provided by the API is equal or higher to this value, KEDA will start scaling out. When the metric is 0 or less, KEDA will scale down to 0. (This value can be a float) +- `activationTargetValue` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds).(Default: `0`, Optional, This value can be a float) +- `parameters` - Parameters that will be used by the search template. It supports multiple params separated by a semicolon character ( `;` ). +- `valueLocation` - [GJSON path notation](https://github.com/tidwall/gjson#path-syntax) to refer to the field in the payload containing the metric value. +- `unsafeSsl` - Skip certificate validation when connecting over HTTPS. (Values: `true`, `false`, Default: `false`, Optional) + +### Authentication Parameters + +You can authenticate by using a username/password or apiKey/cloudID if you're using using ElasticSearch on Elastic Cloud. + +**Password Authentication:** + +- `username` - Username to authenticate with to Elasticsearch cluster. +- `password` - Password for configured user to login to Elasticsearch cluster. + +**Cloud ID and API Key Authentication:** + +[Cloud ID](https://www.elastic.co/guide/en/cloud/current/ec-cloud-id.html) and API Key can be used for Elastic Cloud Service. + +- `cloudID` - CloudID to connect with ElasticSearch on Elastic Cloud. +- `apiKey` - API key to authenticate with ElasticSearch on Elastic Cloud. + +### Examples + +Here is an example of how to deploy a scaled object with the `elasticsearch` scale trigger which uses the search template and `TriggerAuthentication`. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: elasticsearch-secrets +type: Opaque +data: + password: cGFzc3cwcmQh +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-elasticsearch-secret +spec: + secretTargetRef: + - parameter: password + name: elasticsearch-secrets + key: password +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: elasticsearch-scaledobject +spec: + scaleTargetRef: + name: "deployment-name" + triggers: + - type: elasticsearch + metadata: + addresses: "http://localhost:9200" + username: "elastic" + index: "my-index" + searchTemplateName: "my-search-template" + valueLocation: "hits.total.value" + targetValue: "10" + parameters: "dummy_value:1" + authenticationRef: + name: keda-trigger-auth-elasticsearch-secret +``` +Here is an example of how to deploy a scaled object with the `elasticsearch` scale trigger which uses `query`. In this example the transactions will be count that the application has to process based on APM. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: elasticsearch-secrets +type: Opaque +data: + password: cGFzc3cwcmQh +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-elasticsearch-secret +spec: + secretTargetRef: + - parameter: password + name: elasticsearch-secrets + key: password +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: elasticsearch-scaledobject +spec: + scaleTargetRef: + name: "deployment-name" + triggers: + - type: elasticsearch + metadata: + addresses: "http://localhost:9200" + username: "elastic" + index: "my-index" + query: | + { + "size": 0, + "query": { + "bool": { + "must": [ + { + "term": { + "service.name": "my-application" } + }, + { + "term": { + "service.environment": "production" } + }, + { + "range": { + "@timestamp": { + "gte": "now-2m", + "lte": "now-1m" + } + } + } + ] + } + }, + "aggs": { + "transaction_count": { + "cardinality": { + "field": "transaction.id" } + } + } + } + valueLocation: "aggregations.transaction_count.value" + targetValue: "1000" + authenticationRef: + name: keda-trigger-auth-elasticsearch-secret +``` diff --git a/content/docs/2.17/scalers/etcd.md b/content/docs/2.17/scalers/etcd.md new file mode 100644 index 000000000..b28ed7b1b --- /dev/null +++ b/content/docs/2.17/scalers/etcd.md @@ -0,0 +1,171 @@ ++++ +title = "Etcd" +availability = "v2.9+" +maintainer = "Huawei Cloud" +category = "Data & Storage" +description = "Scale applications based on an etcd key-value pair. By watching an etcd key, a passively received push mode, the scaler can activate applications with lower load usage than frequent pull mode" +go_file = "etcd_scaler" ++++ + +### Trigger Specification + +This specification describes the `etcd` trigger that scales based on an etcd key-value pair. + +```yaml + triggers: + - type: etcd + metadata: + endpoints: 172.0.0.1:2379,172.0.0.2:2379,172.0.0.3:2379 + watchKey: length + value: '5.5' + activationValue: '0.5' + watchProgressNotifyInterval: '600' + usernameFromEnv: # Optional + passwordFromEnv: # Optional +``` + +**Parameter list:** + +- `endpoints` - Etcd servers' endpoints information. It supports multiple endpoints split by a comma character (`,`). +- `watchKey` - Name of the etcd key used for the scaler client to get/watch the etcd value from etcd servers. +- `value` - Target relation between the scaled workload and the etcd value. It will be calculated following this formula: relation = (etcd value) / (scaled workload pods). (This value can be a float) +- `activationValue` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds).(Default: `0`, Optional, This value can be a float) +- `watchProgressNotifyInterval` - Set this parameter to the same as `--experimental-watch-progress-notify-interval` parameter of etcd servers. (Default: `600`, units are seconds, Optional) +- `usernameFromEnv` - Provide admin username from env instead of as a secret. (Optional) +- `passwordFromEnv` - Provide admin password from env instead of as a secret. (Optional) + +### Authentication Parameters + +You can use `TriggerAuthentication` CRD to configure the authenticate by tls. It is required you should set `tls` to `enable`. If required for your etcd configuration, you may also provide a `ca`, `cert`, `key` and `keyPassword`. `cert` and `key` must be specified together. + +**Password based authentication:** + +- `username` - Username for authentication. (Optional) +- `password` - Password for authentication. (Optional) + +**Credential based authentication:** + +**TLS:** + +- `tls` - To enable SSL auth for etcd, set this to `enable`. If not set, TLS for etcd is not used. (Values: `enable`, `disable`, Default: `disable`, Optional) +- `ca` - Certificate authority file for TLS client authentication. (Optional) +- `cert` - Certificate for client authentication. (Optional) +- `key` - Key for client authentication. (Optional) +- `keyPassword` - If set the `keyPassword` is used to decrypt the provided `key`. (Optional) + +### Example + +Your etcd cluster no auth: + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: etcd-scaledobject +spec: + scaleTargetRef: + name: my-app-target + pollingInterval: 30 + triggers: + - type: etcd + metadata: + endpoints: 172.0.0.1:2379,172.0.0.2:2379,172.0.0.3:2379 + watchKey: length + value: '5.5' +``` + +Your etcd cluster turn on Password auth: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-etcd-secrets + namespace: default +data: + etcd-username: + etcd-password: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-etcd-credential + namespace: default +spec: + secretTargetRef: + - parameter: username + name: keda-etcd-secrets + key: etcd-username + - parameter: password + name: keda-etcd-secrets + key: etcd-password +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: etcd-scaledobject +spec: + scaleTargetRef: + name: my-app-target + pollingInterval: 30 + triggers: + - type: etcd + metadata: + endpoints: 172.0.0.1:2379,172.0.0.2:2379,172.0.0.3:2379 + watchKey: length + value: '5.5' + authenticationRef: + name: keda-trigger-auth-etcd-credential +``` + +Your etcd cluster turn on SASL/TLS auth: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-etcd-secrets + namespace: default +data: + tls: ZW5hYmxl + ca: + cert: + key: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-etcd-credential + namespace: default +spec: + secretTargetRef: + - parameter: tls + name: keda-etcd-secrets + key: tls + - parameter: ca + name: keda-etcd-secrets + key: ca + - parameter: cert + name: keda-etcd-secrets + key: cert + - parameter: key + name: keda-etcd-secrets + key: key +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: etcd-scaledobject +spec: + scaleTargetRef: + name: my-app-target + pollingInterval: 30 + triggers: + - type: etcd + metadata: + endpoints: 172.0.0.1:2379,172.0.0.2:2379,172.0.0.3:2379 + watchKey: length + value: '5.5' + authenticationRef: + name: keda-trigger-auth-etcd-credential +``` diff --git a/content/docs/2.17/scalers/external-push.md b/content/docs/2.17/scalers/external-push.md new file mode 100644 index 000000000..ffabbc0fe --- /dev/null +++ b/content/docs/2.17/scalers/external-push.md @@ -0,0 +1,100 @@ ++++ +title = "External Push" +availability = "v2.0+" +maintainer = "Microsoft" +category = "Extensibility" +description = "Scale applications based on an external push scaler." +go_file = "external_scaler" ++++ + +### Trigger Specification + +This specification describes the `external-push` trigger for an external push scaler. + +```yaml +triggers: +- type: external-push + metadata: + scalerAddress: external-scaler-service:8080 + caCert : /path/to/tls/ca.pem + tlsCertFile: /path/to/tls/cert.pem # Deprecated. https://github.com/kedacore/keda/issues/4549 + tlsClientCert: /path/to/tls/cert.pem + tlsClientKey: /path/to/tls/key.pem + unsafeSsl: false +``` + +**Parameter list:** + +- `scalerAddress` - Address of the external push scaler implementing `ExternalScaler.StreamIsActive` in externalscaler.proto. Format must be `host:port`. +- `tlsCertFile` - Location of a certificate to use for the GRPC connection to authenticate with. (Optional) +- `unsafeSsl` - Skip certificate validation when connecting over HTTPS. (Values: `true`, `false`, Default: `false`, Optional) + +The entire metadata object is passed to the external scaler in `ScaledObjectRef.scalerMetadata`. + +> For implementing an external scaler, refer to [External Scalers Concept](../concepts/external-scalers.md). + +### Authentication Parameters + +- `caCert` - Certificate Authority (CA) certificate to use for the GRPC connection to authenticate with. (Optional) +- `tlsClientCert` - Client certificate to use for the GRPC connection to authenticate with. (Optional) +- `tlsClientKey` - Client private key to use for the GRPC connection to authenticate with. (Optional) + +### Example + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: external-scaledobject +spec: + scaleTargetRef: + name: keda-node + triggers: + - type: external-push + metadata: + scalerAddress: external-scaler-service:8080 + unsafeSsl: false +``` + +Here is an example of external scaler with certificates + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: certificate +data: + ca.crt: "YOUR_CA_IN_SECRET" + tls.crt: "YOUR_CERTIFICATE_IN_SECRET" + tls.key: "YOUR_KEY_IN_SECRET" +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth +spec: + secretTargetRef: + - parameter: caCert + name: certificate + key: ca.crt + - parameter: tlsClientCert + name: certificate + key: tls.crt + - parameter: tlsClientKey + name: certificate + key: tls.key +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: external-scaledobject +spec: + scaleTargetRef: + name: keda-node + triggers: + - type: external-push + metadata: + scalerAddress: external-scaler-service:8080 + authenticationRef: + name: keda-trigger-auth +``` diff --git a/content/docs/2.17/scalers/external.md b/content/docs/2.17/scalers/external.md new file mode 100644 index 000000000..ec785073d --- /dev/null +++ b/content/docs/2.17/scalers/external.md @@ -0,0 +1,106 @@ ++++ +title = "External" +availability = "v1.0+" +maintainer = "Microsoft" +category = "Extensibility" +description = "Scale applications based on an external scaler." +go_file = "external_scaler" ++++ + +### Trigger Specification + +This specification describes the `external` trigger for an external scaler. + +```yaml +triggers: +- type: external + metadata: + scalerAddress: external-scaler-service:8080 + caCert : /path/to/tls/ca.pem + tlsCertFile: /path/to/tls/cert.pem # Deprecated. https://github.com/kedacore/keda/issues/4549 + tlsClientCert: /path/to/tls/cert.pem + tlsClientKey: /path/to/tls/key.pem + unsafeSsl: false +``` + +**Parameter list:** + +- `scalerAddress` - Address of the external scaler. Format must be `host:port`. +- `tlsCertFile` - Location of a certificate to use for the GRPC connection to authenticate with. (Optional) +- `unsafeSsl` - Skip certificate validation when connecting over HTTPS. (Values: `true`, `false`, Default: `false`, Optional) + +The entire metadata object is passed to the external scaler in `ScaledObjectRef.scalerMetadata`. + +> For implementing an external scaler, refer to [External Scalers Concept](../concepts/external-scalers.md). + +### Authentication Parameters + +- `caCert` - Certificate Authority (CA) certificate to use for the GRPC connection to authenticate with. (Optional) +- `tlsClientCert` - Client certificate to use for the GRPC connection to authenticate with. (Optional) +- `tlsClientKey` - Client private key to use for the GRPC connection to authenticate with. (Optional) + +### Example + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: external-scaledobject +spec: + scaleTargetRef: + name: keda-redis-node + triggers: + - type: external + metadata: + scalerAddress: redis-external-scaler-service:8080 + address: REDIS_HOST + password: REDIS_PASSWORD + listName: mylist + listLength: "5" +``` + +Here is an example of external scaler with certificates + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: certificate +data: + ca.crt: "YOUR_CA_IN_SECRET" + tls.crt: "YOUR_CERTIFICATE_IN_SECRET" + tls.key: "YOUR_KEY_IN_SECRET" +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth +spec: + secretTargetRef: + - parameter: caCert + name: certificate + key: ca.crt + - parameter: tlsClientCert + name: certificate + key: tls.crt + - parameter: tlsClientKey + name: certificate + key: tls.key +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: external-scaledobject +spec: + scaleTargetRef: + name: keda-redis-node + triggers: + - type: external + metadata: + scalerAddress: redis-external-scaler-service:8080 + metricType: mymetric + scalerAddress: mydomain.com:443 + extraKey: "demo" + authenticationRef: + name: keda-trigger-auth +``` \ No newline at end of file diff --git a/content/docs/2.17/scalers/gcp-cloud-tasks.md b/content/docs/2.17/scalers/gcp-cloud-tasks.md new file mode 100644 index 000000000..0c504c8ba --- /dev/null +++ b/content/docs/2.17/scalers/gcp-cloud-tasks.md @@ -0,0 +1,151 @@ ++++ +title = "Google Cloud Platform Cloud Tasks" +availability = "v2.12+" +maintainer = "Community" +description = "Scale applications based on Google Cloud Platform Cloud Tasks." +go_file = "gcp_cloud_tasks_scaler" ++++ + +### Trigger Specification + +This specification describes the `gcp-cloudtasks` trigger for Google Cloud Platform Cloud Tasks. + +```yaml +triggers: +- type: gcp-cloudtasks + metadata: + value: "5" # Optional - Default is 100 + activationValue: "10.5" # Optional - Default is 0 + filterDuration: '1' # Optional - Default is 2 + queueName: "myqueue" # Required + projectID: "myproject" # Required, the project where the queue resides + credentialsFromEnv: GOOGLE_APPLICATION_CREDENTIALS_JSON # Required +``` + +The Google Cloud Platform (GCP) Cloud Tasks trigger allows you to scale based on the number of tasks queued in you queue. + +The `credentialsFromEnv` property maps to the name of an environment variable in the scale target (`scaleTargetRef`) that contains the service account credentials (JSON). KEDA will use those to connect to Google Cloud Platform and collect the required stack driver metrics in order to read the number of messages in the Cloud Task queue. + +- `value` - Target value for the scaler. (Default: `100`, Optional, This value can be a float) +- `activationValue` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional, This value can be a float) +- `queueName` defines the queue that should be monitored. +- `projectID` defines the GCP project where the queue that should be monitored resides. +- `filterDuration` - Duration (in minutes) for filtering metrics. (Default: `2`) + +### Authentication Parameters +You can use `TriggerAuthentication` CRD to configure the authenticate by providing the service account credentials in JSON. + +**Credential based authentication:** + +- `GoogleApplicationCredentials` - Service account credentials in JSON. + +### Example + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: cloudtasks-scaledobject + namespace: keda-cloudtasks-test +spec: + scaleTargetRef: + name: keda-cloudtasks-go + triggers: + - type: gcp-cloudtasks + metadata: + activationValue: "5" + projectID: "myproject" # Required + queueName: "myqueue" # Required + credentialsFromEnv: GOOGLE_APPLICATION_CREDENTIALS_JSON # Required +``` + +### Example using TriggerAuthentication + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-gcp-credentials +spec: + secretTargetRef: + - parameter: GoogleApplicationCredentials + name: cloudtasks-secret # Required. Refers to the name of the secret + key: GOOGLE_APPLICATION_CREDENTIALS_JSON # Required. +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: cloudtasks-scaledobject +spec: + scaleTargetRef: + name: keda-cloudtasks-go + triggers: + - type: gcp-cloudtasks + authenticationRef: + name: keda-trigger-auth-gcp-credentials + metadata: + activationValue: "5" + projectID: "myproject" # Required + queueName: "myqueue" # Required +``` + +**Identity based authentication:** + +You can also use `TriggerAuthentication` CRD to configure the authentication using the associated service account of the running machine in Google Cloud. You only need to create a `TriggerAuthentication` as this example, and reference it in the `ScaledObject`. `ClusterTriggerAuthentication` can also be used if you intend to use it globally in your cluster. + +### Example using TriggerAuthentication with GCP Identity + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-gcp-credentials +spec: + podIdentity: + provider: gcp +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: cloudtasks-scaledobject +spec: + scaleTargetRef: + name: keda-cloudtasks-go + triggers: + - type: gcp-cloudtasks + authenticationRef: + name: keda-trigger-auth-gcp-credentials + metadata: + activationValue: "5" + projectID: "myproject" # Required + queueName: "myqueue" # Required +``` + +## Example using ClusterTriggerAuthentication with GCP Identity + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ClusterTriggerAuthentication +metadata: + name: keda-clustertrigger-auth-gcp-credentials +spec: + podIdentity: + provider: gcp +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: cloudtasks-scaledobject +spec: + scaleTargetRef: + name: keda-cloudtasks-go + triggers: + - type: gcp-cloudtasks + authenticationRef: + name: keda-clustertrigger-auth-gcp-credentials + kind: ClusterTriggerAuthentication + metadata: + activationValue: "5" + projectID: "myproject" # Required + queueName: "myqueue" # Required +``` diff --git a/content/docs/2.17/scalers/gcp-pub-sub.md b/content/docs/2.17/scalers/gcp-pub-sub.md new file mode 100644 index 000000000..9427d9bb9 --- /dev/null +++ b/content/docs/2.17/scalers/gcp-pub-sub.md @@ -0,0 +1,195 @@ ++++ +title = "Google Cloud Platform Pub/Sub" +availability = "v1.0+" +maintainer = "Community" +category = "Messaging" +description = "Scale applications based on Google Cloud Platform Pub/Sub." +go_file = "gcp_pubsub_scaler" ++++ + +### Trigger Specification + +This specification describes the `gcp-pubsub` trigger for Google Cloud Platform Pub/Sub. + +```yaml +triggers: +- type: gcp-pubsub + metadata: + subscriptionSize: "5" # Deprecated, use mode and value fields instead + mode: "SubscriptionSize" # Optional - Default is SubscriptionSize - SubscriptionSize or OldestUnackedMessageAge + aggregation: "sum" # Optional - Only meaningful for distribution-valued metrics + value: "5.5" # Optional - Default is 10 + valueIfNull: '0.0' # Optional - Default is "" + activationValue: "10.5" # Optional - Default is 0 + timeHorizon: "1m" # Optional - Default is 2m and with aggregation 5m + # Exactly one of the subscription or topic name options is required + subscriptionName: "mysubscription" + subscriptionNameFromEnv: "MY_SUBSCRIPTION_FROM_ENV" + topicName: "mytopic" + topicNameFromEnv: "MY_TOPIC_FROM_ENV" + credentialsFromEnv: GOOGLE_APPLICATION_CREDENTIALS_JSON # Required +``` + +The Google Cloud Platform (GCP) Pub/Sub trigger allows you to scale based on any metrics from your Pub/Sub subscription or topic, such as number of messages or oldest unacked message age, etc. + +- `credentialsFromEnv` - This property maps to the name of an environment variable in the scale target (`scaleTargetRef`) that contains the service account credentials (JSON). KEDA will use those to connect to Google Cloud Platform and collect the required stack driver metrics in order to read the number of messages in the Pub/Sub subscription. + +- `mode` - The metric used to scale your workload. It's the `PascalCase` of the official metric name. For example, if you are going to leverage the metric `subscription/pull_request_count`, you will fill the value as `PullRequestCount`. Please refer to https://cloud.google.com/monitoring/api/metrics_gcp#gcp-pubsub. All metrics starting with `subscription/` and `topic/` are supported. (Default: `SubscriptionSize`, aka `NumUndeliveredMessages`) + +- `aggregation` - The aggregation function used to aggregate the metric. (Values: mean, median, variance, stddev, sum, count, percentileX (X: an integer in the range 0 to 100), Required for metrics with value type `DISTRIBUTION`) + + +- `activationValue` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds).(Default: `0`, Optional, This value can be a float) + +- `timeHorizon` - Time range for which you want to retrieve the matrics. (Default: `2m` and Default with aggregation field: `5m`) + +- `valueIfNull` - Value returned if request returns no timeseries.(Default: `""`, Optional, This value can be a float) + +- `subscriptionName` defines the subscription that should be monitored. You can use different formulas: + - Just the subscription name, in which case you will reference a subscription from the current project or the one specified in the credentials file used. + - Use the full link provided by Google, so that you can reference a subscription that is hosted in another project Eg: `projects/myproject/subscriptions/mysubscription`. + +- `subscriptionNameFromEnv` - The name of an environment variable on the scale target that holds the subscription name. The resolved name is processed the same as `subscriptionName`. + +- `topicName` defines the topic that should be monitored. Similar to `susbcriptionName`, you can use different formulas: + - Just the topic name, in which case you will reference a topic from the current project or the one specified in the credentials file used. + - Use the full link provided by Google, so that you can reference a topic that is hosted in another project Eg: `projects/myproject/topics/mytopic`. + +- `topicNameFromEnv` - The name of an environment variable on the scale target that holds the topic name. The resolved name is processed the same as `topicName`. + +Here's an [example](https://github.com/kedacore/sample-go-gcppubsub). + +### Authentication Parameters +You can use `TriggerAuthentication` CRD to configure the authenticate by providing the service account credentials in JSON. + + +**Credential based authentication:** + +- `GoogleApplicationCredentials` - Service account credentials in JSON. + +### Example + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: pubsub-scaledobject + namespace: keda-pubsub-test +spec: + scaleTargetRef: + name: keda-pubsub-go + triggers: + - type: gcp-pubsub + metadata: + mode: "SubscriptionSize" + value: "5" + subscriptionName: "mysubscription" # Required + credentialsFromEnv: GOOGLE_APPLICATION_CREDENTIALS_JSON # Required +``` + +### Example using distribution-valued metric and aggregation + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: pubsub-scaledobject + namespace: keda-pubsub-test +spec: + scaleTargetRef: + name: keda-pubsub-go + triggers: + - type: gcp-pubsub + metadata: + # Example: https://cloud.google.com/pubsub/docs/monitoring#monitoring_message_throughput_2 + mode: "MessageSizes" + aggregation: "count" # Calculate count of messages being published + value: "50" + topicName: "mytopic" # Required + credentialsFromEnv: GOOGLE_APPLICATION_CREDENTIALS_JSON # Required +``` + +### Example using TriggerAuthentication + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-gcp-credentials +spec: + secretTargetRef: + - parameter: GoogleApplicationCredentials + name: pubsub-secret # Required. Refers to the name of the secret + key: GOOGLE_APPLICATION_CREDENTIALS_JSON # Required. +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: pubsub-scaledobject +spec: + scaleTargetRef: + name: keda-pubsub-go + triggers: + - type: gcp-pubsub + authenticationRef: + name: keda-trigger-auth-gcp-credentials + metadata: + subscriptionName: "input" # Required +``` + +**Identity based authentication:** + +You can also use `TriggerAuthentication` CRD to configure the authentication using the associated service account of the running machine in Google Cloud. You only need to create a `TriggerAuthentication` as this example, and reference it in the `ScaledObject`. `ClusterTriggerAuthentication` can also be used if you intend to use it globally in your cluster. + +### Example using TriggerAuthentication with GCP Identity + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-gcp-credentials +spec: + podIdentity: + provider: gcp +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: pubsub-scaledobject +spec: + scaleTargetRef: + name: keda-pubsub-go + triggers: + - type: gcp-pubsub + authenticationRef: + name: keda-trigger-auth-gcp-credentials + metadata: + subscriptionName: "input" # Required +``` + +## Example using ClusterTriggerAuthentication with GCP Identity + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ClusterTriggerAuthentication +metadata: + name: keda-clustertrigger-auth-gcp-credentials +spec: + podIdentity: + provider: gcp +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: pubsub-scaledobject +spec: + scaleTargetRef: + name: keda-pubsub-go + triggers: + - type: gcp-pubsub + authenticationRef: + name: keda-clustertrigger-auth-gcp-credentials + kind: ClusterTriggerAuthentication + metadata: + subscriptionName: "input" # Required +``` diff --git a/content/docs/2.17/scalers/gcp-stackdriver.md b/content/docs/2.17/scalers/gcp-stackdriver.md new file mode 100644 index 000000000..39e425b33 --- /dev/null +++ b/content/docs/2.17/scalers/gcp-stackdriver.md @@ -0,0 +1,137 @@ ++++ +title = "Google Cloud Platform Stackdriver" +availability = "2.7+" +maintainer = "Community" +category = "Metrics" +description = "Scale applications based on a metric obtained from Stackdriver." +go_file = "gcp_stackdriver_scaler" ++++ + +### Trigger Specification + +This specification describes the `gcp-stackdriver` trigger for GCP Stackdriver. It scales based on a metric obtained from issuing a query to Stackdriver. + +```yaml +triggers: +- type: gcp-stackdriver + metadata: + projectId: my-project-id + filter: 'metric.type="storage.googleapis.com/network/received_bytes_count" AND resource.type="gcs_bucket" AND metric.label.method="WriteObject" AND resource.label.bucket_name="my-gcp-bucket"' + targetValue: '100.50' + valueIfNull: '0.0' #Optional - Default is "" + filterDuration: '1' # Optional - Default is 2 + activationTargetValue: "10.5" # Optional - Default is 0 + credentialsFromEnv: GOOGLE_APPLICATION_CREDENTIALS_JSON + alignmentPeriodSeconds: '60' + alignmentAligner: mean + alignmentReducer: none +``` + +**Parameter list:** + +- `projectId` - GCP project Id that contains the metric. +- `filter` - The stackdriver query filter for obtaining the metric. The metric is for the last minute and if multiple values are returned, the first one is used. +- `targetValue` - Average target value to trigger scaling actions. (Default: `5`, Optional, This value can be a float) +- `activationTargetValue` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds).(Default: `0`, Optional, This value can be a float) +- `valueIfNull` - Value return if request return no timeseries.(Default: `""`, Optional, This value can be a float) +- `filterDuration` - Duration (in minutes) for filtering metrics. (Default: `2`) + +The `credentialsFromEnv` property maps to the name of an environment variable in the scale target (`scaleTargetRef`) that contains the service account credentials (JSON). KEDA will use those to connect to Google Cloud Platform and collect the configured stack driver metrics. + +The `alignmentPeriodSeconds`, `alignmentAligner` and `alignmentReducer` properties controls time series aggregation before the metrics are returned. See below for more details. + +### Timeseries alignment properties +It is much better to aggregate the time series values before they are returned from stackdriver instead of getting the raw values. +For that, you must specify a value of 60 or more for the `alignmentPeriodSeconds` property as well as an alignment operation in the `alignmentAligner` property and/or a reducer in the `alignmentReducer` property. + +Valid values for the `alignmentAligner` property are: none, delta, rate, interpolate, next_older, min, max, mean, count, sum, stddev, count_true, count_false, fraction_true, percentile_99, percentile_95, percentile_50, percentile_05 and percent_change. +Valid values for the `alignmentReducer` property are: none, mean, min, max, sum, stddev, count, count_true, count_false, fraction_true, percentile_99, percentile_95, percentile_50 and percentile_05. + +For more information on aggregation, see [here](https://cloud.google.com/monitoring/api/v3/aggregation#aggr-intro). + +### Authentication Parameters +You can use `TriggerAuthentication` CRD to configure the authenticate by providing the service account credentials in JSON. + +**Credential based authentication:** + +- `GoogleApplicationCredentials` - Service account credentials in JSON. + +**Identity based authentication:** + +You can also use `TriggerAuthentication` CRD to configure the authentication using the associated service account of the running machine in Google Cloud. You only need to create a `TriggerAuthentication` as this example, and reference it in the `ScaledObject`. `ClusterTriggerAuthentication` can also be used if you intend to use it globally in your cluster. + +### Examples + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: gcp-stackdriver-scaledobject + namespace: keda-gcp-stackdriver-test +spec: + scaleTargetRef: + name: keda-gcp-stackdriver-go + triggers: + - type: gcp-stackdriver + metadata: + projectId: my-project-id + filter: 'metric.type="storage.googleapis.com/network/received_bytes_count" AND resource.type="gcs_bucket" AND metric.label.method="WriteObject" AND resource.label.bucket_name="my-gcp-bucket"' + targetValue: "5" + credentialsFromEnv: GOOGLE_APPLICATION_CREDENTIALS_JSON +``` + +#### Use TriggerAuthentication with Kubernetes secret + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-gcp-credentials +spec: + secretTargetRef: + - parameter: GoogleApplicationCredentials + name: gcp-stackdriver-secret # Required. Refers to the name of the secret + key: GOOGLE_APPLICATION_CREDENTIALS_JSON # Required. +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: gcp-stackdriver-scaledobject +spec: + scaleTargetRef: + name: keda-gcp-stackdriver-go + triggers: + - type: gcp-stackdriver + authenticationRef: + name: keda-trigger-auth-gcp-credentials + metadata: + projectId: my-project-id + filter: 'metric.type="storage.googleapis.com/network/received_bytes_count" AND resource.type="gcs_bucket" AND metric.label.method="WriteObject" AND resource.label.bucket_name="my-gcp-bucket"' +``` + +#### Use TriggerAuthentication with GCP Identity + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-gcp-credentials +spec: + podIdentity: + provider: gcp +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: gcp-stackdriver-scaledobject +spec: + scaleTargetRef: + name: keda-gcp-stackdriver-go + triggers: + - type: gcp-stackdriver + authenticationRef: + name: keda-trigger-auth-gcp-credentials + metadata: + projectId: my-project-id + filter: 'metric.type="storage.googleapis.com/network/received_bytes_count" AND resource.type="gcs_bucket" AND metric.label.method="WriteObject" AND resource.label.bucket_name="my-gcp-bucket"' +``` diff --git a/content/docs/2.17/scalers/gcp-storage.md b/content/docs/2.17/scalers/gcp-storage.md new file mode 100644 index 000000000..e32b5c912 --- /dev/null +++ b/content/docs/2.17/scalers/gcp-storage.md @@ -0,0 +1,130 @@ ++++ +title = "Google Cloud Platform Storage" +availability = "2.7+" +maintainer = "Community" +category = "Data & Storage" +description = "Scale applications based on the count of objects in a given Google Cloud Storage (GCS) bucket." +go_file = "gcp_storage_scaler" ++++ + +### Trigger Specification + +This specification describes the `gcp-storage` scaler, which scales Kubernetes workloads based on the count of objects in a given Google Cloud Storage (GCS) bucket. This scaler assumes the worker, when run, will process and clear the bucket by deleting/moving objects therein. + +```yaml +triggers: +- type: gcp-storage + metadata: + bucketName: test-bucket + targetObjectCount: '100' + activationTargetObjectCount: '10' # Optional + maxBucketItemsToScan: '1000' + credentialsFromEnv: GOOGLE_APPLICATION_CREDENTIALS_JSON # Optional + credentialsFromEnvFile: GOOGLE_APPLICATION_CREDENTIALS_JSON # Optional + blobPrefix: # Optional. Prefix for the Blob. Use this to specify sub path for the blobs if required. Default : "" + blobDelimiter: # Optional. Delimiter for identifying the blob Prefix. Default: "" +``` + +**Parameter list:** + +- `bucketName` - Name of the bucket in GCS. +- `targetObjectCount` - Average target value to trigger scaling actions. (Default: `100`, Optional) +- `activationTargetObjectCount` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) +- `maxBucketItemsToScan` - When to stop counting how many objects are in the bucket. (Default: `1000`, Optional) +- `blobPrefix` - Prefix for the Blob. Use this to specify sub path for the blobs if required. (Default: `""`, Optional) +- `blobDelimiter` - Delimiter for identifying the blob prefix. (Default: `""`, Optional) +As counting the number of objects involves iterating over their metadata it is advised to set this number to the value of `targetObjectCount` * `maxReplicaCount`. + +The metric name will be generated automatically based on the trigger index and `bucketName`, for example: **s0-gcp-storage-bucketName**. + +You can provide in the metadata either `credentialsFromEnv` or `credentialsFromEnvFile`. +- `credentialsFromEnv` - Set to the name of the environment variable that holds the credential information. +- `credentialsFromEnvFile` - Set to the name of a json file that holds the credential information. + +### Authentication Parameters +You can use `TriggerAuthentication` CRD to configure the authenticate by providing the service account credentials in JSON. + +**Credential based authentication:** + +- `GoogleApplicationCredentials` - Service account credentials in JSON. + +**Identity based authentication:** + +You can also use `TriggerAuthentication` CRD to configure the authentication using the associated service account of the running machine in Google Cloud. You only need to create a `TriggerAuthentication` as this example, and reference it in the `ScaledObject`. `ClusterTriggerAuthentication` can also be used if you intend to use it globally in your cluster. + +### Examples + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: gcp-storage-scaledobject + namespace: keda-gcp-storage-test +spec: + scaleTargetRef: + name: keda-gcp-storage-go + triggers: + - type: gcp-storage + metadata: + bucketName: "Transactions" + targetObjectCount: "5" + credentialsFromEnv: GOOGLE_APPLICATION_CREDENTIALS_JSON +``` + +#### Use TriggerAuthentication with Kubernetes secret + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-gcp-credentials +spec: + secretTargetRef: + - parameter: GoogleApplicationCredentials + name: gcp-storage-secret # Required. Refers to the name of the secret + key: GOOGLE_APPLICATION_CREDENTIALS_JSON # Required. +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: gcp-storage-scaledobject +spec: + scaleTargetRef: + name: keda-gcp-storage-go + triggers: + - type: gcp-storage + authenticationRef: + name: keda-trigger-auth-gcp-credentials + metadata: + bucketName: "Transactions" + targetObjectCount: "5" + blobPrefix: blobsubpath # Default : "" + blobDelimiter: "/" # Default: "" +``` + +#### Use TriggerAuthentication with GCP Identity + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-gcp-credentials +spec: + podIdentity: + provider: gcp +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: gcp-storage-scaledobject +spec: + scaleTargetRef: + name: keda-gcp-storage-go + triggers: + - type: gcp-storage + authenticationRef: + name: keda-trigger-auth-gcp-credentials + metadata: + bucketName: "Transactions" + targetObjectCount: "5" +``` diff --git a/content/docs/2.17/scalers/github-runner.md b/content/docs/2.17/scalers/github-runner.md new file mode 100644 index 000000000..6a8cbeaf1 --- /dev/null +++ b/content/docs/2.17/scalers/github-runner.md @@ -0,0 +1,292 @@ ++++ +title = "Github Runner Scaler" +availability = "v2.10+" +maintainer = "GitHub" +category = "CI/CD" +description = "Scale GitHub Runners based on the number of queued jobs in GitHub Actions" +go_file = "github_runner_scaler" ++++ + +### Trigger Specification + +This specification describes the `github-runner-scaler` trigger that scales based on queued jobs in GitHub Actions. + +```yaml +triggers: + - type: github-runner + metadata: + # Optional: The URL of the GitHub API, defaults to https://api.github.com + githubApiURL: "https://api.github.com" + # Required: The owner of the GitHub repository, or the organization that owns the repository + owner: "{owner}" + # Required: The scope of the runner, can be either "org" (organisation), "ent" (enterprise) or "repo" (repository) + runnerScope: "{runnerScope}" + # Optional: The list of repositories to scale, separated by comma + repos: "{repos}" + # Optional: The list of runner labels to scale on, separated by comma + labels: "{labels}" + # Optional: Not scale on default runner labels ("self-hosted", "linux", "x64"), can be either "true" or "false", defaults to "false" (scale on default runner labels) + noDefaultLabels: "{noDefaultLabels}" + # Optional: The target number of queued jobs to scale on + targetWorkflowQueueLength: "1" # Default 1 + # Optional: The name of the application ID from the GitHub App + applicationID: "{applicatonID}" + # Optional: The name of the installation ID from the GitHub App once installed into Org or repo. + installationID: "{installationID}" + authenticationRef: + name: personalAccessToken or appKey triggerAuthentication Reference +``` + +**Parameter list:** + +- `githubApiURL` - The URL of the GitHub API, defaults to https://api.github.com. You should only need to modify this if you have your own GitHub Appliance. (Optional) +- `owner` - The owner of the GitHub repository, or the organization that owns the repository. (Required) +- `runnerScope` - The scope of the runner, can be either "org", "ent" or "repo". (Required) +- `repos` - The list of repositories to scale, separated by comma. (Optional) +- `labels` - The list of runner labels to scale on, separated by comma. (Optional) +- `noDefaultLabels` - Not scale on default runner labels ("self-hosted", "linux", "x64"). (Values: `true`,`false`, Default: "false", Optional) +- `targetWorkflowQueueLength` - The target number of queued jobs to scale on. (Optional, Default: 1) +- `applicationID` - The name of the application ID from the GitHub App. (Optional, Required if installationID set) +- `installationID` - The name of the installation ID from the GitHub App once installed into Org or repo. (Optional, Required if applicationID set) + +*Parameters from Environment Variables* + +You can access each parameter from above using environment variables. When you specify the parameter in metadata with a suffix of `FromEnv`, +the scaler will use the value from the environment variable. The environment variable must be available to the manifest. e.g. `labelsFromEnv: "RUNNER_LABELS"` will use the environment variable `RUNNER_LABELS` as the source fo the `labels` parameter. + +- `githubApiURLFromEnv` - The URL of the GitHub API, defaults to https://api.github.com. You should only need to modify this if you have your own GitHub Appliance. (Optional) +- `ownerFromEnv` - The owner of the GitHub repository, or the organization that owns the repository. (Required) +- `runnerScopeFromEnv` - The scope of the runner, can be either "org", "ent" or "repo". (Required) +- `reposFromEnv` - The list of repositories to scale, separated by comma. (Optional) +- `labelsFromEnv` - The list of runner labels to scale on, separated by comma. (Optional) +- `noDefaultLabelsFromEnv` - Not scale on default runner labels ("self-hosted", "linux", "x64"), can be either "true" or "false". (Optional) +- `targetWorkflowQueueLengthFromEnv` - The target number of queued jobs to scale on. (Optional, Default: 1) +- `applicationIDFromEnv` - The name of the application ID from the GitHub App. (Optional) (Required if installationID set) +- `installationIDFromEnv` - The name of the installation ID from the GitHub App once installed into Org or repo. (Optional) (Required if applicationID set) + +### Authentication Parameters + +You authenticate with GitHub using a Personal Access Token or a GitHub App private key via `TriggerAuthentication` configuration. + +**Token or Key Authentication:** + +- `personalAccessToken` - The Personal Access Token (PAT) for GitHub from your user. (Optional, Required if GitHub App not used) +- `appKey` - The private key for the GitHub App. This is the contents of the `.pem` file you downloaded when you created the GitHub App. (Optional, Required if applicationID set) + +### Setting up the GitHub App + +You can use the GitHub App to authenticate with GitHub. This is useful if you want a more secure method of authentication with higher rate limits. + +1. Create a GitHub App in your organization or repository. ([docs](https://docs.github.com/en/developers/apps/creating-a-github-app)) +2. Make a note of the Application ID. You will need these to configure the scaler. +3. Disable Webhooks on your GitHub App. +4. Set the permissions for your GitHub App. The following permissions are required: + - **Repository permissions** + - Actions - Read-only + - Administration - Read & Write + - Metadata - Read-only + - **Organization permissions** + - Actions - Read-only + - Metadata - Read-only + - Self-hosted Runners - Read & write +5. Download the private key for the GitHub App. ([docs](https://docs.github.com/en/developers/apps/authenticating-with-github-apps#generating-a-private-key)) +6. Install the GitHub App on your organization or repository. ([docs](https://docs.github.com/en/developers/apps/installing-github-apps)) +7. Make a note of the Installation ID. You will need these to configure the scaler. + +### How does it work? + +The scaler will query the GitHub API to get the number of queued jobs in the specified repositories, subject to filters. If the number of queued jobs is equal to or greater than the `targetWorkflowQueueLength`, the scaler will scale up. + +We provide various options to have granular control over what runners to scale: +- **Repository Filtering** - If no `repos` are specified, the scaler will query all repositories in the specified `owner`. This is useful if you want to scale on all repositories in an organization, but will result in a lot of API calls and affect the Rate Limit. +- **Label-based Filtering** - The `labels` parameter is used to filter the runners that the scaler will scale. It uses the minimum applicable label for the runner. For example, if you have a runner with the labels `golang` and `helm`, and you specify `helm` in the `labels` field on the GitHub Action, the scaler will scale up that runner. By default the scaler will always scale on default labels (`self-hosted`, `linux`, `x64`) in addition to the ones defined in `labels` parameter, scaling on default labels can be disabled by setting `noDefaultLabels` parameter to `true`. + +**API Query Chain** + +The scaler will query the GitHub API in the following order: + +- If no repos are specified: Get the list of repos for the specified owner. +- For each repo: Get the list of workflows runs in the repo. +- For each queued workflow run: Get the list of jobs in the queued workflow run. +- For each job: if the scaler matches, increment the queue length for that scaler. + +**Notes on Rate Limits** + +GitHub Documentation on Rate Limiting [https://docs.github.com/en/rest/overview/resources-in-the-rest-api?apiVersion=2022-11-28#rate-limiting](https://docs.github.com/en/rest/overview/resources-in-the-rest-api?apiVersion=2022-11-28#rate-limiting) + +Example: The GitHub API has a rate limit of standard 5000 requests per hour. The scaler will make 1 request per repository to get the list of workflows, +and 1 request per queued workflow to get the list of jobs. If you have 100 repositories, and 10 queued workflows (across all those repos), the scaler will make 110 requests per scaler check (default: 30 secs). This is 3.6% of the hourly rate limit per 30 seconds. + +Careful design of how you design your repository request layout can help reduce the number of API calls. Usage of the `repos` parameter is recommended to reduce the number of API calls to the GitHub API. + +Note: This does not apply to a hosted appliance as there are no rate limits. + +Additional Note: The GitHub App authentication method has a rate limit of 15000 rather than 5000 per hour. + +**References** + +GitHub's self-hosted runner documentation: [https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners](https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners) + +myoung34's excellent worker on containerised runners: [https://github.com/myoung34/docker-github-actions-runner](https://github.com/myoung34/docker-github-actions-runner) + +### Example + +```yaml +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: github-auth +data: + personalAccessToken: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: github-trigger-auth + namespace: default +spec: + secretTargetRef: + - parameter: personalAccessToken + name: github-auth + key: personalAccessToken +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: github-runner-scaledobject + namespace: default +spec: + scaleTargetRef: + name: gitrunner-deployment + minReplicaCount: 1 + maxReplicaCount: 5 + triggers: + - type: github-runner + metadata: + githubApiURL: "https://api.github.com" + owner: "kedacore" + runnerScope: "repo" + repos: "keda,keda-docs" + labels: "golang,helm" + targetWorkflowQueueLength: "1" + authenticationRef: + name: github-trigger-auth +``` +Alternate example using ScaledJobs and using myoung34's work on containerised runners: +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledJob +metadata: + name: scaledjob-github-runner + namespace: github-runner +spec: + jobTargetRef: + template: + metadata: + labels: + app: scaledjob-github-runner + spec: + containers: + - name: scaledjob-github-runner + image: myoung34/github-runner:2.302.1-ubuntu-focal + imagePullPolicy: Always + env: + - name: EPHEMERAL + value: "true" + - name: DISABLE_RUNNER_UPDATE + value: "true" + - name: REPO_URL + value: "https://github.com/OWNER/REPONAME" + - name: RUNNER_SCOPE + value: "repo" + - name: LABELS + value: "my-label" + - name: ACCESS_TOKEN + valueFrom: + secretKeyRef: + name: {{.SecretName}} + key: personalAccessToken + restartPolicy: Never + minReplicaCount: 0 + maxReplicaCount: 20 + pollingInterval: 30 + triggers: + - type: github-runner + metadata: + owner: OWNER + repos: REPONAME(S) + labelsFromEnv: "LABELS" + runnerScopeFromEnv: "RUNNER_SCOPE" + authenticationRef: + name: github-trigger-auth +``` +GitHub App example using ScaledJobs and using myoung34's work on containerised runners: +```yaml +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: github-auth +data: + appKey: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: github-trigger-auth + namespace: default +spec: + secretTargetRef: + - parameter: appKey + name: github-auth + key: appKey +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledJob +metadata: + name: scaledjob-github-runner + namespace: github-runner +spec: + jobTargetRef: + template: + metadata: + labels: + app: scaledjob-github-runner + spec: + containers: + - name: scaledjob-github-runner + image: myoung34/github-runner:2.302.1-ubuntu-focal + imagePullPolicy: Always + env: + - name: EPHEMERAL + value: "true" + - name: DISABLE_RUNNER_UPDATE + value: "true" + - name: REPO_URL + value: "https://github.com/OWNER/REPONAME" + - name: RUNNER_SCOPE + value: "repo" + - name: LABELS + value: "my-label" + - name: ACCESS_TOKEN + valueFrom: + secretKeyRef: + name: {{.SecretName}} + key: personalAccessToken + restartPolicy: Never + minReplicaCount: 0 + maxReplicaCount: 20 + pollingInterval: 30 + triggers: + - type: github-runner + metadata: + owner: OWNER + repos: REPONAME(S) + labelsFromEnv: "LABELS" + runnerScopeFromEnv: "RUNNER_SCOPE" + applicationID: "1234" + installationID: "5678" + authenticationRef: + name: github-trigger-auth +``` diff --git a/content/docs/2.17/scalers/graphite.md b/content/docs/2.17/scalers/graphite.md new file mode 100644 index 000000000..08fa4d5f0 --- /dev/null +++ b/content/docs/2.17/scalers/graphite.md @@ -0,0 +1,112 @@ ++++ +title = "Graphite" +availability = "v2.5+" +maintainer = "Community" +category = "Metrics" +description = "Scale applications based on metrics in Graphite." +go_file = "graphite_scaler" ++++ + +### Trigger Specification + +This specification describes the `graphite` trigger that scales based on metrics in Graphite. + +```yaml +triggers: +- type: graphite + metadata: + # Required + serverAddress: http://:81 + query: stats.counters.http.hello-world.request.count.count # Note: query must return a vector/scalar single element response + threshold: '10.5' + activationThreshold: '5' + queryTime: '-10Minutes' # Note: Query time in from argv Seconds/Minutes/Hours +``` +**Parameter list:** + +- `serverAddress` - Address of Graphite +- `query` - Query to run. +- `threshold` - Target value to trigger scaling actions. (Default: 100, Optional, This value can be a float) +- `activationThreshold` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds).(Default: `0`, Optional, This value can be a float) +- `queryTime` - Relative time range to execute query against. Please see the [graphite API docs](https://graphite-api.readthedocs.io/en/latest/api.html#from-until) for more information. + +### Authentication Parameters + +Graphite Scaler supports one type of authentication - basic authentication + +**Basic authentication:** +- `authMode`: It must contain `basic` in case of Basic Authentication. Specify this in trigger configuration. +- `username` - This is a required field. Provide the username to be used for basic authentication. +- `password` - Provide the password to be used for authentication. For convenience, this has been marked optional, because many applications implement basic auth with a username as apikey and password as empty. + +### Examples + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: graphite-scaledobject + namespace: default +spec: + scaleTargetRef: + name: my-deployment + triggers: + - type: graphite + metadata: + serverAddress: http://:81 + threshold: '100' + query: maxSeries(keepLastValue(reportd.*.gauge.detect.latest_max_time.value, 1)) + queryTime: '-1Minutes' +``` + +Here is an example of a Graphite Scaler with Basic Authentication, define the `Secret` and `TriggerAuthentication` as follows + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-graphite-secret + namespace: default +data: + username: "dXNlcm5hbWUK" # Must be base64 + password: "cGFzc3dvcmQK" +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-graphite-creds + namespace: default +spec: + secretTargetRef: + - parameter: username + name: keda-graphite-secret + key: username + - parameter: password + name: keda-graphite-secret + key: password +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: graphite-scaledobject + namespace: default + labels: + deploymentName: php-apache-graphite +spec: + cooldownPeriod: 10 + maxReplicaCount: 5 + minReplicaCount: 0 + pollingInterval: 5 + scaleTargetRef: + name: php-apache-graphite + triggers: + - type: graphite + metadata: + authMode: "basic" + query: https_metric + queryTime: -1Hours + serverAddress: http://:81 + threshold: "100" + authenticationRef: + name: keda-graphite-creds +``` diff --git a/content/docs/2.17/scalers/huawei-cloudeye.md b/content/docs/2.17/scalers/huawei-cloudeye.md new file mode 100644 index 000000000..2ddfadaba --- /dev/null +++ b/content/docs/2.17/scalers/huawei-cloudeye.md @@ -0,0 +1,124 @@ ++++ +title = "Huawei Cloudeye" +availability = "v1.1+" +maintainer = "Community" +category = "Metrics" +description = "Scale applications based on a Huawei Cloudeye." +go_file = "huawei_cloudeye_scaler" ++++ + +### Trigger Specification + +This specification describes the `huawei-cloudeye` trigger that scales based on a Huawei Cloudeye. + +```yaml +triggers: +- type: huawei-cloudeye + metadata: + namespace: SYS.ELB + metricName: mb_l7_qps + dimensionName: lbaas_instance_id + dimensionValue: 5e052238-0346-xxb0-86ea-92d9f33e29d2 + targetMetricValue: "5.5" + activationTargetMetricValue: "1.5" + minMetricValue: "1.1" # Deprecated +``` + +**Parameter list:** + +- `namespace` - Namespace of the metric.The format is service.item; service and item must be strings, must start with a letter, can only contain 0-9/a-z/A-Z/_, the total length of service.item is 3, the maximum is 32. +- `metricName` - Name of the metric. +- `dimensionName` - Name of the metric dimension. +- `dimensionValue` - Value of the metric dimension. +- `targetMetricValue` - Target value for your metric. (This value can be a float) +- `activationTargetMetricValue` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds).(Default: `0`, Optional, This value can be a float) +- `minMetricValue` - **Deprecated if favor of `activationTargetMetricValue`** Minimum value for your metric. If the actual value of the metric you get from cloudeye is less than the minimum value, then the scaler is not active. (This value can be a float) +- `metricCollectionTime` - Collection time of the metric in seconds. Equivalent to the earliest start time of the end time. (default: 300) +- `metricFilter` - Aggregation method of the metric. (Values: `average`, `max`, `min`, `sum`, Default: `average`, Optional) +- `metricPeriod` - Granularity of the metric in seconds. (Default: 300, Optional) + +### Authentication Parameters + +You can use `TriggerAuthentication` CRD to configure the authenticate by providing a set of IAM credentials. + +**Credential based authentication:** + +- `IdentityEndpoint` - Identity Endpoint. +- `ProjectID` - Project ID. +- `DomainID` - Id of domain. +- `Domain` - Domain. +- `Region` - Region. +- `Cloud` - Cloud name. (Default: `myhuaweicloud.com`, Optional) +- `AccessKey` - Id of the user. +- `SecretKey` - Access key for the user to authenticate with. + +The user will need access to read data from Huawei Cloudeye. + +### Example + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-huawei-secrets + namespace: keda-test +data: + IdentityEndpoint: + ProjectID: + DomainID: + Region: + Domain: + AccessKey: + SecretKey: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-huawei-credential + namespace: keda-test +spec: + secretTargetRef: + - parameter: IdentityEndpoint # Required. + name: keda-huawei-secrets # Required. + key: IdentityEndpoint # Required. + - parameter: ProjectID # Required. + name: keda-huawei-secrets # Required. + key: ProjectID # Required. + - parameter: DomainID # Required. + name: keda-huawei-secrets # Required. + key: DomainID # Required. + - parameter: Region # Required. + name: keda-huawei-secrets # Required. + key: Region # Required. + - parameter: Domain # Required. + name: keda-huawei-secrets # Required. + key: Domain # Required. + - parameter: AccessKey # Required. + name: keda-huawei-secrets # Required. + key: AccessKey # Required. + - parameter: SecretKey # Required. + name: keda-huawei-secrets # Required. + key: SecretKey # Required. +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: huawei-cloudeye-scaledobject + namespace: keda-test +spec: + scaleTargetRef: + name: nginx-deployment + maxReplicaCount: 5 + minReplicaCount: 2 + triggers: + - type: huawei-cloudeye + metadata: + namespace: SYS.ELB + dimensionName: lbaas_instance_id + dimensionValue: 5e052238-0346-47b0-xxea-92d9f33e29d2 + metricName: mb_l7_qps + targetMetricValue: "100" + minMetricValue: "1" + authenticationRef: + name: keda-trigger-auth-huawei-credential +``` diff --git a/content/docs/2.17/scalers/ibm-mq.md b/content/docs/2.17/scalers/ibm-mq.md new file mode 100644 index 000000000..32d0b0065 --- /dev/null +++ b/content/docs/2.17/scalers/ibm-mq.md @@ -0,0 +1,161 @@ ++++ +title = "IBM MQ" +availability = "v2.0+" +maintainer = "Community" +category = "Messaging" +description = "Scale applications based on IBM MQ Queue" +go_file = "ibmmq_scaler" ++++ + +### Trigger Specification + +This specification describes the `ibmmq` trigger for IBM MQ Queue. + +```yaml +triggers: +- type: ibmmq + metadata: + host: # REQUIRED - IBM MQ Queue Manager Admin REST Endpoint + queueName: # REQUIRED - Your queue name + tlsDisabled: # DEPRECATED: This parameter is deprecated as of KEDA v2.16 in favor of unsafeSsl and will be removed in version v2.18 + queueDepth: # OPTIONAL - Queue depth target for HPA. Default: 20 messages + activationQueueDepth: # OPTIONAL - Activation queue depth target. Default: 0 messages + usernameFromEnv: # OPTIONAL - Provide admin username from env instead of as a secret + passwordFromEnv: # OPTIONAL - Provide admin password from env instead of as a secret + unsafeSsl: "false" # OPTIONAL - Used for skipping certificate check when having self-signed certs 'true'. Default: false +``` + +**Parameter list:** + +- `host` - IBM MQ Queue Manager Admin REST Endpoint. Example URI endpoint structure on IBM cloud `https://example.mq.appdomain.cloud/ibmmq/rest/v2/admin/action/qmgr/QM/mqsc`. +- `queueName` (or `queueNames`) - Name of the queue. It supports multiple queues separated by a comma character ( `,` ). +- `operation` - Operation that will be applied to compute the number of messages. Either `max` (default), `sum`, or `avg`. (Optional) +- `tlsDisabled` - Can be set to 'true' to disable TLS. (DEPRECATED: This parameter is deprecated as of KEDA v2.16 in favor of unsafeSsl and will be removed in version v2.18, Values: `true`, `false` , Default: `false`, Optional) +- `queueDepth` - Queue depth Target for HPA. (Default: `20`, Optional) +- `activationQueueDepth` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) +- `usernameFromEnv` - Provide admin username from env instead of as a secret. (Optional) +- `passwordFromEnv` - Provide admin password from env instead of as a secret. (Optional) +- `unsafeSsl` - Whether to allow unsafe SSL (Values: `true`, `false`, Default: `false`, Optional) + +### Authentication Parameters + +TriggerAuthentication CRD is used to connect and authenticate to IBM MQ: + +**Authentication Parameters** + +- `ADMIN_USER` - REQUIRED - The admin REST endpoint username for your MQ Queue Manager`. +- `ADMIN_PASSWORD` - REQUIRED - The admin REST endpoint API key for your MQ Queue Manager. +- `ca` - Certificate authority file for TLS client authentication. (Optional) +- `cert` - Certificate for client authentication. (Optional) +- `key` - Key for client authentication. (Optional) +- `keyPassword` - If set the keyPassword is used to decrypt the provided key. (Optional) +- `usernameFromEnv` - Provide admin username from env instead of as a secret. (Optional) +- `passwordFromEnv` - Provide admin password from env instead of as a secret. (Optional) + +### Example + +Example with Basic Auth + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-ibmmq-secret +data: + ADMIN_USER: # REQUIRED - Admin Username + ADMIN_PASSWORD: # REQUIRED - Admin Password +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: ibmmq-scaledobject + namespace: default + labels: + deploymentName: ibmmq-deployment +spec: + scaleTargetRef: + name: ibmmq-deployment + pollingInterval: 5 # OPTIONAL - Default: 30 seconds + cooldownPeriod: 30 # OPTIONAL - Default: 300 seconds + maxReplicaCount: 18 # OPTIONAL - Default: 100 + triggers: + - type: ibmmq + metadata: + host: # REQUIRED - IBM MQ Queue Manager Admin REST Endpoint + queueName: # REQUIRED - Your queue name + queueDepth: # OPTIONAL - Queue depth target for HPA. Default: 20 messages + usernameFromEnv: # OPTIONAL - Provide admin username from env instead of as a secret + passwordFromEnv: # OPTIONAL - Provide admin password from env instead of as a secret + authenticationRef: + name: keda-ibmmq-trigger-auth +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-ibmmq-trigger-auth + namespace: default +spec: + secretTargetRef: + - parameter: username + name: keda-ibmmq-secret + key: ADMIN_USER + - parameter: password + name: keda-ibmmq-secret + key: ADMIN_PASSWORD +``` + +Example with Basic Auth and TLS + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-ibmmq-secret +data: + ADMIN_USER: # REQUIRED - Admin Username + ADMIN_PASSWORD: # REQUIRED - Admin Password + cert: + key: +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: ibmmq-scaledobject + namespace: default + labels: + deploymentName: ibmmq-deployment +spec: + scaleTargetRef: + name: ibmmq-deployment + pollingInterval: 5 # OPTIONAL - Default: 30 seconds + cooldownPeriod: 30 # OPTIONAL - Default: 300 seconds + maxReplicaCount: 18 # OPTIONAL - Default: 100 + triggers: + - type: ibmmq + metadata: + host: # REQUIRED - IBM MQ Queue Manager Admin REST Endpoint + queueName: # REQUIRED - Your queue name + queueDepth: # OPTIONAL - Queue depth target for HPA. Default: 20 messages + authenticationRef: + name: keda-ibmmq-trigger-auth +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-ibmmq-trigger-auth + namespace: default +spec: + secretTargetRef: + - parameter: username + name: keda-ibmmq-secret + key: ADMIN_USER + - parameter: password + name: keda-ibmmq-secret + key: ADMIN_PASSWORD + - parameter: cert + name: keda-ibmmq-secret + key: cert + - parameter: key + name: keda-ibmmq-secret + key: key +``` diff --git a/content/docs/2.17/scalers/influxdb.md b/content/docs/2.17/scalers/influxdb.md new file mode 100644 index 000000000..2be2f830b --- /dev/null +++ b/content/docs/2.17/scalers/influxdb.md @@ -0,0 +1,76 @@ ++++ +title = "InfluxDB" +availability = "v2.1+" +maintainer = "Community" +category = "Data & Storage" +description = "Scale applications based on InfluxDB queries" +go_file = "influxdb_scaler" ++++ + +### Trigger Specification + +This specification describes the `influxdb` trigger that scales based on the results of a InfluxDB query. + +```yaml +triggers: + - type: influxdb + metadata: + serverURL: http://influxdb:8086 + organizationName: influx-org + organizationNameFromEnv: INFLUXDB_ORG_NAME # Optional: You can use this instead of `organizationName` parameter. See details in "Parameter List" section + thresholdValue: '4.4' + activationThresholdValue: '6.2' + query: | + from(bucket: "bucket-of-interest") + |> range(start: -12h) + |> filter(fn: (r) => r._measurement == "stat") + authToken: some-auth-token + authTokenFromEnv: INFLUXDB_AUTH_TOKEN # Optional: You can use this instead of `authToken` parameter. See details in "Parameter List" section +``` + +**Parameter list:** + +- `authToken` - Authentication token needed for the InfluxDB client to communicate with an associated server. +- `authTokenFromEnv` - Defines the authorization token, similar to `authToken`, but reads it from an environment variable on the scale target. +- `organizationName` - Organization name needed for the client to locate all information contained in that [organization](https://docs.influxdata.com/influxdb/v2.0/organizations/) such as buckets, tasks, etc. +- `organizationNameFromEnv` - Defines the organization name, similar to `organizationName`, but reads it from an environment variable on the scale target. +- `serverURL` - Holds the url value of the InfluxDB server. +- `thresholdValue` - Provided by the user. This value can vary from use case to use case depending on the data of interest, and is needed to trigger the scaling in/out depending on what value comes back from the query. (This value can be a float) +- `activationThresholdValue` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds).(Default: `0`, Optional, This value can be a float) +- `query` - Flux query that will yield the value for the scaler to compare the `thresholdValue` against. +- `unsafeSsl` - Skip certificate validation when connecting over HTTPS. (Values: `true`, `false`, Default: `false`, Optional) + +### Authentication Parameters + +You can authenticate by using an authorization token. + +**Authorization Token Authentication:** + +- `authToken` - Authorization token for InfluxDB server. + +### Example + +Below is an example of how to deploy a scaled object with the `InfluxDB` scale trigger. + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: influxdb-scaledobject + namespace: my-project +spec: + scaleTargetRef: + name: nginx-worker + triggers: + - type: influxdb + metadata: + serverURL: http://influxdb:8086 + organizationNameFromEnv: INFLUXDB_ORG_NAME + thresholdValue: '4' + activationThresholdValue: '6' + query: | + from(bucket: "bucket-of-interest") + |> range(start: -12h) + |> filter(fn: (r) => r._measurement == "stat") + authTokenFromEnv: INFLUXDB_AUTH_TOKEN +``` diff --git a/content/docs/2.17/scalers/kubernetes-workload.md b/content/docs/2.17/scalers/kubernetes-workload.md new file mode 100644 index 000000000..bb0d18159 --- /dev/null +++ b/content/docs/2.17/scalers/kubernetes-workload.md @@ -0,0 +1,50 @@ ++++ +title = "Kubernetes Workload" +availability = "v2.4+" +maintainer = "Community" +category = "Apps" +description = "Scale applications based on the count of running pods that match the given selectors." +go_file = "kubernetes_workload_scaler" ++++ + +### Trigger Specification + +```yaml +triggers: +- type: kubernetes-workload + metadata: + podSelector: 'app=backend' + value: '0.5' + activationValue: '3.1' +``` + +**Parameter list:** + +- `podSelector` - [Label selector](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors) that will be used to get the pod count. It supports multiple selectors split by a comma character (`,`). It also supports [set-based requirements](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#set-based-requirement) and a mix of them. +- `value` - Target relation between the scaled workload and the amount of pods which matches the selector. It will be calculated following this formula: `relation = (pods which match selector) / (scaled workload pods)`. (This value can be a float) +- `activationValue` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds).(Default: `0`, Optional, This value can be a float) + +> 💡 **Note:** The search scope is limited to the namespace where the `ScaledObject` is deployed. + +The count excludes terminated pods, i.e. [pod status](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#podstatus-v1-core) `phase` equals `Succeeded` or `Failed`. + +### Authentication Parameters + +The own KEDA's identity is used to list the pods, so no extra configuration is needed here. + +### Example + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: workload-scaledobject +spec: + scaleTargetRef: + name: workload-deployment + triggers: + - type: kubernetes-workload + metadata: + podSelector: 'app=backend, deploy notin (critical, monolith)' + value: '3' +``` diff --git a/content/docs/2.17/scalers/liiklus-topic.md b/content/docs/2.17/scalers/liiklus-topic.md new file mode 100644 index 000000000..4ef51a7c6 --- /dev/null +++ b/content/docs/2.17/scalers/liiklus-topic.md @@ -0,0 +1,63 @@ ++++ +title = "Liiklus Topic" +availability = "v1.0+" +maintainer = "Community" +category = "Messaging" +description = "Scale applications based on Liiklus Topic." +go_file = "liiklus_scaler" ++++ + +### Trigger Specification + +This specification describes the `liiklus` trigger for Liiklus Topic. + +```yaml +triggers: +- type: liiklus + metadata: + # Required + address: localhost:6565 + group: my-group + topic: test-topic + # Optional + lagThreshold: "50" + # Optional + activationLagThreshold: "20" + groupVersion: 1 +``` + +**Parameter list:** + +- `address` - Address of the gRPC liiklus API endpoint. +- `group` - Name of consumer group. +- `topic` - Topic to monitor and scale based on `lagThreshold`. +- `lagThreshold` - Value to trigger scaling actions for. (Default: `10`, Optional) +- `activationLagThreshold` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) +- `groupVersion` - Version of the group to consider when looking at messages. See [docs](https://github.com/bsideup/liiklus/blob/22efb7049ebcdd0dcf6f7f5735cdb5af1ae014de/app/src/test/java/com/github/bsideup/liiklus/GroupVersionTest.java). (Default: `0`, Optional) + +### Authentication Parameters + +Not supported yet. + +### Example + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: liiklus-scaledobject + namespace: default +spec: + scaleTargetRef: + name: function-deployment + pollingInterval: 30 + triggers: + - type: liiklus + metadata: + # Required + address: localhost:6565 + group: my-group # Make sure that this consumer group name is the same one as the one that is consuming topics + topic: test-topic + # Optional + lagThreshold: "50" # Default value is set to 10 +``` diff --git a/content/docs/2.17/scalers/loki.md b/content/docs/2.17/scalers/loki.md new file mode 100644 index 000000000..e3e553f1d --- /dev/null +++ b/content/docs/2.17/scalers/loki.md @@ -0,0 +1,161 @@ ++++ +title = "Loki" +availability = "v2.9+" +maintainer = "Community" +category = "Metrics" +description = "Scale applications based on Loki query result." +go_file = "loki_scaler" ++++ + +### Trigger Specification + +This specification describes the `loki` trigger that scales based on a Loki query result. Here is an example of providing values in metadata: + +```yaml +triggers: +- type: loki + metadata: + # Required fields: + serverAddress: http://:3100 # Note: loki server URL + query: sum(rate({filename="/var/log/syslog"}[1m])) # Note: query must return a vector/scalar single element response + threshold: '0.7' + # Optional fields: + activationThreshold: '2.50' + tenantName: Tenant1 # Optional. X-Scope-OrgID header for specifying the tenant name in a multi-tenant setup. + ignoreNullValues: false # Default is `true`, which means ignoring the empty value list from Loki. Set to `false` the scaler will return error when Loki target is lost + unsafeSsl: "false" # Default is `false`, Used for skipping certificate check when having self-signed certs for Loki endpoint +``` + +**Parameter list:** + +- `serverAddress` - URL of Loki server. +- `query` - LogQL query to run. The query must return a vector/scalar single element response. +- `threshold` - Value to start scaling for. (This value can be a float) +- `activationThreshold` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds).(Default: `0`, Optional, This value can be a float) +- `tenantName` - The `X-Scope-OrgID` header for specifying the tenant name in a multi-tenant setup. (Optional) +- `ignoreNullValues` - Value to reporting error when Loki target is lost. (Values: `true`,`false`, Default: `true`, Optional) +- `unsafeSsl` - Used for skipping certificate check e.g: using self-signed certs. (Values: `true`,`false`, Default: `false`, Optional) +- `authModes` - Authentaication mode to be used. (Values: `bearer`,`basic`, Optional) + +### Authentication Parameters + +Loki doesn't provide any kind of authentication out of the box. However, most commonly Loki is configured along with a Basic Auth or Bearer Auth, which are the only valid options for authentication here. + +You can use `TriggerAuthentication` CRD to configure the authentication. Specify `authModes` and other trigger parameters along with secret credentials in `TriggerAuthentication` as mentioned below: + +**Bearer authentication:** +- `authModes`: It must contain `bearer` in case of Bearer Authentication. Specify this in trigger configuration. +- `bearerToken`: The token needed for authentication. + +**Basic authentication:** +- `authModes`: It must contain `basic` in case of Basic Authentication. Specify this in trigger configuration. +- `username` - Provide the username to be used for basic authentication. +- `password` - Provide the password to be used for authentication. (Optional, For convenience this has been marked optional as many applications implement basic auth with a username as apikey and password as empty.) + +### Example + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: loki-scaledobject + namespace: default +spec: + maxReplicaCount: 12 + scaleTargetRef: + name: nginx + triggers: + - type: loki + metadata: + serverAddress: http://:3100 + threshold: '0.7' + query: sum(rate({filename="/var/log/syslog"}[1m])) +``` + +Here is an example of a loki scaler with Bearer Authentication, where the `Secret` and `TriggerAuthentication` are defined as follows: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-loki-secret + namespace: default +data: + bearerToken: "BEARER_TOKEN" +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-loki-creds + namespace: default +spec: + secretTargetRef: + - parameter: bearerToken + name: keda-loki-secret + key: bearerToken +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: loki-scaledobject + namespace: default +spec: + maxReplicaCount: 12 + scaleTargetRef: + name: nginx + triggers: + - type: loki + metadata: + serverAddress: http://:3100 + threshold: '0.7' + query: sum(rate({filename="/var/log/syslog"}[1m])) + authModes: "bearer" + authenticationRef: + name: keda-loki-creds +``` + +Here is an example of a loki scaler with Basic Authentication, where the `Secret` and `TriggerAuthentication` are defined as follows: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-loki-secret + namespace: default +data: + username: dXNlcm5hbWU= + password: cGFzc3dvcmQ= +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-loki-creds + namespace: default +spec: + secretTargetRef: + - parameter: username + name: keda-loki-secret + key: username + - parameter: password + name: keda-loki-secret + key: password +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: loki-scaledobject + namespace: default +spec: + maxReplicaCount: 12 + scaleTargetRef: + name: nginx + triggers: + - type: loki + metadata: + serverAddress: http://:3100 + threshold: '0.7' + query: sum(rate({filename="/var/log/syslog"}[1m])) + authModes: "basic" + authenticationRef: + name: keda-loki-creds +``` diff --git a/content/docs/2.17/scalers/memory.md b/content/docs/2.17/scalers/memory.md new file mode 100644 index 000000000..c95ecf5bc --- /dev/null +++ b/content/docs/2.17/scalers/memory.md @@ -0,0 +1,97 @@ ++++ +title = "Memory" +availability = "v2.0+" +maintainer = "Community" +category = "Apps" +description = "Scale applications based on memory metrics." +go_file = "cpu_memory_scaler" ++++ + +> **Notice:** +> - This scaler **requires prerequisites**. See the 'Prerequisites' section. +> - This scaler can scale to 0 only when user defines at least one additional scaler which is not CPU or Memory (eg. Kafka + Memory, or Prometheus + Memory) and `minReplicaCount` is 0. +> - This scaler only applies to ScaledObject, not to Scaling Jobs. + +### Prerequisites + +KEDA uses standard `cpu` and `memory` metrics from the Kubernetes Metrics Server, which is not installed by default on certain Kubernetes deployments such as EKS on AWS. Additionally, the `resources` section of the relevant Kubernetes Pods must include at least one of `requests` or `limits`. + +- The Kubernetes Metrics Server must be installed. Installation instructions vary based on your Kubernetes provider. +- The configuration for your Kubernetes Pods must include a `resources` section with specified `requests` (or `limits`). See [Resource Management for Pods and Containers](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/). If the resources section is empty (`resources: {}` or similar), KEDA checks if the `defaultRequest` (or `default` for limits) is set in `LimitRange` for the `Container` type in the same namespace. If `defaultRequest` (or `default` for limits) is missing too, the error `missing request for {cpu/memory}` occurs. + +```yaml +# a working example of resources with specified requests +spec: + containers: + - name: app + image: images.my-company.example/app:v4 + resources: + requests: + memory: "128Mi" + cpu: "500m" +``` + +### Trigger Specification + +This specification describes the `memory` trigger that scales based on memory metrics. + +```yaml +triggers: +- type: memory + metricType: Utilization # Allowed types are 'Utilization' or 'AverageValue' + metadata: + type: Utilization # DEPRECATED: This parameter is deprecated in favor of trigger.metricType and will be removed in version v2.18; allowed types are 'Utilization' or 'AverageValue' + value: "60" + containerName: "" # Optional. You can use this to target a specific container in a pod +``` + +**Parameter list:** + +- `type` - Type of metric to use. Options are `Utilization`, or `AverageValue`. +- `value` - Value to trigger scaling actions for: + - When using `Utilization`, the target value is the average of the resource metric across all relevant pods, represented as a percentage of the requested value of the resource for the pods. + - When using `AverageValue`, the target value is the target value of the average of the metric across all relevant pods (quantity). +- `containerName` - Name of the specific container to scale based on its memory, rather than the entire pod. Defaults to empty if not specified. + +> 💡 **NOTE:** `containerName` parameter requires Kubernetes cluster version 1.20 or higher with `HPAContainerMetrics` feature enabled. Please see [container resource metrics](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#container-resource-metrics) for more information. + +> 💡 **NOTE:** The `type` parameter is deprecated in favor of the global `metricType` and will be removed in a future release. Users are advised to use `metricType` instead. + +### Example + +The following example targets memory utilization of entire pod. If the pod has multiple containers, it will be sum of all the containers in it. + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: memory-scaledobject + namespace: default +spec: + scaleTargetRef: + name: my-deployment + triggers: + - type: memory + metricType: Utilization # Allowed types are 'Utilization' or 'AverageValue' + metadata: + value: "50" +``` + +The following example targets memory utilization of a specific container (`foo`) in a pod. + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: memory-scaledobject + namespace: default +spec: + scaleTargetRef: + name: my-deployment + triggers: + - type: memory + metricType: Utilization # Allowed types are 'Utilization' or 'AverageValue' + metadata: + value: "50" + containerName: "foo" +``` diff --git a/content/docs/2.17/scalers/metrics-api.md b/content/docs/2.17/scalers/metrics-api.md new file mode 100644 index 000000000..ae22df53b --- /dev/null +++ b/content/docs/2.17/scalers/metrics-api.md @@ -0,0 +1,307 @@ ++++ +title = "Metrics API" +availability = "v2.0+" +maintainer = "Community" +category = "Metrics" +description = "Scale applications based on a metric provided by an API" +go_file = "metrics_api_scaler" ++++ + +### Trigger Specification + +This specification describes the `metrics-api` trigger that scales based on a metric value provided by an API. + +This scaler allows users to utilize **any existing APIs** as a metric provider. + +Here is an example of trigger configuration using metrics-api scaler: + +```yaml +triggers: +- type: metrics-api + metadata: + targetValue: "8.8" + format: "json" + activationTargetValue: "3.8" + url: "http://api:3232/api/v1/stats" + valueLocation: "components.worker.tasks" +``` + +**Parameter list:** + +- `url` - Full URL of the API operation to call to get the metric value (eg. `http://app:1317/api/v1/stats`). +- `format` - One of the following formats: `json`, `xml`, `yaml`, `prometheus`. (Default: `json`, Optional) +- `valueLocation` - The location of the metric value in the response payload. The value is format specific. + * `json` - [GJSON path notation](https://github.com/tidwall/gjson#path-syntax) to refer to the field in the payload containing the metric value. + * `yaml`, `xml`, `prometheus` - implemented as dot-separated path algorithm for value parsing. + * `{foo: {bar: 42}}` - `foo.bar` will return 42 + * `{foo: [{bar: 42}]}` - `foo.0.bar` will return 42 + * same syntax is used for `yaml`, `xml` and `prometheus` formats. + +- `targetValue` - Target value to scale on. When the metric provided by the API is equal or higher to this value, KEDA will start scaling out. When the metric is 0 or less, KEDA will scale down to 0. (This value can be a float) +- `activationTargetValue` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds).(Default: `0`, Optional, This value can be a float) +- `unsafeSsl` - Skip certificate validation when connecting over HTTPS. (Values: `true`, `false`, Default: `false`, Optional) + + +### Authentication Parameters + +Metrics Scaler API supported four types of authentication - API Key based authentication, basic authentication, TLS +authentication, and Bearer authentication. + +You can use `TriggerAuthentication` CRD to configure the authentication. Specify `authMode` and other trigger parameters + along with secret credentials in `TriggerAuthentication` as mentioned below: + +**API Key based authentication:** +- `authMode`: It must be set to `apiKey` in case of API key Authentication. Specify this in trigger configuration. +- `method` - This specifies the possible methods API Key based authentication supports. Possible values are `header` and `query`. `header` is the default method. Specify this in trigger configuration. +- `keyParamName` - This is either header key or query param used for passing apikey. Default header is `X-API-KEY` and default query param is `api_key`. Specify this in trigger configuration. If your implementation has different key, please specify it here. +- `apiKey` - API Key needed for authentication. + +**Basic authentication:** +- `authMode` - It must be set to `basic` in case of Basic Authentication. Specify this in trigger configuration. +- `username` - Provide the username to be used for basic authentication. +- `password` - Provide the password to be used for authentication. (Optional) + +**TLS authentication:** +- `authMode` - It must be set to `tls` in case of TLS Authentication. Specify this in trigger configuration. +- `ca` - Certificate authority file for TLS client authentication. +- `cert` - Certificate for client authentication. +- `key` - Key for client authentication. (Optional) + +**Bearer authentication:** +- `authMode` - It must be set to `bearer` in case of Bearer Authentication. Specify this in trigger configuration. +- `token` - Token that should be placed in the `Authorization` header. The header will be `Authorization: Bearer {token}`. + +### Example + +Here is a full example of scaled object definition using Metric API trigger: + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: http-scaledobject + namespace: keda + labels: + deploymentName: dummy +spec: + maxReplicaCount: 12 + scaleTargetRef: + name: dummy + triggers: + - type: metrics-api + metadata: + targetValue: "7" + url: "http://api:3232/components/stats" + valueLocation: 'components.worker.tasks' +``` + +When checking current metric Metrics API scaler sends GET request to provided `url` and then uses `valueLocation` +to access the value in response's payload. + +The above example expects that the API endpoint will return response similar to this one: +```json +{ + "components": { + "worker": { + "tasks": 12, + ... + }, + ... + }, + ... +} +``` +Assuming such response, Metrics API trigger will figure out that current metric value is 12. + +> 💡 **NOTE:**The value of the metric can either be an integral unquoted json number type (e.g. 123). Or a quantity (e.g. "123", "1.23", "10Mi"). + +Here is an example of a metric scaler with API Key based authentication, + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-metric-api-secret + namespace: default +data: + apiKey: "APIKEY" +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-metric-api-creds + namespace: default +spec: + secretTargetRef: + - parameter: apiKey + name: keda-metric-api-secret + key: apiKey +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: http-scaledobject + namespace: keda + labels: + deploymentName: dummy +spec: + maxReplicaCount: 12 + scaleTargetRef: + name: dummy + triggers: + - type: metrics-api + metadata: + targetValue: "7" + url: "http://api:3232/components/stats" + valueLocation: 'components.worker.tasks' + authMode: "apiKey" + method: "query" + keyParamName: "QUERY_KEY" + authenticationRef: + name: keda-metric-api-creds +``` + +Here is an example of a metric scaler with Bearer Authentication, define the `Secret` and `TriggerAuthentication` as follows + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-metric-api-secret + namespace: default +data: + token: "PlaceYourTokenHere" +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-metric-api-creds + namespace: default +spec: + secretTargetRef: + - parameter: token + name: keda-metric-api-secret + key: token +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: http-scaledobject + namespace: keda + labels: + deploymentName: dummy +spec: + maxReplicaCount: 12 + scaleTargetRef: + name: dummy + triggers: + - type: metrics-api + metadata: + targetValue: "7" + url: "http://api:3232/components/stats" + valueLocation: 'components.worker.tasks' + authMode: "bearer" + authenticationRef: + name: keda-metric-api-creds +``` + +Here is an example of a metric scaler with Basic Authentication, define the `Secret` and `TriggerAuthentication` as follows + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-metric-api-secret + namespace: default +data: + username: "username" + password: "password" +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-metric-api-creds + namespace: default +spec: + secretTargetRef: + - parameter: username + name: keda-metric-api-secret + key: username + - parameter: password + name: keda-metric-api-secret + key: password +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: http-scaledobject + namespace: keda + labels: + deploymentName: dummy +spec: + maxReplicaCount: 12 + scaleTargetRef: + name: dummy + triggers: + - type: metrics-api + metadata: + targetValue: "7" + url: "http://api:3232/components/stats" + valueLocation: 'components.worker.tasks' + authMode: "basic" + authenticationRef: + name: keda-metric-api-creds +``` + + +Here is an example of a metric scaler with TLS Authentication, define the `Secret` and `TriggerAuthentication` as follows + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-metric-api-secret + namespace: default +data: + cert: "cert" + key: "key" + ca: "ca" +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-metric-api-creds + namespace: default +spec: + secretTargetRef: + - parameter: cert + name: keda-metric-api-secret + key: cert + - parameter: key + name: keda-metric-api-secret + key: key + - parameter: ca + name: keda-metric-api-secret + key: ca +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: http-scaledobject + namespace: keda + labels: + deploymentName: dummy +spec: + maxReplicaCount: 12 + scaleTargetRef: + name: dummy + triggers: + - type: metrics-api + metadata: + targetValue: "7" + url: "http://api:3232/components/stats" + valueLocation: 'components.worker.tasks' + authMode: "tls" + authenticationRef: + name: keda-metric-api-creds +``` diff --git a/content/docs/2.17/scalers/mongodb.md b/content/docs/2.17/scalers/mongodb.md new file mode 100644 index 000000000..2ca3c1068 --- /dev/null +++ b/content/docs/2.17/scalers/mongodb.md @@ -0,0 +1,159 @@ ++++ +title = "MongoDB" +maintainer = "Community" +category = "Data & Storage" +description = "Scale applications based on MongoDB queries." +availability = "v2.1+" +go_file = "mongo_scaler" ++++ + +### Trigger Specification + +This specification describes the `mongodb` trigger that scales based on result of MongoDB query. + +```yaml +triggers: + - type: mongodb + metadata: + # name of an environment variable containing a valid MongoDB connection string + connectionStringFromEnv: MongoDB_CONNECTION_STRING + # Required: database name + dbName: "test" + # Required: collection name + collection: "test_collection" + # Required: query expr, used by filter data + query: '{"region":"eu-1","state":"running","plan":"planA"}' + # Required: according to the number of query result, to scale the TargetRef + queryValue: "1" + # Optional: according to the number of query result, the scaler is active + activationQueryValue: "1" +``` + +Alternatively, you can configure connection parameters explicitly instead of providing a connection string: + +```yaml +triggers: + - type: mongodb + metadata: + # scheme of the MongoDB server. if using MongoDB Altas, you can set it to "mongodb+srv" + scheme: "mongodb" + # host name of the MongoDB server. Example of mongodb service: "mongodb-svc..svc.cluster.local" + host: mongodb-svc.default.svc.cluster.local + # port number of the MongoDB server. + port: "27017" + # username credential for connecting to the MongoDB server + username: test_user + # name of an environment variable containing a valid password for connecting to the MongoDB server + passwordFromEnv: MongoDB_Password + # Required: database name + dbName: "test" + # Required: collection name + collection: "test_collection" + # Required: query expr, used by filter data + query: '{"region":"eu-1","state":"running","plan":"planA"}' + # Required: according to the number of query result, to scale the TargetRef + queryValue: "1" + # Optional: according to the number of query result, the scaler is active + activationQueryValue: "1" +``` + +**Parameter list:** + +The `mongodb` trigger always requires the following information: + +- `dbName` - Name of the database. +- `collection` - Name of the collection. +- `query` - A MongoDB query that should return single numeric value. +- `queryValue` - A threshold that will define when scaling should occur. +- `activationQueryValue` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) + +To connect to the MongoDB server, you can provide either: + +- `connectionStringFromEnv` - The name of an environment variable containing a valid MongoDB connection string for connecting to the MongoDB server. + +Or provide more detailed connection parameters explicitly (a connection string will be generated for you at runtime): + +- `scheme` - The scheme of the MongoDB server, if using MongoDB Atlas, you can set it to `mongodb+srv`. (Default: `mongodb`, Optional) +- `host` - The host name of the MongoDB server. +- `port` - The port number of the MongoDB server. +- `username` - Username to authenticate with to MongoDB database. +- `passwordFromEnv` - The name of an environment variable containing the password credential for connecting to the MongoDB server. + +When configuring with a connection string, you can use this URL format: + +``` +mongodb[+srv]://:@mongodb-svc..svc.cluster.local:27017/ +``` + + +### Authentication Parameters + +As an alternative to environment variables, You can authenticate with the MongoDB server by using connection string or password authentication via `TriggerAuthentication` or `ClusterTriggerAuthentication` configuration. + +**Connection String Authentication:** + +- `connectionString` - Connection string for MongoDB server. + +**Password Authentication:** + +- `scheme` - The scheme of the MongoDB server, if using MongoDB Atlas, you can set it to `mongodb+srv`. (Default: `mongodb`, Optional) +- `host` - The host name of the MongoDB server. +- `port` - The port number of the MongoDB server. +- `username` - Username to authenticate with to MongoDB database. +- `password` - Password for the configured user to login to MongoDB server. +- `dbName` - Name of the database. + +### Example + +Here is an example of how to deploy a scaled Job with the `MongoDB` scale trigger which uses `TriggerAuthentication`. + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledJob +metadata: + name: mongodb-job +spec: + jobTargetRef: + template: + spec: + containers: + - name: mongodb-update + image: 1314520999/mongodb-update:latest + args: + - --dataBase=test + - --collection=test_collection + imagePullPolicy: IfNotPresent + restartPolicy: Never + backoffLimit: 1 + pollingInterval: 30 # Optional. Default: 30 seconds + maxReplicaCount: 30 # Optional. Default: 100 + successfulJobsHistoryLimit: 0 # Optional. Default: 100. How many completed jobs should be kept. + failedJobsHistoryLimit: 10 # Optional. Default: 100. How many failed jobs should be kept. + triggers: + - type: mongodb + metadata: + dbName: "test" + collection: "test_collection" + query: '{"region":"eu-1","state":"running","plan":"planA"}' + queryValue: "1" + authenticationRef: + name: mongodb-trigger +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: mongodb-trigger +spec: + secretTargetRef: + - parameter: connectionString + name: mongodb-secret + key: connect +--- +apiVersion: v1 +kind: Secret +metadata: + name: mongodb-secret +type: Opaque +data: + connect: bW9uZ29kYjovL3Rlc3RfdXNlcjp0ZXN0X3Bhc3N3b3JkQG1vbmdvZGItc3ZjLm1vbmdvREIuc3ZjLmNsdXN0ZXIubG9jYWw6MjcwMTcvdGVzdA== +``` diff --git a/content/docs/2.17/scalers/mssql.md b/content/docs/2.17/scalers/mssql.md new file mode 100644 index 000000000..78b5c9803 --- /dev/null +++ b/content/docs/2.17/scalers/mssql.md @@ -0,0 +1,126 @@ ++++ +title = "MSSQL" +availability = "v2.2+" +maintainer = "Microsoft" +category = "Data & Storage" +description = "Scale applications based on Microsoft SQL Server (MSSQL) query results." +go_file = "mssql_scaler" ++++ + +### Trigger Specification + +This specification describes the `mssql` trigger that scales based on the results of a [Microsoft SQL Server](https://www.microsoft.com/sql-server/) (MSSQL) query result. This trigger supports local [MSSQL containers](https://hub.docker.com/_/microsoft-mssql-server) as well as SQL Server endpoints hosted in the cloud, such as [Azure SQL Database](https://azure.microsoft.com/services/sql-database/). + +```yaml +triggers: +- type: mssql + metadata: + connectionStringFromEnv: MSSQL_CONNECTION_STRING + query: "SELECT COUNT(*) FROM backlog WHERE state='running' OR state='queued'" + targetValue: "5.5" + activationTargetValue: '5' +``` + +> 💡 **NOTE:** The connection string format supported by this scaler has some incompatibilities with connection string formats supported by other platforms, like .NET. For example, the MSSQL instance's port number must be separated into its own `Port` property instead of adding it to the `Server` property. You can learn more about all the supported connection string formats for this mssql scaler [here](https://github.com/denisenkom/go-mssqldb#the-connection-string-can-be-specified-in-one-of-three-formats). + +Alternatively, you configure connection parameters explicitly instead of providing a connection string: + +```yaml +triggers: +- type: mssql + metadata: + username: "kedaUser" + passwordFromEnv: MSSQL_PASSWORD + host: mssqlinst.namespace.svc.cluster.local + port: "1433" # optional + database: test_db_name + query: "SELECT COUNT(*) FROM backlog WHERE state='running' OR state='queued'" + targetValue: 1 +``` + +The `mssql` trigger always requires the following information: + +- `query` - A [T-SQL](https://docs.microsoft.com/sql/t-sql/language-reference) query that returns a single numeric value. This can be a regular query or the name of a stored procedure. +- `targetValue` - A threshold that is used as `targetValue` or `targetAverageValue` (depending on the trigger metric type) in the Horizontal Pod Autoscaler (HPA). (This value can be a float) +- `activationTargetValue` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds).(Default: `0`, Optional, This value can be a float) + +> Note that the query must return a single integer value. If the query has a possibility of returning `null`, a default value can be set using the `COALESCE` function. For example, `SELECT COALESCE(column_name, 0) FROM table_name;`. See [MSSQL documentation](https://learn.microsoft.com/en-us/sql/t-sql/language-elements/coalesce-transact-sql) for more information on the `COALESCE` function. + +To connect to the MSSQL instance, you can provide either: + +- `connectionStringFromEnv` - The name of an environment variable containing a valid MSSQL connection string. + +Or provide more detailed connection parameters explicitly (a connection string will be generated for you at runtime): + +- `host` - The hostname of the MSSQL instance endpoint. +- `port` - The port number of the MSSQL instance endpoint. (Default: 1433, Optional) +- `database` - The name of the database to query. +- `username` - The username credential for connecting to the MSSQL instance. +- `passwordFromEnv` - The name of an environment variable containing the password credential for connecting to the MSSQL instance. + +When configuring with a connection string, you can use either a URL format (note the URL encoding of special characters): + +``` +sqlserver://user1:Password%231@example.database.windows.net:1433?database=AdventureWorks +``` + +Or the more traditional ADO format: + +``` +Server=example.database.windows.net;Port=1433;Database=AdventureWorks;Persist Security Info=False;User ID=user1;Password=Password#1;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30; +``` + +### Authentication parameters + +As an alternative to using environment variables, you can authenticate with the MSSQL instance using connection string or password authentication via `TriggerAuthentication` configuration. + +**Connection string authentication:** + +- `connectionString` - The connection string for the MSSQL instance. + +**Password authentication:** + +- `host` - The hostname of the MSSQL instance endpoint. +- `port` - The port number of the MSSQL instance endpoint. (default 1433) +- `database` - The name of the database to query. +- `username` - The username credential for connecting to the MSSQL instance. +- `password` - The password credential for connecting to the MSSQL instance. + +### Example + +The following is an example of how to deploy a scaled object with the `mssql` scale trigger that uses `TriggerAuthentication` and a connection string. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: mssql-secrets +type: Opaque +data: + mssql-connection-string: U2VydmVyPWV4YW1wbGUuZGF0YWJhc2Uud2luZG93cy5uZXQ7cG9ydD0xNDMzO0RhdGFiYXNlPUFkdmVudHVyZVdvcmtzO1BlcnNpc3QgU2VjdXJpdHkgSW5mbz1GYWxzZTtVc2VyIElEPXVzZXIxO1Bhc3N3b3JkPVBhc3N3b3JkIzE7RW5jcnlwdD1UcnVlO1RydXN0U2VydmVyQ2VydGlmaWNhdGU9RmFsc2U7 # base64 encoded value of MSSQL connectionString of format "Server=example.database.windows.net;port=1433;Database=AdventureWorks;Persist Security Info=False;User ID=user1;Password=Password#1;Encrypt=True;TrustServerCertificate=False;" +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-mssql-secret +spec: + secretTargetRef: + - parameter: connectionString + name: mssql-secrets + key: mssql-connection-string +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: mssql-scaledobject +spec: + scaleTargetRef: + name: consumer # e.g. the name of the resource to scale + triggers: + - type: mssql + metadata: + targetValue: 1 + query: "SELECT COUNT(*) FROM backlog WHERE state='running' OR state='queued'" + authenticationRef: + name: keda-trigger-auth-mssql-secret +``` diff --git a/content/docs/2.17/scalers/mysql.md b/content/docs/2.17/scalers/mysql.md new file mode 100644 index 000000000..0dd3947ab --- /dev/null +++ b/content/docs/2.17/scalers/mysql.md @@ -0,0 +1,98 @@ ++++ +title = "MySQL" +availability = "v1.2+" +maintainer = "Community" +category = "Data & Storage" +description = "Scale applications based on MySQL query result." +go_file = "mysql_scaler" ++++ + +### Trigger Specification + +This specification describes the `mysql` trigger that scales based on result of MySQL query. + +The trigger always requires the following information: + +- `query` - A MySQL query that should return single numeric value. +- `queryValue` - A threshold that is used as `targetValue` or `targetAverageValue` (depending on the trigger metric type) in HPA. (This value can be a float) +- `activationQueryValue` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds).(Default: `0`, Optional, This value can be a float) + +> Note that the query must return a single integer value. If the query has a possibility of returning `null`, a default value can be set using the `COALESCE` function. For example, `SELECT COALESCE(column_name, 0) FROM table_name;`. See [MySQL documentation](https://dev.mysql.com/doc/refman/8.4/en/comparison-operators.html#function_coalesce) for more information on the `COALESCE` function. + +To provide information about how to connect to MySQL you can provide: + +- `connectionStringFromEnv` - MySQL connection string that should point to environment variable with valid value. + +Or provide more detailed information: + +- `host` - The host of the MySQL server. +- `port` - The port of the MySQL server. +- `dbName` - Name of the database. +- `username` - Username to authenticate with to MySQL database. +- `passwordFromEnv` - Password for the given user, this should be blank (no password) or point to an environment variable with the password. + +Some parameters could be provided using environmental variables, instead of setting them directly in metadata. Here is a list of parameters you can use to retrieve values from environment variables: + +- `hostFromEnv` - The host of the MySQL server, similar to `host`, but reads it from an environment variable on the scale target. +- `portFromEnv` - The port of the MySQL server, similar to `port`, but reads it from an environment variable on the scale target. +- `usernameFromEnv` - The username for connecting with host of the MySQL server, similar to `username`, but reads it from an environment variable on the scale target. + +### Authentication Parameters + +You can authenticate by using connection string or password authentication. + +**Connection String Authentication:** + +- `connectionString` - Connection string for MySQL database. + +**Password Authentication:** + +- `host` - The host of the MySQL server. +- `port` - The port of the MySQL server. +- `dbName` - Name of the database. +- `username` - Username to authenticate with to MySQL database. +- `password` - Password for configured user to login to MySQL database. +variables. + +### Example + +Here is an example of how to deploy a scaled object with the `mysql` scale trigger which uses `TriggerAuthentication`. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: mysql-secrets + namespace: my-project +type: Opaque +data: + mysql_conn_str: dXNlckB0Y3AobXlzcWw6MzMwNikvc3RhdHNfZGI= # base64 encoded value of mysql connectionString of format user:password@tcp(mysql:3306)/stats_db +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-mysql-secret + namespace: my-project +spec: + secretTargetRef: + - parameter: connectionString + name: mysql-secrets + key: mysql_conn_str +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: mysql-scaledobject + namespace: my-project +spec: + scaleTargetRef: + name: worker + triggers: + - type: mysql + metadata: + queryValue: "4.4" + activationQueryValue: "5.4" + query: "SELECT CEIL(COUNT(*) / 6) FROM task_instance WHERE state='running' OR state='queued'" + authenticationRef: + name: keda-trigger-auth-mysql-secret +``` diff --git a/content/docs/2.17/scalers/nats-jetstream.md b/content/docs/2.17/scalers/nats-jetstream.md new file mode 100644 index 000000000..f81517a83 --- /dev/null +++ b/content/docs/2.17/scalers/nats-jetstream.md @@ -0,0 +1,70 @@ ++++ +title = "NATS JetStream" +layout = "scaler" +availability = "v2.8+" +maintainer = "Community" +category = "Messaging" +description = "Scale applications based on NATS JetStream." +go_file = "nats_jetstream_scaler" ++++ + +### Trigger Specification + +This specification describes the `nats-jetstream` trigger for NATS JetStream. + +```yaml +triggers: +- type: nats-jetstream + metadata: + natsServerMonitoringEndpoint: "nats.nats.svc.cluster.local:8222" + account: "$G" + stream: "mystream" + consumer: "pull_consumer" + lagThreshold: "10" + activationLagThreshold: "15" + useHttps: "false" +``` + +**Parameter list:** + +- `natsServerMonitoringEndpoint` - Location of the NATS server monitoring endpoint. +- `account` - Name of the NATS account. "$G" is default when no account is configured. +- `stream` - Name of the JS stream within the account. +- `consumer` - Name of the consumer for a given stream. +- `lagThreshold` - Average target value to trigger scaling actions. +- `activationLagThreshold` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) +- `useHttps` - Specifies if the NATS server monitoring endpoint is using HTTPS. (Default: `false`, Optional) + +### Authentication Parameters + +Some parameters of the JetStream details can be pulled from `TriggerAuthentication` objects: + +- `natsServerMonitoringEndpoint` - Location of the NATS Streaming monitoring endpoint. +- `account` - Name of the NATS account. "$G" is default when no account is configured. + +### Example + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: nats-jetstream-scaledobject + namespace: nats-jetstream +spec: + pollingInterval: 3 # Optional. Default: 30 seconds + cooldownPeriod: 10 # Optional. Default: 300 seconds + minReplicaCount: 0 # Optional. Default: 0 + maxReplicaCount: 2 # Optional. Default: 100 + scaleTargetRef: + name: sub + triggers: + - type: nats-jetstream + metadata: + natsServerMonitoringEndpoint: "nats.nats.svc.cluster.local:8222" + account: "$G" + stream: "mystream" + consumer: "pull_consumer" + lagThreshold: "10" + activationLagThreshold: "15" + useHttps: "false" +``` diff --git a/content/docs/2.17/scalers/nats-streaming.md b/content/docs/2.17/scalers/nats-streaming.md new file mode 100644 index 000000000..490082ed3 --- /dev/null +++ b/content/docs/2.17/scalers/nats-streaming.md @@ -0,0 +1,177 @@ ++++ +title = "NATS Streaming" +availability = "v1.0+" +maintainer = "Community" +category = "Messaging" +description = "Scale applications based on NATS Streaming." +go_file = "stan_scaler" ++++ + +### Trigger Specification + +This specification describes the `stan` trigger for NATS Streaming. + +```yaml +triggers: +- type: stan + metadata: + natsServerMonitoringEndpoint: "stan-nats-ss.stan.svc.cluster.local:8222" + queueGroup: "grp1" + durableName: "ImDurable" + subject: "Test" + lagThreshold: "10" + activationLagThreshold: "5" + useHttps: "false" +``` + +**Parameter list:** + +- `natsServerMonitoringEndpoint` - Location of the Nats Streaming monitoring endpoint. +- `queueGroup` - Name of queue group of the subscribers. +- `durableName` - Name of durability used by subscribers. +- `subject` - Name of the channel. +- `lagThreshold` - Average target value to trigger scaling actions. +- `activationLagThreshold` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) +- `useHttps` - Specifies if the NATS Streaming monitoring endpoint is using HTTPS. (Default: `false`, Optional) + +### Authentication Parameters + +You can authenticate with the NATS streaming server by using connection string authentication via `TriggerAuthentication` configuration. + +- `natsServerMonitoringEndpoint` - Location of the NATS Streaming monitoring endpoint. + +**TLS Authentication** + +- `tls` - To enable SSL auth for nats-streaming, set this to `enable`. If not set, TLS for nats-streaming is not used. (Values: `enable`, `disable`, Default: `disable`, Optional) +- `ca` - Certificate authority file for TLS client authentication. (Optional) +- `cert` - Certificate for client authentication. (Optional) +- `key` - Key for client authentication. (Optional) + +### Example without any TriggerAuthentication + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: stan-scaledobject + namespace: example +spec: + pollingInterval: 10 # Optional. Default: 30 seconds + cooldownPeriod: 30 # Optional. Default: 300 seconds + minReplicaCount: 0 # Optional. Default: 0 + maxReplicaCount: 30 # Optional. Default: 100 + scaleTargetRef: + name: gonuts-sub + triggers: + - type: stan + metadata: + natsServerMonitoringEndpoint: "stan-nats-ss.stan.svc.cluster.local:8222" + queueGroup: "grp1" + durableName: "ImDurable" + subject: "Test" + lagThreshold: "10" + useHttps: "false" +``` +#### Example with no TLS + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: stan-secret + namespace: example +type: Opaque +data: + stan_endpoint: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-stan-secret + namespace: example +spec: + secretTargetRef: + - parameter: natsServerMonitoringEndpoint + name: stan-secret + key: stan_endpoint +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: stan-scaledobject + namespace: example +spec: + pollingInterval: 10 # Optional. Default: 30 seconds + cooldownPeriod: 30 # Optional. Default: 300 seconds + minReplicaCount: 0 # Optional. Default: 0 + maxReplicaCount: 30 # Optional. Default: 100 + scaleTargetRef: + name: gonuts-sub + triggers: + - type: stan + metadata: + queueGroup: "grp1" + durableName: "ImDurable" + subject: "Test" + lagThreshold: "10" + authenticationRef: + name: keda-trigger-auth-stan-secret +``` + +#### Example with TLS + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-stan-secrets + namespace: default +data: + tls: enable + ca: + cert: + key: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-stan-secret + namespace: default +spec: + secretTargetRef: + - parameter: tls + name: keda-stan-secrets + key: tls + - parameter: ca + name: keda-stan-secrets + key: ca + - parameter: cert + name: keda-stan-secrets + key: cert + - parameter: key + name: keda-stan-secrets + key: key +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: stan-scaledobject + namespace: example +spec: + pollingInterval: 10 # Optional. Default: 30 seconds + cooldownPeriod: 30 # Optional. Default: 300 seconds + minReplicaCount: 0 # Optional. Default: 0 + maxReplicaCount: 30 # Optional. Default: 100 + scaleTargetRef: + name: gonuts-sub + triggers: + - type: stan + metadata: + queueGroup: "grp1" + durableName: "ImDurable" + subject: "Test" + lagThreshold: "10" + useHttps: "true" + authenticationRef: + name: keda-trigger-auth-stan-secret +``` \ No newline at end of file diff --git a/content/docs/2.17/scalers/new-relic.md b/content/docs/2.17/scalers/new-relic.md new file mode 100644 index 000000000..08bc3b521 --- /dev/null +++ b/content/docs/2.17/scalers/new-relic.md @@ -0,0 +1,108 @@ ++++ +title = "New Relic" +availability = "2.6+" +maintainer = "Community" +category = "Metrics" +description = "Scale applications based on New Relic NRQL" +go_file = "newrelic_scaler" ++++ + +### Trigger Specification + +This specification describes the `new-relic` trigger that scales based on a New Relic metric. + +```yaml +triggers: + - type: new-relic + metadata: + # Required: Account - Subaccount to run the query on + account: '1234567' + # Required: QueryKey - Api key to connect to New Relic + queryKey: "NRAK-xxxxxxxxxxxxxxxxxxxxxxxxxxx" + # Optional: nrRegion - Region to query data for. Default value is US. + region: "US" + # Optional: noDataError - If the query returns no data should this be treated as an error. Default value is false. + noDataError: "true" + # Required: nrql + nrql: "SELECT average(duration) from Transaction where appName='SITE'" + # Required: threshold + threshold: "50.50" + # Optional: activationThreshold - Target value for activating the scaler. + activationThreshold: "20.1" +``` + +**Parameter list:** + +- `account` - The account within New Relic that the request should be targeted against. +- `queryKey` - The API key that will be leveraged to connect to New Relic and make requests. [official documentation](https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/) +- `region` - The region to connect to for the New Relic apis. (Values: `LOCAL`, `EU`, `STAGING`, `US`, Default: `US`, Optional) +- `noDataError` - Should queries that return nodata be treated as an error, if set to false and a query returns nodata, the result be `0`. (Values: `true`, `false`, Default: `false`, Optional) +- `nrql` - The New Relic query that will be run to get the data requested. + + NOTE: The default New Relic time range for a query is last 30 minutes, which can produce unexpected responses. To mimic the behavior of a TIMESERIES query, you need to reduce the scope to 1 minute, this can be achieved by adding `SINCE 1 MINUTE AGO` to the query. + [official documentation](https://docs.newrelic.com/docs/query-your-data/nrql-new-relic-query-language/get-started/introduction-nrql-new-relics-query-language/) [official documentation](https://docs.newrelic.com/docs/query-your-data/nrql-new-relic-query-language/get-started/introduction-nrql-new-relics-query-language/) +- `threshold` - A threshold that is used as the `targetValue` or `targetAverageValue` (depending on the trigger metric type) in the HPA configuration. (This value can be a float) +- `activationThreshold` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds).(Default: `0`, Optional, This value can be a float) + +### Authentication Parameters + +You can use `TriggerAuthentication` CRD to configure the authentication with a `queryKey`. + +**Authentication:** + +- `queryKey` - The API key that will be leveraged to connect to New Relic and make requests. [official documentation](https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/) + +- `account` - The account within New Relic that the request should be targeted against. This can be used to replace the value that would be provided in the trigger. + +- `region` - The region to connect to for the New Relic apis. This can be used to replace the value that would be provided in the trigger. + +### Example + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: new-relic-secret + namespace: my-project +type: Opaque +data: + apiKey: TlJBSy0xMjM0NTY3ODkwMTIzNDU2Nwo= # base64 encoding of the new relic api key NRAK-12345678901234567 + account: MTIzNDU2 # base64 encoding of the new relic account number 123456 + region: VVM= # base64 encoding of the new relic region US +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-new-relic + namespace: my-project +spec: + secretTargetRef: + - parameter: queryKey + name: new-relic-secret + key: apiKey + - parameter: account + name: new-relic-secret + key: account + - parameter: region + name: new-relic-secret + key: region + +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: newrelic-scaledobject + namespace: keda +spec: + maxReplicaCount: 12 + scaleTargetRef: + name: dummy + triggers: + - type: new-relic + metadata: + nrql: "SELECT average(duration) from Transaction where appName='SITE'" + noDataError: "true" + threshold: '1000' + authenticationRef: + name: keda-trigger-auth-new-relic +``` diff --git a/content/docs/2.17/scalers/openstack-metric.md b/content/docs/2.17/scalers/openstack-metric.md new file mode 100644 index 000000000..c50a8ec04 --- /dev/null +++ b/content/docs/2.17/scalers/openstack-metric.md @@ -0,0 +1,174 @@ ++++ +title = "OpenStack Metric" +availability = "v2.3+" +maintainer = "Community" +category = "Metrics" +description = "Scale applications based on a threshold reached by a specific measure from OpenStack Metric API." +go_file = "openstack_metrics_scaler" ++++ + +### Trigger Specification + +This specification describes the `openstack-metric` trigger for OpenStack metrics. +> The OpenStack metric API follows [Gnocchi](https://gnocchi.xyz/) API. Attempt to the fact that Gnocchi API is an open-source time series database embedded in OpenStack system and every parameter on OpenStack Metric API follows its patterns but you don't need to reference anything from Gnocchi. It scales based on a specific measure from a given resource metric. +It's highly recommended to check [Gnocchi](https://gnocchi.xyz/) docs. + +```yaml +triggers: +- type: openstack-metric + metadata: + metricsURL: http://localhost:8041/v1/metric #required + metricID: 003bb589-166d-439d-8c31-cbf098d863de #required + aggregationMethod: "mean" #required + granularity: 300 #required (seconds) + threshold: "1.250" #required + activationThreshold: "0.250" #optional + timeout: 30 #optional +``` + +> Protocol (http or https) should always be provided when specifying URLs + +**Parameter list:** +- `metricsURL` - The URL to check for the metrics API, based. It must contain the hostname, the metric port, the API version, and the resource ID. The pattern is: `http://:///metric`. +- `metricID` - The Id of the intendend metric. +- `aggregationMethod` - The aggregation method that will be used to calculate metrics, it must follows the configured possible metrics derived from gnocchi API like: `mean`, `min`, `max`, `std`, `sum`, `count`, the complete aggregation methods list can be found [here](https://gnocchi.xyz/rest.html#archive-policy). +- `granularity` - The configured granularity from metric collection in seconds. it must follow the same value configured in OpenStack, but it must be coutned in seconds. Sample: If you have a 5 minutes time window granularity defined, so you must input a value of 300 seconds (5*60). +- `threshold` - The target value that, when reached, will scale the application. (This value can be a float) +- `activationThreshold` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds).(Default: `0`, Optional, This value can be a float) +- `timeout` - The timeout, in seconds, for the HTTP client requests that will query the Metric API. (Default: `30`, Optional) + +### Authentication Parameters + +To authenticate, this scaler uses tokens. Tokens are automatically retrieved by the scaler from [Keystone](https://docs.openstack.org/keystone/latest/), the official OpenStack Identity Provider. You can provide your credentials using Secrets either by using the "password" method or the "application credentials" method. Both cases use `TriggerAuthentication`. + +#### Password + +- `authURL` - The Keystone authentication URL. The pattern is: `http://://`. +- `userID` - The OpenStack project user ID. +- `password` - The password for the provided user. +- `projectID` - The OpenStack project ID. + +#### Application Credentials + +- `authURL` - The Keystone authentication URL. The pattern is: `http://://`. +- `appCredentialID` - The Application Credential ID. +- `appCredentialSecret` - The Application Credential secret. + +### Example + +#### Password method + +Here is an example of how to deploy a scaled object with the `openstack-metric` scale trigger which uses `TriggerAuthentication` and the Password method from above. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: openstack-secret-password + namespace: default +type: Opaque +data: + authURL: aHR0cDovL2xvY2FsaG9zdDo1MDAwL3YzLw== + userID: MWYwYzI3ODFiNDExNGQxM2E0NGI4ODk4Zjg1MzQwYmU= + password: YWRtaW4= + projectID: YjE2MWRjNTE4Y2QyNGJkYTg0ZDk0ZDlhMGU3M2ZjODc= +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: metrics-password-trigger-authentication + namespace: default +spec: + secretTargetRef: + - parameter: authURL + name: openstack-secret-password + key: authURL + - parameter: userID + name: openstack-secret-password + key: userID + - parameter: password + name: openstack-secret-password + key: password + - parameter: projectID + name: openstack-secret-password + key: projectID +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: openstack-metric-scaledobject + namespace: default +spec: + scaleTargetRef: + name: my-deployment + pollingInterval: 10 + cooldownPeriod: 10 + minReplicaCount: 0 + triggers: + - type: openstack-metric + metadata: + metricsURL: http://localhost:8041/v1/metric + metricID: faf01aa5-da88-4929-905d-b83fbab46771 + aggregationMethod: "mean" + granularity: 300 + threshold: 1250 + timeout: 30 + authenticationRef: + name: openstack-metric-password-trigger-authentication +``` + +#### Application Credentials method + +You can also use the Application Credentials method. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: openstack-secret-appcredentials + namespace: default +type: Opaque +data: + authURL: aHR0cDovL2xvY2FsaG9zdDo1MDAwL3YzLw== + appCredentialID: OWYyY2UyYWRlYmFkNGQxNzg0NTgwZjE5ZTljMTExZTQ= + appCredentialSecret: LVdSbFJBZW9sMm91Z3VmZzNEVlBqcll6aU9za1pkZ3c4Y180XzRFU1pZREloT0RmajJkOHg0dU5yb3NudVIzWmxDVTZNLTVDT3R5NDFJX3M5R1N5Wnc= +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: openstack-metric-appcredentials-trigger-authentication + namespace: default +spec: + secretTargetRef: + - parameter: authURL + name: openstack-secret-appcredentials + key: authURL + - parameter: appCredentialID + name: openstack-secret-appcredentials + key: appCredentialID + - parameter: appCredentialSecret + name: openstack-secret-appcredentials + key: appCredentialSecret +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: openstack-metric-scaledobject + namespace: default +spec: + scaleTargetRef: + name: my-deployment + pollingInterval: 10 + cooldownPeriod: 10 + minReplicaCount: 0 + triggers: + - type: openstack-metric + metadata: + metricsURL: http://localhost:8041/v1/metric + metricID: faf01aa5-da88-4929-905d-b83fbab46771 + aggregationMethod: "mean" + granularity: 300 + threshold: 1250 + authenticationRef: + name: openstack-metric-appcredentials-trigger-authentication +``` diff --git a/content/docs/2.17/scalers/openstack-swift.md b/content/docs/2.17/scalers/openstack-swift.md new file mode 100644 index 000000000..147132ec5 --- /dev/null +++ b/content/docs/2.17/scalers/openstack-swift.md @@ -0,0 +1,180 @@ ++++ +title = "OpenStack Swift" +availability = "v2.1+" +maintainer = "Community" +category = "Metrics" +description = "Scale applications based on the count of objects in a given OpenStack Swift container." +go_file = "openstack_swift_scaler" ++++ + +### Trigger Specification + +This specification describes the `openstack-swift` trigger for OpenStack Swift containers. It scales based on the count of objects in a given container. + +```yaml +triggers: +- type: openstack-swift + metadata: + containerName: my-container # Required + swiftURL: http://localhost:8080/v1/b161dc815cd24bda84d94d9a0e73cf78 # Optional + objectCount: "2" # Optional + activationObjectCount: "5" # Optional + objectPrefix: "my-prefix" # Optional + objectDelimiter: "/" # Optional + objectLimit: "10" # Optional + onlyFiles: "true" # Optional + timeout: "2" # Optional +``` + +> Please, always provide the protocol (http or https) when specifying URLs. + +**Parameter list:** + +- `swiftURL` - The URL to query the Swift API. If not provided, the scaler will try to find the Swift public URL for a certain region, using the OpenStack catalog, which is returned when requesting an authentication token. (Optional) +- `containerName` - Name of Swift container in an OpenStack account. +- `objectCount` - Average target value to trigger scaling actions. (Default: `2`, Optional) +- `activationObjectCount` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) +- `objectPrefix` - Prefix for the object. Only objects with this prefix will be returned. Use this prefix to specify sub-paths for the objects. (Default: `""`, Optional) +- `objectDelimiter` - Delimiter for identifying the object prefix. It is the character used to split object names. (Default: `""`, Optional) +- `objectLimit` - The maximum number of objects returned by the API. By default, the Swift API only returns up to 10000 names. (Default: `""`, Optional) +- `onlyFiles` - Specifies if the scaler should be triggered only by the number of files, without considering folders. Inside a container, one can have files and folders. Folders (empty or not) are counted as objects, just as files are. If one wants to scale based on only files, this parameter must be set to `true`. (Values: `true`, `false`, Default: `false`, Optional) +- `timeout` - The timeout, in seconds, for the HTTP client requests that will query the Swift API. (Default: `30`, Optional) + +For more information about `prefix`, `delimiter`, and `limit`, please, refer to the [Object Store API](https://docs.openstack.org/api-ref/object-store/). + +### Authentication Parameters + +To authenticate, this scaler uses tokens. Tokens are automatically retrieved by the scaler from [Keystone](https://docs.openstack.org/keystone/latest/), the official OpenStack Identity Provider. You can provide your credentials using Secrets either by using the "password" method or the "application credentials" method. Both cases use `TriggerAuthentication`. + +#### Password + +- `authURL` - The Keystone authentication URL. The scaler supports only Keystone v3. You shouldn't include the `/v3` parameter in your URL path. This is done automatically by the scaler. +- `userID` - The OpenStack project user ID. +- `password` - The password for the provided user. +- `projectID` - The OpenStack project ID. +- `regionName` - The OpenStack region name where the Swift service is available. This parameter is not required and is used only when the `swiftURL` is not provided to the scaler. If the region name is not provided, it will look for the first Swift public URL available in the OpenStack catalog. + +#### Application Credentials + +- `authURL` - The Keystone authentication URL. The scaler supports only Keystone v3. You shouldn't include the `/v3` parameter in your URL path. This is done automatically by the scaler. +- `appCredentialID` - The Application Credential ID. +- `appCredentialSecret` - The Application Credential secret. +- `regionName` - The OpenStack region name where the Swift service is available. This parameter is not required and is used only when the `swiftURL` is not provided to the scaler. If the region name is not provided, it will look for the first Swift public URL available in the OpenStack catalog. + +### Example + +#### Password method + +Here is an example of how to deploy a scaled object with the `openstack-swift` scale trigger which uses `TriggerAuthentication` and the Password method from above. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: openstack-secret-password + namespace: default +type: Opaque +data: + authURL: aHR0cDovL2xvY2FsaG9zdDo1MDAwLw== + userID: MWYwYzI3ODFiNDExNGQxM2E0NGI4ODk4Zjg1MzQwYmU= + password: YWRtaW4= + projectID: YjE2MWRjNTE4Y2QyNGJkYTg0ZDk0ZDlhMGU3M2ZjODc= + regionName: Y2FsaWZvcm5pYS0x +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: swift-password-trigger-authentication + namespace: default +spec: + secretTargetRef: + - parameter: authURL + name: openstack-secret-password + key: authURL + - parameter: userID + name: openstack-secret-password + key: userID + - parameter: password + name: openstack-secret-password + key: password + - parameter: projectID + name: openstack-secret-password + key: projectID + - parameter: regionName + name: openstack-secret-password + key: regionName +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: swift-scaledobject + namespace: default +spec: + scaleTargetRef: + name: my-deployment + pollingInterval: 10 + cooldownPeriod: 10 + minReplicaCount: 0 + triggers: + - type: openstack-swift + metadata: + containerName: my-container + objectCount: "1" + onlyFiles: "true" + authenticationRef: + name: swift-password-trigger-authentication +``` + +#### Application Credentials method + +You can also use the Application Credentials method. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: openstack-secret-appcredentials + namespace: default +type: Opaque +data: + authURL: aHR0cDovL2xvY2FsaG9zdDo1MDAwLw== + appCredentialID: OWYyY2UyYWRlYmFkNGQxNzg0NTgwZjE5ZTljMTExZTQ= + appCredentialSecret: LVdSbFJBZW9sMm91Z3VmZzNEVlBqcll6aU9za1pkZ3c4Y180XzRFU1pZREloT0RmajJkOHg0dU5yb3NudVIzWmxDVTZNLTVDT3R5NDFJX3M5R1N5Wnc= +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: swift-appcredentials-trigger-authentication + namespace: default +spec: + secretTargetRef: + - parameter: authURL + name: openstack-secret-appcredentials + key: authURL + - parameter: appCredentialID + name: openstack-secret-appcredentials + key: appCredentialID + - parameter: appCredentialSecret + name: openstack-secret-appcredentials + key: appCredentialSecret +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: swift-scaledobject + namespace: default +spec: + scaleTargetRef: + name: my-deployment + pollingInterval: 10 + cooldownPeriod: 10 + minReplicaCount: 0 + triggers: + - type: openstack-swift + metadata: + swiftURL: http://localhost:8080/v1/AUTH_b161dc518cd24bda84d94d9a0e73fc87 + containerName: my-container + objectCount: "1" + authenticationRef: + name: swift-appcredentials-trigger-authentication +``` \ No newline at end of file diff --git a/content/docs/2.17/scalers/postgresql.md b/content/docs/2.17/scalers/postgresql.md new file mode 100644 index 000000000..473fb8e4e --- /dev/null +++ b/content/docs/2.17/scalers/postgresql.md @@ -0,0 +1,172 @@ ++++ +title = "PostgreSQL" +availability = "v1.2+" +maintainer = "Community" +category = "Data & Storage" +description = "Scale applications based on a PostgreSQL query." +go_file = "postgresql_scaler" ++++ + +### Trigger Specification + +This specification describes the `postgresql` trigger that scales based on a PostgreSQL query + +The PostgreSQL scaler allows for three connection options: + +A user can offer a full connection string +(often in the form of an environment variable secret). + +- `connectionFromEnv` - PostgreSQL connection string that should point to environment variable with valid value. + +Alternatively, a user can specify individual +arguments (host, userName, password, etc.), and the scaler will form a connection string +internally. + +- `host` - Service URL to postgresql. Note that you should use a full svc URL as KEDA will need to contact postgresql from a different namespace. +- `userName` - Username for postgresql user. +- `passwordFromEnv` Password for postgresql user. +- `port` - Postgresql port. +- `dbName` - Postgresql Database name. +- `sslmode` - SSL policy for communicating with database. + +It is also possible to leverage a `TriggerAuthentication` object having the `azure-workload`'s provider type to connect to an Azure Postgres Flexible Server resource through an UAMI Azure managed identity. +More details and an example are provided down below. + +Finally, a user inserts a query that returns the desired value. + +- `query` - What query to poll postgresql with. Query must return an integer. +- `targetQueryValue` - A threshold that is used as `targetValue` or `targetAverageValue` (depending on the trigger metric type) in HPA. (This value can be a float) +- `activationTargetQueryValue` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds).(Default: `0`, Optional, This value can be a float) + + +> Note that the query must return a single integer value. If the query has a possibility of returning `null`, a default value can be set using the `COALESCE` function. For example, `SELECT COALESCE(column_name, 0) FROM table_name;`. See [PostgreSQL documentation](https://www.postgresql.org/docs/current/functions-conditional.html#FUNCTIONS-COALESCE-NVL-IFNULL) for more information on the `COALESCE` function. + + +This is an example of using a full connection string with `AIRFLOW_CONN_AIRFLOW_DB` set as `postgresql://test@localhost`: + +```yaml +triggers: +- type: postgresql + metadata: + connectionFromEnv: AIRFLOW_CONN_AIRFLOW_DB + query: "SELECT ceil(COUNT(*)::decimal / 16) FROM task_instance WHERE state='running' OR state='queued';" + targetQueryValue: "1.1" + activationTargetQueryValue: "5" +``` + +While this is an example of specifying each parameter: + +```yaml +triggers: +- type: postgresql + metadata: + userName: "kedaUser" + passwordFromEnv: PG_PASSWORD + host: postgres-svc.namespace.cluster.local #use the cluster-wide namespace as KEDA + #lives in a different namespace from your postgres + port: "5432" + dbName: test_db_name + sslmode: disable + query: "SELECT ceil(COUNT(*)::decimal / 16) FROM task_instance WHERE state='running' OR state='queued';" + targetQueryValue: "2.2" +``` + +### Authentication Parameters + +You can authenticate by using a password, or store the password within the connectionString, or leverage Azure Access Token authentication to connect to a Azure Postgres Flexible Server. + +**Connection String Authentication:** + +- `connection` - Connection string for PostgreSQL database. + +**Password Authentication:** + +- `host` - Service URL to PostgreSQL. Note that you should use a fully qualified URL (including the namespace) as KEDA will need to contact PostgreSQL from a different namespace. +- `userName` - Username for PostgreSQL user. +- `password` Password for configured user to login to PostgreSQL database variables. +- `port` - PostgreSQL port. +- `dbName` - PostgreSQL Database name. +- `sslmode` - SSL policy for communicating with database. + +#### Example + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: airflow-worker +spec: + scaleTargetRef: + name: airflow-worker + pollingInterval: 10 # Optional. Default: 30 seconds + cooldownPeriod: 30 # Optional. Default: 300 seconds + maxReplicaCount: 10 # Optional. Default: 100 + triggers: + - type: postgresql + metadata: + connectionFromEnv: AIRFLOW_CONN_AIRFLOW_DB + query: "SELECT ceil(COUNT(*)::decimal / 16) FROM task_instance WHERE state='running' OR state='queued';" + targetQueryValue: 1 +``` + +**Azure Access Token authentication:** + +#### Prerequisites: +- The UAMI should be able to access the Azure Postgres Flexible Server, [refer to this link for more info](https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/how-to-connect-with-managed-identity#create-an-azure-database-for-postgresql-flexible-server-user-for-your-managed-identity). +- The UAMI should be granted access to the table mentioned in the query performed by KEDA. +This can be achieved by: + - creating a group role to allow access to the particular schema where the table queried by KEDA is persisted, and then assign the newly created Postgres user identity from previous step to this group role. + - granting permission on the table queried by KEDA to the newly created Postgres user from previous step, via a query that looks like + `GRANT ALL ON TO "";`. + +Next, a user can specify individual arguments (host, userName, password, etc.), and the scaler will form a connection string internally. An access token, which will act as a password, will be retrieved each time KEDA performs its process. + - `host` - FQDN of the Azure Postgres Flexible Server. + - `userName` - Name of the UAMI Azure identity (``). + - `port` - Postgresql port (the default value is `"5432"`, please have a look at the `Remarks` down below). + - `dbName` - Postgresql Database name. + - `sslmode` - SSL policy for communicating with database (the value should be `require`). + + +#### Remarks + +- While the Azure Postgres Flexible Server resource provides a [`PGBouncer`](https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-pgbouncer) feature which opens a port `6432` to interact with the server, this access token authentication's feature was not working properly while using the `PGBouncer` port, but it worked without issues while using the default server's port. Therefore, KEDA should use the Postgres server's default port, but the other applications (i.e. Airflow, ...) deployed on the same Kubernetes cluster can use the `PGBouncer` port. + + +#### Example + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: azure-pg-flex-auth +spec: + podIdentity: + provider: azure-workload + # Optional-> identityId: + # Optional-> identityTenantId: + +--- + +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: airflow-worker +spec: + scaleTargetRef: + name: airflow-worker + pollingInterval: 10 # Optional. Default: 30 seconds + cooldownPeriod: 30 # Optional. Default: 300 seconds + maxReplicaCount: 10 # Optional. Default: 100 + triggers: + - type: postgresql + authenticationRef: + name: azure-pg-flex-auth + metadata: + host: + port: "5432" + userName: + dbName: + sslmode: require + query: "SELECT ceil(COUNT(*)::decimal / 16) FROM task_instance WHERE state='running' OR state='queued';" + targetQueryValue: 1 +``` \ No newline at end of file diff --git a/content/docs/2.17/scalers/predictkube.md b/content/docs/2.17/scalers/predictkube.md new file mode 100644 index 000000000..f79cf594e --- /dev/null +++ b/content/docs/2.17/scalers/predictkube.md @@ -0,0 +1,113 @@ ++++ +title = "Predictkube" +availability = "v2.6+" +maintainer = "Dysnix" +category = "Metrics" +description = "AI-based predictive scaling based on Prometheus metrics & PredictKube SaaS." +go_file = "predictkube_scaler" ++++ + +PredictKube is an open-source project with the SAAS part consisting of an AI model that requires API to connect to the project for the main functions of predicting and scaling. + +To make our AI model access your data and make a prediction based on it, please use the API key we'll send to your e-mail. + +Review our [Privacy Policy](https://predictkube.com/privacy-policy) to see how your data circulates in and out PredictKube. + +### Trigger Specification + +This specification describes the `predictkube` trigger that scales based on a predicting load based on `prometheus` metrics. + +```yaml +triggers: +- type: predictkube + metadata: + # Required fields: + predictHorizon: "2h" + historyTimeWindow: "7d" + prometheusAddress: http://:9090 + query: sum(rate(http_requests_total{deployment="my-deployment"}[2m])) + queryStep: "2m" + threshold: '100.50' + activationThreshold: '50.1' +``` + +**Parameter list:** + +- `predictHorizon` - Prediction time interval. It is usually equal to the cool-down period of your application. +- `historyTimeWindow` - Time range for which to request metrics from Prometheus. We recommend using minimum 7-14 days time window as historical data. +- `prometheusAddress` - Address of Prometheus server. +- `query` - Predict the query that will yield the value for the scaler to compare against the `threshold`. The query must return a vector/scalar single element response. +- `queryStep` - The maximum time between two slices within the boundaries for QML range query, used in the query. +- `threshold` - Value to start scaling for. (This value can be a float) +- `activationThreshold`- Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds).(Default: `0`, Optional, This value can be a float) + +### Authentication Parameters + +Predictkube Scaler supports one type of authentication - authentication by API key. +Prometheus used in Predictkube Scaler supports all authentication methods that are available in Prometheus Scaler. + +**Auth gateway based authentication:** + +- `apiKey` - API key previously issued for this tenant. You can get your API key by clicking on any **GET API KEY** button on the [website of PredictKube](https://predictkube.com/) + +### Example + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: predictkube-secrets + namespace: some-namespace +type: Opaque +data: + apiKey: # Required: base64 encoded value of PredictKube apiKey + bearerToken: "BEARER_TOKEN" # Optional: bearer authentication for Prometheus + ca: "CUSTOM_CA_CERT" # Optional: certificate authority file for TLS client authentication for Prometheus +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-predictkube-secret + namespace: some-namespace +spec: + secretTargetRef: + # Required: API key for your predictkube account + - parameter: apiKey + name: predictkube-secrets + key: apiKey + # Optional: might be required if you're using bearer auth for Promethrus + - parameter: bearerToken + name: keda-prom-secret + key: bearerToken + # Optional: might be required if you're using a custom CA for Promethrus + - parameter: ca + name: keda-prom-secret + key: ca +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: predictkube-scaledobject + namespace: some-namespace +spec: + scaleTargetRef: + name: my-deployment + kind: StatefulSet + pollingInterval: 30 + cooldownPeriod: 7200 + minReplicaCount: 3 + maxReplicaCount: 50 + triggers: + - type: predictkube + metadata: + predictHorizon: "2h" + historyTimeWindow: "7d" + prometheusAddress: http://:9090 + query: sum(rate(http_requests_total{deployment="my-deployment"}[2m])) # Note: query must return a vector/scalar single element response + queryStep: "2m" # Note: query step duration for range prometheus queries + threshold: "100" + authModes: "bearer" # might be required if you're using bearer auth for Promethrus + authenticationRef: + name: keda-trigger-auth-predictkube-secret +``` diff --git a/content/docs/2.17/scalers/prometheus.md b/content/docs/2.17/scalers/prometheus.md new file mode 100644 index 000000000..e72cf3668 --- /dev/null +++ b/content/docs/2.17/scalers/prometheus.md @@ -0,0 +1,525 @@ ++++ +title = "Prometheus" +availability = "v1.0+" +maintainer = "Community" +category = "Metrics" +description = "Scale applications based on Prometheus." +go_file = "prometheus_scaler" ++++ + +### Trigger Specification + +This specification describes the `prometheus` trigger that scales based on a Prometheus. + +```yaml +triggers: +- type: prometheus + metadata: + # Required fields: + serverAddress: http://:9090 + query: sum(rate(http_requests_total{deployment="my-deployment"}[2m])) # Note: query must return a vector/scalar single element response + threshold: '100.50' + activationThreshold: '5.5' + # Optional fields: + namespace: example-namespace # for namespaced queries, eg. Thanos + customHeaders: X-Client-Id=cid,X-Tenant-Id=tid,X-Organization-Id=oid # Optional. Custom headers to include in query. In case of auth header, use the custom authentication or relevant authModes. + ignoreNullValues: false # Default is `true`, which means ignoring the empty value list from Prometheus. Set to `false` the scaler will return error when Prometheus target is lost + queryParameters: key-1=value-1,key-2=value-2 + unsafeSsl: "false" # Default is `false`, Used for skipping certificate check when having self-signed certs for Prometheus endpoint + +``` + +**Parameter list:** + +- `serverAddress` - Address of Prometheus server. If using VictoriaMetrics cluster version, set full URL to Prometheus querying API, e.g. `http://:8481/select/0/prometheus` +- `query` - Query to run. +- `threshold` - Value to start scaling for. (This value can be a float) +- `activationThreshold` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds).(Default: `0`, Optional, This value can be a float) +- `namespace` - A namespace that should be used for namespaced queries. These are required by some highly available Prometheus setups, such as [Thanos](https://thanos.io). (Optional) +- `customHeaders` - Custom headers to include while querying the prometheus endpoint. In case of authentication headers, use custom authentication or relevant `authModes` instead. (Optional) +- `ignoreNullValues` - Value to reporting error when Prometheus target is lost (Values: `true`,`false`, Default: `true`, Optional) +- `queryParameters` - A comma-separated list of query Parameters to include while querying the Prometheus endpoint. (Optional) +- `unsafeSsl` - Used for skipping certificate check e.g: using self-signed certs (Values: `true`,`false`, Default: `false`, Optional) + +### Authentication Parameters + +Prometheus Scaler supports various types of authentication to help you integrate with Prometheus. + +You can use `TriggerAuthentication` CRD to configure the authentication. It is possible to specify multiple authentication types i.e. `authModes: "tls,basic"` Specify `authModes` and other trigger parameters along with secret credentials in `TriggerAuthentication` as mentioned below: + +**Bearer authentication:** +- `authModes`: It must contain `bearer` in case of Bearer Authentication. Specify this in trigger configuration. +- `bearerToken`: The token needed for authentication. This is a required field. + +**Basic authentication:** +- `authModes`: It must contain `basic` in case of Basic Authentication. Specify this in trigger configuration. +- `username` - This is a required field. Provide the username to be used for basic authentication. +- `password` - Provide the password to be used for authentication. For convenience, this has been marked optional, because many applications implement basic auth with a username as apikey and password as empty. + +**TLS authentication:** +- `authModes`: It must contain `tls` in case of TLS Authentication. Specify this in trigger configuration. +- `ca` - Certificate authority file for TLS client authentication. +- `cert` - Certificate for client authentication. This is a required field. +- `key` - Key for client authentication. Optional. This is a required field. + +**Custom authentication:** +- `authModes`: It must contain `custom` in case of Custom Authentication. Specify this in trigger configuration. +- `customAuthHeader`: Custom Authorization Header name to be used. This is required field. +- `customAuthValue`: Custom Authorization Header value. This is required field. + +> 💡 **NOTE:**It's also possible to set the CA certificate regardless of the selected `authModes` (also without any authentication). This might be useful if you are using an enterprise CA. + +### Integrating Cloud offerings + +#### Amazon Managed Service for Prometheus + +Amazon Web Services (AWS) offers a [managed service for Prometheus](https://aws.amazon.com/prometheus/) that provides a scalable and secure Prometheus deployment. The Prometheus scaler can be used to run Prometheus queries against this managed service. + +- [EKS Pod Identity](https://aws.amazon.com/about-aws/whats-new/2023/11/amazon-eks-pod-identity/) provider can be used in `authenticationRef` - see later in example. TriggerAuthentication and Secret are also supported authentication methods. +- Create Amazon Managed Service for Prometheus [workspace](https://docs.aws.amazon.com/prometheus/latest/userguide/AMP-onboard-create-workspace.html) in your AWS account +- Retrieve the Prometheus query endpoint URL from the [AWS managed Prometheus Workspace](https://docs.aws.amazon.com/prometheus/latest/userguide/AMP-onboard-create-workspace.html). This endpoint will be used to send queries. +- Configure Prometheus scaler to use the workspace endpoint and an authentication method like EKS Pod Identity. + +Using the managed service eliminates the operational burden of running your own Prometheus servers. Queries can be executed against a fully managed, auto-scaling Prometheus deployment on AWS. Costs scale linearly with usage. + +To gain a better understanding of creating a Prometheus trigger for Amazon Managed Service for Prometheus, refer to [this example](#example-amazon-managed-service-for-prometheus-amp). + +#### Azure Monitor Managed Service for Prometheus + +Azure has a [managed service for Prometheus](https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/prometheus-metrics-overview) and Prometheus scaler can be used to run prometheus query against that. +- [Azure AD Workload Identity](https://azure.github.io/azure-workload-identity/docs/) provider can be used in `authenticationRef` - see later in example. +- `Monitoring Data Reader` role needs to be assigned to workload identity (or pod identity) on the `Azure Monitor Workspace`. +- No other auth (via `authModes`) can be provided with Azure Pod/Workload Identity Auth. +- Prometheus query endpoint can be retreived from [Azure Monitor Workspace](https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/azure-monitor-workspace-overview) that was configured to ingest prometheus metrics. + +To gain a better understanding of creating a Prometheus trigger for Azure Monitor Managed Service for Prometheus, refer to [this example](#example-azure-monitor-managed-service-for-prometheus). + +#### Google Managed Service for Prometheus + +Google Cloud Platform provides a comprehensive [managed service for Prometheus](https://cloud.google.com/stackdriver/docs/managed-prometheus), enabling you to effortlessly export and query Prometheus metrics. +By utilizing Prometheus scaler, you can seamlessly integrate it with the GCP managed service and handle authentication using the GCP workload identity mechanism. + +See the follwowing steps to configure the scaler integration. + +- Setup [GCP Workload Identity](./../authentication-providers/gcp-workload-identity) on KEDA operator; +- Assign the [Monitoring Viewer](https://cloud.google.com/iam/docs/understanding-roles#monitoring.viewer) role (namely `roles/monitoring.viewer`) to the Google Service Account on Identity Access and Management (IAM). +- No other auth (via `authModes`) should be provided other than GCP workload identity auth; +- Prometheus server address should follow the Google's Monitoring API for [Prometheus HTTP API](https://cloud.google.com/stackdriver/docs/managed-prometheus/query#api-prometheus): + - Example: `https://monitoring.googleapis.com/v1/projects/GOOGLE_PROJECT_ID/location/global/prometheus` - where `GOOGLE_PROJECT_ID` should be replaced by your Google project ID. + +To gain a better understanding of creating a Prometheus trigger for Google Managed Prometheus, refer to [this example](#example-google-managed-prometheus). + +### Examples + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: prometheus-scaledobject + namespace: default +spec: + scaleTargetRef: + name: my-deployment + triggers: + - type: prometheus + metadata: + serverAddress: http://:9090 + threshold: '100' + query: sum(rate(http_requests_total{deployment="my-deployment"}[2m])) +``` + +#### Example: Bearer Authentication + +Here is an example of a prometheus scaler with Bearer Authentication, define the `Secret` and `TriggerAuthentication` as follows + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-prom-secret + namespace: default +data: + bearerToken: "BEARER_TOKEN" + ca: "CUSTOM_CA_CERT" +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-prom-creds + namespace: default +spec: + secretTargetRef: + - parameter: bearerToken + name: keda-prom-secret + key: bearerToken + # might be required if you're using a custom CA + - parameter: ca + name: keda-prom-secret + key: ca +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: prometheus-scaledobject + namespace: keda + labels: + deploymentName: dummy +spec: + maxReplicaCount: 12 + scaleTargetRef: + name: dummy + triggers: + - type: prometheus + metadata: + serverAddress: http://:9090 + threshold: '100' + query: sum(rate(http_requests_total{deployment="my-deployment"}[2m])) + authModes: "bearer" + authenticationRef: + name: keda-prom-creds +``` + +#### Example: Basic Authentication + +Here is an example of a prometheus scaler with Basic Authentication, define the `Secret` and `TriggerAuthentication` as follows + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-prom-secret + namespace: default +data: + username: "dXNlcm5hbWUK" # Must be base64 + password: "cGFzc3dvcmQK" +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-prom-creds + namespace: default +spec: + secretTargetRef: + - parameter: username + name: keda-prom-secret + key: username + - parameter: password + name: keda-prom-secret + key: password +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: prometheus-scaledobject + namespace: keda + labels: + deploymentName: dummy +spec: + maxReplicaCount: 12 + scaleTargetRef: + name: dummy + triggers: + - type: prometheus + metadata: + serverAddress: http://:9090 + threshold: '100' + query: sum(rate(http_requests_total{deployment="my-deployment"}[2m])) + authModes: "basic" + authenticationRef: + name: keda-prom-creds +``` + +#### Example: TLS Authentication + +Here is an example of a prometheus scaler with TLS Authentication, define the `Secret` and `TriggerAuthentication` as follows + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-prom-secret + namespace: default +data: + cert: "cert" + key: "key" + ca: "ca" +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-prom-creds + namespace: default +spec: + secretTargetRef: + - parameter: cert + name: keda-prom-secret + key: cert + - parameter: key + name: keda-prom-secret + key: key + - parameter: ca + name: keda-prom-secret + key: ca +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: prometheus-scaledobject + namespace: keda + labels: + deploymentName: dummy +spec: + maxReplicaCount: 12 + scaleTargetRef: + name: dummy + triggers: + - type: prometheus + metadata: + serverAddress: http://:9090 + threshold: '100' + query: sum(rate(http_requests_total{deployment="my-deployment"}[2m])) + authModes: "tls" + authenticationRef: + name: keda-prom-creds +``` + +#### Example: TLS & Basic Authentication + +Here is an example of a prometheus scaler with TLS and Basic Authentication, define the `Secret` and `TriggerAuthentication` as follows + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-prom-secret + namespace: default +data: + cert: "cert" + key: "key" + ca: "ca" + username: "username" + password: "password" +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-prom-creds + namespace: default +spec: + secretTargetRef: + - parameter: cert + name: keda-prom-secret + key: cert + - parameter: key + name: keda-prom-secret + key: key + - parameter: ca + name: keda-prom-secret + key: ca + - parameter: username + name: keda-prom-secret + key: username + - parameter: password + name: keda-prom-secret + key: password +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: prometheus-scaledobject + namespace: keda + labels: + deploymentName: dummy +spec: + maxReplicaCount: 12 + scaleTargetRef: + name: dummy + triggers: + - type: prometheus + metadata: + serverAddress: http://:9090 + threshold: '100' + query: sum(rate(http_requests_total{deployment="my-deployment"}[2m])) + authModes: "tls,basic" + authenticationRef: + name: keda-prom-creds +``` + +#### Example: Custom Authentication + +Here is an example of a prometheus scaler with Custom Authentication, define the `Secret` and `TriggerAuthentication` as follows + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-prom-secret + namespace: default +data: + customAuthHeader: "X-AUTH-TOKEN" + customAuthValue: "auth-token" +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-prom-creds + namespace: default +spec: + secretTargetRef: + - parameter: customAuthHeader + name: keda-prom-secret + key: customAuthHeader + - parameter: customAuthValue + name: keda-prom-secret + key: customAuthValue +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: prometheus-scaledobject + namespace: keda + labels: + deploymentName: dummy +spec: + maxReplicaCount: 12 + scaleTargetRef: + name: dummy + triggers: + - type: prometheus + metadata: + serverAddress: http://:9090 + threshold: '100' + query: sum(rate(http_requests_total{deployment="my-deployment"}[2m])) + authModes: "custom" + authenticationRef: + name: keda-prom-creds +``` + +#### Example: Azure Monitor Managed Service for Prometheus + +Here is an example of a prometheus scaler with Azure Pod Identity and Azure Workload Identity, define the `TriggerAuthentication` and `ScaledObject` as follows + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: azure-managed-prometheus-trigger-auth +spec: + podIdentity: + provider: azure-workload + identityId: # Optional. Default: Identity linked with the label set when installing KEDA. +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: azure-managed-prometheus-scaler +spec: + scaleTargetRef: + name: deployment-name-to-be-scaled + minReplicaCount: 1 + maxReplicaCount: 20 + triggers: + - type: prometheus + metadata: + serverAddress: https://test-azure-monitor-workspace-name-9ksc.eastus.prometheus.monitor.azure.com + query: sum(rate(http_requests_total{deployment="my-deployment"}[2m])) # Note: query must return a vector/scalar single element response + threshold: '100.50' + activationThreshold: '5.5' + authenticationRef: + name: azure-managed-prometheus-trigger-auth +``` +#### Example: Amazon Managed Service for Prometheus (AMP) + +Below is an example showcasing the use of Prometheus scaler with AWS EKS Pod Identity. Please note that in this particular example, the Deployment is named as `keda-deploy`. Also replace the AwsRegion and AMP WorkspaceId for your requirements. + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-aws-credentials +spec: + podIdentity: + provider: aws +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: keda-deploy + labels: + app: keda-deploy +spec: + replicas: 0 + selector: + matchLabels: + app: keda-deploy + template: + metadata: + labels: + app: keda-deploy + spec: + containers: + - name: nginx + image: nginxinc/nginx-unprivileged + ports: + - containerPort: 80 +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: keda-so + labels: + app: keda-deploy +spec: + scaleTargetRef: + name: keda-deploy + maxReplicaCount: 2 + minReplicaCount: 0 + cooldownPeriod: 1 + advanced: + horizontalPodAutoscalerConfig: + behavior: + scaleDown: + stabilizationWindowSeconds: 15 + triggers: + - type: prometheus + authenticationRef: + name: keda-trigger-auth-aws-credentials + metadata: + awsRegion: {{.AwsRegion}} + serverAddress: "https://aps-workspaces.{{.AwsRegion}}.amazonaws.com/workspaces/{{.WorkspaceID}}" + query: "vector(100)" + threshold: "50.0" + identityOwner: operator +``` + +#### Example: Google Managed Prometheus + +Below is an example showcasing the use of Prometheus scaler with GCP Workload Identity. Please note that in this particular example, the Google project ID has been set as `my-google-project`. + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ClusterTriggerAuthentication +metadata: + name: google-workload-identity-auth +spec: + podIdentity: + provider: gcp +--- +apiVersion: keda.sh/v1alpha1 +metadata: + name: google-managed-prometheus-scaler +spec: + scaleTargetRef: + name: deployment-name-to-be-scaled + minReplicaCount: 1 + maxReplicaCount: 20 + triggers: + - type: prometheus + metadata: + serverAddress: https://monitoring.googleapis.com/v1/projects/my-google-project/location/global/prometheus + query: sum(rate(http_requests_total{deployment="my-deployment"}[2m])) + threshold: '50.0' + authenticationRef: + kind: ClusterTriggerAuthentication + name: google-workload-identity-auth +``` diff --git a/content/docs/2.17/scalers/pulsar.md b/content/docs/2.17/scalers/pulsar.md new file mode 100644 index 000000000..b0c8edc11 --- /dev/null +++ b/content/docs/2.17/scalers/pulsar.md @@ -0,0 +1,343 @@ ++++ +title = "Apache Pulsar" +availability = "v2.8" +maintainer = "Community" +category = "Messaging" +description = "Scale applications based on an Apache Pulsar topic subscription." +go_file = "pulsar_scaler" +layout = "scaler" ++++ + +### Trigger Specification + +This specification describes the `pulsar` trigger for an Apache Pulsar topic. + +```yaml +triggers: +- type: pulsar + metadata: + adminURL: http://localhost:80 + topic: persistent://public/default/my-topic + isPartitionedTopic: false + subscription: sub1 + msgBacklogThreshold: '5' + activationMsgBacklogThreshold: '2' + authModes: "" +``` + +**Parameter list:** + +- `adminURL` - Stats URL of the admin API for your topic. +- `topic` - Pulsar topic. format of `persistent://{tenant}/{namespace}/{topicName}` +- `isPartitionedTopic` - Whether the `topic` is partitioned. When `true`, the `msgBacklogThreshold` will be the cumulative subscription backlog across partitions. (default: `false`, Optional) +- `subscription` - Name of the topic subscription +- `msgBacklogThreshold` - Average target value to trigger scaling actions. (default: 10) +- `activationMsgBacklogThreshold` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) +- `authModes` - a comma separated list of authentication modes to use. (Values: `bearer`, `tls`, `basic`, `oauth`, Default: `""`, Optional, `tls,bearer` or `tls,basic` are valid combinations and would indicate mutual TLS to secure the connection and then `bearer` or `basic` headers should be added to the HTTP request) +- `oauthTokenURI` - The OAuth Access Token URI for the OAuth provider, used when `authModes` is set to `oauth`. Ignored if provided from `authenticationRef`. (Optional) +- `scope` - A comma separated lists of OAuth scopes, used when `authModes` is set to `oauth`. Ignored if provided from `authenticationRef`. (Optional) +- `clientID` - Client ID for the OAuth provider, used when `authModes` is set to `oauth`. Ignored if provided from `authenticationRef`. (Optional) +- `endpointParams` - Additional parameters as URL-encoded query string for requests to the token endpoint for the OAuth provider, used when `authModes` is set to `oauth`. Ignored if provided from `authenticationRef`. (Optional) + +### Authentication Parameters + +The authentication is defined in `authModes`. The associated configuration parameters are specified in the `TriggerAuthentication` spec. + +**Bearer Auth** + +When configuring Bearer Authentication (Token Auth), configure the following: + +- `bearerToken`: This token will be sent as a header in the form `Authorization: Bearer `. + +**Basic Auth** +When configuring Basic Authentication, configure the following: + +- `username`: the username +- `password`: the password (optional) + +**TLS:** + +When configuring mutual TLS authentication, configure the following: + +- `ca`: The trusted root Certificate authority used to validate the server's certificate. +- `cert`: Certificate for client authentication. +- `key`: Key for client authentication. + +**OAuth 2** + +When configuring OAuth Authentication, configure the following: +- `oauthTokenURI` - The OAuth Access Token URI for the OAuth provider. (Optional) +- `scope` - A comma separated lists of OAuth scopes. (Optional) +- `clientID`: Client ID for the OAuth provider. (Optional) +- `clientSecret`: Client secret for the OAuth provider. (Optional) +- `endpointParams`: Additional parameters as URL-encoded query string for requests to the token endpoint for the OAuth provider. (Optional) + +These can also be configured in the trigger metadata except the `clientSecret` + +### TLS with custom CA Certificates + +When configuring a trusted root CA that is not well known, it is sufficient to specify the `ca` field on the `TriggerAuthentication` resource. See [Bearer Token with TLS via custom trusted CA Certificate](#bearer-token-with-tls-via-custom-trusted-ca-certificate) for an example. + +Before 2.9.0, it was necessary to configure `tls: enable` on the `ScaledObject`. That was removed in a backwards compatible way, so you can remove that field now. + +### Examples + +#### No Authentication and No TLS + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: pulsar-scaledobject + namespace: default +spec: + scaleTargetRef: + name: pulsar-consumer + pollingInterval: 30 + triggers: + - type: pulsar + metadata: + adminURL: http://localhost:80 + topic: persistent://public/default/my-topic + isPartitionedTopic: false + subscription: sub1 + msgBacklogThreshold: '5' +``` + +#### Only Mutual TLS Authentication + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-pulsar-secrets + namespace: default +data: + ca: + cert: + key: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-pulsar-credential + namespace: default +spec: + secretTargetRef: + - parameter: ca + name: keda-pulsar-secrets + key: ca + - parameter: cert + name: keda-pulsar-secrets + key: cert + - parameter: key + name: keda-pulsar-secrets + key: key +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: pulsar-scaledobject + namespace: default +spec: + scaleTargetRef: + name: pulsar-consumer + pollingInterval: 30 + triggers: + - type: pulsar + metadata: + authModes: "tls" + adminURL: https://localhost:8443 + topic: persistent://public/default/my-topic + subscription: sub1 + msgBacklogThreshold: '5' + authenticationRef: + name: keda-trigger-auth-pulsar-credential +``` + +#### Bearer Token with TLS via custom trusted CA Certificate + +In order to enable Pulsar's Token Authentication feature, you can use the following example. Note that this example +also utilizes a custom CA Certificate for TLS support. Because TLS Authentication is not used in this example, the +`authModes` field only has `bearer`. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-pulsar-secrets + namespace: default +data: + ca: + token: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-pulsar-credential + namespace: default +spec: + secretTargetRef: + - parameter: ca + name: keda-pulsar-secrets + key: ca + - parameter: bearerToken + name: keda-pulsar-secrets + key: token +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: pulsar-scaledobject + namespace: default +spec: + scaleTargetRef: + name: pulsar-consumer + pollingInterval: 30 + triggers: + - type: pulsar + metadata: + authModes: "bearer" + adminURL: https://localhost:8443 + topic: persistent://public/default/my-topic + subscription: sub1 + msgBacklogThreshold: '5' + authenticationRef: + name: keda-trigger-auth-pulsar-credential +``` + +#### Basic Auth with TLS Relying on Well Known Root CA + +In order to enable Pulsar's Token Authentication feature, you can use the following example. Note that this example +also utilizes a custom CA Certificate for TLS support. Because TLS Authentication is not used in this example, the +`authModes` field only has `bearer`. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-pulsar-secrets + namespace: default +data: + username: + password: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-pulsar-credential + namespace: default +spec: + secretTargetRef: + - parameter: username + name: keda-pulsar-secrets + key: admin + - parameter: password + name: keda-pulsar-secrets + key: password +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: pulsar-scaledobject + namespace: default +spec: + scaleTargetRef: + name: pulsar-consumer + pollingInterval: 30 + triggers: + - type: pulsar + metadata: + authModes: "bearer" + adminURL: https://pulsar.com:8443 + topic: persistent://public/default/my-topic + subscription: sub1 + msgBacklogThreshold: '5' + authenticationRef: + name: keda-trigger-auth-pulsar-credential +``` + +#### OAuth Authentication + +In order to enable Pulsar's OAuth Authentication feature, you can use the following example. Note that only client credentials flow is supported. +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-pulsar-secrets + namespace: default +data: + oauthTokenURI: + scope: + clientID: + clientSecret: + endpointParams: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-pulsar-credential + namespace: default +spec: + secretTargetRef: + - parameter: oauthTokenURI + name: keda-pulsar-secrets + key: oauthTokenURI + - parameter: scope + name: keda-pulsar-secrets + key: scope + - parameter: clientID + name: keda-pulsar-secrets + key: clientID + - parameter: clientSecret + name: keda-pulsar-secrets + key: clientSecret + - parameter: endpointParams + name: keda-pulsar-secrets + key: endpointParams +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: pulsar-scaledobject + namespace: default +spec: + scaleTargetRef: + name: pulsar-consumer + pollingInterval: 30 + triggers: + - type: pulsar + metadata: + authModes: "oauth" + adminURL: https://pulsar.com:8443 + topic: persistent://public/default/my-topic + subscription: sub1 + msgBacklogThreshold: '5' + authenticationRef: + name: keda-trigger-auth-pulsar-credential +``` +You can also use the following example without `authenticationRef` if your OAuth provider supports. + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: pulsar-scaledobject + namespace: default +spec: + scaleTargetRef: + name: pulsar-consumer + pollingInterval: 30 + triggers: + - type: pulsar + metadata: + authModes: "oauth" + adminURL: https://pulsar.com:8443 + topic: persistent://public/default/my-topic + subscription: sub1 + msgBacklogThreshold: '5' + oauthTokenURI: http://oauth.com/oauth2/token + scope: + clientID: + endpointParams: +``` \ No newline at end of file diff --git a/content/docs/2.17/scalers/rabbitmq-queue.md b/content/docs/2.17/scalers/rabbitmq-queue.md new file mode 100644 index 000000000..e2cc20e20 --- /dev/null +++ b/content/docs/2.17/scalers/rabbitmq-queue.md @@ -0,0 +1,420 @@ ++++ +title = "RabbitMQ Queue" +availability = "v1.0+" +maintainer = "Microsoft" +category = "Messaging" +description = "Scale applications based on RabbitMQ Queue." +go_file = "rabbitmq_scaler" ++++ + +### Trigger Specification + +This specification describes the `rabbitmq` trigger for RabbitMQ Queue. + +```yaml +triggers: +- type: rabbitmq + metadata: + host: amqp://localhost:5672/vhost # Optional. If not specified, it must be done by using TriggerAuthentication. + protocol: auto # Optional. Specifies protocol to use, either amqp or http, or auto to autodetect based on the `host` value. Default value is auto. + mode: QueueLength # QueueLength or MessageRate + value: "100.50" # message backlog or publish/sec. target per instance + activationValue: "10.5" # Optional. Activation threshold + queueName: testqueue + vhostName: / # Optional. If not specified, use the vhost in the `host` connection string. Required for Azure AD Workload Identity authorization (see bellow) + # Alternatively, you can use existing environment variables to read configuration from: + # See details in "Parameter list" section + hostFromEnv: RABBITMQ_HOST # Optional. You can use this instead of `host` parameter + usernameFromEnv: RABBITMQ_USERNAME # Optional. You can use this instead of TriggerAuthentication + passwordFromEnv: RABBITMQ_PASSWORD # Optional. You can use this instead of TriggerAuthentication + unsafeSsl: true +``` + +**Parameter list:** + +- `host` - Host of RabbitMQ with format `://:/vhost`. If the protocol is HTTP than the host may follow this format `http://://`. In example the resolved host value could be `amqp://guest:password@localhost:5672/vhost` or `http://guest:password@localhost:15672/path/vhost`. If the host doesn't contain vhost than the trailing slash is required in this case like `http://guest:password@localhost:5672/`. When using a username/password consider using `hostFromEnv` or a TriggerAuthentication. +- `queueName` - Name of the queue to read message from. +- `mode` - QueueLength to trigger on number of messages in the queue. MessageRate to trigger on the published rate into the queue. (Values: `QueueLength`, `MessageRate`) +- `value` - Message backlog or Publish/sec. rate to trigger on. (This value can be a float when `mode: MessageRate`) +- `activationValue` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds).(Default: `0`, Optional, This value can be a float) +- `protocol` - Protocol to be used for communication. (Values: `auto`, `http`, `amqp`, Default: `auto`, Optional) +- `vhostName` - Vhost to use for the connection, overrides any vhost set in the connection string from `host`/`hostFromEnv`. (Optional / Required if Azure AD Workload Identity authorization is used) +- `queueLength` - DEPRECATED! Use `mode: QueueLength` and `value: ##` instead. Target value for queue length passed to the scaler. Example: if one pod can handle 10 messages, set the queue length target to 10. If the actual number of messages in the queue is 30, the scaler scales to 3 pods. Default is 20 unless `publishRate` is specified, in which case `queueLength` is disabled for this trigger. +- `useRegex` - This parameter allows to use regex (in `queueName` parameter) to select queue instead of full name. (Values: `true`, `false`, Default: `false`, Optional, Only applies to hosts that use the `http` protocol) +- `pageSize` - This parameter allows setting page size. (Default: `100`, Optional, Only applies when `useRegex` is `true`) +- `operation` - Operation that will be applied to compute the number of messages in case of `useRegex` enabled. Either `sum` (default),`max`, or `avg`. (Optional) +- `timeout` - Timeout **for this specific trigger**. This value will override the value defined in `KEDA_HTTP_DEFAULT_TIMEOUT`. (Optional, Only applies to hosts that use the `http` protocol) +- `excludeUnacknowledged` - Set to `true` to specify that the `QueueLength` value should exclude unacknowledged messages (Ready messages only). (Values: `true`, `false`, Default: `false`, Optional, Only applies to hosts that use the `http` protocol) +- `unsafeSsl` - Whether to allow unsafe SSL (Values: `true`, `false`, Default: `false` ) + +Some parameters could be provided using environmental variables, instead of setting them directly in metadata. Here is a list of parameters you can use to retrieve values from environment variables: + +- `hostFromEnv` - The host and port of the RabbitMQ server, similar to `host`, but reads it from an environment variable on the scale target. +- `usernameFromEnv` - The username to use to connect to the broker's management endpoint. +- `passwordFromEnv` - The password to use to connect to the broker's management endpoint. + +> 💡 **Note:** `host`/`hostFromEnv` has an optional vhost name after the host slash which will be used to scope API request. + +> 💡 **Note:** When using `host`/`hostFromEnv` or TriggerAuthentication, the supplied password cannot contain special characters. + +> 💡 **Note:** `mode: MessageRate` requires protocol `http`. + +> 💡 **Note:** `useRegex: "true"` requires protocol `http`. + +> ⚠ **Important:** if you have unacknowledged messages and want to have these counted for the scaling to happen, make sure to utilize the `http` REST API interface which allows for these to be counted. + +> ⚠ **Important:** If scaling against both is desired then the `ScaledObject` should have two triggers, one for `mode: QueueLength` and the other for `mode: MessageRate`. HPA will scale based on the largest result considering each of the two triggers independently. + +### Authentication Parameters + +TriggerAuthentication CRD is used to connect and authenticate to RabbitMQ: + +- For AMQP, the URI should look similar to `amqp://guest:password@localhost:5672/vhost`. +- For HTTP, the URI should look similar to `http://guest:password@localhost:15672/path/vhost`. + +> See the [RabbitMQ Ports](https://www.rabbitmq.com/networking.html#ports) section for more details on how to configure the ports. + +**Username and Password based authentication:** + +This allows sensitive credentials to be stored and managed separately from the connection string. + +- `username` - The username to use to connect to the broker's management endpoint. +- `password` - The password to use to connect to the broker's management endpoint. + +> 💡 **Note:** If username or password are set in TriggerAuthentication or environment variables, they will override any credentials provided in the host. + +**TLS authentication:** + +- `tls` - To enable SSL auth for RabbitMQ, set this to `enable`. If not set, TLS for RabbitMQ is not used. (Values: `enable`, `disable`, Default: `disable`, Optional) +- `ca` - Certificate authority file for TLS client authentication. (Optional) +- `cert` - Certificate for client authentication. (Optional) +- `key` - Key for client authentication. (Optional) + +> Using RabbitMQ host with amqps will require enabling the tls settings and passing the required parameters. + +**Azure Workload Identity authentication:** + +For RabbitMQ with OIDC support (>= 3.11) you can use TriggerAuthentication CRD with `podIdentity.provider = azure-workload` and with parameter `workloadIdentityResource` which would hold application identifier of App Registraion in Azure AD. In this case `username:password` part in host URI should be ommited and `vHostName` has to be set explicitly in `ScaledObject`. Only HTTP protocol is supported for AKS Workload Identity currently. + +### Example + +#### AMQP protocol: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-rabbitmq-secret +data: + host: # base64 encoded value of format amqp://guest:password@localhost:5672/vhost +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-rabbitmq-conn + namespace: default +spec: + secretTargetRef: + - parameter: host + name: keda-rabbitmq-secret + key: host +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: rabbitmq-scaledobject + namespace: default +spec: + scaleTargetRef: + name: rabbitmq-deployment + triggers: + - type: rabbitmq + metadata: + protocol: amqp + queueName: testqueue + mode: QueueLength + value: "20" + authenticationRef: + name: keda-trigger-auth-rabbitmq-conn +``` + +#### AMQP protocol with user/password auth: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-rabbitmq-secret +data: + host: # base64 encoded value of format amqp://localhost:5672/vhost (no username/password) + username: # base64 encoded value of username + password: # base64 encoded value of password +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-rabbitmq-conn + namespace: default +spec: + secretTargetRef: + - parameter: host + name: keda-rabbitmq-secret + key: host + - parameter: username + name: keda-rabbitmq-secret + key: username + - parameter: password + name: keda-rabbitmq-secret + key: password +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: rabbitmq-scaledobject + namespace: default +spec: + scaleTargetRef: + name: rabbitmq-deployment + triggers: + - type: rabbitmq + metadata: + protocol: amqp + queueName: testqueue + mode: QueueLength + value: "20" + authenticationRef: + name: keda-trigger-auth-rabbitmq-conn +``` + +#### AMQPS protocol with TLS auth: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-rabbitmq-secret +data: + host: # base64 encoded value of format amqps://guest:password@localhost:5672/vhost + tls: "enable" + ca: + cert: + key: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-rabbitmq-conn + namespace: default +spec: + secretTargetRef: + - parameter: host + name: keda-rabbitmq-secret + key: host + - parameter: tls + name: keda-rabbitmq-secret + key: tls + - parameter: ca + name: keda-rabbitmq-secret + key: ca + - parameter: cert + name: keda-rabbitmq-secret + key: cert + - parameter: key + name: keda-rabbitmq-secret + key: key +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: rabbitmq-scaledobject + namespace: default +spec: + scaleTargetRef: + name: rabbitmq-deployment + triggers: + - type: rabbitmq + metadata: + protocol: amqp + queueName: testqueue + mode: QueueLength + value: "20" + authenticationRef: + name: keda-trigger-auth-rabbitmq-conn +``` + +#### HTTP protocol (`QueueLength`): + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-rabbitmq-secret +data: + host: # base64 encoded value of format http://guest:password@localhost:15672/path/vhost +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-rabbitmq-conn + namespace: default +spec: + secretTargetRef: + - parameter: host + name: keda-rabbitmq-secret + key: host +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: rabbitmq-scaledobject + namespace: default +spec: + scaleTargetRef: + name: rabbitmq-deployment + triggers: + - type: rabbitmq + metadata: + protocol: http + queueName: testqueue + mode: QueueLength + value: "20" + authenticationRef: + name: keda-trigger-auth-rabbitmq-conn +``` + +#### HTTP protocol (`MessageRate` and `QueueLength`): + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-rabbitmq-secret +data: + host: # base64 encoded value of format http://guest:password@localhost:15672/path/vhost +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-rabbitmq-conn + namespace: default +spec: + secretTargetRef: + - parameter: host + name: keda-rabbitmq-secret + key: host +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: rabbitmq-scaledobject + namespace: default +spec: + scaleTargetRef: + name: rabbitmq-deployment + triggers: + - type: rabbitmq + metadata: + protocol: http + queueName: testqueue + mode: QueueLength + value: "20" + authenticationRef: + name: keda-trigger-auth-rabbitmq-conn + - type: rabbitmq + metadata: + protocol: http + queueName: testqueue + mode: MessageRate + value: "100" + authenticationRef: + name: keda-trigger-auth-rabbitmq-conn +``` + +#### HTTP protocol (`QueueLength`) and using regex (`useRegex`): + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-rabbitmq-secret +data: + host: # base64 encoded value of format http://guest:password@localhost:15672/path/vhost +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-rabbitmq-conn + namespace: default +spec: + secretTargetRef: + - parameter: host + name: keda-rabbitmq-secret + key: host +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: rabbitmq-scaledobject + namespace: default +spec: + scaleTargetRef: + name: rabbitmq-deployment + triggers: + - type: rabbitmq + metadata: + protocol: http + queueName: ^.*incoming$ + mode: QueueLength + value: "20" + useRegex: "true" + operation: max + authenticationRef: + name: keda-trigger-auth-rabbitmq-conn +``` + +#### HTTP protocol (`QueueLength`) with Azure Workload Identity: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: keda-rabbitmq-secret +data: + host: # base64 encoded value of format http://localhost:15672/ !! no password !! + clientId: # base64 encoded value of Client ID (same as for Rabbit's auth_oauth2.resource_server_id) +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-rabbitmq-conn + namespace: default +spec: + podIdentity: + provider: azure-workload + secretTargetRef: + - parameter: host + name: keda-rabbitmq-secret + key: host + - parameter: workloadIdentityResource + name: keda-rabbitmq-secret + key: clientId +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: rabbitmq-scaledobject + namespace: default +spec: + scaleTargetRef: + name: rabbitmq-deployment + triggers: + - type: rabbitmq + metadata: + protocol: http + vHostName: / + queueName: testqueue + mode: QueueLength + value: "20" + authenticationRef: + name: keda-trigger-auth-rabbitmq-conn +``` diff --git a/content/docs/2.17/scalers/redis-cluster-lists.md b/content/docs/2.17/scalers/redis-cluster-lists.md new file mode 100644 index 000000000..6e6c240e9 --- /dev/null +++ b/content/docs/2.17/scalers/redis-cluster-lists.md @@ -0,0 +1,125 @@ ++++ +title = "Redis Lists (supports Redis Cluster)" +availability = "v2.1+" +maintainer = "Community" +category = "Data & Storage" +description = "Redis Lists scaler with support for Redis Cluster topology" +go_file = "redis_scaler" ++++ + +### Trigger Specification + +This specification describes the `redis-cluster` trigger that scales based on the length of a list in a Redis Cluster. + +```yaml +triggers: +- type: redis-cluster + metadata: + addresses: localhost:6379 # Comma separated list of the format host:port + usernameFromEnv: REDIS_USERNAME # optional + passwordFromEnv: REDIS_PASSWORD + listName: mylist # Required + listLength: "5" # Required + activationListLength: "5" # optional + enableTLS: "false" # optional + unsafeSsl: "false" # optional + databaseIndex: "0" # optional + # Alternatively, you can use existing environment variables to read configuration from: + # See details in "Parameter list" section + addressesFromEnv: REDIS_ADDRESSES # Optional. You can use this instead of `addresses` parameter +``` + +**Parameter list:** + +- `addresses` - Comma separated list of hosts and ports of the Redis Cluster nodes. +- `hosts` - Comma separated list of hosts of the Redis Cluster nodes. Alternative to `addresses` and requires `ports` to be configured as well. +- `ports` - Comma separated list of corresponding ports for the hosts of the Redis Cluster nodes. Alternative to `addresses` and requires `hosts` to be configured as well. +- `usernameFromEnv` - Environment variable to read the authentication username from to authenticate with the Redis server. +- `passwordFromEnv` - Environment variable to read the authentication password from to authenticate with the Redis server. + - Both the hostname, username and password fields need to be set to the names of the environment variables in the target deployment that contain the host name, username and password respectively. +- `listName` - Name of the Redis List that you want to monitor. +- `listLength` - Average target value to trigger scaling actions. +- `activationListLength` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) +- `enableTLS` - Allow a connection to a redis queue using tls. (Values: `true`, `false`, Default: `false`, Optional) +- `unsafeSsl` - Used for skipping certificate check e.g: using self-signed certs. (Values: `true`,`false`, Default: `false`, Optional, This requires `enableTLS: true`) +- `databaseIndex` - Index of Redis database to use. If not specified, the default value is 0. + +Some parameters could be provided using environmental variables, instead of setting them directly in metadata. Here is a list of parameters you can use to retrieve values from environment variables: + +- `addressesFromEnv` - The hosts and their respective ports of the Redis Cluster nodes, similar to `addresses`, but reads it from an environment variable on the scale target. +- `hostsFromEnv` - The hosts of the Redis Cluster nodes, similar to `hosts`, but reads it from an environment variable on the scale target. +- `portsFromEnv` - The corresponding ports for the hosts of the Redis Cluster nodes, similar to `ports`, but reads it from an environment variable on the scale target. + +### Authentication Parameters + +You can authenticate by using a password. + +**Connection Authentication:** + +- `addresses` - Comma separated list of host:port format. +- `hosts` - Comma separated list of hostname of the Redis Cluster nodes. If specified, the `ports` should also be specified. +- `ports` - Comma separated list of ports of the Redis Cluster nodes. If specified, the `hosts` should also be specified. + +**Authentication:** + +- `username` - Redis username to authenticate with. +- `password` - Redis password to authenticate with. + +**TLS:** + +Parameters used for configuring TLS authentication. Note this can not be used together with `enableTLS` and `unsafeSsl` on the `ScaledObject`, which is used to define using insecure TLS with skipping certificate check. + +- `tls` - To enable SSL auth for Redis, set this to `enable`. If not set, TLS for Redis is not used. (Values: `enable`, `disable`, Default: `disable`, Optional) +- `ca` - Certificate authority file for TLS authentication. (Optional) +- `cert` - Certificate for client authentication. (Optional) +- `key` - Key for client authentication. (Optional) +- `keyPassword` - If set the `keyPassword` is used to decrypt the provided `key`. (Optional) + +### Example + +Here is an example of how to deploy a scaled object with the `redis-cluster` scale trigger which uses `TriggerAuthentication`. + +You can also provide the `usernameFromEnv` and `passwordFromEnv` on the `ScaledObject` directly. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: votes-db-secret + namespace: my-project +type: Opaque +data: + redis_username: YWRtaW4= + redis_password: YWRtaW4= +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-redis-secret + namespace: my-project +spec: + secretTargetRef: + - parameter: username + name: votes-db-secret + key: redis_username + - parameter: password + name: votes-db-secret + key: redis_password +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: redis-scaledobject + namespace: my-project +spec: + scaleTargetRef: + name: votes + triggers: + - type: redis-cluster + metadata: + addresses: node1:6379, node2:6379, node3:6379 + listName: mylist + listLength: "10" + authenticationRef: + name: keda-trigger-auth-redis-secret +``` diff --git a/content/docs/2.17/scalers/redis-cluster-streams.md b/content/docs/2.17/scalers/redis-cluster-streams.md new file mode 100644 index 000000000..be57aa5f7 --- /dev/null +++ b/content/docs/2.17/scalers/redis-cluster-streams.md @@ -0,0 +1,252 @@ ++++ +title = "Redis Streams (supports Redis Cluster)" +availability = "v2.1+" +maintainer = "Community" +category = "Data & Storage" +description = "Redis Streams scaler with support for Redis Cluster topology" +go_file = "redis_streams_scaler" ++++ + +### Trigger Specification + +Redis 5.0 introduced [Redis Streams](https://redis.io/topics/streams-intro) which is an append-only log data structure. + +One of its features includes [`Consumer Groups`](https://redis.io/topics/streams-intro#consumer-groups), that allows a group of clients to co-operate consuming a different portion of the same stream of messages. + +There are three ways to configure `redis-streams` trigger: +1. Based on the *Pending Entries List* (see [`XPENDING`](https://redis.io/commands/xpending)) for a specific Consumer Group of a Redis Stream +2. Based on the *Stream Length* (see [`XLEN`](https://redis.io/commands/xlen)) +3. Based on the *Consumer Group Lag* (see [`XINFO GROUPS`](https://redis.io/commands/xinfo-groups/)). This is the only configuration that supports scaling down to 0. IMPORTANT: Redis 7+ is required for this feature to run. + + +```yaml +triggers: +- type: redis-cluster-streams + metadata: + addresses: localhost:6379 # Required if hosts and ports are not provided. Format - comma separated list of host:port + hosts: localhost # Comma separated lists of hosts. Required if address is not provided + ports: "6379" # Comma separated lists of ports. Required if addresses are not provided and hosts has been provided. + usernameFromEnv: REDIS_USERNAME # optional (can also use authenticationRef) + passwordFromEnv: REDIS_PASSWORD # optional (can also use authenticationRef) + stream: my-stream # Required - name of the Redis Stream + consumerGroup: my-consumer-group # optional - name of consumer group associated with Redis Stream + pendingEntriesCount: "10" # optional - number of entries in the Pending Entries List for the specified consumer group in the Redis Stream + streamLength: "50" # optional - Redis stream length, alternative to pendingEntriesCount scaler trigger + lagCount: "5" # optional - number of lagging entries in the consumer group, alternative to pendingEntriesCount scaler trigger + activationLagCount: "3" # required if lagCount is provided - lag count at which scaler triggers + enableTLS: "false" # optional + unsafeSsl: "false" # optional + databaseIndex: "0" # optional + # Alternatively, you can use existing environment variables to read configuration from: + # See details in "Parameter list" section + addressesFromEnv: REDIS_ADDRESSES # Optional. You can use this instead of `addresses` parameter + hostsFromEnv: REDIS_HOSTS # Optional. You can use this instead of `hosts` parameter + portsFromEnv: REDIS_PORTS # Optional. You can use this instead of `ports` parameter +``` + +**Parameter list:** + +- `addresses` - Comma separated list of hosts and ports of Redis Cluster nodes in the format `host:port` for example `node1:6379, node2:6379, node3:6379`. + +> As an alternative to the `addresses` field, the user can specify `hosts` and `ports` parameters. + +- `hosts` - Comma separated list of hosts of Redis Cluster nodes. + +> It is not required if `addresses` has been provided. + +- `ports`: Comma separated list of ports for corresponding hosts of Redis Cluster nodes. + +> It is only to be used along with the `hosts`/`hostsFromEnv` attribute and not required if `addresses` has been provided. + +- `usernameFromEnv` - Name of the environment variable your deployment uses to get the Redis username. (Optional) +- `passwordFromEnv` - Name of the environment variable your deployment uses to get the Redis password. (Optional) + +- `stream` - Name of the Redis Stream. +- `consumerGroup` - Name of the Consumer group associated with Redis Stream. +> Setting the `consumerGroup` causes the scaler to operate on `pendingEntriesCount`. Lack of `consumerGroup` will cause the scaler to be based on `streamLength` +- `pendingEntriesCount` - Threshold for the number of `Pending Entries List`. This is the average target value to scale the workload. (Default: `5`, Optional) +- `streamLength` - Threshold for stream length, alternative average target value to scale workload. (Default: `5`, Optional) +- `lagCount` - Threshold for the consumer group lag number, alternative average target value to scale workload. (Default: `5`, Optional) +- `activationLagCount` - Lag count threshold at which to start scaling. Any average lag count below this value will not trigger the scaler. (Default: `0`, Optional) +- `enableTLS` - Allow a connection to Redis using tls. (Values: `true`, `false`, Default: `false`, Optional) +- `unsafeSsl` - Used for skipping certificate check e.g: using self-signed certs. (Values: `true`,`false`, Default: `false`, Optional, This requires `enableTLS: true`) +- `databaseIndex` - Index of Redis database to use. If not specified, the default value is 0. + +Some parameters could be provided using environmental variables, instead of setting them directly in metadata. Here is a list of parameters you can use to retrieve values from environment variables: + +- `addressesFromEnv` - The hosts and corresponding ports of Redis Cluster nodes, similar to `addresses`, but reads it from an environment variable on the scale target. Name of the environment variable your deployment uses to get the URLs of Redis Cluster nodes. The resolved hosts should follow a format like `node1:6379, node2:6379, node3:6379 ...`. +- `hostsFromEnv` - The hosts of the Redis Cluster nodes, similar to `hosts`, but reads it from an environment variable on the scale target. +- `portsFromEnv` - The corresponding ports for the hosts of Redis Cluster nodes, similar to `ports`, but reads it from an environment variable on the scale target. + +### Authentication Parameters + +The scaler supports two modes of authentication: + +#### Using username/password authentication + +Use the `username` and `password` field in the `metadata` to specify the name of an environment variable that your deployment uses to get the Redis username/password. + +This is usually resolved from a `Secret V1` or a `ConfigMap V1` collections. `env` and `envFrom` are both supported. + +Here is an example: + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: redis-streams-scaledobject + namespace: default +spec: + scaleTargetRef: + name: redis-streams-consumer + pollingInterval: 15 + cooldownPeriod: 200 + maxReplicaCount: 25 + minReplicaCount: 1 + triggers: + - type: redis-cluster-streams + metadata: + addressesFromEnv: REDIS_ADDRESSES + usernameFromEnv: REDIS_USERNAME # name of the environment variable in the Deployment + passwordFromEnv: REDIS_PASSWORD # name of the environment variable in the Deployment + stream: my-stream + consumerGroup: consumer-group-1 + pendingEntriesCount: "10" +``` + +#### Using `TriggerAuthentication` + +**TLS:** + +Parameters used for configuring TLS authentication. Note this can not be used together with `enableTLS` and `unsafeSsl` on the `ScaledObject`, which is used to define using insecure TLS with skipping certificate check. + +- `tls` - To enable SSL auth for Redis, set this to `enable`. If not set, TLS for Redis is not used. (Values: `enable`, `disable`, Default: `disable`, Optional) +- `ca` - Certificate authority file for TLS authentication. (Optional) +- `cert` - Certificate for client authentication. (Optional) +- `key` - Key for client authentication. (Optional) +- `keyPassword` - If set the `keyPassword` is used to decrypt the provided `key`. (Optional) + +**Authentication:** + +- `username` - Redis username to authenticate with. +- `password` - Redis password to authenticate with. + +You can use `TriggerAuthentication` CRD to configure the authentication. For example: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: redis-streams-auth +type: Opaque +data: + redis_username: + redis_password: + tls: "enable" + ca: + cert: + key: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-redis-stream-triggerauth +spec: + secretTargetRef: + - parameter: username + name: redis-streams-auth # name of the Secret + key: redis_username # name of the key in the Secret + - parameter: password + name: redis-streams-auth # name of the Secret + key: redis_password # name of the key in the Secret + - parameter: tls + name: redis-streams-auth + key: tls + - parameter: ca + name: redis-streams-auth + key: ca + - parameter: cert + name: redis-streams-auth + key: cert + - parameter: key + name: redis-streams-auth + key: key +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: redis-streams-scaledobject + namespace: default +spec: + scaleTargetRef: + name: redis-streams-consumer + pollingInterval: 15 + cooldownPeriod: 200 + maxReplicaCount: 25 + minReplicaCount: 1 + triggers: + - type: redis-cluster-streams + metadata: + address: node1:6379, node2:6379, node3:6379 + stream: my-stream + consumerGroup: consumer-group-1 + pendingEntriesCount: "10" + authenticationRef: + name: keda-redis-stream-triggerauth # name of the TriggerAuthentication resource +``` + +#### Using `streamLength` + +To scale based on redis stream `XLEN` don't set `consumerGroup`. Example: + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: redis-streams-scaledobject + namespace: default +spec: + scaleTargetRef: + name: redis-streams-consumer + pollingInterval: 20 + cooldownPeriod: 200 + maxReplicaCount: 10 + minReplicaCount: 1 + triggers: + - type: redis-cluster-streams + metadata: + addressesFromEnv: REDIS_ADDRESSES + usernameFromEnv: REDIS_USERNAME # name of the environment variable in the Deployment + passwordFromEnv: REDIS_PASSWORD # name of the environment variable in the Deployment + stream: my-stream + streamLength: "50" +``` + +#### Using `lagCount` + +To scale based on redis stream `XINFO GROUPS`, be sure to set `activationLagCount`. Example: + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: redis-streams-scaledobject + namespace: default +spec: + scaleTargetRef: + name: redis-streams-consumer + pollingInterval: 15 + cooldownPeriod: 200 + maxReplicaCount: 25 + minReplicaCount: 1 + triggers: + - type: redis-cluster-streams + metadata: + addressesFromEnv: REDIS_ADDRESSES + usernameFromEnv: REDIS_USERNAME # name of the environment variable in the Deployment + passwordFromEnv: REDIS_PASSWORD # name of the environment variable in the Deployment + stream: my-stream + consumerGroup: consumer-group-1 + lagCount: "10" + activationLagCount: "3" +``` diff --git a/content/docs/2.17/scalers/redis-lists.md b/content/docs/2.17/scalers/redis-lists.md new file mode 100644 index 000000000..e8b0ad83f --- /dev/null +++ b/content/docs/2.17/scalers/redis-lists.md @@ -0,0 +1,125 @@ ++++ +title = "Redis Lists" +availability = "v1.0+" +maintainer = "Community" +category = "Data & Storage" +description = "Scale applications based on Redis Lists." +go_file = "redis_scaler" ++++ + +### Trigger Specification + +This specification describes the `redis` trigger that scales based on the length of a list in Redis. + +```yaml +triggers: +- type: redis + metadata: + address: localhost:6379 # Format must be host:port + usernameFromEnv: REDIS_USERNAME # optional + passwordFromEnv: REDIS_PASSWORD + listName: mylist # Required + listLength: "5" # Required + activationListLength: "5" # optional + enableTLS: "false" # optional + unsafeSsl: "false" # optional + databaseIndex: "0" # optional + # Alternatively, you can use existing environment variables to read configuration from: + # See details in "Parameter list" section + addressFromEnv: REDIS_HOST # Optional. You can use this instead of `address` parameter +``` + +**Parameter list:** + +- `address` - The host and port of the Redis server. +- `host` - The host of the Redis server. Alternative to `address` and requires `port` to be configured as well. +- `port` - The port of the Redis server. Alternative to `address` and requires `host` to be configured as well. +- `usernameFromEnv` - Environment variable to read the authentication username from to authenticate with the Redis server. +- `passwordFromEnv` - Environment variable to read the authentication password from to authenticate with the Redis server. + - Both the hostname, username and password fields need to be set to the names of the environment variables in the target deployment that contain the host name, username and password respectively. +- `listName` - Name of the Redis List that you want to monitor. +- `listLength` - Average target value to trigger scaling actions. +- `activationListLength` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) +- `enableTLS` - Allow a connection to a redis queue using tls. (Values: `true`, `false`, Default: `false`, Optional) +- `unsafeSsl` - Used for skipping certificate check e.g: using self-signed certs. (Values: `true`,`false`, Default: `false`, Optional, This requires `enableTLS: true`) +- `databaseIndex` - Index of Redis database to use. If not specified, the default value is 0. + +Some parameters could be provided using environmental variables, instead of setting them directly in metadata. Here is a list of parameters you can use to retrieve values from environment variables: + +- `addressFromEnv` - The host and port of the Redis server, similar to `address`, but reads it from an environment variable on the scale target. +- `hostFromEnv` - The host of the Redis server, similar to `host`, but reads it from an environment variable on the scale target. +- `portFromEnv` - The port of the Redis server, similar to `port`, but reads it from an environment variable on the scale target. + +### Authentication Parameters + +You can authenticate by using a username (optional) and password. + +**Connection Authentication:** + +- `address` - The hostname and port for the Redis server (host:port format). +- `host` - The hostname of the Redis server. If specified, the `port` should also be specified. +- `port` - The port of the Redis server. If specified, the `host` should also be specified. + +**TLS:** + +Parameters used for configuring TLS authentication. Note this can not be used together with `enableTLS` and `unsafeSsl` on the `ScaledObject`, which is used to define using insecure TLS with skipping certificate check. + +- `tls` - To enable SSL auth for Redis, set this to `enable`. If not set, TLS for Redis is not used. (Values: `enable`, `disable`, Default: `disable`, Optional) +- `ca` - Certificate authority file for TLS authentication. (Optional) +- `cert` - Certificate for client authentication. (Optional) +- `key` - Key for client authentication. (Optional) +- `keyPassword` - If set the `keyPassword` is used to decrypt the provided `key`. (Optional) + +**Authentication:** + +- `username` - Redis username to authenticate with. +- `password` - Redis password to authenticate with. + +### Example + +Here is an example of how to deploy a scaled object with the `redis` scale trigger which uses `TriggerAuthentication`. + +You can also provide the `usernameFromEnv` and `passwordFromEnv` on the `ScaledObject` directly. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: votes-db-secret + namespace: my-project +type: Opaque +data: + redis_username: YWRtaW4= + redis_password: YWRtaW4= +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-redis-secret + namespace: my-project +spec: + secretTargetRef: + - parameter: username + name: votes-db-secret + key: redis_username + - parameter: password + name: votes-db-secret + key: redis_password +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: redis-scaledobject + namespace: my-project +spec: + scaleTargetRef: + name: votes + triggers: + - type: redis + metadata: + address: localhost:6379 + listName: mylist + listLength: "10" + authenticationRef: + name: keda-trigger-auth-redis-secret +``` diff --git a/content/docs/2.17/scalers/redis-sentinel-lists.md b/content/docs/2.17/scalers/redis-sentinel-lists.md new file mode 100644 index 000000000..1ce8b9805 --- /dev/null +++ b/content/docs/2.17/scalers/redis-sentinel-lists.md @@ -0,0 +1,140 @@ ++++ +title = "Redis Lists (supports Redis Sentinel)" +availability = "v2.5+" +maintainer = "Community" +category = "Data & Storage" +description = "Redis Lists scaler with support for Redis Sentinel topology" +go_file = "redis_scaler" ++++ + +### Trigger Specification + +This specification describes the `redis-sentinel` trigger that scales based on the length of a list in a Redis Sentinel setup. + +```yaml +triggers: +- type: redis-sentinel + metadata: + addresses: localhost:26379 # Comma separated list of the format host:port + usernameFromEnv: REDIS_USERNAME # optional + passwordFromEnv: REDIS_PASSWORD # optional + sentinelUsernameFromEnv: REDIS_SENTINEL_USERNAME # optional + sentinelPasswordFromEnv: REDIS_SENTINEL_PASSWORD # optional + sentinelMasterFromEnv: REDIS_SENTINEL_MASTER # optional + listName: mylist # Required + listLength: "5" # Required + activationListLength: "5" # optional + enableTLS: "false" # optional + unsafeSsl: "false" # optional + databaseIndex: "0" # optional + # Alternatively, you can use existing environment variables to read configuration from: + # See details in "Parameter list" section + addressesFromEnv: REDIS_ADDRESSES # Optional. You can use this instead of `addresses` parameter +``` + +**Parameter list:** + +- `addresses` - Comma separated list of hosts and ports of the Redis Sentinel nodes. +- `hosts` - Comma separated list of hosts of the Redis Sentinel nodes. Alternative to `addresses` and requires `ports` to be configured as well. +- `ports` - Comma separated list of corresponding ports for the hosts of the Redis Sentinel nodes. Alternative to `addresses` and requires `hosts` to be configured as well. +- `usernameFromEnv` - Environment variable to read the authentication username from to authenticate with the Redis server. +- `passwordFromEnv` - Environment variable to read the authentication password from to authenticate with the Redis server. + - Both the hostname, username and password fields need to be set to the names of the environment variables in the target deployment that contain the host name, username and password respectively. +- `sentinelUsernameFromEnv` - Environment variable to read the authentication username from to authenticate with the Redis Sentinel server. +- `sentinelPasswordFromEnv` - Environment variable to read the authentication password from to authenticate with the Redis Sentinel server. +- sentinelMaster - The name of the primary (still referred to as the 'master' in Sentinel) to get the Redis server address for. +- `listName` - Name of the Redis List that you want to monitor. +- `listLength` - Average target value to trigger scaling actions. +- `activationListLength` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) +- `enableTLS` - Allow a connection to a redis queue using tls. (Values: `true`, `false`, Default: `false`, Optional) +- `unsafeSsl` - Used for skipping certificate check e.g: using self-signed certs. (Values: `true`,`false`, Default: `false`, Optional, This requires `enableTLS: true`) +- `databaseIndex` - Index of Redis database to use. If not specified, the default value is 0. + +Some parameters could be provided using environmental variables, instead of setting them directly in metadata. Here is a list of parameters you can use to retrieve values from environment variables: + +- `addressesFromEnv` - The hosts and their respective ports of the Redis Sentinel nodes, similar to `addresses`, but reads it from an environment variable on the scale target. +- `hostsFromEnv` - The hosts of the Redis Sentinel nodes, similar to `hosts`, but reads it from an environment variable on the scale target. +- `portsFromEnv` - The corresponding ports for the hosts of the Redis Sentinel nodes, similar to `ports`, but reads it from an environment variable on the scale target. +- `sentinelMasterFromEnv` - The name of the primary (still referred to as the 'master' in Sentinel) to get the Redis server address for; similar to `sentinelMaster`, but reads it from an environment variable on the scale target. + +### Authentication Parameters + +You can authenticate by using a password. + +**Connection Authentication:** + +- `addresses` - Comma separated list of host:port format. +- `hosts` - Comma separated list of hostname of the Redis Sentinel nodes. If specified, the `ports` should also be specified. +- `ports` - Comma separated list of ports of the Redis Sentinel nodes. If specified, the `hosts` should also be specified. +- `sentinelMaster` - The name of the primary (still referred to as the 'master' in Sentinel) to get the Redis server address for. + +**Authentication:** + +- `username` - Redis username to authenticate with. +- `password` - Redis password to authenticate with. + +**Sentinel Authentication:** + +- `sentinelUsername` - Redis Sentinel username to authenticate with. +- `sentinelPassword` - Redis Sentinel password to authenticate with. + +**TLS:** + +Parameters used for configuring TLS authentication. Note this can not be used together with `enableTLS` and `unsafeSsl` on the `ScaledObject`, which is used to define using insecure TLS with skipping certificate check. + +- `tls` - To enable SSL auth for Redis, set this to `enable`. If not set, TLS for Redis is not used. (Values: `enable`, `disable`, Default: `disable`, Optional) +- `ca` - Certificate authority file for TLS authentication. (Optional) +- `cert` - Certificate for client authentication. (Optional) +- `key` - Key for client authentication. (Optional) +- `keyPassword` - If set the `keyPassword` is used to decrypt the provided `key`. (Optional) + + +### Example + +Here is an example of how to deploy a scaled object with the `redis-sentinel` scale trigger which uses `TriggerAuthentication`. + +You can also provide the `usernameFromEnv` and `passwordFromEnv` on the `ScaledObject` directly. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: votes-db-secret + namespace: my-project +type: Opaque +data: + redis_username: YWRtaW4= + redis_password: YWRtaW4= +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-redis-secret + namespace: my-project +spec: + secretTargetRef: + - parameter: username + name: votes-db-secret + key: redis_username + - parameter: password + name: votes-db-secret + key: redis_password +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: redis-scaledobject + namespace: my-project +spec: + scaleTargetRef: + name: votes + triggers: + - type: redis-sentinel + metadata: + addresses: node1:26379, node2:26379, node3:26379 + listName: mylist + listLength: "10" + sentinelMaster: "myprimary" + authenticationRef: + name: keda-trigger-auth-redis-secret +``` diff --git a/content/docs/2.17/scalers/redis-sentinel-streams.md b/content/docs/2.17/scalers/redis-sentinel-streams.md new file mode 100644 index 000000000..95db3210d --- /dev/null +++ b/content/docs/2.17/scalers/redis-sentinel-streams.md @@ -0,0 +1,260 @@ ++++ +title = "Redis Streams (supports Redis Sentinel)" +availability = "v2.5+" +maintainer = "Community" +category = "Data & Storage" +description = "Redis Streams scaler with support for Redis Sentinel topology" +go_file = "redis_streams_scaler" ++++ + +### Trigger Specification + +Redis 5.0 introduced [Redis Streams](https://redis.io/topics/streams-intro) which is an append-only log data structure. + +One of its features includes [`Consumer Groups`](https://redis.io/topics/streams-intro#consumer-groups), that allows a group of clients to co-operate consuming a different portion of the same stream of messages. + +There are three ways to configure `redis-streams` trigger: +1. Based on the *Pending Entries List* (see [`XPENDING`](https://redis.io/commands/xpending)) for a specific Consumer Group of a Redis Stream +2. Based on the *Stream Length* (see [`XLEN`](https://redis.io/commands/xlen)) +3. Based on the *Consumer Group Lag* (see [`XINFO GROUPS`](https://redis.io/commands/xinfo-groups/)). This is the only configuration that supports scaling down to 0. IMPORTANT: Redis 7+ is required for this feature to run. + + +```yaml +triggers: +- type: redis-sentinel-streams + metadata: + addresses: localhost:26379 # Required if hosts and ports are not provided. Format - comma separated list of host:port + hosts: localhost # Comma separated lists of hosts. Required if address is not provided + ports: "26379" # Comma separated lists of ports. Required if addresses are not provided and hosts has been provided. + usernameFromEnv: REDIS_USERNAME # optional (can also use authenticationRef) + passwordFromEnv: REDIS_PASSWORD # optional (can also use authenticationRef) + stream: my-stream # Required - name of the Redis Stream + consumerGroup: my-consumer-group # optional - name of consumer group associated with Redis Stream + pendingEntriesCount: "10" # optional - number of entries in the Pending Entries List for the specified consumer group in the Redis Stream + streamLength: "50" # optional - Redis stream length, alternative to pendingEntriesCount scaler trigger + lagCount: "5" # optional - number of lagging entries in the consumer group, alternative to pendingEntriesCount scaler trigger + activationLagCount: "3" # required if lagCount is provided - lag count at which scaler triggers + enableTLS: "false" # optional + unsafeSsl: "false" # optional + databaseIndex: "0" # optional + # Alternatively, you can use existing environment variables to read configuration from: + # See details in "Parameter list" section + addressesFromEnv: REDIS_ADDRESSES # Optional. You can use this instead of `addresses` parameter + hostsFromEnv: REDIS_HOSTS # Optional. You can use this instead of `hosts` parameter + portsFromEnv: REDIS_PORTS # Optional. You can use this instead of `ports` parameter +``` + +**Parameter list:** + +- `addresses` - Comma separated list of hosts and ports of Redis Sentinel nodes in the format `host:port` for example `node1:26379, node2:26379, node3:26379`. + +> As an alternative to the `addresses` field, the user can specify `hosts` and `ports` parameters. + +- `hosts` - Comma separated list of hosts of Redis Sentinel nodes. + +> It is not required if `addresses` has been provided. + +- `ports`: Comma separated list of ports for corresponding hosts of Redis Sentinel nodes. + +> It is only to be used along with the `hosts`/`hostsFromEnv` attribute and not required if `addresses` has been provided. + +- `usernameFromEnv` - Name of the environment variable your deployment uses to get the Redis username. (Optional) +- `passwordFromEnv` - Name of the environment variable your deployment uses to get the Redis password. (Optional) + +- `sentinelUsernameFromEnv` - Name of the environment variable your deployment uses to get the Redis Sentinel username. (Optional) +- `sentinelPasswordFromEnv` - Name of the environment variable your deployment uses to get the Redis Sentinel password. (Optional) + +- `sentinelMaster` - The name of the primary (still referred to as the 'master' in Sentinel) in Sentinel to get the Redis server address for. +- `stream` - Name of the Redis Stream. +- `consumerGroup` - Name of the Consumer group associated with Redis Stream. +> Setting the `consumerGroup` causes the scaler to operate on `pendingEntriesCount`. Lack of `consumerGroup` will cause the scaler to be based on `streamLength` +- `pendingEntriesCount` - Threshold for the number of `Pending Entries List`. This is the average target value to scale the workload. (Default: `5`, Optional) +- `streamLength` - Threshold for stream length, alternative average target value to scale workload. (Default: `5`, Optional) +- `lagCount` - Threshold for the consumer group lag number, alternative average target value to scale workload. (Default: `5`, Optional) +- `activationLagCount` - Lag count threshold at which to start scaling. Any average lag count below this value will not trigger the scaler. (Default: `0`, Optional) +- `enableTLS` - Allow a connection to Redis using tls. (Values: `true`, `false`, Default: `false`, Optional) +- `unsafeSsl` - Used for skipping certificate check e.g: using self-signed certs. (Values: `true`,`false`, Default: `false`, Optional, This requires `enableTLS: true`) +- `databaseIndex` - Index of Redis database to use. If not specified, the default value is 0. + +Some parameters could be provided using environmental variables, instead of setting them directly in metadata. Here is a list of parameters you can use to retrieve values from environment variables: + +- `addressesFromEnv` - The hosts and corresponding ports of Redis Sentinel nodes, similar to `addresses`, but reads it from an environment variable on the scale target. Name of the environment variable your deployment uses to get the URLs of Redis Sentinel nodes. The resolved hosts should follow a format like `node1:26379, node2:26379, node3:26379 ...`. +- `hostsFromEnv` - The hosts of the Redis Sentinel nodes, similar to `hosts`, but reads it from an environment variable on the scale target. +- `portsFromEnv` - The corresponding ports for the hosts of Redis Sentinel nodes, similar to `ports`, but reads it from an environment variable on the scale target. +- `sentinelMasterFromEnv` - The name of the primary (still referred to as the 'master' in Sentinel) in Sentinel to get the Redis server address for, similar to `sentinelMaster`, but reads it from an environment variable on the scale target. + +### Authentication Parameters + +The scaler supports two modes of authentication: + +#### Using username/password authentication + +Use the `username` and `password` field in the `metadata` to specify the name of an environment variable that your deployment uses to get the Redis username/password. + +This is usually resolved from a `Secret V1` or a `ConfigMap V1` collections. `env` and `envFrom` are both supported. + +Here is an example: + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: redis-streams-scaledobject + namespace: default +spec: + scaleTargetRef: + name: redis-streams-consumer + pollingInterval: 15 + cooldownPeriod: 200 + maxReplicaCount: 25 + minReplicaCount: 1 + triggers: + - type: redis-sentinel-streams + metadata: + addressesFromEnv: REDIS_ADDRESSES + usernameFromEnv: REDIS_USERNAME # name of the environment variable in the Deployment + passwordFromEnv: REDIS_PASSWORD # name of the environment variable in the Deployment + stream: my-stream + consumerGroup: consumer-group-1 + pendingEntriesCount: "10" + sentinelMaster: "myprimary" +``` + +#### Using `TriggerAuthentication` + +**TLS:** + +Parameters used for configuring TLS authentication. Note this can not be used together with `enableTLS` and `unsafeSsl` on the `ScaledObject`, which is used to define using insecure TLS with skipping certificate check. + +- `tls` - To enable SSL auth for Redis, set this to `enable`. If not set, TLS for Redis is not used. (Values: `enable`, `disable`, Default: `disable`, Optional) +- `ca` - Certificate authority file for TLS authentication. (Optional) +- `cert` - Certificate for client authentication. (Optional) +- `key` - Key for client authentication. (Optional) +- `keyPassword` - If set the `keyPassword` is used to decrypt the provided `key`. (Optional) + +**Authentication:** + +- `username` - Redis username to authenticate with. +- `password` - Redis password to authenticate with. + +You can use `TriggerAuthentication` CRD to configure the authentication. For example: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: redis-streams-auth +type: Opaque +data: + redis_username: + redis_password: + tls: "enable" + ca: + cert: + key: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-redis-stream-triggerauth +spec: + secretTargetRef: + - parameter: username + name: redis-streams-auth # name of the Secret + key: redis_username # name of the key in the Secret + - parameter: password + name: redis-streams-auth # name of the Secret + key: redis_password # name of the key in the Secret + - parameter: tls + name: redis-streams-auth + key: tls + - parameter: ca + name: redis-streams-auth + key: ca + - parameter: cert + name: redis-streams-auth + key: cert + - parameter: key + name: redis-streams-auth + key: key +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: redis-streams-scaledobject + namespace: default +spec: + scaleTargetRef: + name: redis-streams-consumer + pollingInterval: 15 + cooldownPeriod: 200 + maxReplicaCount: 25 + minReplicaCount: 1 + triggers: + - type: redis-sentinel-streams + metadata: + address: node1:26379, node2:26379, node3:26379 + stream: my-stream + consumerGroup: consumer-group-1 + pendingEntriesCount: "10" + sentinelMaster: "myprimary" + authenticationRef: + name: keda-redis-stream-triggerauth # name of the TriggerAuthentication resource +``` + +#### Using `streamLength` + +To scale based on redis stream `XLEN` don't set `consumerGroup`. Example: + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: redis-streams-scaledobject + namespace: default +spec: + scaleTargetRef: + name: redis-streams-consumer + pollingInterval: 20 + cooldownPeriod: 200 + maxReplicaCount: 10 + minReplicaCount: 1 + triggers: + - type: redis-sentinel-streams + metadata: + addressesFromEnv: REDIS_ADDRESSES + usernameFromEnv: REDIS_USERNAME # name of the environment variable in the Deployment + passwordFromEnv: REDIS_PASSWORD # name of the environment variable in the Deployment + stream: my-stream + streamLength: "50" + sentinelMaster: "myprimary" +``` + +#### Using `lagCount` + +To scale based on redis stream `XINFO GROUPS`, be sure to set `activationLagCount`. Example: + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: redis-streams-scaledobject + namespace: default +spec: + scaleTargetRef: + name: redis-streams-consumer + pollingInterval: 15 + cooldownPeriod: 200 + maxReplicaCount: 25 + minReplicaCount: 1 + triggers: + - type: redis-cluster-streams + metadata: + addressesFromEnv: REDIS_ADDRESSES + usernameFromEnv: REDIS_USERNAME # name of the environment variable in the Deployment + passwordFromEnv: REDIS_PASSWORD # name of the environment variable in the Deployment + stream: my-stream + consumerGroup: consumer-group-1 + lagCount: "10" + activationLagCount: "3" +``` diff --git a/content/docs/2.17/scalers/redis-streams.md b/content/docs/2.17/scalers/redis-streams.md new file mode 100644 index 000000000..0c8436acb --- /dev/null +++ b/content/docs/2.17/scalers/redis-streams.md @@ -0,0 +1,252 @@ ++++ +title = "Redis Streams" +availability = "v1.5+" +maintainer = "Community" +category = "Data & Storage" +description = "Scale applications based on Redis Streams." +go_file = "redis_streams_scaler" ++++ + +### Trigger Specification + +Redis 5.0 introduced [Redis Streams](https://redis.io/topics/streams-intro) which is an append-only log data structure. + +One of its features includes [`Consumer Groups`](https://redis.io/topics/streams-intro#consumer-groups), that allows a group of clients to co-operate consuming a different portion of the same stream of messages. + +There are three ways to configure `redis-streams` trigger: +1. Based on the *Pending Entries List* (see [`XPENDING`](https://redis.io/commands/xpending)) for a specific Consumer Group of a Redis Stream +2. Based on the *Stream Length* (see [`XLEN`](https://redis.io/commands/xlen)) +3. Based on the *Consumer Group Lag* (see [`XINFO GROUPS`](https://redis.io/commands/xinfo-groups/)). This is the only configuration that supports scaling down to 0. IMPORTANT: Redis 7+ is required for this feature to run. + + +```yaml +triggers: +- type: redis-streams + metadata: + address: localhost:6379 # Required if host and port are not provided. Format - host:port + host: localhost # Required if address is not provided + port: "6379" # Required if address is not provided and host has been provided. + usernameFromEnv: REDIS_USERNAME # optional (can also use authenticationRef) + passwordFromEnv: REDIS_PASSWORD # optional (can also use authenticationRef) + stream: my-stream # Required - name of the Redis Stream + consumerGroup: my-consumer-group # optional - name of consumer group associated with Redis Stream + pendingEntriesCount: "10" # optional - number of entries in the Pending Entries List for the specified consumer group in the Redis Stream + streamLength: "50" # optional - Redis stream length, alternative to pendingEntriesCount scaler trigger + lagCount: "5" # optional - number of lagging entries in the consumer group, alternative to pendingEntriesCount scaler trigger + activationLagCount: "3" # required if lagCount is provided - lag count at which scaler triggers + enableTLS: "false" # optional + unsafeSsl: "false" # optional + databaseIndex: "0" # optional + # Alternatively, you can use existing environment variables to read configuration from: + # See details in "Parameter list" section + addressFromEnv: REDIS_ADDRESS # Optional. You can use this instead of `address` parameter + hostFromEnv: REDIS_HOST # Optional. You can use this instead of `host` parameter + portFromEnv: REDIS_PORT # Optional. You can use this instead of `port` parameter +``` + +**Parameter list:** + +- `address` - The host and port of the Redis server in the format `host:port`, for example `my-redis:6379`. + +> As an alternative to the `address` field, the user can specify `host` and `port` parameters. + +- `host` - The host of the Redis server. + +> It is not required if `address` has been provided. + +- `port` - The port of the Redis server. + +> It is only to be used along with the `host`/`hostFromEnv` attribute and not required if `address` has been provided. + +- `usernameFromEnv` - Name of the environment variable your deployment uses to get the Redis username. (Optional) +- `passwordFromEnv` - Name of the environment variable your deployment uses to get the Redis password. (Optional) + +- `stream` - Name of the Redis Stream. +- `consumerGroup` - Name of the Consumer group associated with Redis Stream. +> Setting the `consumerGroup` causes the scaler to operate on `pendingEntriesCount`. Lack of `consumerGroup` will cause the scaler to be based on `streamLength` +- `pendingEntriesCount` - Threshold for the number of `Pending Entries List`. This is the average target value to scale the workload. (Default: `5`, Optional) +- `streamLength` - Threshold for stream length, alternative average target value to scale workload. (Default: `5`, Optional) +- `lagCount` - Threshold for the consumer group lag number, alternative average target value to scale workload. (Default: `5`, Optional) +- `activationLagCount` - Lag count threshold at which to start scaling. Any average lag count below this value will not trigger the scaler. (Default: `0`, Optional) +- `enableTLS` - Allow a connection to Redis using tls. (Values: `true`, `false`, Default: `false`, Optional) +- `unsafeSsl` - Used for skipping certificate check e.g: using self-signed certs. (Values: `true`,`false`, Default: `false`, Optional, This requires `enableTLS: true`) +- `databaseIndex` - Index of Redis database to use. If not specified, the default value is 0. + +Some parameters could be provided using environmental variables, instead of setting them directly in metadata. Here is a list of parameters you can use to retrieve values from environment variables: + +- `addressFromEnv` - The host and port of the Redis server, similar to `address`, but reads it from an environment variable on the scale target. Name of the environment variable your deployment uses to get the Redis server URL. The resolved host should follow a format like `my-redis:6379`. +- `hostFromEnv` - The host of the Redis server, similar to `host`, but reads it from an environment variable on the scale target. +- `portFromEnv` - The port of the Redis server, similar to `port`, but reads it from an environment variable on the scale target. + +### Authentication Parameters + +The scaler supports two modes of authentication: + +#### Using username/password authentication + +Use the `username` and `password` field in the `metadata` to specify the name of an environment variable that your deployment uses to get the Redis username/password. + +This is usually resolved from a `Secret V1` or a `ConfigMap V1` collections. `env` and `envFrom` are both supported. + +Here is an example: + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: redis-streams-scaledobject + namespace: default +spec: + scaleTargetRef: + name: redis-streams-consumer + pollingInterval: 15 + cooldownPeriod: 200 + maxReplicaCount: 25 + minReplicaCount: 1 + triggers: + - type: redis-streams + metadata: + addressFromEnv: REDIS_HOST + usernameFromEnv: REDIS_USERNAME # name of the environment variable in the Deployment + passwordFromEnv: REDIS_PASSWORD # name of the environment variable in the Deployment + stream: my-stream + consumerGroup: consumer-group-1 + pendingEntriesCount: "10" +``` + +#### Using `TriggerAuthentication` + +**TLS:** + +Parameters used for configuring TLS authentication. Note this can not be used together with `enableTLS` and `unsafeSsl` on the `ScaledObject`, which is used to define using insecure TLS with skipping certificate check. + +- `tls` - To enable SSL auth for Redis, set this to `enable`. If not set, TLS for Redis is not used. (Values: `enable`, `disable`, Default: `disable`, Optional) +- `ca` - Certificate authority file for TLS authentication. (Optional) +- `cert` - Certificate for client authentication. (Optional) +- `key` - Key for client authentication. (Optional) +- `keyPassword` - If set the `keyPassword` is used to decrypt the provided `key`. (Optional) + +**Authentication:** + +- `username` - Redis username to authenticate with. +- `password` - Redis password to authenticate with. + +You can use `TriggerAuthentication` CRD to configure the authentication. For example: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: redis-streams-auth +type: Opaque +data: + redis_username: + redis_password: + tls: "enable" + ca: + cert: + key: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-redis-stream-triggerauth +spec: + secretTargetRef: + - parameter: username + name: redis-streams-auth # name of the Secret + key: redis_username # name of the key in the Secret + - parameter: password + name: redis-streams-auth # name of the Secret + key: redis_password # name of the key in the Secret + - parameter: tls + name: redis-streams-auth + key: tls + - parameter: ca + name: redis-streams-auth + key: ca + - parameter: cert + name: redis-streams-auth + key: cert + - parameter: key + name: redis-streams-auth + key: key +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: redis-streams-scaledobject + namespace: default +spec: + scaleTargetRef: + name: redis-streams-consumer + pollingInterval: 15 + cooldownPeriod: 200 + maxReplicaCount: 25 + minReplicaCount: 1 + triggers: + - type: redis-streams + metadata: + address: localhost:6379 + stream: my-stream + consumerGroup: consumer-group-1 + pendingEntriesCount: "10" + authenticationRef: + name: keda-redis-stream-triggerauth # name of the TriggerAuthentication resource +``` + +#### Using `streamLength` + +To scale based on redis stream `XLEN` don't set `consumerGroup`. Example: + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: redis-streams-scaledobject + namespace: default +spec: + scaleTargetRef: + name: redis-streams-consumer + pollingInterval: 20 + cooldownPeriod: 200 + maxReplicaCount: 10 + minReplicaCount: 1 + triggers: + - type: redis-streams + metadata: + addressFromEnv: REDIS_HOST + usernameFromEnv: REDIS_USERNAME # name of the environment variable in the Deployment + passwordFromEnv: REDIS_PASSWORD # name of the environment variable in the Deployment + stream: my-stream + streamLength: "50" +``` + +#### Using `lagCount` + +To scale based on redis stream `XINFO GROUPS`, be sure to set `activationLagCount`. Example: + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: redis-streams-scaledobject + namespace: default +spec: + scaleTargetRef: + name: redis-streams-consumer + pollingInterval: 15 + cooldownPeriod: 200 + maxReplicaCount: 25 + minReplicaCount: 1 + triggers: + - type: redis-streams + metadata: + addressFromEnv: REDIS_HOST + usernameFromEnv: REDIS_USERNAME # name of the environment variable in the Deployment + passwordFromEnv: REDIS_PASSWORD # name of the environment variable in the Deployment + stream: my-stream + consumerGroup: consumer-group-1 + lagCount: "10" + activationLagCount: "3" +``` \ No newline at end of file diff --git a/content/docs/2.17/scalers/selenium-grid-scaler.md b/content/docs/2.17/scalers/selenium-grid-scaler.md new file mode 100644 index 000000000..6ce9d281a --- /dev/null +++ b/content/docs/2.17/scalers/selenium-grid-scaler.md @@ -0,0 +1,233 @@ ++++ +title = "Selenium Grid Scaler" +availability = "v2.4+" +maintainer = "Volvo Cars, SeleniumHQ" +category = "Testing" +description = "Scales Selenium browser nodes based on number of requests waiting in session queue" +go_file = "selenium_grid_scaler" ++++ + +### Trigger Specification + +This specification describes the `selenium-grid` trigger that scales browser nodes based on number of requests in session queue and the max sessions per grid. + +The scaler creates one browser node per pending request in session queue, divided by the max amount of sessions that can run in parallel. You will have to create one trigger per browser capability that you would like to support in your Selenium Grid. + +The below is an example trigger configuration for chrome node. + +```yaml +triggers: + - type: selenium-grid + metadata: + url: 'http://selenium-hub:4444/graphql' # Required. Can be ommitted if specified via TriggerAuthentication/ClusterTriggerAuthentication. + browserName: 'chrome' # Required + browserVersion: '91.0' # Optional. Only required when supporting multiple versions of browser in your Selenium Grid. + unsafeSsl : 'true' # Optional + activationThreshold: 5 # Optional + platformName: 'Linux' # Optional +``` + +**Parameter list:** + +- `url` - Graphql url of your Selenium Grid. Refer to the Selenium Grid's documentation [here](https://www.selenium.dev/documentation/en/grid/grid_4/graphql_support/) to for more info. If endpoint requires authentication, you can use `TriggerAuthentication` to provide the credentials instead of embedding in the URL. +- `browserName` - Name of browser that usually gets passed in the browser capability. Refer to the [Selenium Grid's](https://www.selenium.dev/documentation/en/getting_started_with_webdriver/browsers/) and [WebdriverIO's](https://webdriver.io/docs/options/#capabilities) documentation for more info. +- `sessionBrowserName` - Name of the browser when it is an active session, only set if `BrowserName` changes between the queue and the active session. See the Edge example below for further detail. (Optional) +- `browserVersion` - Version of browser that usually gets passed in the browser capability. Refer to the [Selenium Grid's](https://www.selenium.dev/documentation/en/getting_started_with_webdriver/browsers/) and [WebdriverIO's](https://webdriver.io/docs/options/#capabilities) documentation for more info. (Optional) +- `unsafeSsl` - Skip certificate validation when connecting over HTTPS. (Values: `true`, `false`, Default: `false`, Optional) +- `activationThreshold` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) +- `platformName` - Name of the browser platform. Refer to the [Selenium Grid's](https://www.selenium.dev/documentation/en/getting_started_with_webdriver/browsers/) and [WebdriverIO's](https://webdriver.io/docs/options/#capabilities) documentation for more info. (Default: `Linux`, Optional) +- `nodeMaxSessions` - Number of maximum sessions that can run in parallel on a Node. Update this parameter align with node config `--max-sessions` (`SE_NODE_MAX_SESSIONS`) to have the correct scaling behavior. (Default: `1`, Optional). + +**Trigger Authentication** +- `username` - Username for basic authentication in GraphQL endpoint instead of embedding in the URL. (Optional) +- `password` - Password for basic authentication in GraphQL endpoint instead of embedding in the URL. (Optional) +- `authType` - Type of authentication to be used. This can be set to `Bearer` or `OAuth2` in case Selenium Grid behind an Ingress proxy with other authentication types. (Optional) +- `accessToken` - Access token. This is required when `authType` is set a value. (Optional) + +### Example + +Here is a full example of scaled object definition using Selenium Grid trigger: + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: selenium-grid-chrome-scaledobject + namespace: keda + labels: + deploymentName: selenium-chrome-node +spec: + maxReplicaCount: 8 + scaleTargetRef: + name: selenium-chrome-node + triggers: + - type: selenium-grid + metadata: + url: 'http://selenium-hub:4444/graphql' + browserName: 'chrome' +``` + +The above example will create Chrome browser nodes equal to the requests pending in session queue for Chrome browser. + +Similarly for Firefox + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: selenium-grid-firefox-scaledobject + namespace: keda + labels: + deploymentName: selenium-firefox-node +spec: + maxReplicaCount: 8 + scaleTargetRef: + name: selenium-firefox-node + triggers: + - type: selenium-grid + metadata: + url: 'http://selenium-hub:4444/graphql' + browserName: 'firefox' +``` + +Similarly for Edge. Note that for Edge you must set the `sessionBrowserName` to `msedge` inorder for scaling to work properly. + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: selenium-grid-edge-scaledobject + namespace: keda + labels: + deploymentName: selenium-edge-node +spec: + maxReplicaCount: 8 + scaleTargetRef: + name: selenium-edge-node + triggers: + - type: selenium-grid + metadata: + url: 'http://selenium-hub:4444/graphql' + browserName: 'MicrosoftEdge' + sessionBrowserName: 'msedge' +``` + +In case you want to scale from 0 (`minReplicaCount: 0`), and browser nodes are configured different `--max-sessions` greater than 1, you can set `nodeMaxSessions` for scaler align with number of slots available per node to have the correct scaling behavior. + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: selenium-grid-chrome-scaledobject + namespace: keda + labels: + deploymentName: selenium-chrome-node +spec: + maxReplicaCount: 8 + scaleTargetRef: + name: selenium-chrome-node + triggers: + - type: selenium-grid + metadata: + url: 'http://selenium-hub:4444/graphql' + browserName: 'chrome' + nodeMaxSessions: 4 +``` + +If you are supporting multiple versions of browser capability in your Selenium Grid, You should create one scaler for every browser version and pass the `browserVersion` in the metadata. + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: selenium-grid-chrome-91-scaledobject + namespace: keda + labels: + deploymentName: selenium-chrome-node-91 +spec: + maxReplicaCount: 8 + scaleTargetRef: + name: selenium-chrome-node-91 + triggers: + - type: selenium-grid + metadata: + url: 'http://selenium-hub:4444/graphql' + browserName: 'chrome' + browserVersion: '91.0' +``` + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: selenium-grid-chrome-90-scaledobject + namespace: keda + labels: + deploymentName: selenium-chrome-node-90 +spec: + maxReplicaCount: 8 + scaleTargetRef: + name: selenium-chrome-node-90 + triggers: + - type: selenium-grid + metadata: + url: 'http://selenium-hub:4444/graphql' + browserName: 'chrome' + browserVersion: '90.0' +``` + +### Authentication Parameters + +It is possible to specify the Graphql url of your Selenium Grid using authentication parameters. This useful if you have enabled Selenium Grid's Basic HTTP Authentication and would like to keep your credentials secure. + +- `url` - Graphql url of your Selenium Grid. Refer to the Selenium Grid's documentation [here](https://www.selenium.dev/documentation/en/grid/grid_4/graphql_support/) for more info. +- `username` - Username for basic authentication in GraphQL endpoint instead of embedding in the URL. (Optional) +- `password` - Password for basic authentication in GraphQL endpoint instead of embedding in the URL. (Optional) + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: selenium-grid-secret + namespace: keda +type: Opaque +data: + graphql-url: base64 encoded value of GraphQL URL + graphql-username: base64 encoded value of GraphQL Username + graphql-password: base64 encoded value of GraphQL Password +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-selenium-grid-secret + namespace: keda +spec: + secretTargetRef: + - parameter: url + name: selenium-grid-secret + key: graphql-url + - parameter: username + name: selenium-grid-secret + key: graphql-username + - parameter: password + name: selenium-grid-secret + key: graphql-password +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: selenium-grid-chrome-scaledobject + namespace: keda + labels: + deploymentName: selenium-chrome-node +spec: + maxReplicaCount: 8 + scaleTargetRef: + name: selenium-chrome-node + triggers: + - type: selenium-grid + metadata: + browserName: 'chrome' + authenticationRef: + name: keda-trigger-auth-selenium-grid-secret +``` diff --git a/content/docs/2.17/scalers/solace-pub-sub.md b/content/docs/2.17/scalers/solace-pub-sub.md new file mode 100644 index 000000000..c8a3e8ba5 --- /dev/null +++ b/content/docs/2.17/scalers/solace-pub-sub.md @@ -0,0 +1,143 @@ ++++ +title = "Solace PubSub+ Event Broker" +availability = "2.4+" +maintainer = "Community" +category = "Messaging" +description = "Scale applications based on Solace PubSub+ Event Broker Queues" +go_file = "solace_scaler" ++++ + +### Trigger Specification + +This specification describes the `solace-event-queue` trigger that scales based on a Solace PubSub+ Event Broker queue. + +```yaml +triggers: +- type: solace-event-queue + metadata: + solaceSempBaseURL: http://solace_broker:8080 + messageVpn: message-vpn + queueName: queue_name + messageCountTarget: '100' + messageSpoolUsageTarget: '100' ### Megabytes (MB) + messageReceiveRateTarget: '50' ### Messages/second over last 1 minute interval + activationMessageCountTarget: '10' + activationMessageSpoolUsageTarget: '5' ### Megabytes (MB) + activationMessageReceiveRateTarget: '1' ### Messages/second over last 1 minute interval + username: semp-user + password: semp-pwd + usernameFromEnv: ENV_VAR_USER + passwordFromEnv: ENV_VAR_PWD +``` + +**Parameter list:** + +- `solaceSempBaseURL` - Solace SEMP Endpoint in format: `://:`. +- `messageVpn` - Message VPN hosted on the Solace broker. +- `queueName` - Message Queue to be monitored. +- `messageCountTarget` - The target number of messages manageable by a pod. The scaler will cause the replicas to increase if the queue message backlog is greater than the target value per active replica. +- `activationMessageCountTarget` - Target message count oberved on a queue for activating the scaler (scaling from 0->1 or 1->0 replicas). Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) +- `messageSpoolUsageTarget` - Integer value expressed in Megabytes (MB). The target spool usage manageable by a pod. The scaler will cause the replicas to increase if the queue spool usage is greater than the target value per active replica. +- `activationMessageSpoolUsageTarget` - Target message spool backlog (data stored in a queue expressed in Megabytes) for activating the scaler (scaling from 0->1 or 1->0 replicas). Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) +- `messageReceiveRateTarget` - Target number of messages/second managable by a replica. +- `activationMessageReceiveRateTarget` - Target number of messages per second delivered to a queue for activating the scaler (scaling from 0->1 or 1->0 replicas). Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) +- `username` - User account with access to Solace SEMP RESTful endpoint. +- `password` - Password for the user account. +- `usernameFromEnv` - Environment variable set with SEMP user account. +- `passwordFromEnv` - Environment variable set with password for the user account. + +**Parameter Requirements:** + +- Parameters resolving the target queue are all **required:** `solaceSempBaseURL`, `messageVpn`, `queueName` +- **At least** one of `messageCountTarget`, `messageSpoolUsageTarget`, or `messageReceiveRateTarget` is **required.** If one or more values are present, the metric value resulting in the highest desired replicas will be used. (Standard KEDA/HPA behavior) +- The Solace PubSub+ Scaler polls the Solace SEMP REST API to monitor target queues. Currently, the scaler supports basic authentication. `username` and `password` are **required** for the `solace-event-queue` trigger to function. These values can be set in several different ways: + - `username` and `password` may be set directly in the trigger metadata + - Use TriggerAuthentication record. See [Authentication Parameters](#authentication-parameters) below. + - Alternatively, credentials may be passed from environment variables identified by `usernameFromEnv` and `passwordFromEnv` metadata fields. The values of these fields are the names of environment variables that must be available to the scaler at run-time. + +**Important Notes about Metric Configuration:** + +> 💡 **Note:** `messageCountTarget` provides good reactivity to changes in demand based on the queue undelivered backlog. The monitored queue value from SEMPv2 is `collections.msgs.count` + +> 💡 **Note:** `messageReceiveRateTarget` provides the ability to achieve consumer stability under constant load. The value monitored from SEMPv2 is `data.averageRxMsgRate`, which is the average message delivery rate to a queue over a one minute period. + +> 💡 **Important:** For best results, both `messageCountTarget` and `messageReceiveRateTarget` should be specified to configure a Solace Scaler. A combined approach capitalizes on the best characteristics of each metric. + +> 💡 **Note:** Configured by itself, `messageCountTarget` will make consumer scaling reactive but may introduce [flapping](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#flapping): the constant creation and destruction of replicas as the system tries to achieve a steady state. +>> For example, a Solace consumer app scaled by KEDA experiences an increase in the rate of message delivery to its source queue, resulting in a backlog (high message count). Based on the `messageCountTarget`, KEDA increases the number of replicas and the backlog is cleared. With the backlog reduced, KEDA scales-in the application, reducing the number of replicas. If the rate of message delivery remains high, the application may not be able to maintain the backlog with the lower number of replicas, causing KEDA to scale-out the workload again. The backlog is again cleared - and the pattern repeats. Using the `messageReceiveRateTarget` as an additional metric can be used to identify a suitable replica count to handle the inbound message rate while keeping the backlog clear and the application performant. + +> 💡 **Note:** Configured by itself, `messageReceiveRateTarget` cannot scale consumers based on queue backlog. + +> 💡 **Activation Values:** `activationMessageCountTarget`, `activationMessageSpoolUsageTarget`, and `activationMessageReceiveRateTarget` are assumed to be `0` (zero) if not specified. + +### Authentication Parameters + +You can use TriggerAuthentication CRD to configure the username and password to connect to the management endpoint. + +**Username and Password based authentication:** +- `username` - Required. The username to use to connect to the Solace PubSub+ Event Broker's SEMP endpoint. +- `password` - Required. The password to use to connect to the Solace PubSub+ Event Broker's SEMP endpoint. + +### Example + +The objects in the example below are declared in `namespace=solace`. It is not required to do so. If you do define a namespace for the configuration objects, then they should all be delcared in the same namespace. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: solace-secret + namespace: solace + labels: + app: solace-consumer +type: Opaque +data: + SEMP_USER: YWRtaW4= + SEMP_PASSWORD: S2VkYUxhYkFkbWluUHdkMQ== +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: solace-scaled-object + namespace: solace +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: solace-consumer + pollingInterval: 20 + cooldownPeriod: 60 + minReplicaCount: 0 + maxReplicaCount: 10 + triggers: + - type: solace-event-queue + metadata: + ## Connection Details + solaceSempBaseURL: http://broker-pubsubplus.solace.svc.cluster.local:8080 + messageVpn: test_vpn + queueName: SCALED_CONSUMER_QUEUE1 + ## Scaler Details, average values per replica + messageCountTarget: '50' + messageSpoolUsageTarget: '100000' + messageReceiveRateTarget: '20' + ## Activation - Scale from 0 replicas to active workload if one of the conditions is met + activationMessageCountTarget: '5' + activationMessageSpoolUsageTarget: '2' + activationMessageReceiveRateTarget: '5' + authenticationRef: + name: solace-trigger-auth +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: solace-trigger-auth + namespace: solace +spec: + secretTargetRef: + - parameter: username + name: solace-secret + key: SEMP_USER + - parameter: password + name: solace-secret + key: SEMP_PASSWORD +``` diff --git a/content/docs/2.17/scalers/solr.md b/content/docs/2.17/scalers/solr.md new file mode 100644 index 000000000..9eaac8f65 --- /dev/null +++ b/content/docs/2.17/scalers/solr.md @@ -0,0 +1,83 @@ ++++ +title = "Solr" +availability = "v2.11+" +maintainer = "Community" +category = "Metrics" +description = "Scale applications based on Solr query results." +go_file = "solr_scaler" ++++ + +### Trigger Specification + +This specification describes the solr trigger that scales based on the outputs of a Solr query. + +```yaml +triggers: + - type: solr + metadata: + host: "http://solr-service.solr-ns.svc.cluster.local:8983" + query: "*:*" + collection: "my_core" + targetQueryValue: "1" + activationTargetQueryValue : "3" +``` + +**Parameter list:** + +- `host` - The hostname for connecting to the Solr service. +- `query`- A Solr query that should return single numeric value. (Default: `*:*`, Optional) +- `collection` - Your collection name on Solr. +- `targetQueryValue` - A threshold that is used as targetValue or targetAverageValue (depending on the trigger metric type) in HPA. (This value can be a float) +- `activationTargetQueryValue` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional) + +### Authentication Parameters + +You can authenticate by using a password via TriggerAuthentication configuration. + +**Credential based authentication:** + +- `username` - Username for configured user to login to the Solr instance. +- `password` - Password for configured user to login to the Solr instance. + +### Example + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: solr-secret +type: Opaque +data: + solr_username: SOLR_USERNAME + solr_password: SOLR_PASSWORD +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: trigger-auth-solr +spec: + secretTargetRef: + - parameter: username + name: solr-secret + key: solr_username + - parameter: password + name: solr-secret + key: solr_password +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: solr-scaledobject +spec: + scaleTargetRef: + name: nginx-deploy + triggers: + - type: solr + metadata: + host: "http://solr-service.solr-ns.svc.cluster.local:8983" + query: "*:*" + collection: "my_core" + targetQueryValue: "1" + authenticationRef: + name: trigger-auth-solr +``` diff --git a/content/docs/2.17/scalers/splunk.md b/content/docs/2.17/scalers/splunk.md new file mode 100644 index 000000000..beee29ea8 --- /dev/null +++ b/content/docs/2.17/scalers/splunk.md @@ -0,0 +1,292 @@ ++++ +title = "Splunk" +availability = "v2.15+" +maintainer = "Community" +category = "Data & Storage" +description = "Scale applications based on Splunk saved search results." +go_file = "splunk_scaler" ++++ + +### Trigger Specification + +This specification describes the `splunk` trigger that scales based on the result of a [saved search](https://docs.splunk.com/Documentation/Splunk/9.2.1/Search/Savingsearches). + +The trigger always requires the following information: + +```yaml +triggers: + - type: splunk + metadata: + host: https://splunk.default.svc.cluster.local:8089 + targetValue: "1" + activationValue: "10" + savedSearchName: my-saved-search-name + valueField: count +``` + +**Parameter list:** + +- `host` - Search API host and port. Example: `https://localhost:8089`. +- `unsafeSsl` - Whether to trust invalid certificates or not. (Values: `"true"`, `"false"`, Default: `"false"`, Optional) +- `targetValue` - Value to reach to start scaling (This value can be a integer or float). +- `activationValue` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). +- `savedSearchName` - Name of saved search that returns metric data for scaling. +- `valueField` - The name of the field in the search results containing the metric value. Example: `index=_internal | tail | stats count`, the `valueField` is `count`. + +### Authentication Parameters + +You can authenticate by using a username/password or an API token. You will need to use `TriggerAuthentication` CRD to configure the authentication. + +> **Note:** +> +> `TriggerAuthentication` is required to use this scaler due to the hard requirement of providing a `username` for the Splunk API. + +**Parameter list:** + +- `username` - Splunk username authorized to access the search API. +- `apiToken` - Splunk API token for supplied `username`. Conflicts with `password`. +- `password` - Password for supplied `username`. Conflicts with `apiToken`. + +The user will need access to the saved search. + +### Examples + +### Username/password + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: splunk-creds +data: + username: YWRtaW4= # "admin" + password: cGFzc3dvcmQ= # "password" +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: splunk-auth +spec: + secretTargetRef: + - parameter: username + name: splunk-creds + key: username + - parameter: password + name: splunk-creds + key: password +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: splunk-scaledobject +spec: + pollingInterval: 15 + minReplicaCount: 1 + maxReplicaCount: 3 + scaleTargetRef: + name: nginx + triggers: + - type: splunk + authenticationRef: + name: splunk-auth + metadata: + host: https://splunk.default.svc.cluster.local:8089 + targetValue: "11" + activationValue: "15" + savedSearchName: my-saved-search-name + valueField: count +``` + +### API Token + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: splunk-creds +data: + username: YWRtaW4= # admin + apiToken: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: splunk-auth +spec: + secretTargetRef: + - parameter: username + name: splunk-creds + key: username + - parameter: apiToken + name: splunk-creds + key: apiToken +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: splunk-scaledobject +spec: + pollingInterval: 15 + minReplicaCount: 1 + maxReplicaCount: 3 + scaleTargetRef: + name: nginx + triggers: + - type: splunk + authenticationRef: + name: splunk-auth + metadata: + host: https://splunk.default.svc.cluster.local:8089 + targetValue: "11" + activationValue: "15" + savedSearchName: my-saved-search-name + valueField: count +``` + +### Full example using Splunk deployment + +```yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: splunkconf +data: + default.yml: | + splunk: + conf: + - key: savedsearches + value: + directory: /opt/splunk/etc/users/admin/search/local + content: + my-saved-search-name: + action.email.useNSSubject: 1 + action.webhook.enable_allowlist: 0 + alert.track: 0 + cron_schedule: '*/1 * * * *' + dispatch.earliest_time: -15m + dispatch.latest_time: now + display.general.type: statistics + display.page.search.tab: statistics + display.visualizations.show: 0 + enableSched: 1 + request.ui_dispatch_app: search + request.ui_dispatch_view: search + search: index=_internal | tail | stats count +--- +apiVersion: v1 +kind: Secret +metadata: + name: splunk-creds +data: + username: YWRtaW4= # "admin" + password: cGFzc3dvcmQ= # "password" +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: splunk-auth +spec: + secretTargetRef: + - parameter: username + name: splunk-creds + key: username + - parameter: password + name: splunk-creds + key: password +--- +apiVersion: v1 +kind: Service +metadata: + name: splunk +spec: + ports: + - port: 8000 + targetPort: web + name: web-svc + - port: 8089 + targetPort: 8089 + name: api-svc + selector: + app: splunk +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: splunk + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: splunk + template: + metadata: + labels: + app: splunk + spec: + containers: + - name: splunk + image: splunk/splunk:9.2 + ports: + - containerPort: 8000 + name: web + - containerPort: 8089 + name: api + env: + - name: SPLUNK_START_ARGS + value: --accept-license + - name: SPLUNK_PASSWORD + value: password + volumeMounts: + - name: splunkconf-volume + mountPath: /tmp/defaults + volumes: + - name: splunkconf-volume + configMap: + name: splunkconf +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx +spec: + selector: + matchLabels: + app: nginx + replicas: 1 + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx + ports: + - containerPort: 8080 +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: splunk-scaledobject +spec: + pollingInterval: 15 + minReplicaCount: 1 + maxReplicaCount: 3 + scaleTargetRef: + name: nginx + triggers: + - type: splunk + authenticationRef: + name: splunk-auth + metadata: + host: https://splunk.default.svc.cluster.local:8089 + unsafeSsl: "true" + targetValue: "5" + activationValue: "5" + savedSearchName: my-saved-search-name + valueField: count +``` diff --git a/content/docs/2.17/setupscaler.md b/content/docs/2.17/setupscaler.md new file mode 100644 index 000000000..a8efd7078 --- /dev/null +++ b/content/docs/2.17/setupscaler.md @@ -0,0 +1,120 @@ ++++ +title = "Setup Autoscaling with KEDA" +weight = 500 +description = "Procedure to Setup a Scaler in KEDA" ++++ + +## Prerequisites + +1. **Kubernetes Cluster**: + - Ensure you have a running Kubernetes cluster set up and accessible. + - If you don't have a cluster yet, follow the [official Kubernetes documentation](https://kubernetes.io/docs/setup/) to create a new cluster suitable for your environment (local machine, cloud provider, etc.). + +2. **KEDA Installation**: + - KEDA needs to be installed on your Kubernetes cluster before you can use it. + - Follow the [KEDA installation guide](https://keda.sh/docs/2.14/deploy/) carefully, including any prerequisites specific to your Kubernetes setup. + - The installation guide provides instructions for different installation methods (e.g., YAML, Helm charts, etc.). Choose the method that suits your needs. + +3. **kubectl**: + - The `kubectl` command-line tool is required to interact with your Kubernetes cluster. + - Follow the [official kubectl installation guide](https://kubernetes.io/docs/tasks/tools/#kubectl) to install `kubectl` on your operating system. + - Once installed, configure `kubectl` to communicate with your Kubernetes cluster by following the cluster-specific instructions provided by your Kubernetes setup. + +## Step 1: Identify the Scaler You Need + +KEDA supports various scalers that correspond to different event sources or triggers. Determining the right scaler is crucial for scaling your application based on the desired event source. + +1. Visit the [KEDA Scalers documentation](https://keda.sh/docs/2.14/scalers/) and browse through the list of available scalers. +2. Identify the scaler that matches the event source you want to use for scaling your application. For example: + - If you want to scale based on incoming HTTP traffic, you would need the [HTTP Add-on](https://kedacore.github.io/http-add-on/). + + > **Note:** + > The HTTP Add-on is still in beta stage and may not provide the full functionality or stability expected in a production environment. + + - If you want to scale based on messages in a RabbitMQ queue, you would need the **RabbitMQ scaler**. + - If you want to scale based on a cron schedule, you would need the **Cron scaler**. +3. Open the documentation page for your chosen scaler and familiarize yourself with its specific requirements and configuration options. + +## Step 2: Install the Required Scaler (if needed) + +Some scalers are part of the core KEDA installation, while others need to be installed separately as add-ons. + +1. Refer to the documentation of your chosen scaler to check if it needs to be installed separately. +2. If the scaler needs to be installed separately, follow the installation instructions provided in the scaler's documentation carefully. + - The installation process typically involves running a command (e.g., `helm install` for Helm charts) or applying YAML manifests using `kubectl`. +3. Verify that the scaler has been installed successfully by checking the output of the installation process or by running any provided verification commands. + +## Step 3: Create a ScaledObject Configuration File + +KEDA uses a custom resource called `ScaledObject` to define how your application should be scaled based on the chosen event source or trigger. + +1. Create a new file (e.g., `scaledobject.yaml`) in a text editor or using the command line. +2. Define the `ScaledObject` configuration in this file, following the structure and examples provided in the documentation of your chosen scaler. +3. Typically, the `ScaledObject` configuration includes the following sections: + - `metadata`: Specifies the name and namespace for the `ScaledObject`. + - `spec.scaleTargetRef`: Identifies the Kubernetes deployment or other resource that should be scaled. + - `spec.pollingInterval` (optional): Specifies how often KEDA should check for scaling events (defaults to 15 seconds). + - `spec.cooldownPeriod` (optional): Specifies the cool-down period in seconds after a scaling event (defaults to 300 seconds). + - `spec.maxReplicaCount` (optional): Specifies the maximum number of replicas to scale up to (defaults to 100). + - `spec.triggers`: Defines the specific configuration for your chosen scaler, including any required parameters or settings. +4. Refer to the scaler's documentation for detailed explanations and examples of the `triggers` section and any other required or optional configuration settings. +5. Save the `scaledobject.yaml` file after making the necessary modifications. + +## Step 4: Apply the ScaledObject Configuration + +Once you have created the `ScaledObject` configuration file, apply it to your Kubernetes cluster using `kubectl`: + +1. Open a terminal or command prompt and navigate to the directory containing the `scaledobject.yaml` file. +2. Run the following command to apply the `ScaledObject` configuration: + + ```bash + kubectl apply -f scaledobject.yaml + ``` + + ```plaintext + scaledobject.keda.sh/ created + ``` + +3. Verify that the `ScaledObject` has been created successfully by running: + + ```bash + kubectl get scaledobjects + ``` + + This should display the `ScaledObject` you just created. + + ```plaintext + NAME SCALETARGETKIND SCALETARGETNAME MIN MAX TRIGGERS AUTHENTICATION READY ACTIVE FALLBACK AGE + Deployment 1 10 cpu True False 10s + ``` + +After applying the `ScaledObject` configuration, KEDA will start monitoring the specified event source and scale your application accordingly, based on the configurations you provided. + +## Step 5: Monitor Scaling Events + +You can monitor the scaling events and logs generated by KEDA using the following commands: + +1. List all `ScaledObjects` in your cluster: + + ```bash + kubectl get scaledobjects + ``` + + This will show you the current state of your `ScaledObject` and the number of replicas. + + ```plaintext + NAME SCALETARGETKIND SCALETARGETNAME MIN MAX TRIGGERS AUTHENTICATION READY ACTIVE FALLBACK AGE + Deployment 1 10 cpu True False 10s + ``` + +2. View the logs of the KEDA operator: + + ```bash + kubectl logs -n keda -l app=keda-operator + ``` + + The KEDA operator logs will show you detailed information about scaling events, decisions made by KEDA based on the event source, and any errors or warnings. + + ```plaintext + {"level":"info","ts":,"logger":"scalehandler","msg":"Successfully scaled deployment","scaledobject.Namespace":"","scaledobject.Name":"","scaler":} + ``` diff --git a/content/docs/2.17/troubleshooting.md b/content/docs/2.17/troubleshooting.md new file mode 100644 index 000000000..9bc0b05b1 --- /dev/null +++ b/content/docs/2.17/troubleshooting.md @@ -0,0 +1,6 @@ ++++ +title = "Troubleshooting" +description = "How to address commonly encountered KEDA issues" ++++ + +{{< troubleshooting >}}