feat(core): Introduce AWS secrets manager as external secrets store (#…

…8982)

packages/cli/package.json

@@ -61,6 +61,7 @@
   ],
   "devDependencies": {
     "@redocly/cli": "^1.6.0",
+    "@types/aws4": "^1.5.1",
     "@types/basic-auth": "^1.1.3",
     "@types/bcryptjs": "^2.4.2",
     "@types/compression": "1.0.1",
@@ -103,6 +104,7 @@
     "@rudderstack/rudder-sdk-node": "2.0.7",
     "@sentry/integrations": "7.87.0",
     "@sentry/node": "7.87.0",
+    "aws4": "1.11.0",
     "axios": "1.6.7",
     "basic-auth": "2.0.1",
     "bcryptjs": "2.4.3",

packages/cli/src/ExternalSecrets/ExternalSecretsManager.ee.ts

@@ -204,7 +204,7 @@ export class ExternalSecretsManager {
 		return Object.keys(this.providers);
 	}
 
-	getSecret(provider: string, name: string): IDataObject | undefined {
+	getSecret(provider: string, name: string) {
 		return this.getProvider(provider)?.getSecret(name);
 	}
 

packages/cli/src/ExternalSecrets/ExternalSecretsProviders.ee.ts

@@ -2,10 +2,12 @@ import type { SecretsProvider } from '@/Interfaces';
 import { Service } from 'typedi';
 import { InfisicalProvider } from './providers/infisical';
 import { VaultProvider } from './providers/vault';
+import { AwsSecretsManager } from './providers/aws-secrets/aws-secrets-manager';
 
 @Service()
 export class ExternalSecretsProviders {
 	providers: Record<string, { new (): SecretsProvider }> = {
+		awsSecretsManager: AwsSecretsManager,
 		infisical: InfisicalProvider,
 		vault: VaultProvider,
 	};

packages/cli/src/ExternalSecrets/providers/aws-secrets/aws-secrets-client.ts

@@ -0,0 +1,151 @@
+import axios from 'axios';
+import * as aws4 from 'aws4';
+import type { AxiosRequestConfig } from 'axios';
+import type { Request as Aws4Options } from 'aws4';
+import type {
+	AwsSecretsManagerContext,
+	ConnectionTestResult,
+	Secret,
+	SecretsNamesPage,
+	SecretsPage,
+	AwsSecretsClientSettings,
+} from './types';
+
+export class AwsSecretsClient {
+	private settings: AwsSecretsClientSettings = {
+		region: '',
+		host: '',
+		url: '',
+		accessKeyId: '',
+		secretAccessKey: '',
+	};
+
+	constructor(settings: AwsSecretsManagerContext['settings']) {
+		const { region, accessKeyId, secretAccessKey } = settings;
+
+		this.settings = {
+			region,
+			host: `secretsmanager.${region}.amazonaws.com`,
+			url: `https://secretsmanager.${region}.amazonaws.com`,
+			accessKeyId,
+			secretAccessKey,
+		};
+	}
+
+	/**
+	 * Check whether the client can connect to AWS Secrets Manager.
+	 */
+	async checkConnection(): ConnectionTestResult {
+		try {
+			await this.fetchSecretsNamesPage();
+			return [true];
+		} catch (e) {
+			const error = e instanceof Error ? e : new Error(`${e}`);
+			return [false, error.message];
+		}
+	}
+
+	/**
+	 * Fetch all secrets from AWS Secrets Manager.
+	 */
+	async fetchAllSecrets() {
+		const secrets: Secret[] = [];
+
+		const allSecretsNames = await this.fetchAllSecretsNames();
+
+		const batches = this.batch(allSecretsNames);
+
+		for (const batch of batches) {
+			const page = await this.fetchSecretsPage(batch);
+
+			secrets.push(
+				...page.SecretValues.map((s) => ({ secretName: s.Name, secretValue: s.SecretString })),
+			);
+		}
+
+		return secrets;
+	}
+
+	private batch<T>(arr: T[], size = 20): T[][] {
+		return Array.from({ length: Math.ceil(arr.length / size) }, (_, index) =>
+			arr.slice(index * size, (index + 1) * size),
+		);
+	}
+
+	private toRequestOptions(
+		action: 'ListSecrets' | 'BatchGetSecretValue',
+		body: string,
+	): Aws4Options {
+		return {
+			method: 'POST',
+			service: 'secretsmanager',
+			region: this.settings.region,
+			host: this.settings.host,
+			headers: {
+				'X-Amz-Target': `secretsmanager.${action}`,
+				'Content-Type': 'application/x-amz-json-1.1',
+			},
+			body,
+		};
+	}
+
+	/**
+	 * @doc https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_BatchGetSecretValue.html
+	 */
+	private async fetchSecretsPage(secretsNames: string[], nextToken?: string) {
+		const body = JSON.stringify(
+			nextToken
+				? { SecretIdList: secretsNames, NextToken: nextToken }
+				: { SecretIdList: secretsNames },
+		);
+
+		const options = this.toRequestOptions('BatchGetSecretValue', body);
+		const { headers } = aws4.sign(options, this.settings);
+
+		const config: AxiosRequestConfig = {
+			method: 'POST',
+			url: this.settings.url,
+			headers,
+			data: body,
+		};
+
+		const response = await axios.request<SecretsPage>(config);
+
+		return response.data;
+	}
+
+	private async fetchAllSecretsNames() {
+		const names: string[] = [];
+
+		let nextToken: string | undefined;
+
+		do {
+			const page = await this.fetchSecretsNamesPage(nextToken);
+			names.push(...page.SecretList.map((s) => s.Name));
+			nextToken = page.NextToken;
+		} while (nextToken);
+
+		return names;
+	}
+
+	/**
+	 * @doc https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_ListSecrets.html
+	 */
+	private async fetchSecretsNamesPage(nextToken?: string) {
+		const body = JSON.stringify(nextToken ? { NextToken: nextToken } : {});
+
+		const options = this.toRequestOptions('ListSecrets', body);
+		const { headers } = aws4.sign(options, this.settings);
+
+		const config: AxiosRequestConfig = {
+			method: 'POST',
+			url: this.settings.url,
+			headers,
+			data: body,
+		};
+
+		const response = await axios.request<SecretsNamesPage>(config);
+
+		return response.data;
+	}
+}

packages/cli/src/ExternalSecrets/providers/aws-secrets/aws-secrets-manager.ts

@@ -0,0 +1,131 @@
+import { AwsSecretsClient } from './aws-secrets-client';
+import { UnknownAuthTypeError } from '@/errors/unknown-auth-type.error';
+import { EXTERNAL_SECRETS_NAME_REGEX } from '@/ExternalSecrets/constants';
+import type { SecretsProvider, SecretsProviderState } from '@/Interfaces';
+import type { INodeProperties } from 'n8n-workflow';
+import type { AwsSecretsManagerContext } from './types';
+
+export class AwsSecretsManager implements SecretsProvider {
+	name = 'awsSecretsManager';
+
+	displayName = 'AWS Secrets Manager';
+
+	state: SecretsProviderState = 'initializing';
+
+	properties: INodeProperties[] = [
+		{
+			displayName:
+				'Need help filling out these fields? <a href="https://docs.n8n.io/external-secrets/#connect-n8n-to-your-secrets-store" target="_blank">Open docs</a>',
+			name: 'notice',
+			type: 'notice',
+			default: '',
+			noDataExpression: true,
+		},
+		{
+			displayName: 'Region',
+			name: 'region',
+			type: 'string',
+			default: '',
+			required: true,
+			placeholder: 'e.g. eu-west-3',
+			noDataExpression: true,
+		},
+		{
+			displayName: 'Authentication Method',
+			name: 'authMethod',
+			type: 'options',
+			options: [
+				{
+					name: 'IAM User',
+					value: 'iamUser',
+					description:
+						'Credentials for IAM user having <code>secretsmanager:ListSecrets</code> and <code>secretsmanager:BatchGetSecretValue</code> permissions. <a href="https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html" target="_blank">Learn more</a>',
+				},
+			],
+			default: 'iamUser',
+			required: true,
+			noDataExpression: true,
+		},
+		{
+			displayName: 'Access Key ID',
+			name: 'accessKeyId',
+			type: 'string',
+			default: '',
+			required: true,
+			placeholder: 'e.g. ACHXUQMBAQEVTE2RKMWP',
+			noDataExpression: true,
+			displayOptions: {
+				show: {
+					authMethod: ['iamUser'],
+				},
+			},
+		},
+		{
+			displayName: 'Secret Access Key',
+			name: 'secretAccessKey',
+			type: 'string',
+			default: '',
+			required: true,
+			placeholder: 'e.g. cbmjrH/xNAjPwlQR3i/1HRSDD+esQX/Lan3gcmBc',
+			typeOptions: { password: true },
+			noDataExpression: true,
+			displayOptions: {
+				show: {
+					authMethod: ['iamUser'],
+				},
+			},
+		},
+	];
+
+	private cachedSecrets: Record<string, string> = {};
+
+	private client: AwsSecretsClient;
+
+	async init(context: AwsSecretsManagerContext) {
+		this.assertAuthType(context);
+
+		this.client = new AwsSecretsClient(context.settings);
+	}
+
+	async test() {
+		return await this.client.checkConnection();
+	}
+
+	async connect() {
+		const [wasSuccessful] = await this.test();
+
+		this.state = wasSuccessful ? 'connected' : 'error';
+	}
+
+	async disconnect() {
+		return;
+	}
+
+	async update() {
+		const secrets = await this.client.fetchAllSecrets();
+
+		const supportedSecrets = secrets.filter((s) => EXTERNAL_SECRETS_NAME_REGEX.test(s.secretName));
+
+		this.cachedSecrets = Object.fromEntries(
+			supportedSecrets.map((s) => [s.secretName, s.secretValue]),
+		);
+	}
+
+	getSecret(name: string) {
+		return this.cachedSecrets[name];
+	}
+
+	hasSecret(name: string) {
+		return name in this.cachedSecrets;
+	}
+
+	getSecretNames() {
+		return Object.keys(this.cachedSecrets);
+	}
+
+	private assertAuthType(context: AwsSecretsManagerContext) {
+		if (context.settings.authMethod === 'iamUser') return;
+
+		throw new UnknownAuthTypeError(context.settings.authMethod);
+	}
+}

packages/cli/src/ExternalSecrets/providers/aws-secrets/types.ts

@@ -0,0 +1,50 @@
+import type { SecretsProviderSettings } from '@/Interfaces';
+
+export type SecretsNamesPage = {
+	NextToken?: string;
+	SecretList: SecretName[];
+};
+
+export type SecretsPage = {
+	NextToken?: string;
+	SecretValues: SecretValue[];
+};
+
+type SecretName = {
+	ARN: string;
+	CreatedDate: number;
+	LastAccessedDate: number;
+	LastChangedDate: number;
+	Name: string;
+	Tags: string[];
+};
+
+type SecretValue = {
+	ARN: string;
+	CreatedDate: number;
+	Name: string;
+	SecretString: string;
+	VersionId: string;
+};
+
+export type Secret = {
+	secretName: string;
+	secretValue: string;
+};
+
+export type ConnectionTestResult = Promise<[boolean] | [boolean, string]>;
+
+export type AwsSecretsManagerContext = SecretsProviderSettings<{
+	region: string;
+	authMethod: 'iamUser';
+	accessKeyId: string;
+	secretAccessKey: string;
+}>;
+
+export type AwsSecretsClientSettings = {
+	region: string;
+	host: string;
+	url: string;
+	accessKeyId: string;
+	secretAccessKey: string;
+};

packages/cli/src/Interfaces.ts

@@ -669,7 +669,7 @@ export abstract class SecretsProvider {
 	abstract disconnect(): Promise<void>;
 	abstract update(): Promise<void>;
 	abstract test(): Promise<[boolean] | [boolean, string]>;
-	abstract getSecret(name: string): IDataObject | undefined;
+	abstract getSecret(name: string): unknown;
 	abstract hasSecret(name: string): boolean;
 	abstract getSecretNames(): string[];
 }

packages/cli/src/SecretsHelpers.ts

@@ -1,4 +1,4 @@
-import type { IDataObject, SecretsHelpersBase } from 'n8n-workflow';
+import type { SecretsHelpersBase } from 'n8n-workflow';
 import { Service } from 'typedi';
 import { ExternalSecretsManager } from './ExternalSecrets/ExternalSecretsManager.ee';
 
@@ -19,7 +19,7 @@ export class SecretsHelper implements SecretsHelpersBase {
 		}
 	}
 
-	getSecret(provider: string, name: string): IDataObject | undefined {
+	getSecret(provider: string, name: string) {
 		return this.service.getSecret(provider, name);
 	}