From 37f2de467b6eefb957bf31c0987e5d06e7f7a7e6 Mon Sep 17 00:00:00 2001
From: Andreas Motl <andreas.motl@crate.io>
Date: Tue, 6 Aug 2024 11:40:16 +0200
Subject: [PATCH] AWS DMS: Add first working version

---
 doc/io/dms/index.md      | 53 ++++++++++++++++++++++++++++++++++++++++
 doc/io/dms/managed.md    | 21 ++++++++++++++++
 doc/io/dms/standalone.md | 26 ++++++++++++++++++++
 doc/io/index.md          |  1 +
 4 files changed, 101 insertions(+)
 create mode 100644 doc/io/dms/index.md
 create mode 100644 doc/io/dms/managed.md
 create mode 100644 doc/io/dms/standalone.md

diff --git a/doc/io/dms/index.md b/doc/io/dms/index.md
new file mode 100644
index 00000000..59fcee51
--- /dev/null
+++ b/doc/io/dms/index.md
@@ -0,0 +1,53 @@
+# AWS DMS Processor
+
+## About
+[AWS Database Migration Service] (AWS DMS) is a managed migration and replication
+service that helps move your database and analytics workloads quickly, securely,
+and with minimal downtime and zero data loss.
+
+The data migration pipeline supports one-shot full-load operations, and continuous
+replication based on change data capture (CDC).
+
+## Details
+A full-load-and-CDC pipeline using AWS DMS and CrateDB will use [Amazon Kinesis]
+Data Streams [as a DMS target], combined with a CrateDB-specific downstream
+processor element.
+
+## Coverage
+AWS DMS supports migration between 20-plus database and analytics engines, either
+on-premises, or per EC2 instance databases.
+
+- Amazon Aurora
+- Amazon DocumentDB
+- Amazon S3
+- IBM Db2 for Linux, UNIX, and Windows versions 9.7 and higher
+- IBM Db2 for z/OS version 12
+- MariaDB versions 10.0 and higher
+- Microsoft Azure SQL Database
+- Microsoft SQL Server versions 2005 and higher
+- MongoDB versions 3.x and higher
+- MySQL versions 5.5 and higher
+- Oracle versions 10.2 and higher
+- PostgreSQL versions 9.4 and higher
+- SAP Adaptive Server Enterprise (ASE) versions 12.5 and higher
+
+AWS DMS also supports the MySQL/MariaDB and PostgreSQL variants on AWS RDS,
+Microsoft Azure, and Google Cloud. [Sources for AWS DMS] displays all the
+compatibility details on one page.
+
+## Usage
+Depending on your needs and requirements, CrateDB and CrateDB Cloud support
+different ways to configure AMS DMS using CrateDB as a CDC consolidation
+database.
+```{toctree}
+:maxdepth: 2
+
+standalone
+managed
+```
+
+
+[Amazon Kinesis]: https://aws.amazon.com/kinesis/
+[as a DMS target]: https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Target.Kinesis.html
+[AWS Database Migration Service]: https://aws.amazon.com/dms/
+[Sources for AWS DMS]: https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Introduction.Sources.html
diff --git a/doc/io/dms/managed.md b/doc/io/dms/managed.md
new file mode 100644
index 00000000..e0800449
--- /dev/null
+++ b/doc/io/dms/managed.md
@@ -0,0 +1,21 @@
+# AWS DMS Managed
+
+## About
+Conduct a data migration from any source supported by AWS DMS into a database
+table on [CrateDB Cloud], exclusively using managed infrastructure components.
+
+:::{note}
+This is a work in progress. Please contact our data engineers to get started.
+:::
+
+## Configuration
+1. Set up a DMS instance to replicate data to an Amazon Kinesis Data Stream.
+2. Take a note about the AWS ARN of that Kinesis Data Stream,
+   for example `arn:aws:kinesis:eu-central-1:831394476016:stream/testdrive`.
+3. Reach out to CrateDB support, to make CrateDB Cloud connect to your data
+   stream, in order to converge it into your CrateDB Cloud instance.
+
+
+[to an Amazon Kinesis Data Stream]: https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Target.Kinesis.html
+[CrateDB]: https://cratedb.com/docs/guide/home/
+[CrateDB Cloud]: https://cratedb.com/docs/cloud/
diff --git a/doc/io/dms/standalone.md b/doc/io/dms/standalone.md
new file mode 100644
index 00000000..98d21f16
--- /dev/null
+++ b/doc/io/dms/standalone.md
@@ -0,0 +1,26 @@
+# AWS DMS Standalone
+
+## About
+Relay an AWS DMS data stream from Amazon Kinesis into a [CrateDB] table using
+a one-stop command `ctk load table kinesis+dms://...`.
+
+You can use it in order to facilitate convenient data transfers to be used
+within data pipelines or ad hoc operations. It can be used as a CLI interface,
+and as a library.
+
+## Install
+Install the CrateDB Toolkit package.
+```shell
+pip install --upgrade 'cratedb-toolkit[kinesis]'
+```
+
+## Usage
+1. Set up a DMS instance, replicating data to Amazon Kinesis.
+2. Transfer data from Kinesis Data Stream into CrateDB database table.
+```shell
+export CRATEDB_SQLALCHEMY_URL=crate://crate@localhost:4200/testdrive/demo
+ctk load table kinesis+dms://arn:aws:kinesis:eu-central-1:831394476016:stream/testdrive
+```
+
+
+[CrateDB]: https://cratedb.com/docs/guide/home/
diff --git a/doc/io/index.md b/doc/io/index.md
index c85b5ed0..5ccfb3f7 100644
--- a/doc/io/index.md
+++ b/doc/io/index.md
@@ -86,6 +86,7 @@ ctk shell --command="SELECT * FROM data_weather LIMIT 10;" --format=json
 :maxdepth: 2
 :hidden:
 
+AWS DMS <dms/index>
 DynamoDB <dynamodb/index>
 InfluxDB <influxdb/index>
 MongoDB <mongodb/index>