From e20285150d4bc67fd3bd3a8ea0eb71c519aee622 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Mon, 9 Sep 2024 16:32:51 +0300 Subject: [PATCH] Add deprecated message --- docs/resources/source_mysql.md | 6 +- docs/resources/source_postgres.md | 4 +- pkg/resources/resource_source_mysql.go | 6 +- pkg/resources/resource_source_postgres.go | 4 +- .../guides/materialize_source_table.md.tmpl | 181 ++++++++++++++++++ 5 files changed, 191 insertions(+), 10 deletions(-) create mode 100644 templates/guides/materialize_source_table.md.tmpl diff --git a/docs/resources/source_mysql.md b/docs/resources/source_mysql.md index 1b3e4bef..49abbb0b 100644 --- a/docs/resources/source_mysql.md +++ b/docs/resources/source_mysql.md @@ -57,12 +57,12 @@ resource "materialize_source_mysql" "test" { - `comment` (String) **Public Preview** Comment on an object in the database. - `database_name` (String) The identifier for the source database in Materialize. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. - `expose_progress` (Block List, Max: 1) The name of the progress collection for the source. If this is not specified, the collection will be named `_progress`. (see [below for nested schema](#nestedblock--expose_progress)) -- `ignore_columns` (List of String, Deprecated) Ignore specific columns when reading data from MySQL. Can only be updated in place when also updating a corresponding `table` attribute. +- `ignore_columns` (List of String, Deprecated) Ignore specific columns when reading data from MySQL. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table resource instead. - `ownership_role` (String) The owernship role of the object. - `region` (String) The region to use for the resource connection. If not set, the default region is used. - `schema_name` (String) The identifier for the source schema in Materialize. Defaults to `public`. -- `table` (Block Set, Deprecated) Specify the tables to be included in the source. If not specified, all tables are included. (see [below for nested schema](#nestedblock--table)) -- `text_columns` (List of String, Deprecated) Decode data as text for specific columns that contain MySQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. +- `table` (Block Set, Deprecated) Specify the tables to be included in the source. Deprecated: Use the new materialize_source_table resource instead. (see [below for nested schema](#nestedblock--table)) +- `text_columns` (List of String, Deprecated) Decode data as text for specific columns that contain MySQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table resource instead. ### Read-Only diff --git a/docs/resources/source_postgres.md b/docs/resources/source_postgres.md index 22a49587..d677aa34 100644 --- a/docs/resources/source_postgres.md +++ b/docs/resources/source_postgres.md @@ -62,8 +62,8 @@ resource "materialize_source_postgres" "example_source_postgres" { - `ownership_role` (String) The owernship role of the object. - `region` (String) The region to use for the resource connection. If not set, the default region is used. - `schema_name` (String) The identifier for the source schema in Materialize. Defaults to `public`. -- `table` (Block Set, Deprecated) Creates subsources for specific tables in the Postgres connection. (see [below for nested schema](#nestedblock--table)) -- `text_columns` (List of String, Deprecated) Decode data as text for specific columns that contain PostgreSQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. +- `table` (Block Set, Deprecated) Creates subsources for specific tables in the Postgres connection. Deprecated: Use the new materialize_source_table resource instead. (see [below for nested schema](#nestedblock--table)) +- `text_columns` (List of String, Deprecated) Decode data as text for specific columns that contain PostgreSQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table resource instead. ### Read-Only diff --git a/pkg/resources/resource_source_mysql.go b/pkg/resources/resource_source_mysql.go index 64266c76..01dc5dd1 100644 --- a/pkg/resources/resource_source_mysql.go +++ b/pkg/resources/resource_source_mysql.go @@ -27,21 +27,21 @@ var sourceMySQLSchema = map[string]*schema.Schema{ ForceNew: true, }), "ignore_columns": { - Description: "Ignore specific columns when reading data from MySQL. Can only be updated in place when also updating a corresponding `table` attribute.", + Description: "Ignore specific columns when reading data from MySQL. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table resource instead.", Deprecated: "Use the new materialize_source_table resource instead.", Type: schema.TypeList, Elem: &schema.Schema{Type: schema.TypeString}, Optional: true, }, "text_columns": { - Description: "Decode data as text for specific columns that contain MySQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute.", + Description: "Decode data as text for specific columns that contain MySQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table resource instead.", Deprecated: "Use the new materialize_source_table resource instead.", Type: schema.TypeList, Elem: &schema.Schema{Type: schema.TypeString}, Optional: true, }, "table": { - Description: "Specify the tables to be included in the source. If not specified, all tables are included.", + Description: "Specify the tables to be included in the source. Deprecated: Use the new materialize_source_table resource instead.", Deprecated: "Use the new materialize_source_table resource instead.", Type: schema.TypeSet, Optional: true, diff --git a/pkg/resources/resource_source_postgres.go b/pkg/resources/resource_source_postgres.go index 9b057a22..bea0cad5 100644 --- a/pkg/resources/resource_source_postgres.go +++ b/pkg/resources/resource_source_postgres.go @@ -33,14 +33,14 @@ var sourcePostgresSchema = map[string]*schema.Schema{ ForceNew: true, }, "text_columns": { - Description: "Decode data as text for specific columns that contain PostgreSQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute.", + Description: "Decode data as text for specific columns that contain PostgreSQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table resource instead.", Deprecated: "Use the new materialize_source_table resource instead.", Type: schema.TypeList, Elem: &schema.Schema{Type: schema.TypeString}, Optional: true, }, "table": { - Description: "Creates subsources for specific tables in the Postgres connection.", + Description: "Creates subsources for specific tables in the Postgres connection. Deprecated: Use the new materialize_source_table resource instead.", Deprecated: "Use the new materialize_source_table resource instead.", Type: schema.TypeSet, Optional: true, diff --git a/templates/guides/materialize_source_table.md.tmpl b/templates/guides/materialize_source_table.md.tmpl new file mode 100644 index 00000000..eaf224f5 --- /dev/null +++ b/templates/guides/materialize_source_table.md.tmpl @@ -0,0 +1,181 @@ +--- +page_title: "Source versioning: migrating to `materialize_source_table` Resource" +subcategory: "" +description: |- + +--- + +# Source versioning: migrating to `materialize_source_table` Resource + +In previous versions of the Materialize Terraform provider, source tables were defined within the source resource itself and were considered subsources of the source rather than separate entities. + +This guide will walk you through the process of migrating your existing source table definitions to the new `materialize_source_table` resource. + +## Old Approach + +Previously, source tables were defined directly within the source resource: + +### Example: MySQL Source + +```hcl +resource "materialize_source_mysql" "mysql_source" { + name = "mysql_source" + cluster_name = "cluster_name" + + mysql_connection { + name = materialize_connection_mysql.mysql_connection.name + } + + table { + upstream_name = "mysql_table1" + upstream_schema_name = "shop" + name = "mysql_table1_local" + } +} +``` + +The same approach was used for other source types such as Postgres and the load generator sources. + +## New Approach + +The new approach separates source definitions and table definitions. You will now create the source without specifying the tables, and then define each table using the `materialize_source_table` resource. + +## Manual Migration Process + +This manual migration process requires users to create new source tables using the new `materialize_source_table` resource and then remove the old ones. + +### Step 1: Define `materialize_source_table` Resources + +Before making any changes to your existing source resources, create new `materialize_source_table` resources for each table that is currently defined within your sources. This ensures that the tables are preserved during the migration: + +```hcl +resource "materialize_source_table" "mysql_table_from_source" { + name = "mysql_table1_from_source" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_mysql.mysql_source.name + // Define the schema and database for the source if needed + } + + upstream_name = "mysql_table1" + upstream_schema_name = "shop" + + ignore_columns = ["about"] +} +``` + +### Step 2: Apply the Changes + +Run `terraform plan` and `terraform apply` to create the new `materialize_source_table` resources. This step ensures that the tables are defined separately from the source and are not removed from Materialize. + +> **Note:** This will start an ingestion process for the newly created source tables. + +### Step 3: Remove Table Blocks from Source Resources + +Once the new `materialize_source_table` resources are successfully created, you can safely remove the `table` blocks from your existing source resources: + +```hcl +resource "materialize_source_mysql" "mysql_source" { + name = "mysql_source" + cluster_name = "cluster_name" + + mysql_connection { + name = materialize_connection_mysql.mysql_connection.name + } +} +``` + +This will drop the old tables from the source resources. + +### Step 4: Update Terraform State + +After removing the `table` blocks from your source resources, run `terraform plan` and `terraform apply` again to update the Terraform state and apply the changes. + +### Step 5: Verify the Migration + +After applying the changes, verify that your tables are still correctly set up in Materialize by checking the table definitions using Materialize’s SQL commands. + +During the migration, you can use both the old `table` blocks and the new `materialize_source_table` resources simultaneously. This allows for a gradual transition until the old method is fully deprecated. + +## Automated Migration Process (TBD) + +> **Note:** This will still not work as the previous source tables are considered subsources of the source and are missing from the `mz_tables` table in Materialize so we can't import them directly without recreating them. + +Once the migration on the Materialize side has been implemented, a more automated migration process will be available. The steps will include: + +### Step 1: Define `materialize_source_table` Resources + +First, define the new `materialize_source_table` resources for each table: + +```hcl +resource "materialize_source_table" "mysql_table_from_source" { + name = "mysql_table1_from_source" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_mysql.mysql_source.name + // Define the schema and database for the source if needed + } + + upstream_name = "mysql_table1" + upstream_schema_name = "shop" + + ignore_columns = ["about"] +} +``` + +### Step 2: Modify the Existing Source Resource + +Next, modify the existing source resource by removing the `table` blocks and adding an `ignore_changes` directive for the `table` attribute. This prevents Terraform from trying to delete the tables: + +```hcl +resource "materialize_source_mysql" "mysql_source" { + name = "mysql_source" + cluster_name = "cluster_name" + + mysql_connection { + name = materialize_connection_mysql.mysql_connection.name + } + + lifecycle { + ignore_changes = [table] + } +} +``` + +- **`lifecycle { ignore_changes = [table] }`**: This directive tells Terraform to ignore changes to the `table` attribute, preventing it from trying to delete tables that were previously defined in the source resource. + +### Step 3: Import the Existing Tables + +You can then import the existing tables into the new `materialize_source_table` resources without disrupting your existing setup: + +```bash +terraform import materialize_source_table.mysql_table_from_source : +``` + +Replace `` with the actual region and `` with the table ID. You can find the table ID by querying the `mz_tables` table. + +### Step 4: Run Terraform Plan and Apply + +Finally, run `terraform plan` and `terraform apply` to ensure that everything is correctly set up without triggering any unwanted deletions. + +This approach allows you to migrate your tables safely without disrupting your existing setup. + +## Importing Existing Tables + +To import existing tables into your Terraform state using the manual migration process, use the following command: + +```bash +terraform import materialize_source_table.table_name : +``` + +Ensure you replace `` with the region where the table is located and `` with the ID of the table. + +> **Note:** The `upstream_name` and `upstream_schema_name` attributes are not yet implemented on the Materialize side, so the import process will not work until these changes are made. + +## Future Improvements + +The Kafka and Webhooks sources are currently being implemented. Once these changes, the migration process will be updated to include them.