-
Notifications
You must be signed in to change notification settings - Fork 409
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Exporter] **Breaking changes** Use new query and alert resources ins…
…tead of legacy ones This change replaces legacy `databricks_sql_query` and `databricks_sql_alert` with new resources `databricks_query` and `databricks_alert`. Also, services `sql-queries` and `sql-alerts` are renamed to `queries` and `alerts`. Other changes include: * Improve performance of Lakeview dashboards scan by using bigger page size * Generalize `isMatchingCatalogAndSchema` implementation for use in multiple resources where attribute names could be different * Generalize handling of `/Workspace` prefix when emitting notebooks, workspace files and directories.
- Loading branch information
Showing
15 changed files
with
295 additions
and
373 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -56,20 +56,6 @@ func getJSONObject(filename string) any { | |
| return obj | ||
| } | ||
|
|
||
| func getJSONArray(filename string) any { | ||
| data, err := os.ReadFile(filename) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
| var obj []any | ||
| err = json.Unmarshal(data, &obj) | ||
| if err != nil { | ||
| fmt.Printf("[ERROR] error! file=%s err=%v\n", filename, err) | ||
| fmt.Printf("[ERROR] data=%s\n", string(data)) | ||
| } | ||
| return obj | ||
| } | ||
|
|
||
| func workspaceConfKeysToURL() string { | ||
| keys := make([]string, 0, len(workspaceConfKeys)) | ||
| for k := range workspaceConfKeys { | ||
|
|
@@ -379,14 +365,14 @@ var emptySqlDashboards = qa.HTTPFixture{ | |
|
|
||
| var emptySqlQueries = qa.HTTPFixture{ | ||
| Method: "GET", | ||
| Resource: "/api/2.0/preview/sql/queries?page_size=100", | ||
| Resource: "/api/2.0/sql/queries?page_size=100", | ||
| Response: map[string]any{}, | ||
| ReuseRequest: true, | ||
| } | ||
|
|
||
| var emptySqlAlerts = qa.HTTPFixture{ | ||
| Method: "GET", | ||
| Resource: "/api/2.0/preview/sql/alerts", | ||
| Resource: "/api/2.0/sql/alerts?page_size=100", | ||
| Response: []tfsql.AlertEntity{}, | ||
| ReuseRequest: true, | ||
| } | ||
|
|
@@ -447,7 +433,7 @@ var emptyMetastoreList = qa.HTTPFixture{ | |
|
|
||
| var emptyLakeviewList = qa.HTTPFixture{ | ||
| Method: "GET", | ||
| Resource: "/api/2.0/lakeview/dashboards?page_size=100", | ||
| Resource: "/api/2.0/lakeview/dashboards?page_size=1000", | ||
| Response: sdk_dashboards.ListDashboardsResponse{}, | ||
| ReuseRequest: true, | ||
| } | ||
|
|
@@ -1015,6 +1001,16 @@ func TestImportingClusters(t *testing.T) { | |
| }, | ||
| }, | ||
| }, | ||
| { | ||
| Method: "GET", | ||
| Resource: "/api/2.0/preview/scim/v2/Users?attributes=id%2CuserName&count=100&startIndex=1", | ||
| ReuseRequest: true, | ||
| Response: scim.UserList{ | ||
| Resources: []scim.User{ | ||
| {ID: "123", DisplayName: "[email protected]", UserName: "[email protected]"}, | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
| func(ctx context.Context, client *common.DatabricksClient) { | ||
| os.Setenv("EXPORTER_PARALLELISM_default", "1") | ||
|
|
@@ -1950,16 +1946,21 @@ func TestImportingSqlObjects(t *testing.T) { | |
| }, | ||
| { | ||
| Method: "GET", | ||
| Resource: "/api/2.0/preview/sql/queries?page_size=100", | ||
| Response: getJSONObject("test-data/get-sql-queries.json"), | ||
| Resource: "/api/2.0/sql/queries?page_size=100", | ||
| Response: getJSONObject("test-data/get-queries.json"), | ||
| ReuseRequest: true, | ||
| }, | ||
| { | ||
| Method: "GET", | ||
| Resource: "/api/2.0/preview/sql/queries/16c4f969-eea0-4aad-8f82-03d79b078dcc", | ||
| Response: getJSONObject("test-data/get-sql-query.json"), | ||
| Resource: "/api/2.0/sql/queries/16c4f969-eea0-4aad-8f82-03d79b078dcc?", | ||
| Response: getJSONObject("test-data/get-query.json"), | ||
| ReuseRequest: true, | ||
| }, | ||
| { | ||
| Method: "GET", | ||
| Resource: "/api/2.0/preview/sql/queries/16c4f969-eea0-4aad-8f82-03d79b078dcc", | ||
| Response: getJSONObject("test-data/get-sql-query.json"), | ||
| }, | ||
| { | ||
| Method: "GET", | ||
| Resource: "/api/2.0/permissions/sql/queries/16c4f969-eea0-4aad-8f82-03d79b078dcc?", | ||
|
|
@@ -1972,14 +1973,14 @@ func TestImportingSqlObjects(t *testing.T) { | |
| }, | ||
| { | ||
| Method: "GET", | ||
| Resource: "/api/2.0/preview/sql/alerts", | ||
| Response: getJSONArray("test-data/get-sql-alerts.json"), | ||
| Resource: "/api/2.0/sql/alerts?page_size=100", | ||
| Response: getJSONObject("test-data/get-alerts.json"), | ||
| ReuseRequest: true, | ||
| }, | ||
| { | ||
| Method: "GET", | ||
| Resource: "/api/2.0/preview/sql/alerts/3cf91a42-6217-4f3c-a6f0-345d489051b9?", | ||
| Response: getJSONObject("test-data/get-sql-alert.json"), | ||
| Resource: "/api/2.0/sql/alerts/3cf91a42-6217-4f3c-a6f0-345d489051b9?", | ||
| Response: getJSONObject("test-data/get-alert.json"), | ||
| }, | ||
| { | ||
| Method: "GET", | ||
|
|
@@ -1993,18 +1994,44 @@ func TestImportingSqlObjects(t *testing.T) { | |
|
|
||
| ic := newImportContext(client) | ||
| ic.Directory = tmpDir | ||
| ic.enableListing("sql-dashboards,sql-queries,sql-endpoints,sql-alerts") | ||
| ic.enableServices("sql-dashboards,sql-queries,sql-alerts,sql-endpoints,access,notebooks") | ||
| ic.enableListing("sql-dashboards,queries,sql-endpoints,alerts") | ||
| ic.enableServices("sql-dashboards,queries,alerts,sql-endpoints,access") | ||
|
|
||
| err := ic.Run() | ||
| assert.NoError(t, err) | ||
|
|
||
| // check the generated HCL for SQL Warehouses | ||
| content, err := os.ReadFile(tmpDir + "/sql-endpoints.tf") | ||
| assert.NoError(t, err) | ||
| contentStr := string(content) | ||
| assert.True(t, strings.Contains(contentStr, `enable_serverless_compute = false`)) | ||
| assert.True(t, strings.Contains(contentStr, `resource "databricks_sql_endpoint" "test" {`)) | ||
| assert.False(t, strings.Contains(contentStr, `tags {`)) | ||
| // check the generated HCL for SQL Dashboards | ||
| content, err = os.ReadFile(tmpDir + "/sql-dashboards.tf") | ||
| assert.NoError(t, err) | ||
| contentStr = string(content) | ||
| assert.True(t, strings.Contains(contentStr, `resource "databricks_sql_dashboard" "test_9cb0c8f5_6262_4a1f_a741_2181de76028f" {`)) | ||
| assert.True(t, strings.Contains(contentStr, `dashboard_id = databricks_sql_dashboard.test_9cb0c8f5_6262_4a1f_a741_2181de76028f.id`)) | ||
| assert.True(t, strings.Contains(contentStr, `resource "databricks_sql_widget" "rd4dd2082685" {`)) | ||
| assert.True(t, strings.Contains(contentStr, `resource "databricks_sql_visualization" "chart_16c4f969_eea0_4aad_8f82_03d79b078dcc_1a062d3a_eefe_11eb_9559_dc7cd9c86087"`)) | ||
| // check the generated HCL for Qieries | ||
| content, err = os.ReadFile(tmpDir + "/queries.tf") | ||
| assert.NoError(t, err) | ||
| contentStr = string(content) | ||
| assert.True(t, strings.Contains(contentStr, `resource "databricks_query" "jobs_per_day_per_status_last_30_days_16c4f969_eea0_4aad_8f82_03d79b078dcc"`)) | ||
| assert.True(t, strings.Contains(contentStr, `warehouse_id = databricks_sql_endpoint.test.id`)) | ||
| assert.True(t, strings.Contains(contentStr, `owner_user_name = "[email protected]"`)) | ||
| assert.True(t, strings.Contains(contentStr, `display_name = "Jobs per day per status last 30 days"`)) | ||
| // check the generated HCL for Alerts | ||
| content, err = os.ReadFile(tmpDir + "/alerts.tf") | ||
| assert.NoError(t, err) | ||
| contentStr = string(content) | ||
| assert.True(t, strings.Contains(contentStr, `resource "databricks_alert" "test_alert_3cf91a42_6217_4f3c_a6f0_345d489051b9"`)) | ||
| assert.True(t, strings.Contains(contentStr, `query_id = databricks_query.jobs_per_day_per_status_last_30_days_16c4f969_eea0_4aad_8f82_03d79b078dcc.id`)) | ||
| assert.True(t, strings.Contains(contentStr, `display_name = "Test Alert"`)) | ||
| assert.True(t, strings.Contains(contentStr, `op = "GREATER_THAN"`)) | ||
| assert.True(t, strings.Contains(contentStr, `owner_user_name = "[email protected]"`)) | ||
| }) | ||
| } | ||
|
|
||
|
|
@@ -2795,7 +2822,7 @@ func TestImportingLakeviewDashboards(t *testing.T) { | |
| noCurrentMetastoreAttached, | ||
| { | ||
| Method: "GET", | ||
| Resource: "/api/2.0/lakeview/dashboards?page_size=100", | ||
| Resource: "/api/2.0/lakeview/dashboards?page_size=1000", | ||
| Response: sdk_dashboards.ListDashboardsResponse{ | ||
| Dashboards: []sdk_dashboards.Dashboard{ | ||
| { | ||
|
|
||
Oops, something went wrong.