Skip to content

Commit

Permalink
Add integration test for default Python bundle initialization
Browse files Browse the repository at this point in the history
  • Loading branch information
@denik
denik committed Dec 18, 2024
1 parent 3c3c172 commit ef5f905
Show file tree
Hide file tree
Showing 10 changed files with 638 additions and 2 deletions.
11 changes: 9 additions & 2 deletions go.mod
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,22 @@
module github.com/databricks/cli

go 1.23
go 1.23.0

toolchain go1.23.2
toolchain go1.23.3

require (
github.com/Masterminds/semver/v3 v3.3.1 // MIT
github.com/briandowns/spinner v1.23.1 // Apache 2.0
github.com/databricks/databricks-sdk-go v0.54.0 // Apache 2.0
github.com/elliotchance/orderedmap/v3 v3.0.0 // MIT
github.com/fatih/color v1.18.0 // MIT
github.com/ghodss/yaml v1.0.0 // MIT + NOTICE
github.com/google/uuid v1.6.0 // BSD-3-Clause
github.com/hashicorp/go-version v1.7.0 // MPL 2.0
github.com/hashicorp/hc-install v0.9.0 // MPL 2.0
github.com/hashicorp/terraform-exec v0.21.0 // MPL 2.0
github.com/hashicorp/terraform-json v0.23.0 // MPL 2.0
github.com/hexops/gotextdiff v1.0.3 // BSD 3-Clause "New" or "Revised" License
github.com/manifoldco/promptui v0.9.0 // BSD-3-Clause
github.com/mattn/go-isatty v0.0.20 // MIT
github.com/nwidger/jsoncolor v0.3.2 // MIT
Expand All @@ -23,6 +25,7 @@ require (
github.com/spf13/cobra v1.8.1 // Apache 2.0
github.com/spf13/pflag v1.0.5 // BSD-3-Clause
github.com/stretchr/testify v1.10.0 // MIT
github.com/wI2L/jsondiff v0.6.1 // MIT
golang.org/x/exp v0.0.0-20240222234643-814bf88cf225
golang.org/x/mod v0.22.0
golang.org/x/oauth2 v0.24.0
Expand Down Expand Up @@ -56,6 +59,10 @@ require (
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/tidwall/gjson v1.18.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
github.com/tidwall/sjson v1.2.5 // indirect
github.com/zclconf/go-cty v1.15.0 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
Expand Down
16 changes: 16 additions & 0 deletions go.sum
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

106 changes: 106 additions & 0 deletions integration/bundle/init_default_python_test.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
package bundle_test

import (
"encoding/json"
"os"
"path/filepath"
"testing"

"github.com/databricks/cli/integration/internal/acc"
"github.com/databricks/cli/internal/testcli"
"github.com/databricks/cli/internal/testutil"
"github.com/databricks/cli/libs/python/pythontest"
"github.com/stretchr/testify/require"
)

var pythonVersions = []string{
"3.8",
"3.9",
"3.10",
"3.11",
"3.12",
"3.13",
}

var pythonVersionsShort = []string{
"3.9",
"3.12",
}

var extraInstalls = map[string][]string{
"3.12": {"setuptools"},
"3.13": {"setuptools"},
}

func TestDefaultPython(t *testing.T) {
versions := pythonVersions
if testing.Short() {
versions = pythonVersionsShort
}

for _, pythonVersion := range versions {
t.Run(pythonVersion, func(t *testing.T) {
testDefaultPython(t, pythonVersion)
})
}
}

func testDefaultPython(t *testing.T, pythonVersion string) {
ctx, wt := acc.WorkspaceTest(t)

uniqueProjectId := testutil.RandomName("")
ctx, replacements := testcli.WithReplacementsMap(ctx)
replacements.Set(uniqueProjectId, "$UNIQUE_PRJ")

testcli.PrepareReplacements(t, replacements, wt.W)

user, err := wt.W.CurrentUser.Me(ctx)
require.NoError(t, err)
if user != nil {
testcli.PrepareReplacementsUser(t, replacements, *user)
}

tmpDir1 := pythontest.RequirePythonVENV(t, ctx, pythonVersion, true)
extras, ok := extraInstalls[pythonVersion]
if ok {
args := append([]string{"pip", "install"}, extras...)
testutil.RunCommand(t, "uv", args...)
}

projectName := "project_name_" + uniqueProjectId

initConfig := map[string]string{
"project_name": projectName,
"include_notebook": "yes",
"include_python": "yes",
"include_dlt": "yes",
}
b, err := json.Marshal(initConfig)
require.NoError(t, err)
err = os.WriteFile(filepath.Join(tmpDir1, "config.json"), b, 0o644)
require.NoError(t, err)

testcli.RequireOutput(t, ctx, []string{"bundle", "init", "default-python", "--config-file", "config.json"}, "testdata/default_python/bundle_init.txt")
testutil.Chdir(t, projectName)

testcli.RequireOutput(t, ctx, []string{"bundle", "validate"}, "testdata/default_python/bundle_validate.txt")

testcli.RequireOutput(t, ctx, []string{"bundle", "deploy"}, "testdata/default_python/bundle_deploy.txt")
t.Cleanup(func() {
// Delete the stack
testcli.RequireSuccessfulRun(t, ctx, "bundle", "destroy", "--auto-approve")
})

ignoredFields := []string{
"/resources/jobs/project_name_$UNIQUE_PRJ_job/email_notifications",
"/resources/jobs/project_name_$UNIQUE_PRJ_job/job_clusters/0/new_cluster/node_type_id",
"/resources/jobs/project_name_$UNIQUE_PRJ_job/url",
"/resources/pipelines/project_name_$UNIQUE_PRJ_pipeline/catalog",
"/resources/pipelines/project_name_$UNIQUE_PRJ_pipeline/url",
"/workspace/current_user/externalId",
"/workspace/current_user/groups",
"/workspace/current_user/name/familyName",
}

testcli.RequireOutputJQ(t, ctx, []string{"bundle", "summary", "--output", "json"}, "testdata/default_python/bundle_summary.txt", ignoredFields)
}
6 changes: 6 additions & 0 deletions integration/bundle/testdata/default_python/bundle_deploy.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
Building project_name_$UNIQUE_PRJ...
Uploading project_name_$UNIQUE_PRJ-0.0.1+<NUMID>.<NUMID>-py3-none-any.whl...
Uploading bundle files to /Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/files...
Deploying resources...
Updating deployment state...
Deployment complete!
8 changes: 8 additions & 0 deletions integration/bundle/testdata/default_python/bundle_init.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@

Welcome to the default Python template for Databricks Asset Bundles!
Workspace to use (auto-detected, edit in 'project_name_$UNIQUE_PRJ/databricks.yml'): https://$DATABRICKS_HOST

✨ Your new project has been created in the 'project_name_$UNIQUE_PRJ' directory!

Please refer to the README.md file for "getting started" instructions.
See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html.
185 changes: 185 additions & 0 deletions integration/bundle/testdata/default_python/bundle_summary.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,185 @@
{
"bundle": {
"name": "project_name_$UNIQUE_PRJ",
"target": "dev",
"environment": "dev",
"terraform": {
"exec_path": "/tmp/.../terraform"
},
"git": {
"bundle_root_path": ".",
"inferred": true
},
"mode": "development",
"deployment": {
"lock": {
"enabled": false
}
}
},
"include": [
"resources/project_name_$UNIQUE_PRJ.job.yml",
"resources/project_name_$UNIQUE_PRJ.pipeline.yml"
],
"workspace": {
"host": "https://$DATABRICKS_HOST",
"current_user": {
"active": true,
"displayName": "$USERNAME",
"emails": [
{
"primary": true,
"type": "work",
"value": "$USERNAME"
}
],
"groups": [
{
"$ref": "Groups/$USER.Groups[0]",
"display": "team.engineering",
"type": "direct",
"value": "$USER.Groups[0]"
}
],
"id": "$USER.Id",
"name": {
"familyName": "$USERNAME",
"givenName": "$USERNAME"
},
"schemas": [
"urn:ietf:params:scim:schemas:core:2.0:User",
"urn:ietf:params:scim:schemas:extension:workspace:2.0:User"
],
"short_name": "$USERNAME",
"userName": "$USERNAME"
},
"root_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev",
"file_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/files",
"resource_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/resources",
"artifact_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/artifacts",
"state_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/state"
},
"resources": {
"jobs": {
"project_name_$UNIQUE_PRJ_job": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"email_notifications": {
"on_failure": [
"$USERNAME"
]
},
"format": "MULTI_TASK",
"id": "<NUMID>",
"job_clusters": [
{
"job_cluster_key": "job_cluster",
"new_cluster": {
"autoscale": {
"max_workers": 4,
"min_workers": 1
},
"node_type_id": "i3.xlarge",
"spark_version": "15.4.x-scala2.12"
}
}
],
"max_concurrent_runs": 4,
"name": "[dev $USERNAME] project_name_$UNIQUE_PRJ_job",
"queue": {
"enabled": true
},
"tags": {
"dev": "$USERNAME"
},
"tasks": [
{
"job_cluster_key": "job_cluster",
"notebook_task": {
"notebook_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/files/src/notebook"
},
"task_key": "notebook_task"
},
{
"depends_on": [
{
"task_key": "notebook_task"
}
],
"pipeline_task": {
"pipeline_id": "${resources.pipelines.project_name_$UNIQUE_PRJ_pipeline.id}"
},
"task_key": "refresh_pipeline"
},
{
"depends_on": [
{
"task_key": "refresh_pipeline"
}
],
"job_cluster_key": "job_cluster",
"libraries": [
{
"whl": "dist/*.whl"
}
],
"python_wheel_task": {
"entry_point": "main",
"package_name": "project_name_$UNIQUE_PRJ"
},
"task_key": "main_task"
}
],
"trigger": {
"pause_status": "PAUSED",
"periodic": {
"interval": 1,
"unit": "DAYS"
}
},
"url": "https://$DATABRICKS_HOST/jobs/<NUMID>?o=<NUMID>"
}
},
"pipelines": {
"project_name_$UNIQUE_PRJ_pipeline": {
"catalog": "main",
"configuration": {
"bundle.sourcePath": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/files/src"
},
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/state/metadata.json"
},
"development": true,
"id": "<UUID>",
"libraries": [
{
"notebook": {
"path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/files/src/dlt_pipeline"
}
}
],
"name": "[dev $USERNAME] project_name_$UNIQUE_PRJ_pipeline",
"target": "project_name_$UNIQUE_PRJ_dev",
"url": "https://$DATABRICKS_HOST/pipelines/<UUID>?o=<NUMID>"
}
}
},
"sync": {
"paths": [
"."
]
},
"presets": {
"name_prefix": "[dev $USERNAME] ",
"pipelines_development": true,
"trigger_pause_status": "PAUSED",
"jobs_max_concurrent_runs": 4,
"tags": {
"dev": "$USERNAME"
}
}
}
8 changes: 8 additions & 0 deletions integration/bundle/testdata/default_python/bundle_validate.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
Name: project_name_$UNIQUE_PRJ
Target: dev
Workspace:
Host: https://$DATABRICKS_HOST
User: $USERNAME
Path: /Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev

Validation OK!
Loading

0 comments on commit ef5f905

Please sign in to comment.