diff --git a/acceptance/.gitignore b/acceptance/.gitignore new file mode 100644 index 0000000000..378eac25d3 --- /dev/null +++ b/acceptance/.gitignore @@ -0,0 +1 @@ +build diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 91ad09e9e1..89d1598553 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -71,6 +71,10 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { cwd, err := os.Getwd() require.NoError(t, err) + // Download terraform and provider and create config; this also creates build directory. + RunCommand(t, []string{filepath.Join(cwd, "install_terraform.py")}, ".") + + buildDir := filepath.Join(cwd, "build") coverDir := os.Getenv("CLI_GOCOVERDIR") if coverDir != "" { @@ -87,7 +91,7 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { t.Setenv("CMD_SERVER_URL", cmdServer.URL) execPath = filepath.Join(cwd, "bin", "callserver.py") } else { - execPath = BuildCLI(t, cwd, coverDir) + execPath = BuildCLI(t, buildDir, coverDir) } t.Setenv("CLI", execPath) @@ -117,11 +121,21 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { homeDir := t.TempDir() // Do not read user's ~/.databrickscfg t.Setenv(env.HomeEnvVar(), homeDir) - - // Prevent CLI from downloading terraform in each test: - t.Setenv("DATABRICKS_TF_EXEC_PATH", tempHomeDir) } + terraformrcPath := filepath.Join(buildDir, ".terraformrc") + t.Setenv("TF_CLI_CONFIG_FILE", terraformrcPath) + t.Setenv("DATABRICKS_TF_CLI_CONFIG_FILE", terraformrcPath) + repls.Set(terraformrcPath, "DATABRICKS_TF_CLI_CONFIG_FILE") + + terraformExecPath := filepath.Join(buildDir, "terraform") + t.Setenv("DATABRICKS_TF_EXEC_PATH", terraformExecPath) + t.Setenv("TERRAFORM", terraformExecPath) + repls.Set(terraformExecPath, "$TERRAFORM") + + // do it last so that full paths match first: + repls.Set(buildDir, "$BUILD_DIR") + workspaceClient, err := databricks.NewWorkspaceClient() require.NoError(t, err) @@ -350,13 +364,12 @@ func readMergedScriptContents(t *testing.T, dir string) string { return strings.Join(prepares, "\n") } -func BuildCLI(t *testing.T, cwd, coverDir string) string { - execPath := filepath.Join(cwd, "build", "databricks") +func BuildCLI(t *testing.T, buildDir, coverDir string) string { + execPath := filepath.Join(buildDir, "databricks") if runtime.GOOS == "windows" { execPath += ".exe" } - start := time.Now() args := []string{ "go", "build", "-mod", "vendor", @@ -374,20 +387,8 @@ func BuildCLI(t *testing.T, cwd, coverDir string) string { args = append(args, "-buildvcs=false") } - cmd := exec.Command(args[0], args[1:]...) - cmd.Dir = ".." - out, err := cmd.CombinedOutput() - elapsed := time.Since(start) - t.Logf("%s took %s", args, elapsed) - require.NoError(t, err, "go build failed: %s: %s\n%s", args, err, out) - if len(out) > 0 { - t.Logf("go build output: %s: %s", args, out) - } - - // Quick check + warm up cache: - cmd = exec.Command(execPath, "--version") - out, err = cmd.CombinedOutput() - require.NoError(t, err, "%s --version failed: %s\n%s", execPath, err, out) + RunCommand(t, args, "..") + RunCommand(t, []string{execPath, "--version"}, ".") return execPath } @@ -525,3 +526,17 @@ func getUVDefaultCacheDir(t *testing.T) string { return cacheDir + "/uv" } } + +func RunCommand(t *testing.T, args []string, dir string) { + start := time.Now() + cmd := exec.Command(args[0], args[1:]...) + cmd.Dir = dir + out, err := cmd.CombinedOutput() + elapsed := time.Since(start) + t.Logf("%s took %s", args, elapsed) + + require.NoError(t, err, "%s failed: %s\n%s", args, err, out) + if len(out) > 0 { + t.Logf("%s output: %s", args, out) + } +} diff --git a/acceptance/build/.gitignore b/acceptance/build/.gitignore deleted file mode 100644 index a48b4db254..0000000000 --- a/acceptance/build/.gitignore +++ /dev/null @@ -1 +0,0 @@ -databricks diff --git a/acceptance/bundle/variables/git-branch/output.txt b/acceptance/bundle/variables/git-branch/output.txt index fb3ab805ac..5e7664f619 100644 --- a/acceptance/bundle/variables/git-branch/output.txt +++ b/acceptance/bundle/variables/git-branch/output.txt @@ -11,7 +11,7 @@ "name": "git", "target": "prod", "terraform": { - "exec_path": "$TMPHOME" + "exec_path": "$TERRAFORM" } }, "sync": { @@ -61,7 +61,7 @@ Validation OK! "name": "git", "target": "dev", "terraform": { - "exec_path": "$TMPHOME" + "exec_path": "$TERRAFORM" } }, "sync": { diff --git a/acceptance/bundle/variables/prepend-workspace-var/output.txt b/acceptance/bundle/variables/prepend-workspace-var/output.txt index fcaa25b4a0..ed6c2b2af2 100644 --- a/acceptance/bundle/variables/prepend-workspace-var/output.txt +++ b/acceptance/bundle/variables/prepend-workspace-var/output.txt @@ -7,7 +7,7 @@ }, "target": "dev", "terraform": { - "exec_path": "$TMPHOME" + "exec_path": "$TERRAFORM" } }, "resources": { diff --git a/acceptance/install_terraform.py b/acceptance/install_terraform.py new file mode 100755 index 0000000000..21e74e0dfa --- /dev/null +++ b/acceptance/install_terraform.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +""" +Script to set up terraform and databricks terraform provider in a local directory: + +- Download terraform. +- Download databricks provider. +- Write a .terraformrc config file that uses this directory. +- The config file contains env vars that need to be set so that databricks CLI uses this terraform and provider. +""" + +import os +import platform +import zipfile +import argparse +from pathlib import Path +from urllib.request import urlretrieve + +os_name = platform.system().lower() + +current_arch = platform.machine().lower() +arch_mapping = { + "x86_64": "amd64", + "amd64": "amd64", + "arm64": "arm64", + "aarch64": "arm64", +} +arch = arch_mapping.get(current_arch, current_arch) + +terraform_version = "1.5.5" +terraform_file = f"terraform_{terraform_version}_{os_name}_{arch}.zip" +terraform_url = f"https://releases.hashicorp.com/terraform/{terraform_version}/{terraform_file}" +terraform_binary = "terraform.exe" if os_name == "windows" else "terraform" + + +def retrieve(url, path): + if not path.exists(): + print(f"Downloading {url} -> {path}") + urlretrieve(url, path) + + +def read_version(path): + for line in path.open(): + if "ProviderVersion" in line: + # Expecting 'const ProviderVersion = "1.64.1"' + items = line.strip().split() + assert len(items) >= 3, items + assert items[-3:-1] == ["ProviderVersion", "="], items + version = items[-1].strip('"') + assert version, items + return version + raise SystemExit(f"Could not find ProviderVersion in {path}") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--targetdir", default="build", type=Path) + parser.add_argument("--provider-version") + args = parser.parse_args() + target = args.targetdir + + if not args.provider_version: + version_file = Path(__file__).parent.parent / "bundle/internal/tf/codegen/schema/version.go" + assert version_file.exists(), version_file + terraform_provider_version = read_version(version_file) + print(f"Read version {terraform_provider_version} from {version_file}") + else: + terraform_provider_version = args.provider_version + + terraform_provider_file = f"terraform-provider-databricks_{terraform_provider_version}_{os_name}_{arch}.zip" + terraform_provider_url = f"https://github.com/databricks/terraform-provider-databricks/releases/download/v{terraform_provider_version}/{terraform_provider_file}" + + target.mkdir(exist_ok=True) + + zip_path = target / terraform_file + terraform_path = target / terraform_binary + terraform_provider_path = target / terraform_provider_file + + retrieve(terraform_url, zip_path) + retrieve(terraform_provider_url, terraform_provider_path) + + if not terraform_path.exists(): + print(f"Extracting {zip_path} -> {terraform_path}") + + with zipfile.ZipFile(zip_path, "r") as zip_ref: + zip_ref.extractall(target) + + terraform_path.chmod(0o755) + + tfplugins_path = target / "tfplugins" + provider_dir = Path( + tfplugins_path / f"registry.terraform.io/databricks/databricks/{terraform_provider_version}/{os_name}_{arch}" + ) + if not provider_dir.exists(): + print(f"Extracting {terraform_provider_path} -> {provider_dir}") + os.makedirs(provider_dir, exist_ok=True) + with zipfile.ZipFile(terraform_provider_path, "r") as zip_ref: + zip_ref.extractall(provider_dir) + + files = list(provider_dir.iterdir()) + assert files, provider_dir + + for f in files: + f.chmod(0o755) + + terraformrc_path = target / ".terraformrc" + if not terraformrc_path.exists(): + print(f"Writing {terraformrc_path} (see it for instructions)") + + terraformrc_path.write_text(f"""# Set these env variables before running databricks cli: +# export DATABRICKS_TF_CLI_CONFIG_FILE={terraformrc_path.absolute()} +# export DATABRICKS_TF_EXEC_PATH={terraform_path.absolute()} + +provider_installation {{ + filesystem_mirror {{ + path = "{tfplugins_path.absolute()}" + include = ["registry.terraform.io/databricks/databricks"] + }} +}} +""") + + +if __name__ == "__main__": + main() diff --git a/acceptance/terraform/main.tf b/acceptance/terraform/main.tf new file mode 100644 index 0000000000..93f665ff4a --- /dev/null +++ b/acceptance/terraform/main.tf @@ -0,0 +1,25 @@ +terraform { + required_providers { + databricks = { + source = "databricks/databricks" + version = "1.64.1" + } + } + + required_version = "= 1.5.5" +} + +provider "databricks" { + # Optionally, specify the Databricks host and token + # host = "https://" + # token = "" +} + +data "databricks_current_user" "me" { + # Retrieves the current user's information +} + +output "username" { + description = "Username" + value = "${data.databricks_current_user.me.user_name}" +} diff --git a/acceptance/terraform/output.txt b/acceptance/terraform/output.txt new file mode 100644 index 0000000000..8ce96d0904 --- /dev/null +++ b/acceptance/terraform/output.txt @@ -0,0 +1,69 @@ + +>>> $TERRAFORM init -no-color +Timestamp [INFO] Terraform version: 1.5.5 +Timestamp [DEBUG] using github.com/hashicorp/go-tfe v1.26.0 +Timestamp [DEBUG] using github.com/hashicorp/hcl/v2 v2.16.2 +Timestamp [DEBUG] using github.com/hashicorp/terraform-svchost v0.1.0 +Timestamp [DEBUG] using github.com/zclconf/go-cty v1.12.2 +Timestamp [INFO] Go runtime version: go1.20.7 +Timestamp [INFO] CLI args: []string{"$TERRAFORM", "init", "-no-color"} +Timestamp [DEBUG] Attempting to open CLI config file: DATABRICKS_TF_CLI_CONFIG_FILE +Timestamp [INFO] Loading CLI configuration from DATABRICKS_TF_CLI_CONFIG_FILE +Timestamp [DEBUG] Not reading CLI config directory because config location is overridden by environment variable +Timestamp [DEBUG] Explicit provider installation configuration is set +Timestamp [INFO] CLI command args: []string{"init", "-no-color"} + +Initializing the backend... +Timestamp [DEBUG] New state was assigned lineage "[UUID]" +Timestamp [DEBUG] checking for provisioner in "." +Timestamp [DEBUG] checking for provisioner in "$BUILD_DIR" + +Initializing provider plugins... +- Finding databricks/databricks versions matching "1.64.1"... +- Installing databricks/databricks v1.64.1... +- Installed databricks/databricks v1.64.1 (unauthenticated) + +Terraform has created a lock file .terraform.lock.hcl to record the provider +selections it made above. Include this file in your version control repository +so that Terraform can guarantee to make the same selections by default when +you run "terraform init" in the future. + + +Warning: Incomplete lock file information for providers + +Due to your customized provider installation methods, Terraform was forced to +calculate lock file checksums locally for the following providers: + - databricks/databricks + +The current .terraform.lock.hcl file only includes checksums for +darwin_arm64, so Terraform running on another platform will fail to install +these providers. + +To calculate additional checksums for another platform, run: + terraform providers lock -platform=linux_amd64 +(where linux_amd64 is the platform to generate) + +Terraform has been successfully initialized! + +You may now begin working with Terraform. Try running "terraform plan" to see +any changes that are required for your infrastructure. All Terraform commands +should now work. + +If you ever set or change modules or backend configuration for Terraform, +rerun this command to reinitialize your working directory. If you forget, other +commands will detect it and remind you to do so if necessary. + +>>> $TERRAFORM plan -no-color +data.databricks_current_user.me: Reading... +data.databricks_current_user.me: Read complete after 0s [id=$USER.Id] + +Changes to Outputs: + + username = "$USERNAME" + +You can apply this plan to save these new output values to the Terraform +state, without changing any real infrastructure. + +───────────────────────────────────────────────────────────────────────────── + +Note: You didn't use the -out option to save this plan, so Terraform can't +guarantee to take exactly these actions if you run "terraform apply" now. diff --git a/acceptance/terraform/script b/acceptance/terraform/script new file mode 100644 index 0000000000..a78bb7dc2c --- /dev/null +++ b/acceptance/terraform/script @@ -0,0 +1,5 @@ +export TF_LOG=DEBUG +trace $TERRAFORM init -no-color +export TF_LOG=WARN +trace $TERRAFORM plan -no-color +rm -fr .terraform.lock.hcl .terraform diff --git a/acceptance/terraform/test.toml b/acceptance/terraform/test.toml new file mode 100644 index 0000000000..da94548411 --- /dev/null +++ b/acceptance/terraform/test.toml @@ -0,0 +1,3 @@ +[[Repls]] +Old = '\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d+\+\d+' +New = 'Timestamp'