acc: Use real (cached) terraform in acceptance tests

acceptance/.gitignore

@@ -0,0 +1 @@
+build

acceptance/acceptance_test.go

@@ -71,6 +71,10 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 	cwd, err := os.Getwd()
 	require.NoError(t, err)
 
+	// Download terraform and provider and create config; this also creates build directory.
+	RunCommand(t, []string{filepath.Join(cwd, "install_terraform.py")}, ".")
+
+	buildDir := filepath.Join(cwd, "build")
 	coverDir := os.Getenv("CLI_GOCOVERDIR")
 
 	if coverDir != "" {
@@ -87,7 +91,7 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 		t.Setenv("CMD_SERVER_URL", cmdServer.URL)
 		execPath = filepath.Join(cwd, "bin", "callserver.py")
 	} else {
-		execPath = BuildCLI(t, cwd, coverDir)
+		execPath = BuildCLI(t, buildDir, coverDir)
 	}
 
 	t.Setenv("CLI", execPath)
@@ -117,11 +121,21 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 		homeDir := t.TempDir()
 		// Do not read user's ~/.databrickscfg
 		t.Setenv(env.HomeEnvVar(), homeDir)
-
-		// Prevent CLI from downloading terraform in each test:
-		t.Setenv("DATABRICKS_TF_EXEC_PATH", tempHomeDir)
 	}
 
+	terraformrcPath := filepath.Join(buildDir, ".terraformrc")
+	t.Setenv("TF_CLI_CONFIG_FILE", terraformrcPath)
+	t.Setenv("DATABRICKS_TF_CLI_CONFIG_FILE", terraformrcPath)
+	repls.Set(terraformrcPath, "DATABRICKS_TF_CLI_CONFIG_FILE")
+
+	terraformExecPath := filepath.Join(buildDir, "terraform")
+	t.Setenv("DATABRICKS_TF_EXEC_PATH", terraformExecPath)
+	t.Setenv("TERRAFORM", terraformExecPath)
+	repls.Set(terraformExecPath, "$TERRAFORM")
+
+	// do it last so that full paths match first:
+	repls.Set(buildDir, "$BUILD_DIR")
+
 	workspaceClient, err := databricks.NewWorkspaceClient()
 	require.NoError(t, err)
 
@@ -350,13 +364,12 @@ func readMergedScriptContents(t *testing.T, dir string) string {
 	return strings.Join(prepares, "\n")
 }
 
-func BuildCLI(t *testing.T, cwd, coverDir string) string {
-	execPath := filepath.Join(cwd, "build", "databricks")
+func BuildCLI(t *testing.T, buildDir, coverDir string) string {
+	execPath := filepath.Join(buildDir, "databricks")
 	if runtime.GOOS == "windows" {
 		execPath += ".exe"
 	}
 
-	start := time.Now()
 	args := []string{
 		"go", "build",
 		"-mod", "vendor",
@@ -374,20 +387,8 @@ func BuildCLI(t *testing.T, cwd, coverDir string) string {
 		args = append(args, "-buildvcs=false")
 	}
 
-	cmd := exec.Command(args[0], args[1:]...)
-	cmd.Dir = ".."
-	out, err := cmd.CombinedOutput()
-	elapsed := time.Since(start)
-	t.Logf("%s took %s", args, elapsed)
-	require.NoError(t, err, "go build failed: %s: %s\n%s", args, err, out)
-	if len(out) > 0 {
-		t.Logf("go build output: %s: %s", args, out)
-	}
-
-	// Quick check + warm up cache:
-	cmd = exec.Command(execPath, "--version")
-	out, err = cmd.CombinedOutput()
-	require.NoError(t, err, "%s --version failed: %s\n%s", execPath, err, out)
+	RunCommand(t, args, "..")
+	RunCommand(t, []string{execPath, "--version"}, ".")
 	return execPath
 }
 
@@ -525,3 +526,17 @@ func getUVDefaultCacheDir(t *testing.T) string {
 		return cacheDir + "/uv"
 	}
 }
+
+func RunCommand(t *testing.T, args []string, dir string) {
+	start := time.Now()
+	cmd := exec.Command(args[0], args[1:]...)
+	cmd.Dir = dir
+	out, err := cmd.CombinedOutput()
+	elapsed := time.Since(start)
+	t.Logf("%s took %s", args, elapsed)
+
+	require.NoError(t, err, "%s failed: %s\n%s", args, err, out)
+	if len(out) > 0 {
+		t.Logf("%s output: %s", args, out)
+	}
+}

acceptance/bundle/variables/git-branch/output.txt

@@ -11,7 +11,7 @@
     "name": "git",
     "target": "prod",
     "terraform": {
-      "exec_path": "$TMPHOME"
+      "exec_path": "$TERRAFORM"
     }
   },
   "sync": {
@@ -61,7 +61,7 @@ Validation OK!
     "name": "git",
     "target": "dev",
     "terraform": {
-      "exec_path": "$TMPHOME"
+      "exec_path": "$TERRAFORM"
     }
   },
   "sync": {

acceptance/bundle/variables/prepend-workspace-var/output.txt

@@ -7,7 +7,7 @@
     },
     "target": "dev",
     "terraform": {
-      "exec_path": "$TMPHOME"
+      "exec_path": "$TERRAFORM"
     }
   },
   "resources": {

acceptance/install_terraform.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+"""
+Script to set up terraform and databricks terraform provider in a local directory:
+
+- Download terraform.
+- Download databricks provider.
+- Write a .terraformrc config file that uses this directory.
+- The config file contains env vars that need to be set so that databricks CLI uses this terraform and provider.
+"""
+
+import os
+import platform
+import zipfile
+import argparse
+from pathlib import Path
+from urllib.request import urlretrieve
+
+os_name = platform.system().lower()
+
+current_arch = platform.machine().lower()
+arch_mapping = {
+    "x86_64": "amd64",
+    "amd64": "amd64",
+    "arm64": "arm64",
+    "aarch64": "arm64",
+}
+arch = arch_mapping.get(current_arch, current_arch)
+
+terraform_version = "1.5.5"
+terraform_file = f"terraform_{terraform_version}_{os_name}_{arch}.zip"
+terraform_url = f"https://releases.hashicorp.com/terraform/{terraform_version}/{terraform_file}"
+terraform_binary = "terraform.exe" if os_name == "windows" else "terraform"
+
+
+def retrieve(url, path):
+    if not path.exists():
+        print(f"Downloading {url} -> {path}")
+        urlretrieve(url, path)
+
+
+def read_version(path):
+    for line in path.open():
+        if "ProviderVersion" in line:
+            # Expecting 'const ProviderVersion = "1.64.1"'
+            items = line.strip().split()
+            assert len(items) >= 3, items
+            assert items[-3:-1] == ["ProviderVersion", "="], items
+            version = items[-1].strip('"')
+            assert version, items
+            return version
+    raise SystemExit(f"Could not find ProviderVersion in {path}")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--targetdir", default="build", type=Path)
+    parser.add_argument("--provider-version")
+    args = parser.parse_args()
+    target = args.targetdir
+
+    if not args.provider_version:
+        version_file = Path(__file__).parent.parent / "bundle/internal/tf/codegen/schema/version.go"
+        assert version_file.exists(), version_file
+        terraform_provider_version = read_version(version_file)
+        print(f"Read version {terraform_provider_version} from {version_file}")
+    else:
+        terraform_provider_version = args.provider_version
+
+    terraform_provider_file = f"terraform-provider-databricks_{terraform_provider_version}_{os_name}_{arch}.zip"
+    terraform_provider_url = f"https://github.com/databricks/terraform-provider-databricks/releases/download/v{terraform_provider_version}/{terraform_provider_file}"
+
+    target.mkdir(exist_ok=True)
+
+    zip_path = target / terraform_file
+    terraform_path = target / terraform_binary
+    terraform_provider_path = target / terraform_provider_file
+
+    retrieve(terraform_url, zip_path)
+    retrieve(terraform_provider_url, terraform_provider_path)
+
+    if not terraform_path.exists():
+        print(f"Extracting {zip_path} -> {terraform_path}")
+
+        with zipfile.ZipFile(zip_path, "r") as zip_ref:
+            zip_ref.extractall(target)
+
+        terraform_path.chmod(0o755)
+
+    tfplugins_path = target / "tfplugins"
+    provider_dir = Path(
+        tfplugins_path / f"registry.terraform.io/databricks/databricks/{terraform_provider_version}/{os_name}_{arch}"
+    )
+    if not provider_dir.exists():
+        print(f"Extracting {terraform_provider_path} -> {provider_dir}")
+        os.makedirs(provider_dir, exist_ok=True)
+        with zipfile.ZipFile(terraform_provider_path, "r") as zip_ref:
+            zip_ref.extractall(provider_dir)
+
+        files = list(provider_dir.iterdir())
+        assert files, provider_dir
+
+        for f in files:
+            f.chmod(0o755)
+
+    terraformrc_path = target / ".terraformrc"
+    if not terraformrc_path.exists():
+        print(f"Writing {terraformrc_path} (see it for instructions)")
+
+        terraformrc_path.write_text(f"""# Set these env variables before running databricks cli:
+# export DATABRICKS_TF_CLI_CONFIG_FILE={terraformrc_path.absolute()}
+# export DATABRICKS_TF_EXEC_PATH={terraform_path.absolute()}
+
+provider_installation {{
+    filesystem_mirror {{
+        path = "{tfplugins_path.absolute()}"
+        include = ["registry.terraform.io/databricks/databricks"]
+    }}
+}}
+""")
+
+
+if __name__ == "__main__":
+    main()

acceptance/terraform/main.tf

@@ -0,0 +1,25 @@
+terraform {
+  required_providers {
+    databricks = {
+      source  = "databricks/databricks"
+      version = "1.64.1"
+    }
+  }
+
+  required_version = "= 1.5.5"
+}
+
+provider "databricks" {
+  # Optionally, specify the Databricks host and token
+  # host  = "https://<your-databricks-instance>"
+  # token = "<YOUR_PERSONAL_ACCESS_TOKEN>"
+}
+
+data "databricks_current_user" "me" {
+  # Retrieves the current user's information
+}
+
+output "username" {
+  description = "Username"
+  value       = "${data.databricks_current_user.me.user_name}"
+}

acceptance/terraform/output.txt

@@ -0,0 +1,69 @@
+
+>>> $TERRAFORM init -no-color
+Timestamp [INFO]  Terraform version: 1.5.5
+Timestamp [DEBUG] using github.com/hashicorp/go-tfe v1.26.0
+Timestamp [DEBUG] using github.com/hashicorp/hcl/v2 v2.16.2
+Timestamp [DEBUG] using github.com/hashicorp/terraform-svchost v0.1.0
+Timestamp [DEBUG] using github.com/zclconf/go-cty v1.12.2
+Timestamp [INFO]  Go runtime version: go1.20.7
+Timestamp [INFO]  CLI args: []string{"$TERRAFORM", "init", "-no-color"}
+Timestamp [DEBUG] Attempting to open CLI config file: DATABRICKS_TF_CLI_CONFIG_FILE
+Timestamp [INFO]  Loading CLI configuration from DATABRICKS_TF_CLI_CONFIG_FILE
+Timestamp [DEBUG] Not reading CLI config directory because config location is overridden by environment variable
+Timestamp [DEBUG] Explicit provider installation configuration is set
+Timestamp [INFO]  CLI command args: []string{"init", "-no-color"}
+
+Initializing the backend...
+Timestamp [DEBUG] New state was assigned lineage "[UUID]"
+Timestamp [DEBUG] checking for provisioner in "."
+Timestamp [DEBUG] checking for provisioner in "$BUILD_DIR"
+
+Initializing provider plugins...
+- Finding databricks/databricks versions matching "1.64.1"...
+- Installing databricks/databricks v1.64.1...
+- Installed databricks/databricks v1.64.1 (unauthenticated)
+
+Terraform has created a lock file .terraform.lock.hcl to record the provider
+selections it made above. Include this file in your version control repository
+so that Terraform can guarantee to make the same selections by default when
+you run "terraform init" in the future.
+
+
+Warning: Incomplete lock file information for providers
+
+Due to your customized provider installation methods, Terraform was forced to
+calculate lock file checksums locally for the following providers:
+  - databricks/databricks
+
+The current .terraform.lock.hcl file only includes checksums for
+darwin_arm64, so Terraform running on another platform will fail to install
+these providers.
+
+To calculate additional checksums for another platform, run:
+  terraform providers lock -platform=linux_amd64
+(where linux_amd64 is the platform to generate)
+
+Terraform has been successfully initialized!
+
+You may now begin working with Terraform. Try running "terraform plan" to see
+any changes that are required for your infrastructure. All Terraform commands
+should now work.
+
+If you ever set or change modules or backend configuration for Terraform,
+rerun this command to reinitialize your working directory. If you forget, other
+commands will detect it and remind you to do so if necessary.
+
+>>> $TERRAFORM plan -no-color
+data.databricks_current_user.me: Reading...
+data.databricks_current_user.me: Read complete after 0s [id=$USER.Id]
+
+Changes to Outputs:
+  + username = "$USERNAME"
+
+You can apply this plan to save these new output values to the Terraform
+state, without changing any real infrastructure.
+
+─────────────────────────────────────────────────────────────────────────────
+
+Note: You didn't use the -out option to save this plan, so Terraform can't
+guarantee to take exactly these actions if you run "terraform apply" now.

acceptance/terraform/script

@@ -0,0 +1,5 @@
+export TF_LOG=DEBUG
+trace $TERRAFORM init -no-color
+export TF_LOG=WARN
+trace $TERRAFORM plan -no-color
+rm -fr .terraform.lock.hcl .terraform

acceptance/terraform/test.toml

@@ -0,0 +1,3 @@
+[[Repls]]
+Old = '\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d+\+\d+'
+New = 'Timestamp'