Merge pull request #232 from dflook/partial_backend_config

Fix reading backend config var files

.github/workflows/test-apply.yaml

@@ -1025,3 +1025,28 @@ jobs:
             echo "::error:: failure-reason not set correctly"
             exit 1
           fi
+
+  partial_backend_fingerprint:
+    runs-on: ubuntu-latest
+    name: Get comment using partial fingerprint
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Plan
+        uses: dflook/[email protected]
+        with:
+          path: tests/workflows/test-apply/partial_backend
+          backend_config_file: tests/workflows/test-apply/partial_backend/backend_config
+          backend_config: key=${{ github.run_id }}${{ github.run_attempt }}
+
+      - name: Apply
+        uses: ./terraform-apply
+        with:
+          path: tests/workflows/test-apply/partial_backend
+          backend_config_file: tests/workflows/test-apply/partial_backend/backend_config
+          backend_config: key=${{ github.run_id }}${{ github.run_attempt }}

.github/workflows/test-plan.yaml

@@ -784,3 +784,55 @@ jobs:
         uses: ./terraform-plan
         with:
           label: test-plan default_path
+
+  path_does_not_exist:
+    runs-on: ubuntu-latest
+    name: Error with invalid paths
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: path
+        uses: ./terraform-plan
+        id: path
+        continue-on-error: true
+        with:
+          path: nope
+          add_github_comment: false
+
+      - name: var_file
+        uses: ./terraform-plan
+        id: var_file
+        continue-on-error: true
+        with:
+          path: tests/workflows/test-plan/plan
+          var_file: |
+              var_file/doesnt/exist.tfvars   
+              var_file/doesnt/exist2.tfvars
+          add_github_comment: false
+
+      - name: backend_config_file
+        uses: ./terraform-plan
+        id: backend_config_file
+        continue-on-error: true
+        with:
+          path: tests/workflows/test-plan/plan
+          backend_config_file: backend_config/doesnt/exist.tfvars
+          add_github_comment: false
+
+      - name: Check invalid
+        run: |
+          if [[ "${{ steps.path.outcome }}" != "failure" ]]; then
+            echo "Non existant path did not fail correctly"
+            exit 1
+          fi
+
+          if [[ "${{ steps.var_file.outcome }}" != "failure" ]]; then
+            echo "Non existant var_file did not fail correctly"
+            exit 1
+          fi
+          
+          if [[ "${{ steps.backend_config_file.outcome }}" != "failure" ]]; then
+            echo "Non existant backend_config_file did not fail correctly"
+            exit 1
+          fi          

image/actions.sh

@@ -139,6 +139,12 @@ function set-init-args() {
 
     if [[ -n "$INPUT_BACKEND_CONFIG_FILE" ]]; then
         for file in $(echo "$INPUT_BACKEND_CONFIG_FILE" | tr ',' '\n'); do
+
+            if [[ ! -f "$file" ]]; then
+                error_log "Path does not exist: \"$file\""
+                exit 1
+            fi
+
             INIT_ARGS="$INIT_ARGS -backend-config=$(relative_to "$INPUT_PATH" "$file")"
         done
     fi
@@ -291,6 +297,12 @@ function set-plan-args() {
 
     if [[ -n "$INPUT_VAR_FILE" ]]; then
         for file in $(echo "$INPUT_VAR_FILE" | tr ',' '\n'); do
+
+            if [[ ! -f "$file" ]]; then
+                error_log "Path does not exist: \"$file\""
+                exit 1
+            fi
+
             PLAN_ARGS="$PLAN_ARGS -var-file=$(relative_to "$INPUT_PATH" "$file")"
         done
     fi

image/setup.py

@@ -2,7 +2,7 @@
 
 setup(
     name='terraform-github-actions',
-    version='1.0.0',
+    version='1.31.1',
     packages=find_packages('src'),
     package_dir={'': 'src'},
     package_data={'terraform_version': ['backend_constraints.json']},

image/src/github_pr_comment/__main__.py

@@ -16,7 +16,7 @@
 from github_actions.env import GithubEnv
 from github_actions.find_pr import find_pr, WorkflowException
 from github_actions.inputs import PlanPrInputs
-from github_pr_comment.backend_config import complete_config
+from github_pr_comment.backend_config import complete_config, partial_config
 from github_pr_comment.backend_fingerprint import fingerprint
 from github_pr_comment.cmp import plan_cmp, remove_warnings, remove_unchanged_attributes
 from github_pr_comment.comment import find_comment, TerraformComment, update_comment, serialize, deserialize
@@ -271,7 +271,7 @@ def get_pr() -> PrUrl:
 
     return cast(PrUrl, pr_url)
 
-def get_comment(action_inputs: PlanPrInputs, backend_fingerprint: bytes) -> TerraformComment:
+def get_comment(action_inputs: PlanPrInputs, backend_fingerprint: bytes, backup_fingerprint: bytes) -> TerraformComment:
     if 'comment' in step_cache:
         return deserialize(step_cache['comment'])
 
@@ -283,7 +283,6 @@ def get_comment(action_inputs: PlanPrInputs, backend_fingerprint: bytes) -> Terr
 
     headers = {
         'workspace': os.environ.get('INPUT_WORKSPACE', 'default'),
-        'backend': comment_hash(backend_fingerprint, pr_url)
     }
 
     if backend_type := os.environ.get('TERRAFORM_BACKEND_TYPE'):
@@ -302,7 +301,12 @@ def get_comment(action_inputs: PlanPrInputs, backend_fingerprint: bytes) -> Terr
         debug(f'Plan modifier: {plan_modifier}')
         headers['plan_modifier'] = hashlib.sha256(canonicaljson.encode_canonical_json(plan_modifier)).hexdigest()
 
-    return find_comment(github, issue_url, username, headers, legacy_description)
+    backup_headers = headers.copy()
+
+    headers['backend'] = comment_hash(backend_fingerprint, pr_url)
+    backup_headers['backend'] = comment_hash(backup_fingerprint, pr_url)
+
+    return find_comment(github, issue_url, username, headers, backup_headers, legacy_description)
 
 def is_approved(proposed_plan: str, comment: TerraformComment) -> bool:
 
@@ -357,11 +361,13 @@ def main() -> int:
 
     module = load_module(Path(action_inputs.get('INPUT_PATH', '.')))
 
-    backend_type, backend_config = complete_config(action_inputs, module)
+    backend_type, backend_config = partial_config(action_inputs, module)
+    partial_backend_fingerprint = fingerprint(backend_type, backend_config, os.environ)
 
+    backend_type, backend_config = complete_config(action_inputs, module)
     backend_fingerprint = fingerprint(backend_type, backend_config, os.environ)
 
-    comment = get_comment(action_inputs, backend_fingerprint)
+    comment = get_comment(action_inputs, backend_fingerprint, partial_backend_fingerprint)
 
     status = cast(Status, os.environ.get('STATUS', ''))
 

image/src/github_pr_comment/backend_config.py

@@ -1,15 +1,16 @@
 import re
+import sys
 from typing import Tuple, Any
-
+from pathlib import Path
 from github_actions.debug import debug
 from github_actions.inputs import InitInputs
-from terraform.module import TerraformModule
+from terraform.module import TerraformModule, load_backend_config_file
 
 BackendConfig = dict[str, Any]
 BackendType = str
 
 
-def partial_backend_config(module: TerraformModule) -> Tuple[BackendType, BackendConfig]:
+def read_module_backend_config(module: TerraformModule) -> Tuple[BackendType, BackendConfig]:
     """Return the backend config specified in the terraform module."""
 
     for terraform in module.get('terraform', []):
@@ -23,8 +24,8 @@ def partial_backend_config(module: TerraformModule) -> Tuple[BackendType, Backen
     return 'local', {}
 
 
-def read_backend_config_vars(init_inputs: InitInputs) -> BackendConfig:
-    """Read any backend config from input variables."""
+def read_backend_config_files(init_inputs: InitInputs) -> BackendConfig:
+    """Read any backend config from backend config files."""
 
     config: BackendConfig = {}
 
@@ -34,18 +35,59 @@ def read_backend_config_vars(init_inputs: InitInputs) -> BackendConfig:
         except Exception as e:
             debug(f'Failed to load backend config file {path}')
             debug(str(e))
+            sys.stderr.write(f'Failed to load backend config file {path}\n')
+            sys.exit(1)
+
+    return config
+
+def read_backend_config_input(init_inputs: InitInputs) -> BackendConfig:
+    """Read any backend config from input variables."""
+
+    config: BackendConfig = {}
 
     for backend_var in init_inputs.get('INPUT_BACKEND_CONFIG', '').replace(',', '\n').splitlines():
         if match := re.match(r'(.*)\s*=\s*(.*)', backend_var):
             config[match.group(1)] = match.group(2)
 
     return config
 
+def partial_config(action_inputs: InitInputs, module: TerraformModule) -> Tuple[BackendType, BackendConfig]:
+    """
+    A partial backend config for the terraform module
+
+    This includes any values from the backend block in the terraform module,
+    & any values from the backend_config input.
+
+    This doesn't read from backend config files. Old versions didn't read from backend config files
+    and so created incorrect fingerprints. This is still used to match PR comments created using these old
+    versions, until enough time has passed that we don't need to use old comments.
+    """
+
+    backend_type, config = read_module_backend_config(module)
+
+    for key, value in read_backend_config_input(action_inputs).items():
+        config[key] = value
+
+    return backend_type, config
+
 
 def complete_config(action_inputs: InitInputs, module: TerraformModule) -> Tuple[BackendType, BackendConfig]:
-    backend_type, config = partial_backend_config(module)
+    """
+    The complete backend config for the terraform module
+
+    This includes any values from the backend block in the terraform module,
+    any values from backend config files & any values from the backend_config input.
+
+    This doesn't include any tfc credentials (not needed for fingerprinting).
+    It also doesn't include any config values inferred by the provider.
+    """
+
+    backend_type, config = read_module_backend_config(module)
+
+    for key, value in read_backend_config_files(action_inputs).items():
+        config[key] = value
 
-    for key, value in read_backend_config_vars(action_inputs).items():
+    for key, value in read_backend_config_input(action_inputs).items():
         config[key] = value
 
     return backend_type, config

image/src/github_pr_comment/backend_fingerprint.py

@@ -176,6 +176,7 @@ def fingerprint(backend_type: BackendType, backend_config: BackendConfig, env) -
         'remote': fingerprint_remote,
         'artifactory': fingerprint_artifactory,
         'azurerm': fingerprint_azurerm,
+        'azure': fingerprint_azurerm,
         'consul': fingerprint_consul,
         'cloud': fingerprint_cloud,
         'cos': fingerprint_cos,

image/src/github_pr_comment/comment.py

@@ -12,6 +12,13 @@
 except (ValueError, KeyError):
     collapse_threshold = 10
 
+from pkg_resources import get_distribution, DistributionNotFound
+
+try:
+    version = get_distribution('terraform-github-actions').version
+except DistributionNotFound:
+    version = '0.0.0'
+
 class TerraformComment:
     """
     Represents a Terraform PR comment
@@ -218,7 +225,7 @@ def matching_headers(comment: TerraformComment, headers: dict[str, str]) -> bool
 
     return True
 
-def find_comment(github: GithubApi, issue_url: IssueUrl, username: str, headers: dict[str, str], legacy_description: str) -> TerraformComment:
+def find_comment(github: GithubApi, issue_url: IssueUrl, username: str, headers: dict[str, str], backup_headers: dict[str, str], legacy_description: str) -> TerraformComment:
     """
     Find a github comment that matches the given headers
 
@@ -235,8 +242,10 @@ def find_comment(github: GithubApi, issue_url: IssueUrl, username: str, headers:
     """
 
     debug(f"Searching for comment with {headers=}")
+    debug(f"Or backup headers {backup_headers=}")
 
     backup_comment = None
+    legacy_comment = None
 
     for comment_payload in github.paged_get(issue_url + '/comments'):
         if comment_payload['user']['login'] != username:
@@ -251,31 +260,49 @@ def find_comment(github: GithubApi, issue_url: IssueUrl, username: str, headers:
                     debug(f'Found comment that matches headers {comment.headers=} ')
                     return comment
 
-                debug(f"Didn't match comment with {comment.headers=}")
+                if matching_headers(comment, backup_headers):
+                    debug(f'Found comment that matches backup headers {comment.headers=} ')
+                    backup_comment = comment
+                else:
+                    debug(f"Didn't match comment with {comment.headers=}")
 
             else:
                 # Match by description only
 
-                if comment.description == legacy_description and backup_comment is None:
-                    debug(f'Found backup comment that matches legacy description {comment.description=}')
-                    backup_comment = comment
+                if comment.description == legacy_description and legacy_comment is None:
+                    debug(f'Found comment that matches legacy description {comment.description=}')
+                    legacy_comment = comment
                 else:
                     debug(f"Didn't match comment with {comment.description=}")
 
     if backup_comment is not None:
-        debug('Found comment matching legacy description')
+        debug('Using comment matching backup headers')
 
-        # Insert known headers into legacy comment
+        # Use the backup comment but update the headers
         return TerraformComment(
             issue_url=backup_comment.issue_url,
             comment_url=backup_comment.comment_url,
-            headers={k: v for k, v in headers.items() if v is not None},
+            headers=backup_comment.headers | headers,
             description=backup_comment.description,
             summary=backup_comment.summary,
             body=backup_comment.body,
             status=backup_comment.status
         )
 
+    if legacy_comment is not None:
+        debug('Using comment matching legacy description')
+
+        # Insert known headers into legacy comment
+        return TerraformComment(
+            issue_url=legacy_comment.issue_url,
+            comment_url=legacy_comment.comment_url,
+            headers={k: v for k, v in headers.items() if v is not None},
+            description=legacy_comment.description,
+            summary=legacy_comment.summary,
+            body=legacy_comment.body,
+            status=legacy_comment.status
+        )
+
     debug('No existing comment exists')
     return TerraformComment(
         issue_url=issue_url,
@@ -299,10 +326,13 @@ def update_comment(
     status: str = None
 ) -> TerraformComment:
 
+    new_headers = headers if headers is not None else comment.headers
+    new_headers['version'] = version
+
     new_comment = TerraformComment(
         issue_url=comment.issue_url,
         comment_url=comment.comment_url,
-        headers=headers if headers is not None else comment.headers,
+        headers=new_headers,
         description=description if description is not None else comment.description,
         summary=summary if summary is not None else comment.summary,
         body=body if body is not None else comment.body,

image/src/terraform/hcl.py

@@ -14,7 +14,8 @@ def try_load(path: Path) -> dict:
     try:
         with open(path) as f:
             return hcl2.load(f)
-    except:
+    except Exception as e:
+        debug(e)
         return {}
 
 

tests/github_pr_comment/test_file.tfvars

@@ -0,0 +1 @@
+hello = "world"