Add begin_line and add_line meta parameters. Migrate tests.

amplify-education · Jan 13, 2023 · fc8805e · fc8805e
1 parent ac6d89f
commit fc8805e
Show file tree

Hide file tree

Showing 16 changed files with 134 additions and 47 deletions.
diff --git a/hcl2/__main__.py b/hcl2/__main__.py
@@ -18,7 +18,6 @@
 from lark import UnexpectedCharacters, UnexpectedToken
 
 from . import load
-from .parser import hcl2
 from .version import __version__
 
 
@@ -53,7 +52,7 @@ def main():
                 else open(args.OUT_PATH, "w", encoding="utf-8")
             )
             print(args.PATH, file=sys.stderr, flush=True)
-            json.dump(hcl2.parse(in_file.read()), out_file)
+            json.dump(load(in_file), out_file)
             if args.OUT_PATH is None:
                 out_file.write("\n")
                 out_file.close()

diff --git a/hcl2/api.py b/hcl2/api.py
@@ -2,6 +2,7 @@
 from typing import TextIO
 
 from hcl2.parser import hcl2
+from hcl2.transformer import DictTransformer
 
 
 def load(file: TextIO) -> dict:
@@ -15,7 +16,5 @@ def loads(text: str) -> dict:
     # Lark doesn't support a EOF token so our grammar can't look for "new line or end of file"
     # This means that all blocks must end in a new line even if the file ends
     # Append a new line as a temporary fix
-    # Ignoring type as the type-annotation of Lark.parse() claims that it always returns a Tree,
-    # but in the docs of the parse() said that it returns whatever the supplied transformer returns.
-    # We supply DictTransformer so the return type is Dict.
-    return hcl2.parse(text + "\n")  # type: ignore
+    tree = hcl2.parse(text + "\n")
+    return DictTransformer().transform(tree)
diff --git a/hcl2/parser.py b/hcl2/parser.py
@@ -3,8 +3,6 @@
 
 from lark import Lark
 
-from hcl2.transformer import DictTransformer
-
 
 PARSER_FILE = Path(__file__).absolute().resolve().parent / ".lark_cache.bin"
 
@@ -14,5 +12,5 @@
     parser="lalr",
     cache=str(PARSER_FILE),  # Disable/Delete file to effect changes to the grammar
     rel_to=__file__,
-    transformer=DictTransformer(),
+    propagate_positions=True,
 )
diff --git a/hcl2/transformer.py b/hcl2/transformer.py
@@ -4,11 +4,21 @@
 from collections import namedtuple
 from typing import List, Dict, Any
 
-from lark.visitors import Transformer, Discard, _DiscardType
+from lark.tree import Meta
+from lark.visitors import Transformer, Discard, _DiscardType, v_args
+
 
 HEREDOC_PATTERN = re.compile(r"<<([a-zA-Z][a-zA-Z0-9._-]+)\n([\s\S]*)\1", re.S)
 HEREDOC_TRIM_PATTERN = re.compile(r"<<-([a-zA-Z][a-zA-Z0-9._-]+)\n([\s\S]*)\1", re.S)
 
+START_LINE = "__start_line__"
+END_LINE = "__end_line__"
+
+NO_BLOCK_LABEL_TYPES = {"locals", "terraform"}
+ONE_BLOCK_LABEL_TYPES = {"module", "provider", "variable"}
+TWO_BLOCK_LABEL_TYPES = {"data", "resource"}
+
+
 Attribute = namedtuple("Attribute", ("key", "value"))
 
 
@@ -96,7 +106,8 @@ def arguments(self, args: List) -> List:
     def new_line_and_or_comma(self, args: List) -> _DiscardType:
         return Discard
 
-    def block(self, args: List) -> Dict:
+    @v_args(meta=True)
+    def block(self, meta: Meta, args: List) -> Dict:
         args = self.strip_new_line_tokens(args)
 
         # if the last token is a string instead of an object then the block is empty
@@ -113,10 +124,22 @@ def block(self, args: List) -> Dict:
 
         current_level[self.strip_quotes(args[-2])] = args[-1]
 
-        return result
+        if args[0] in TWO_BLOCK_LABEL_TYPES:
+            label_1 = self.strip_quotes(args[1])
+            label_2 = self.strip_quotes(args[2])
+            result[args[0]][label_1][label_2][START_LINE] = meta.line
+            result[args[0]][label_1][label_2][END_LINE] = meta.end_line
+
+        if args[0] in ONE_BLOCK_LABEL_TYPES:
+            label_1 = self.strip_quotes(args[1])
+            result[args[0]][label_1][START_LINE] = meta.line
+            result[args[0]][label_1][END_LINE] = meta.end_line
 
-    def one_line_block(self, args: List) -> Dict:
-        return self.block(args)
+        if args[0] in NO_BLOCK_LABEL_TYPES:
+            result[args[0]][START_LINE] = meta.line
+            result[args[0]][END_LINE] = meta.end_line
+
+        return result
 
     def attribute(self, args: List) -> Attribute:
         key = str(args[0])

diff --git a/test/helpers/terraform-config-json/backend.json b/test/helpers/terraform-config-json/backend.json
@@ -2,19 +2,25 @@
   "provider": [
     {
       "aws": {
-        "region": "${var.region}"
+        "region": "${var.region}",
+        "__start_line__": 2,
+        "__end_line__": 7
       }
     },
     {
       "aws": {
         "region": "${var.backup_region}",
-        "alias": "backup"
+        "alias": "backup",
+        "__start_line__": 7,
+        "__end_line__": 15
       }
     }
   ],
   "terraform": [
     {
-      "required_version": "0.12"
+      "required_version": "0.12",
+      "__start_line__": 15,
+      "__end_line__": 17
     },
     {
       "backend": [
@@ -34,7 +40,9 @@
             "source": "hashicorp/template"
           }
         }
-      ]
+      ],
+      "__start_line__": 17,
+      "__end_line__": 32
     }
   ]
 }
diff --git a/test/helpers/terraform-config-json/cloudwatch.json b/test/helpers/terraform-config-json/cloudwatch.json
@@ -4,23 +4,29 @@
       "aws_cloudwatch_event_rule": {
         "aws_cloudwatch_event_rule": {
           "name": "name",
-          "event_pattern": "    {\n      \"foo\": \"bar\"\n    }"
+          "event_pattern": "    {\n      \"foo\": \"bar\"\n    }",
+          "__start_line__": 1,
+          "__end_line__": 10
         }
       }
     },
     {
       "aws_cloudwatch_event_rule": {
         "aws_cloudwatch_event_rule2": {
           "name": "name",
-          "event_pattern": "{\n  \"foo\": \"bar\"\n}"
+          "event_pattern": "{\n  \"foo\": \"bar\"\n}",
+          "__start_line__": 10,
+          "__end_line__": 19
         }
       }
     },
     {
       "aws_cloudwatch_event_rule": {
         "aws_cloudwatch_event_rule2": {
           "name": "name",
-          "event_pattern": "${jsonencode(var.cloudwatch_pattern_deploytool)}"
+          "event_pattern": "${jsonencode(var.cloudwatch_pattern_deploytool)}",
+          "__start_line__": 19,
+          "__end_line__": 24
         }
       }
     }

diff --git a/test/helpers/terraform-config-json/data_sources.json b/test/helpers/terraform-config-json/data_sources.json
@@ -4,7 +4,9 @@
       "terraform_remote_state": {
         "map": {
           "for_each": "${{for s3_bucket_key in data.aws_s3_bucket_objects.remote_state_objects.keys : regex(local.remote_state_regex, s3_bucket_key)[\"account_alias\"] => s3_bucket_key if length(regexall(local.remote_state_regex, s3_bucket_key)) > 0}}",
-          "backend": "s3"
+          "backend": "s3",
+          "__start_line__": 1,
+          "__end_line__": 10
         }
       }
     }

diff --git a/test/helpers/terraform-config-json/empty-heredoc.json b/test/helpers/terraform-config-json/empty-heredoc.json
@@ -1 +1,3 @@
-{"bar": ""}
+{
+  "bar": ""
+}
diff --git a/test/helpers/terraform-config-json/iam.json b/test/helpers/terraform-config-json/iam.json
@@ -19,7 +19,9 @@
               ],
               "resources": "${aws_s3_bucket.bucket.*.arn.bar}"
             }
-          ]
+          ],
+          "__start_line__": 1,
+          "__end_line__": 21
         }
       }
     },
@@ -33,7 +35,9 @@
               ],
               "resources": "${[for bucket_name in local.buckets_to_proxy : \"arn:aws:s3:::${bucket_name}/*\" if substr(bucket_name, 0, 1) == \"l\"]}"
             }
-          ]
+          ],
+          "__start_line__": 21,
+          "__end_line__": 39
         }
       }
     }

diff --git a/test/helpers/terraform-config-json/locals_embedded_function.json b/test/helpers/terraform-config-json/locals_embedded_function.json
@@ -1,7 +1,9 @@
 {
   "locals": [
     {
-      "function_test": "${var.basename}-${var.forwarder_function_name}_${md5(\"${var.vpc_id}${data.aws_region.current.name}\")}"
+      "function_test": "${var.basename}-${var.forwarder_function_name}_${md5(\"${var.vpc_id}${data.aws_region.current.name}\")}",
+      "__start_line__": 1,
+      "__end_line__": 5
     }
   ]
 }
diff --git a/test/helpers/terraform-config-json/locals_embedded_interpolation.json b/test/helpers/terraform-config-json/locals_embedded_interpolation.json
@@ -1,7 +1,9 @@
 {
   "locals": [
     {
-      "embedded_interpolation": "${module.special_constants.aws_accounts[\"aaa-${local.foo}-${local.bar}\"]}/us-west-2/key_foo"
+      "embedded_interpolation": "${module.special_constants.aws_accounts[\"aaa-${local.foo}-${local.bar}\"]}/us-west-2/key_foo",
+      "__start_line__": 1,
+      "__end_line__": 5
     }
   ]
 }
diff --git a/test/helpers/terraform-config-json/locals_embedded_multi_function_nested.json b/test/helpers/terraform-config-json/locals_embedded_multi_function_nested.json
@@ -2,7 +2,9 @@
   "locals": [
     {
       "multi_function": "${substr(split(\"-\", \"us-west-2\")[0], 0, 1)}",
-      "multi_function_embedded": "${substr(split(\"-\", \"us-west-2\")[0], 0, 1)}"
+      "multi_function_embedded": "${substr(split(\"-\", \"us-west-2\")[0], 0, 1)}",
+      "__start_line__": 3,
+      "__end_line__": 8
     }
   ]
 }
diff --git a/test/helpers/terraform-config-json/multiline_expressions.json b/test/helpers/terraform-config-json/multiline_expressions.json
@@ -3,28 +3,36 @@
     {
       "null_resource": {
         "multiline_comment_multiline": {
-          "triggers": []
+          "triggers": [],
+          "__start_line__": 1,
+          "__end_line__": 8
         }
       }
     },
     {
       "null_resource": {
         "multiline_comment_single_line_before_closing_bracket": {
-          "triggers": []
+          "triggers": [],
+          "__start_line__": 8,
+          "__end_line__": 13
         }
       }
     },
     {
       "null_resource": {
         "multiline_comment_single_line_between_brackets": {
-          "triggers": []
+          "triggers": [],
+          "__start_line__": 13,
+          "__end_line__": 19
         }
       }
     },
     {
       "null_resource": {
         "multiline_comment_single_line_after_opening_bracket": {
-          "triggers": []
+          "triggers": [],
+          "__start_line__": 19,
+          "__end_line__": 24
         }
       }
     },
@@ -33,7 +41,9 @@
         "multiline_comment_multiple_single_element": {
           "triggers": [
             2
-          ]
+          ],
+          "__start_line__": 24,
+          "__end_line__": 32
         }
       }
     }
@@ -43,13 +53,17 @@
       "some_var2": {
         "description": "description",
         "type": "${string}",
-        "default": "${cidrsubnets(\"10.0.0.0/24\", 2, 2)}"
+        "default": "${cidrsubnets(\"10.0.0.0/24\", 2, 2)}",
+        "__start_line__": 32,
+        "__end_line__": 50
       }
     },
     {
       "some_var2": {
         "description": "description",
-        "default": "${concat([{'1': '1'}], [{'2': '2'}])}"
+        "default": "${concat([{'1': '1'}], [{'2': '2'}])}",
+        "__start_line__": 50,
+        "__end_line__": 62
       }
     }
   ]

diff --git a/test/helpers/terraform-config-json/route_table.json b/test/helpers/terraform-config-json/route_table.json
@@ -6,7 +6,9 @@
           "count": "${var.tgw_name == \"\" ? 0 : var.number_of_az}",
           "route_table_id": "${aws_route_table.rt[count.index].id}",
           "destination_cidr_block": "10.0.0.0/8",
-          "transit_gateway_id": "${data.aws_ec2_transit_gateway.tgw[0].id}"
+          "transit_gateway_id": "${data.aws_ec2_transit_gateway.tgw[0].id}",
+          "__start_line__": 1,
+          "__end_line__": 13
         }
       }
     },
@@ -16,7 +18,9 @@
           "count": "${var.tgw_name == \"\" ? 0 : var.number_of_az}",
           "route_table_id": "${aws_route_table.rt[count.index].id}",
           "destination_cidr_block": "10.0.0.0/8",
-          "transit_gateway_id": "${data.aws_ec2_transit_gateway.tgw[0].id}"
+          "transit_gateway_id": "${data.aws_ec2_transit_gateway.tgw[0].id}",
+          "__start_line__": 13,
+          "__end_line__": 20
         }
       }
     }

diff --git a/test/helpers/terraform-config-json/s3.json b/test/helpers/terraform-config-json/s3.json
@@ -25,7 +25,9 @@
             {
               "enabled": true
             }
-          ]
+          ],
+          "__start_line__": 1,
+          "__end_line__": 25
         }
       }
     }
@@ -36,7 +38,9 @@
         "source": "s3_bucket_name",
         "name": "audit",
         "account": "${var.account}",
-        "region": "${var.region}"
+        "region": "${var.region}",
+        "__start_line__": 25,
+        "__end_line__": 33
       }
     }
   ]