transformer overhaul draft

.coveragerc

@@ -3,7 +3,10 @@ branch = true
 omit =
     hcl2/__main__.py
     hcl2/lark_parser.py
+    hcl2/version.py
+    hcl2/__init__.py
+    hcl2/rules/__init__.py
 
 [report]
 show_missing = true
-fail_under = 80
+fail_under = 90

hcl2/__init__.py

@@ -8,11 +8,21 @@
 from .api import (
     load,
     loads,
+    dump,
+    dumps,
     parse,
     parses,
+    parse_to_tree,
+    parses_to_tree,
+    from_dict,
+    from_json,
+    reconstruct,
     transform,
-    reverse_transform,
-    writes,
+    serialize,
 )
 
 from .builder import Builder
+from .deserializer import DeserializerOptions
+from .formatter import FormatterOptions
+from .rules.base import StartRule
+from .utils import SerializationOptions

hcl2/__main__.py

@@ -19,6 +19,7 @@
 from lark import UnexpectedCharacters, UnexpectedToken
 
 from . import load
+from .utils import SerializationOptions
 from .version import __version__
 
 
@@ -58,7 +59,8 @@ def main():
                 else open(args.OUT_PATH, "w", encoding="utf-8")
             )
             print(args.PATH, file=sys.stderr, flush=True)
-            json.dump(load(in_file, with_meta=args.with_meta), out_file)
+            options = SerializationOptions(with_meta=True) if args.with_meta else None
+            json.dump(load(in_file, serialization_options=options), out_file)
             if args.OUT_PATH is None:
                 out_file.write("\n")
                 out_file.close()

hcl2/api.py

@@ -1,67 +1,205 @@
-"""The API that will be exposed to users of this package"""
-from typing import TextIO
+"""The API that will be exposed to users of this package.
+
+Follows the json module convention: load/loads for reading, dump/dumps for writing.
+Also exposes intermediate pipeline stages for advanced usage.
+"""
+
+import json as _json
+from typing import TextIO, Optional
 
 from lark.tree import Tree
-from hcl2.parser import parser, reconstruction_parser
-from hcl2.transformer import DictTransformer
-from hcl2.reconstructor import HCLReconstructor, HCLReverseTransformer
 
+from hcl2.deserializer import BaseDeserializer, DeserializerOptions
+from hcl2.formatter import BaseFormatter, FormatterOptions
+from hcl2.parser import parser as _get_parser
+from hcl2.reconstructor import HCLReconstructor
+from hcl2.rules.base import StartRule
+from hcl2.transformer import RuleTransformer
+from hcl2.utils import SerializationOptions
+
+
+# ---------------------------------------------------------------------------
+# Primary API: load / loads / dump / dumps
+# ---------------------------------------------------------------------------
+
+
+def load(
+    file: TextIO,
+    *,
+    serialization_options: Optional[SerializationOptions] = None,
+) -> dict:
+    """Load a HCL2 file and return a Python dict.
+
+    :param file: File with HCL2 content.
+    :param serialization_options: Options controlling serialization behavior.
+    """
+    return loads(file.read(), serialization_options=serialization_options)
+
+
+def loads(
+    text: str,
+    *,
+    serialization_options: Optional[SerializationOptions] = None,
+) -> dict:
+    """Load HCL2 from a string and return a Python dict.
+
+    :param text: HCL2 text.
+    :param serialization_options: Options controlling serialization behavior.
+    """
+    tree = parses(text)
+    return serialize(tree, serialization_options=serialization_options)
+
+
+def dump(
+    data: dict,
+    file: TextIO,
+    *,
+    deserializer_options: Optional[DeserializerOptions] = None,
+    formatter_options: Optional[FormatterOptions] = None,
+) -> None:
+    """Write a Python dict as HCL2 to a file.
+
+    :param data: Python dict (as produced by :func:`load`).
+    :param file: Writable text file.
+    :param deserializer_options: Options controlling deserialization behavior.
+    :param formatter_options: Options controlling formatting behavior.
+    """
+    file.write(dumps(data, deserializer_options=deserializer_options, formatter_options=formatter_options))
+
+
+def dumps(
+    data: dict,
+    *,
+    deserializer_options: Optional[DeserializerOptions] = None,
+    formatter_options: Optional[FormatterOptions] = None,
+) -> str:
+    """Convert a Python dict to an HCL2 string.
+
+    :param data: Python dict (as produced by :func:`load`).
+    :param deserializer_options: Options controlling deserialization behavior.
+    :param formatter_options: Options controlling formatting behavior.
+    """
+    tree = from_dict(data, deserializer_options=deserializer_options, formatter_options=formatter_options)
+    return reconstruct(tree)
+
+
+# ---------------------------------------------------------------------------
+# Parsing: HCL text -> LarkElement tree or raw Lark tree
+# ---------------------------------------------------------------------------
+
+
+def parse(file: TextIO, *, discard_comments: bool = False) -> StartRule:
+    """Parse a HCL2 file into a LarkElement tree.
+
+    :param file: File with HCL2 content.
+    :param discard_comments: If True, discard comments during transformation.
+    """
+    return parses(file.read(), discard_comments=discard_comments)
+
+
+def parses(text: str, *, discard_comments: bool = False) -> StartRule:
+    """Parse a HCL2 string into a LarkElement tree.
+
+    :param text: HCL2 text.
+    :param discard_comments: If True, discard comments during transformation.
+    """
+    lark_tree = parses_to_tree(text)
+    return transform(lark_tree, discard_comments=discard_comments)
+
+
+def parse_to_tree(file: TextIO) -> Tree:
+    """Parse a HCL2 file into a raw Lark parse tree.
 
-def load(file: TextIO, with_meta=False) -> dict:
-    """Load a HCL2 file.
-    :param file: File with hcl2 to be loaded as a dict.
-    :param with_meta: If set to true then adds `__start_line__` and `__end_line__`
-    parameters to the output dict. Default to false.
+    :param file: File with HCL2 content.
     """
-    return loads(file.read(), with_meta=with_meta)
+    return parses_to_tree(file.read())
+
 
+def parses_to_tree(text: str) -> Tree:
+    """Parse a HCL2 string into a raw Lark parse tree.
 
-def loads(text: str, with_meta=False) -> dict:
-    """Load HCL2 from a string.
-    :param text: Text with hcl2 to be loaded as a dict.
-    :param with_meta: If set to true then adds `__start_line__` and `__end_line__`
-    parameters to the output dict. Default to false.
+    :param text: HCL2 text.
     """
-    # append new line as a workaround for https://github.com/lark-parser/lark/issues/237
+    # Append newline as workaround for https://github.com/lark-parser/lark/issues/237
     # Lark doesn't support EOF token so our grammar can't look for "new line or end of file"
-    # This means that all blocks must end in a new line even if the file ends
-    # Append a new line as a temporary fix
-    tree = parser().parse(text + "\n")
-    return DictTransformer(with_meta=with_meta).transform(tree)
+    return _get_parser().parse(text + "\n")
+
+
+# ---------------------------------------------------------------------------
+# Intermediate pipeline stages
+# ---------------------------------------------------------------------------
 
 
-def parse(file: TextIO) -> Tree:
-    """Load HCL2 syntax tree from a file.
-    :param file: File with hcl2 to be loaded as a dict.
+def from_dict(
+    data: dict,
+    *,
+    deserializer_options: Optional[DeserializerOptions] = None,
+    formatter_options: Optional[FormatterOptions] = None,
+    format: bool = True,
+) -> StartRule:
+    """Convert a Python dict into a LarkElement tree.
+
+    :param data: Python dict (as produced by :func:`load`).
+    :param deserializer_options: Options controlling deserialization behavior.
+    :param formatter_options: Options controlling formatting behavior.
+    :param format: If True (default), apply formatting to the tree.
     """
-    return parses(file.read())
+    deserializer = BaseDeserializer(deserializer_options)
+    tree = deserializer.load_python(data)
+    if format:
+        formatter = BaseFormatter(formatter_options)
+        formatter.format_tree(tree)
+    return tree
+
 
+def from_json(
+    text: str,
+    *,
+    deserializer_options: Optional[DeserializerOptions] = None,
+    formatter_options: Optional[FormatterOptions] = None,
+    format: bool = True,
+) -> StartRule:
+    """Convert a JSON string into a LarkElement tree.
 
-def parses(text: str) -> Tree:
-    """Load HCL2 syntax tree from a string.
-    :param text: Text with hcl2 to be loaded as a dict.
+    :param text: JSON string.
+    :param deserializer_options: Options controlling deserialization behavior.
+    :param formatter_options: Options controlling formatting behavior.
+    :param format: If True (default), apply formatting to the tree.
     """
-    return reconstruction_parser().parse(text)
+    data = _json.loads(text)
+    return from_dict(data, deserializer_options=deserializer_options, formatter_options=formatter_options, format=format)
+
 
+def reconstruct(tree) -> str:
+    """Convert a LarkElement tree (or raw Lark tree) to an HCL2 string.
 
-def transform(ast: Tree, with_meta=False) -> dict:
-    """Convert an HCL2 AST to a dictionary.
-    :param ast: HCL2 syntax tree, output from `parse` or `parses`
-    :param with_meta: If set to true then adds `__start_line__` and `__end_line__`
-        parameters to the output dict. Default to false.
+    :param tree: A :class:`StartRule` (LarkElement tree) or :class:`lark.Tree`.
     """
-    return DictTransformer(with_meta=with_meta).transform(ast)
+    reconstructor = HCLReconstructor()
+    if isinstance(tree, StartRule):
+        tree = tree.to_lark()
+    return reconstructor.reconstruct(tree)
 
 
-def reverse_transform(hcl2_dict: dict) -> Tree:
-    """Convert a dictionary to an HCL2 AST.
-    :param hcl2_dict: a dictionary produced by `load` or `transform`
+def transform(lark_tree: Tree, *, discard_comments: bool = False) -> StartRule:
+    """Transform a raw Lark parse tree into a LarkElement tree.
+
+    :param lark_tree: Raw Lark tree from :func:`parse_to_tree` or :func:`parse_string_to_tree`.
+    :param discard_comments: If True, discard comments during transformation.
     """
-    return HCLReverseTransformer().transform(hcl2_dict)
+    return RuleTransformer(discard_new_line_or_comments=discard_comments).transform(lark_tree)
+
 
+def serialize(
+    tree: StartRule,
+    *,
+    serialization_options: Optional[SerializationOptions] = None,
+) -> dict:
+    """Serialize a LarkElement tree to a Python dict.
 
-def writes(ast: Tree) -> str:
-    """Convert an HCL2 syntax tree to a string.
-    :param ast: HCL2 syntax tree, output from `parse` or `parses`
+    :param tree: A :class:`StartRule` (LarkElement tree).
+    :param serialization_options: Options controlling serialization behavior.
     """
-    return HCLReconstructor(reconstruction_parser()).reconstruct(ast)
+    if serialization_options is not None:
+        return tree.serialize(options=serialization_options)
+    return tree.serialize()

hcl2/builder.py

@@ -3,18 +3,16 @@
 
 from collections import defaultdict
 
-from hcl2.const import START_LINE_KEY, END_LINE_KEY
+from hcl2.const import IS_BLOCK
 
 
 class Builder:
     """
     The `hcl2.Builder` class produces a dictionary that should be identical to the
-    output of `hcl2.load(example_file, with_meta=True)`. The `with_meta` keyword
-    argument is important here. HCL "blocks" in the Python dictionary are
-    identified by the presence of `__start_line__` and `__end_line__` metadata
-    within them. The `Builder` class handles adding that metadata. If that metadata
-    is missing, the `hcl2.reconstructor.HCLReverseTransformer` class fails to
-    identify what is a block and what is just an attribute with an object value.
+    output of `hcl2.load(example_file)`. HCL "blocks" in the Python dictionary are
+    identified by the presence of `__is_block__: True` markers within them.
+    The `Builder` class handles adding that marker. If that marker is missing,
+    the deserializer fails to distinguish blocks from regular object attributes.
     """
 
     def __init__(self, attributes: Optional[dict] = None):
@@ -49,8 +47,7 @@ def build(self):
 
         body.update(
             {
-                START_LINE_KEY: -1,
-                END_LINE_KEY: -1,
+                IS_BLOCK: True,
                 **self.attributes,
             }
         )
@@ -79,7 +76,7 @@ def _add_nested_blocks(
         """Add nested blocks defined within another `Builder` instance to the `block` dictionary"""
         nested_block = nested_blocks_builder.build()
         for key, value in nested_block.items():
-            if key not in (START_LINE_KEY, END_LINE_KEY):
+            if key != IS_BLOCK:
                 if key not in block.keys():
                     block[key] = []
                 block[key].extend(value)

hcl2/const.py

@@ -2,3 +2,4 @@
 
 START_LINE_KEY = "__start_line__"
 END_LINE_KEY = "__end_line__"
+IS_BLOCK = "__is_block__"