Add ANTLR4 grammar and C++ parser

.gitignore

@@ -117,3 +117,6 @@ dmypy.json
 
 # Emacs temporary files
 *~
+
+# Generated files
+/fasm/parser/tags.py

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "third_party/googletest"]
+	path = third_party/googletest
+	url = https://github.com/google/googletest.git

README.md

@@ -1,8 +1,33 @@
 ## FPGA Assembly (FASM) Parser and Generation library
 
-This library provides a textX grammer for the FASM file format some basic
+This repository documents the FASM file format and provides parsing libraries and simple tooling for working with FASM files.
+
+It provides both a pure Python parser based on `textx` and a significantly faster C parser based on `ANTLR`. The library will try and use the ANTLR parser first and fall back to the `textx` parser if the compiled module is not found.
+
+Which parsers are supported by your currently install can be found via `python3 -c "import fasm.parser as p; print(p.available)`. The currently in use parser can be found via `fasm.parser.implementation`.
+
+It is highly recommended to use the ANTLR parser as it is about 15 times faster.
+
 functions for parsing and generating FASM files.
 
+## Build Instructions
+
+CMake is required, and ANTLR has a few dependencies:
+
+    sudo apt install cmake default-jre-headless uuid-dev libantlr4-runtime-dev
+
+Pull dependencies in `third_party`:
+
+    git submodule update --init
+
+Build:
+
+    make build
+
+Test with:
+
+    python setup.py test
+
 ## FPGA Assembly (FASM)
 
 FPGA Assembly is a file format designed by the

fasm/__init__.py

@@ -10,157 +10,8 @@
 # SPDX-License-Identifier: ISC
 
 from __future__ import print_function
-import textx
-import os.path
-from collections import namedtuple
-import enum
-
-
-class ValueFormat(enum.Enum):
-    PLAIN = 0
-    VERILOG_DECIMAL = 1
-    VERILOG_HEX = 2
-    VERILOG_BINARY = 3
-    VERILOG_OCTAL = 4
-
-
-# Python version of a SetFasmFeature line.
-# feature is a string
-# start and end are ints.  When FeatureAddress is missing, start=None and
-# end=None.
-# value is an int.
-#
-# When FeatureValue is missing, value=1.
-# value_format determines what to output the value.
-# Should be a ValueFormat or None.
-# If None, value must be 1 and the value will be omited.
-SetFasmFeature = namedtuple(
-    'SetFasmFeature', 'feature start end value value_format')
-
-Annotation = namedtuple('Annotation', 'name value')
-
-# Python version of FasmLine.
-# set_feature should be a SetFasmFeature or None.
-# annotations should be a tuple of Annotation or None.
-# comment should a string or None.
-FasmLine = namedtuple('FasmLine', 'set_feature annotations comment')
-
-
-def assert_max_width(width, value):
-    """ asserts if the value is greater than the width. """
-    assert value < (2**width), (width, value)
-
-
-def verilog_value_to_int(verilog_value):
-    """ Convert VerilogValue model to width, value, value_format """
-    width = None
-
-    if verilog_value.plain_decimal:
-        return width, int(verilog_value.plain_decimal), ValueFormat.PLAIN
-
-    if verilog_value.width:
-        width = int(verilog_value.width)
-
-    if verilog_value.hex_value:
-        value = int(verilog_value.hex_value.replace('_', ''), 16)
-        value_format = ValueFormat.VERILOG_HEX
-    elif verilog_value.binary_value:
-        value = int(verilog_value.binary_value.replace('_', ''), 2)
-        value_format = ValueFormat.VERILOG_BINARY
-    elif verilog_value.decimal_value:
-        value = int(verilog_value.decimal_value.replace('_', ''), 10)
-        value_format = ValueFormat.VERILOG_DECIMAL
-    elif verilog_value.octal_value:
-        value = int(verilog_value.octal_value.replace('_', ''), 8)
-        value_format = ValueFormat.VERILOG_OCTAL
-    else:
-        assert False, verilog_value
-
-    if width is not None:
-        assert_max_width(width, value)
-
-    return width, value, value_format
-
-
-def set_feature_model_to_tuple(set_feature_model):
-    start = None
-    end = None
-    value = 1
-    address_width = 1
-    value_format = None
-
-    if set_feature_model.feature_address:
-        if set_feature_model.feature_address.address2:
-            end = int(set_feature_model.feature_address.address1, 10)
-            start = int(set_feature_model.feature_address.address2, 10)
-            address_width = end - start + 1
-        else:
-            start = int(set_feature_model.feature_address.address1, 10)
-            end = None
-            address_width = 1
-
-    if set_feature_model.feature_value:
-        width, value, value_format = verilog_value_to_int(
-            set_feature_model.feature_value)
-
-        if width is not None:
-            assert width <= address_width
-
-        assert value < (2**address_width), (value, address_width)
-
-    return SetFasmFeature(
-        feature=set_feature_model.feature,
-        start=start,
-        end=end,
-        value=value,
-        value_format=value_format,
-    )
-
-
-def get_fasm_metamodel():
-    return textx.metamodel_from_file(
-        file_name=os.path.join(os.path.dirname(__file__), 'fasm.tx'),
-        skipws=False)
-
-
-def fasm_model_to_tuple(fasm_model):
-    """ Converts FasmFile model to list of FasmLine named tuples. """
-    if not fasm_model:
-        return
-
-    for fasm_line in fasm_model.lines:
-        set_feature = None
-        annotations = None
-        comment = None
-
-        if fasm_line.set_feature:
-            set_feature = set_feature_model_to_tuple(fasm_line.set_feature)
-
-        if fasm_line.annotations:
-            annotations = tuple(
-                Annotation(
-                    name=annotation.name,
-                    value=annotation.value if annotation.value else '')
-                for annotation in fasm_line.annotations.annotations)
-
-        if fasm_line.comment:
-            comment = fasm_line.comment.comment
-
-        yield FasmLine(
-            set_feature=set_feature,
-            annotations=annotations,
-            comment=comment,
-        )
-
-
-def parse_fasm_string(s):
-    """ Parse FASM string, returning list of FasmLine named tuples."""
-    return fasm_model_to_tuple(get_fasm_metamodel().model_from_str(s))
-
-
-def parse_fasm_filename(filename):
-    """ Parse FASM file, returning list of FasmLine named tuples."""
-    return fasm_model_to_tuple(get_fasm_metamodel().model_from_file(filename))
+from fasm.model import ValueFormat, SetFasmFeature, FasmLine
+from fasm.parser import parse_fasm_filename, parse_fasm_string
 
 
 def fasm_value_to_str(value, width, value_format):

fasm/model.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2017-2020  The SymbiFlow Authors.
+#
+# Use of this source code is governed by a ISC-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/ISC
+#
+# SPDX-License-Identifier: ISC
+
+from collections import namedtuple
+import enum
+
+
+class ValueFormat(enum.Enum):
+    """ Number format used for a FASM value. """
+    PLAIN = 0
+    VERILOG_DECIMAL = 1
+    VERILOG_HEX = 2
+    VERILOG_BINARY = 3
+    VERILOG_OCTAL = 4
+
+
+ValueFormat.PLAIN.__doc__ = \
+    "A decimal number without size or radix e.g. 42"
+ValueFormat.VERILOG_DECIMAL.__doc__ = \
+    "A decimal number with optional size e.g. 8'd42"
+ValueFormat.VERILOG_HEX.__doc__ = \
+    "A hexadecimal number with optional size e.g. 8'h2a"
+ValueFormat.VERILOG_BINARY.__doc__ = \
+    "A binary number with optional size e.g. 8'b00101010"
+ValueFormat.VERILOG_OCTAL.__doc__ = \
+    "An octal number with optional size e.g. 8'o52"
+
+SetFasmFeature = namedtuple(
+    'SetFasmFeature', 'feature start end value value_format')
+SetFasmFeature.__doc__ = """
+Python version of a SetFasmFeature line such as:
+  feature[31:0] = 42
+
+feature is a string e.g. 'feature'
+
+start and end are ints e.g 31, 0
+When FeatureAddress is missing, start=None and
+end=None.
+
+value is an int e.g. 42
+When FeatureValue is missing, value=1.
+
+value_format determines how to output the value e.g. ValueFormat.PLAIN
+It should be a ValueFormat or None.
+If it is None, the value must be 1 and the value will
+be omitted from output.
+"""
+SetFasmFeature.feature.__doc__ = "Feature name (string)"
+SetFasmFeature.start.__doc__ = \
+    "Starting value of the feature range (int or None)"
+SetFasmFeature.end.__doc__ = \
+    "Ending value of the feature range (int or None)"
+SetFasmFeature.value.__doc__ = \
+    "FeatureValue describing the value, or None"
+SetFasmFeature.value_format.__doc__ = \
+    "ValueFormat describing the format of the value, or None."
+
+Annotation = namedtuple('Annotation', 'name value')
+Annotation.__doc__ = """
+Python version of an Annotation, such as:
+  { name = "value" }
+
+Both name and value are strings (not None),
+holding the name and value, respectively.
+"""
+Annotation.name.__doc__ = "Annotation name (string)"
+Annotation.value.__doc__ = "Annotation value (string)"
+
+FasmLine = namedtuple('FasmLine', 'set_feature annotations comment')
+FasmLine.__doc__ = """
+Python version of a FasmLine such as:
+feature[31:0] = 42 { name = "value" } # comment
+
+set_feature should be a SetFasmFeature or None.
+annotations should be a list of Annotation or None.
+comment should a string or None, e.g. " comment"
+"""
+FasmLine.set_feature.__doc__ = "SetFasmFeature or None"
+FasmLine.annotations.__doc__ = "List of Annotation or None"
+FasmLine.comment.__doc__ = \
+    "String or none containing the line comment (after '#')"

fasm/parser/__init__.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2017-2020  The SymbiFlow Authors.
+#
+# Use of this source code is governed by a ISC-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/ISC
+#
+# SPDX-License-Identifier: ISC
+
+from warnings import warn
+
+available = []
+""" List of parser submodules available. Strings should match module names. """
+
+try:
+    from fasm.parser.antlr import \
+        parse_fasm_filename, parse_fasm_string, implementation
+    available.append('antlr')
+except ImportError as e:
+    warn(
+        '\nFalling back on slower textX parser implementation:\n'
+        '  ImportError: {}\n'
+        'Please install all dependencies and reinstall with:\n'
+        '  pip uninstall\n'
+        '  pip install -v fasm'.format(e), RuntimeWarning)
+    from fasm.parser.textx import \
+        parse_fasm_filename, parse_fasm_string, implementation
+
+# The textx parser is available as a fallback.
+available.append('textx')