Update QONNX parsing for 1.0

hls4ml/backends/fpga/fpga_backend.py

@@ -13,6 +13,8 @@
     LSTM,
     Activation,
     BatchNormalization,
+    BatchNormOnnx,
+    Conv,
     Conv1D,
     Conv2D,
     Dense,
@@ -22,8 +24,11 @@
     GarNetStack,
     GlobalPooling1D,
     GlobalPooling2D,
+    MatMul,
+    Merge,
     Pooling1D,
     Pooling2D,
+    Quant,
     SeparableConv1D,
     SeparableConv2D,
     SimpleRNN,
@@ -63,14 +68,25 @@ def __init__(self, name):
             LSTM,
             GRU,
             Dot,
+            Conv,
+            MatMul,
         ]
 
         for layer in accum_layers:
             attrs = self.attribute_map.get(layer, [])
             attrs.append(TypeAttribute('accum'))
             self.attribute_map[layer] = attrs
 
-        rf_layers = accum_layers + [BatchNormalization, Activation, Embedding, GarNet, GarNetStack]
+        rf_layers = accum_layers + [
+            BatchNormalization,
+            Activation,
+            Embedding,
+            GarNet,
+            GarNetStack,
+            Quant,
+            BatchNormOnnx,
+            Merge,
+        ]
 
         for layer in rf_layers:
             attrs = self.attribute_map.get(layer, [])

hls4ml/converters/__init__.py

@@ -10,6 +10,7 @@
 from hls4ml.converters.keras_to_hls import get_supported_keras_layers  # noqa: F401
 from hls4ml.converters.keras_to_hls import parse_keras_model  # noqa: F401
 from hls4ml.converters.keras_to_hls import keras_to_hls, register_keras_layer_handler
+from hls4ml.converters.onnx_to_hls import parse_onnx_model  # noqa: F401
 from hls4ml.model import ModelGraph
 from hls4ml.utils.config import create_config
 from hls4ml.utils.symbolic_utils import LUTFunction

hls4ml/converters/keras/reshape.py

@@ -11,8 +11,8 @@ def parse_flatten_layer(keras_layer, input_names, input_shapes, data_reader):
     layer = parse_default_keras_layer(keras_layer, input_names)
 
     layer['class_name'] = 'Reshape'
-    layer['target_shape'] = [input_shapes[0][0], np.prod(input_shapes[0][1:])]
-    output_shape = layer['target_shape']
+    layer['target_shape'] = [np.prod(input_shapes[0][1:])]  # target shape has no batch dimension
+    output_shape = input_shapes[0][:1] + layer['target_shape']
 
     return layer, output_shape
 

hls4ml/converters/onnx/convolution.py

@@ -1,85 +1,72 @@
-from hls4ml.converters.onnx_to_hls import (
-    compute_pads_1d,
-    compute_pads_2d,
-    get_onnx_attribute,
-    get_onnx_input_name,
-    onnx_handler,
-)
-from hls4ml.converters.utils import compute_padding_1d, compute_padding_2d
+import numpy as np
+
+from hls4ml.converters.onnx_to_hls import get_onnx_attribute, onnx_handler
 
 
 @onnx_handler('Conv')
-def parse_conv_layer(reader, node, inputs_map, input_shapes, graph, config):
+def parse_conv_layer(node, input_names, input_shapes, graph):
     layer = {}
     layer['name'] = node.name
-    layer['data_format'] = 'channels_first'  # ONNX's default is channel first
-    layer['inputs'] = get_onnx_input_name(node, graph)
-    reader.add_input(layer['name'], node.input)
+    if node.domain != 'qonnx.custom_op.channels_last':
+        raise RuntimeError("Please convert the model to channels-last format with qonnx-to-channels-last")
+    layer['data_format'] = 'channels_last'  # QONNX needs to be channels-last.
+    layer['inputs'] = input_names
+    layer['outputs'] = node.output
 
     strides = get_onnx_attribute(node, 'strides')
     kernel_shape = get_onnx_attribute(node, 'kernel_shape')
-
-    if len(input_shapes[0]) == 3:  # Conv1D
-        layer['class_name'] = 'Conv1D'
-
-        layer['in_width'] = input_shapes[0][2]
-        layer['n_chan'] = input_shapes[0][1]
-        layer['filt_width'] = kernel_shape[0]
-        layer['n_filt'] = reader.get_weights_data(layer['name'], 'kernel').shape[2]
-        layer['stride_width'] = strides[0]
-        pads = compute_pads_1d(node, layer)
-
+    # Note:  currently don't have support for auto_pad.
+    pads = get_onnx_attribute(node, 'pads')
+    dilations = get_onnx_attribute(node, 'dilations')
+    if dilations is None:
+        dilations = [1] * len(layer['kernel_shape'])
+
+    if get_onnx_attribute(node, 'group') != 1:
+        raise ValueError("Only 1 group supported corrently")
+
+    layer['in_width'] = input_shapes[0][-2]
+    layer['n_chan'] = input_shapes[0][-1]
+    layer['n_filt'] = input_shapes[1][0]
+
+    layer['n_dim'] = len(input_shapes[0]) - 2  # 2 comes from channels and batch dimentions
+    if layer['n_dim'] not in (1, 2):
+        raise ValueError("Only 1D and 2D convolutions are supported")
+    layer['class_name'] = 'Conv'
+
+    # set some values needed later
+    if layer['n_dim'] == 1:
+        # this is 1D convolution
+        full_width = layer['in_width'] + pads[0] + pads[1]
+        eff_kernel_width = kernel_shape[0] * dilations[0]
+        layer['out_width'] = int(np.ceil((full_width - eff_kernel_width + 1) / strides[0]))
+        # for compatibility interpret some variables
         layer['pad_left'] = pads[0]
         layer['pad_right'] = pads[1]
-
-        if all(x == 0 for x in pads):  # No padding, i.e., 'VALID' padding
-            layer['padding'] = 'valid'
-        else:
-            layer['padding'] = 'same'
-
-        (layer['out_width'], _, _) = compute_padding_1d(
-            layer['padding'], layer['in_width'], layer['stride_width'], layer['filt_width']
-        )
-
-        output_shape = [input_shapes[0][0], layer['n_filt'], layer['out_width']]
-
-    elif len(input_shapes[0]) == 4:  # Conv2D
-        layer['class_name'] = 'Conv2D'
-
-        layer['in_height'] = input_shapes[0][2]
-        layer['in_width'] = input_shapes[0][3]
-        layer['n_chan'] = input_shapes[0][1]
-
+        layer['filt_width'] = kernel_shape[0]
+        layer['stride_width'] = strides[0]
+        layer['dilation_width'] = dilations[0]
+    else:
+        # 2d
+        layer['in_height'] = input_shapes[0][-3]
+        full_height = layer['in_height'] + pads[0] + pads[2]
+        eff_kernel_height = kernel_shape[0] * dilations[0]
+        out_height = int(np.ceil((full_height - eff_kernel_height + 1) / strides[0]))
+        layer['out_height'] = out_height
+
+        full_width = input_shapes[0][-2] + pads[1] + pads[3]
+        eff_kernel_width = kernel_shape[1] * dilations[1]
+        out_width = int(np.ceil((full_width - eff_kernel_width + 1) / strides[1]))
+        layer['out_width'] = out_width
+        # for compatibility interpret some variables
+        layer['pad_top'] = pads[0]
+        layer['pad_left'] = pads[1]
+        layer['pad_bottom'] = pads[2]
+        layer['pad_right'] = pads[3]
         layer['filt_height'] = kernel_shape[0]
         layer['filt_width'] = kernel_shape[1]
-
-        layer['n_filt'] = next(
-            (x.type.tensor_type.shape.dim[1].dim_value for x in graph.value_info if x.name == node.output[0]), None
-        )
         layer['stride_height'] = strides[0]
         layer['stride_width'] = strides[1]
-        pads = compute_pads_2d(node, layer)
-
-        layer['pad_top'] = pads[0]
-        layer['pad_bottom'] = pads[2]
-        layer['pad_left'] = pads[1]
-        layer['pad_right'] = pads[3]
-
-        if all(x == 0 for x in pads):  # No padding, i.e., 'VALID' padding in Keras/Tensorflow
-            layer['padding'] = 'valid'
-        else:  # Only 'valid' and 'same' padding are available in Keras
-            layer['padding'] = 'same'
-
-        (layer['out_height'], layer['out_width'], _, _, _, _) = compute_padding_2d(
-            layer['padding'],
-            layer['in_height'],
-            layer['in_width'],
-            layer['stride_height'],
-            layer['stride_width'],
-            layer['filt_height'],
-            layer['filt_width'],
-        )
-
-        output_shape = [input_shapes[0][0], layer['n_filt'], layer['out_height'], layer['out_width']]
+        layer['dilation_height'] = dilations[0]
+        layer['dilation_width'] = dilations[1]
 
-    return layer, output_shape
+    return layer

hls4ml/converters/onnx/core.py

@@ -1,28 +1,20 @@
-from hls4ml.converters.onnx_to_hls import get_onnx_attribute, get_onnx_input_name, onnx_handler
+import numpy as np
 
+from hls4ml.converters.onnx_to_hls import get_onnx_attribute, onnx_handler
 
-@onnx_handler(*['Gemm', 'MatMul'])
-def parse_gemm_layer(reader, node, inputs_map, input_shapes, graph, config):
+
+@onnx_handler('MatMul')
+def parse_matmul_layer(node, input_names, input_shapes, graph):
     layer = {}
 
-    layer['class_name'] = 'Dense'
+    layer['class_name'] = 'MatMul'
     layer['name'] = node.name
-    layer['inputs'] = get_onnx_input_name(node, graph)
-
-    tran_weight = get_onnx_attribute(node, 'transB', 0)
-    reader.add_input(layer['name'], node.input, tran_weight)
-
-    weights_shape = reader.get_weights_data(layer['name'], 'kernel').shape
-    layer['n_in'] = weights_shape[0]
-    layer['n_out'] = weights_shape[1]
-
-    output_shape = input_shapes[0][:]
-    output_shape[-1] = layer['n_out']
+    layer['inputs'] = input_names
+    layer['outputs'] = list(node.output)
 
-    return layer, output_shape
+    return layer
 
 
-# ------------------Global paras for activations
 # TODO: repair HardSigmoid support
 # https://github.com/fastmachinelearning/hls4ml/issues/409
 activation_layers = [
@@ -37,7 +29,7 @@ def parse_gemm_layer(reader, node, inputs_map, input_shapes, graph, config):
     'Softmax',
     'Softsign',
     'Softplus',
-    'Clip',
+    # 'Clip',
 ]
 
 activation_map = {
@@ -53,70 +45,89 @@ def parse_gemm_layer(reader, node, inputs_map, input_shapes, graph, config):
     'Softmax': 'Softmax',
     'Softsign': 'Activation',
     'Softplus': 'Activation',
-    'Clip': 'Clip',
+    # 'Clip': 'Clip',
 }
 # ---------
 
 
 @onnx_handler(*activation_layers)
-def parse_activation_layer(reader, node, inputs_map, input_shapes, graph, config):
+def parse_activation_layer(node, input_names, input_shapes, graph):
     layer = {}
 
     layer['name'] = node.name
     layer['class_name'] = activation_map[node.op_type]
     layer['activation'] = node.op_type.lower()
-    layer['inputs'] = get_onnx_input_name(node, graph)
+    layer['inputs'] = input_names
+    layer['outputs'] = list(node.output)
 
     if layer['class_name'] != 'Activation':
         if layer['class_name'] == 'Softmax':
             layer['activation'] = 'softmax'
+            layer['axis'] = get_onnx_attribute(node, 'axis', -1)
 
         elif layer['class_name'] in ['ELU', 'LeakyReLU', 'ThresholdedReLU']:
             layer['activation'] = layer['class_name']
             layer['activ_param'] = get_onnx_attribute(node, 'alpha', 0.01)
 
-        elif layer['class_name'] == 'Clip':
-            clip_min_node = [x for x in graph.initializer if x.name in node.input]
-            clip_min = clip_min_node[0].float_data[0]
+        # # Don't yet support Clip
+        # elif layer['class_name'] == 'Clip':
+        #     clip_min_node = [x for x in graph.initializer if x.name in input_names]
+        #     clip_min = clip_min_node[0].float_data[0]
 
-            # Check if it's relu or not
-            if clip_min == 0.0:
-                layer['class_name'] = 'Activation'
-                layer['activation'] = 'ReLU'
-            else:
-                raise Exception('Clip with min != 0 is not supported yet!')
+        #     # Check if it's relu or not
+        #     if clip_min == 0.0:
+        #         layer['class_name'] = 'Activation'
+        #         layer['activation'] = 'ReLU'
+        #     else:
+        #         raise Exception('Clip with min != 0 is not supported yet!')
 
         else:
             layer['activation'] = layer['class_name']
             layer['class_name'] = 'Activation'
 
-    return layer, [shape for shape in input_shapes[0]]
+    return layer
 
 
 @onnx_handler('BatchNormalization')
-def parse_batchnorm_layer(reader, node, inputs_map, input_shapes, graph, config):
+def parse_batchnorm_layer(node, input_names, input_shapes, graph):
     layer = {}
 
-    layer['class_name'] = 'BatchNormalization'
-    layer['data_format'] = 'channels_first'
+    layer['class_name'] = 'BatchNormOnnx'
     layer['name'] = node.name
-    layer['inputs'] = get_onnx_input_name(node, graph)
+    layer['inputs'] = input_names
+    layer['outputs'] = list(node.output)
 
     # Other attributes
-    layer['epsilon'] = get_onnx_attribute(node, 'epsilon')
-    layer['momentum'] = get_onnx_attribute(node, 'momentum')
+    layer['epsilon'] = get_onnx_attribute(node, 'epsilon', 1e-05)
+    # layer['momentum'] = get_onnx_attribute(node, 'momentum', 0.9)  # not used
 
-    reader.add_input(layer['name'], node.input)
-
-    in_size = 1
-    for dim in input_shapes[0][1:]:
-        in_size *= dim
-
-    layer['n_in'] = layer['n_out'] = in_size
+    layer['n_in'] = layer['n_out'] = np.prod(input_shapes[0][1:])
 
     if len(input_shapes[0]) == 2:
         layer['n_filt'] = -1
     elif len(input_shapes[0]) > 2:
-        layer['n_filt'] = input_shapes[0][1]  # Always channel first for onnx
+        if node.domain != 'qonnx.custom_op.channels_last':
+            raise RuntimeError("Please convert the model to channels-last format with qonnx-to-channels-last")
+        layer['data_format'] = 'channels_last'  # QONNX needs to be channels-last.
+        layer['n_filt'] = input_shapes[0][-1]
+    else:
+        raise RuntimeError(f"Unexpected input shape: {input_shapes[0]}")
+
+    return layer
+
+
+@onnx_handler('Quant')
+def parse_quant_layer(node, input_names, input_shapes, graph):
+    layer = {}
+
+    layer['class_name'] = 'Quant'
+    layer['name'] = node.name
+    layer['inputs'] = input_names
+    layer['outputs'] = list(node.output)
+
+    # Other attributes
+    layer['narrow'] = bool(get_onnx_attribute(node, 'narrow'))
+    layer['rounding_mode'] = get_onnx_attribute(node, 'rounding_mode')
+    layer['signed'] = bool(get_onnx_attribute(node, 'signed'))
 
-    return layer, [shape for shape in input_shapes[0]]
+    return layer