Add support for Array-Type in JSON

Fixes: #35
amplab · Jan 20, 2019 · 5831a45 · 5831a45
1 parent 260efe6
commit 5831a45
Show file tree

Hide file tree

Showing 3 changed files with 94 additions and 9 deletions.
diff --git a/serde/src/main/java/edu/berkeley/cs/succinct/DataType.java b/serde/src/main/java/edu/berkeley/cs/succinct/DataType.java
@@ -9,7 +9,11 @@ public enum DataType {
   LONG(4),
   FLOAT(5),
   DOUBLE(6),
-  STRING(7);
+  STRING(7),
+  STRINGARRAY(8),
+  LONGARRAY(9),
+  BYTEARRAY(10),
+  BOOLARRAY(11);
 
   private final int order;
 

diff --git a/serde/src/main/java/edu/berkeley/cs/succinct/block/json/JsonBlockSerializer.java b/serde/src/main/java/edu/berkeley/cs/succinct/block/json/JsonBlockSerializer.java
@@ -2,6 +2,7 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.fasterxml.jackson.databind.node.ValueNode;
 import edu.berkeley.cs.succinct.DataType;
@@ -67,7 +68,36 @@ private void flattenJsonTree(String currentPath, JsonNode jsonNode, ByteArrayOut
         flattenJsonTree(pathPrefix + entry.getKey(), entry.getValue(), out);
       }
     } else if (jsonNode.isArray()) {
-      throw new SerializationException("Arrays in JSON are not supported yet.");
+      ArrayNode arrayNode = (ArrayNode) jsonNode;
+      DataType primitiveArrayType = getNodeType(arrayNode.get(0));
+      DataType jsonArrayType = getArrayNodeType(primitiveArrayType);
+      for (int i = 0; i < arrayNode.size(); i++) {
+        if (getNodeType(arrayNode.get(i)) != primitiveArrayType)
+          throw new SerializationException("Multi Type Arrays in JSON are not supported yet.");
+        else if (!arrayNode.get(i).isValueNode())
+          throw new SerializationException("Non primitive types in Arrays in JSON are not supported yet.");
+      }
+      for (int i = 0; i < arrayNode.size(); i++) {
+        ValueNode valueNode = (ValueNode) arrayNode.get(i);
+        if (!fieldMapping.containsField(currentPath)) {
+          fieldMapping.put(currentPath, delimiters[currentDelimiterIdx++], jsonArrayType);
+        } else {
+          DataType existingType = fieldMapping.getDataType(currentPath);
+          DataType newType = primitiveArrayType;
+          if (existingType != newType) {
+            DataType encapsulatingType = DataType.encapsulatingType(existingType, newType);
+            fieldMapping.updateType(currentPath, getArrayNodeType(encapsulatingType));
+          }
+        }
+        try {
+          byte fieldByte = fieldMapping.getDelimiter(currentPath);
+          out.write(fieldByte);
+          out.write(valueNode.asText().getBytes());
+          out.write(fieldByte);
+        } catch (IOException e) {
+          throw new SerializationException(e.getMessage());
+        }
+      }
     } else if (jsonNode.isValueNode()) {
       ValueNode valueNode = (ValueNode) jsonNode;
       if (!fieldMapping.containsField(currentPath)) {
@@ -108,4 +138,12 @@ private DataType getNodeType(JsonNode node) {
       throw new UnsupportedOperationException("JSON DataType not supported.");
     }
   }
+
+  private DataType getArrayNodeType(DataType fieldType) {
+    if (fieldType == DataType.STRING) return DataType.STRINGARRAY;
+    else if (fieldType == DataType.LONG) return DataType.LONGARRAY;
+    else if (fieldType == DataType.BYTE) return DataType.BYTEARRAY;
+    else if (fieldType == DataType.BOOLEAN) return DataType.BOOLARRAY;
+    else throw new UnsupportedOperationException("JSON DataType not supported.");
+  }
 }
diff --git a/serde/src/main/java/edu/berkeley/cs/succinct/object/deserializer/JsonDeserializer.java b/serde/src/main/java/edu/berkeley/cs/succinct/object/deserializer/JsonDeserializer.java
@@ -7,9 +7,7 @@
 import edu.berkeley.cs.succinct.PrimitiveDeserializer;
 import edu.berkeley.cs.succinct.block.json.FieldMapping;
 
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.*;
 
 public class JsonDeserializer implements ObjectDeserializer<String> {
 
@@ -42,23 +40,53 @@ private String mapToJsonString(Map<String, Object> jsonMap) throws Deserializati
     return jsonString;
   }
 
-  private Map<String, Object> deserializeToMap(byte[] data) throws DeserializationException {
+  private Map<String, Object> deserializeToMap(byte[] data) throws DeserializationException, IllegalArgumentException {
     Map<String, Object> jsonMap = new HashMap<String, Object>();
 
     int curOffset = 0;
     while (curOffset < data.length) {
       byte curDelimiter = data[curOffset++];
       String fieldKey = fieldMapping.getField(curDelimiter);
       DataType fieldType = fieldMapping.getDataType(fieldKey);
-      byte[] fieldValueBytes = extractField(data, curOffset, curDelimiter);
-      Object fieldValue = PrimitiveDeserializer.deserializePrimitive(fieldValueBytes, fieldType);
+      Object fieldValue;
+      int totalLength;
+      if (isArray(fieldType)) {
+        ArrayList<byte []> fieldValuesBytes = extractArrayField(data, curOffset, curDelimiter);
+        DataType typeinArray = getPrimitiveFromArray(fieldType);
+        ArrayList<Object> resultSet = new ArrayList<>();
+        totalLength = 0;
+        for (byte [] fieldValueBytes : fieldValuesBytes) {
+          resultSet.add(PrimitiveDeserializer.deserializePrimitive(fieldValueBytes, typeinArray));
+          totalLength += fieldValueBytes.length + 2; // Skip the field data, and the delimiter at end and start of next field
+        }
+        fieldValue = resultSet.toArray();
+        totalLength -= 1; //  Remove last additional append of delimiter
+      }
+      else {
+        byte[] fieldValueBytes = extractField(data, curOffset, curDelimiter);
+        fieldValue = PrimitiveDeserializer.deserializePrimitive(fieldValueBytes, fieldType);
+        totalLength = fieldValueBytes.length + 1; // Skip the field data, and the end delimiter
+      }
       add(jsonMap, fieldKey, fieldValue);
-      curOffset += (fieldValueBytes.length + 1); // Skip the field data, and the end delimiter
+      curOffset += totalLength;
     }
 
     return jsonMap;
   }
 
+  private DataType getPrimitiveFromArray(DataType fieldType) throws IllegalArgumentException {
+    if (fieldType == DataType.STRINGARRAY) return DataType.STRING;
+    else if (fieldType == DataType.LONGARRAY) return DataType.LONG;
+    else if (fieldType == DataType.BYTEARRAY) return DataType.BYTE;
+    else if (fieldType == DataType.BOOLARRAY) return DataType.BOOLEAN;
+    else throw new IllegalArgumentException("Called getPrimitiveFromArray on a non Array DataType");
+  }
+
+  private Boolean isArray(DataType fieldType) {
+    return fieldType == DataType.STRINGARRAY || fieldType == DataType.LONGARRAY ||
+            fieldType == DataType.BYTEARRAY || fieldType == DataType.BOOLARRAY;
+  }
+
   private void add(Map<String, Object> map, String key, Object value) {
     if (key.contains(".")) {
       String[] keySplits = key.split("\\.", 2);
@@ -84,4 +112,19 @@ private byte[] extractField(byte[] data, int startOffset, byte delimiter) {
     return Arrays.copyOfRange(data, startOffset, i);
   }
 
+  private ArrayList<byte []> extractArrayField(byte[] data, int startOffset, byte delimiter) {
+    int i = startOffset;
+    ArrayList<byte []> res = new ArrayList<>();
+    while (true) {
+      while (data[i] != delimiter) {
+        i++;
+      }
+      res.add(Arrays.copyOfRange(data, startOffset, i));
+      i++;
+      if(data.length <= i || data[i] != delimiter) break;
+      else i++;
+    }
+    return res;
+  }
+
 }