replaced substitution type with binary, to be more robust

core/src/main/java/org/polypheny/db/adapter/RelationalModifyDelegate.java

@@ -90,6 +90,7 @@
 import org.polypheny.db.transaction.Statement;
 import org.polypheny.db.type.PolyType;
 import org.polypheny.db.util.Pair;
+import org.polypheny.db.util.Triple;
 
 public class RelationalModifyDelegate extends RelationalScanDelegate implements Modifiable {
 
@@ -324,37 +325,37 @@ public void dropTable( Context context, long allocId ) {
     public void createGraph( Context context, LogicalGraph logical, AllocationGraph allocation ) {
 
         PhysicalTable node = createSubstitution( context, logical, allocation, "_node_", List.of(
-                Pair.of( "id", GraphType.ID_SIZE ),
-                Pair.of( "label", GraphType.LABEL_SIZE ) ) );
+                Triple.of( "id", GraphType.ID_SIZE, PolyType.VARCHAR ),
+                Triple.of( "label", GraphType.LABEL_SIZE, PolyType.VARCHAR ) ) );
 
         PhysicalTable nProperties = createSubstitution( context, logical, allocation, "_nProperties_", List.of(
-                Pair.of( "id", GraphType.ID_SIZE ),
-                Pair.of( "key", GraphType.KEY_SIZE ),
-                Pair.of( "value", GraphType.VALUE_SIZE ) ) );
+                Triple.of( "id", GraphType.ID_SIZE, PolyType.VARCHAR ),
+                Triple.of( "key", GraphType.KEY_SIZE, PolyType.VARCHAR ),
+                Triple.of( "value", GraphType.VALUE_SIZE, PolyType.VARCHAR ) ) );
 
         PhysicalTable edge = createSubstitution( context, logical, allocation, "_edge_", List.of(
-                Pair.of( "id", GraphType.ID_SIZE ),
-                Pair.of( "label", GraphType.LABEL_SIZE ),
-                Pair.of( "_l_id_", GraphType.ID_SIZE ),
-                Pair.of( "_r_id_", GraphType.ID_SIZE ) ) );
+                Triple.of( "id", GraphType.ID_SIZE, PolyType.VARCHAR ),
+                Triple.of( "label", GraphType.LABEL_SIZE, PolyType.VARCHAR ),
+                Triple.of( "_l_id_", GraphType.ID_SIZE, PolyType.VARCHAR ),
+                Triple.of( "_r_id_", GraphType.ID_SIZE, PolyType.VARCHAR ) ) );
 
         PhysicalTable eProperties = createSubstitution( context, logical, allocation, "_eProperties_", List.of(
-                Pair.of( "id", GraphType.ID_SIZE ),
-                Pair.of( "key", GraphType.KEY_SIZE ),
-                Pair.of( "value", GraphType.VALUE_SIZE ) ) );
+                Triple.of( "id", GraphType.ID_SIZE, PolyType.VARCHAR ),
+                Triple.of( "key", GraphType.KEY_SIZE, PolyType.VARCHAR ),
+                Triple.of( "value", GraphType.VALUE_SIZE, PolyType.VARCHAR ) ) );
 
         catalog.getAllocRelations().put( allocation.id, Pair.of( allocation, List.of( node.id, nProperties.id, edge.id, eProperties.id ) ) );
     }
 
 
-    private PhysicalTable createSubstitution( Context context, LogicalEntity logical, AllocationEntity allocation, String name, List<Pair<String, Integer>> nameLength ) {
+    private PhysicalTable createSubstitution( Context context, LogicalEntity logical, AllocationEntity allocation, String name, List<Triple<String, Integer, PolyType>> nameLength ) {
         IdBuilder builder = IdBuilder.getInstance();
         LogicalTable table = new LogicalTable( builder.getNewLogicalId(), name + logical.id, logical.namespaceId, logical.entityType, null, logical.modifiable );
         List<LogicalColumn> columns = new ArrayList<>();
 
         int i = 0;
-        for ( Pair<String, Integer> col : nameLength ) {
-            LogicalColumn column = new LogicalColumn( builder.getNewFieldId(), col.getLeft(), table.id, table.namespaceId, i, PolyType.VARCHAR, null, col.right, null, null, null, false, Collation.getDefaultCollation(), null );
+        for ( Triple<String, Integer, PolyType> col : nameLength ) {
+            LogicalColumn column = new LogicalColumn( builder.getNewFieldId(), col.getLeft(), table.id, table.namespaceId, i, col.getRight(), null, col.getMiddle(), null, null, null, false, Collation.getDefaultCollation(), null );
             columns.add( column );
             i++;
         }
@@ -382,7 +383,7 @@ public void dropGraph( Context context, AllocationGraph allocation ) {
 
     @Override
     public void createCollection( Context context, LogicalCollection logical, AllocationCollection allocation ) {
-        PhysicalTable physical = createSubstitution( context, logical, allocation, "_doc_", List.of( Pair.of( DocumentType.DOCUMENT_ID, DocumentType.ID_SIZE ), Pair.of( DocumentType.DOCUMENT_DATA, DocumentType.DATA_SIZE ) ) );
+        PhysicalTable physical = createSubstitution( context, logical, allocation, "_doc_", List.of( Triple.of( DocumentType.DOCUMENT_ID, DocumentType.ID_SIZE, PolyType.VARBINARY ), Triple.of( DocumentType.DOCUMENT_DATA, DocumentType.DATA_SIZE, PolyType.VARBINARY ) ) );
         catalog.getAllocRelations().put( allocation.id, Pair.of( allocation, List.of( physical.id ) ) );
     }
 
@@ -394,9 +395,6 @@ public void dropCollection( Context context, AllocationCollection allocation ) {
     }
 
 
-
-
-
     private List<AlgNode> attachRelationalDoc( LogicalDocumentModify alg, Statement statement, CatalogEntity collectionTable, LogicalQueryInformation queryInformation, long adapterId ) {
         RoutedAlgBuilder builder = attachDocUpdate( alg.getInput(), statement, collectionTable, RoutedAlgBuilder.create( statement, alg.getCluster() ), queryInformation, adapterId );
         RexBuilder rexBuilder = alg.getCluster().getRexBuilder();

core/src/main/java/org/polypheny/db/algebra/core/document/DocumentValues.java

@@ -24,6 +24,7 @@
 import org.polypheny.db.algebra.AbstractAlgNode;
 import org.polypheny.db.algebra.AlgNode;
 import org.polypheny.db.algebra.logical.relational.LogicalValues;
+import org.polypheny.db.algebra.type.AlgDataTypeFactory;
 import org.polypheny.db.algebra.type.DocumentType;
 import org.polypheny.db.catalog.entity.CatalogEntity;
 import org.polypheny.db.catalog.snapshot.Snapshot;
@@ -32,6 +33,8 @@
 import org.polypheny.db.rex.RexBuilder;
 import org.polypheny.db.rex.RexLiteral;
 import org.polypheny.db.schema.trait.ModelTrait;
+import org.polypheny.db.type.PolyType;
+import org.polypheny.db.type.entity.PolyBinary;
 import org.polypheny.db.type.entity.PolyString;
 import org.polypheny.db.type.entity.document.PolyDocument;
 
@@ -85,9 +88,9 @@ protected static ImmutableList<ImmutableList<RexLiteral>> relationalize( List<Po
                     throw new RuntimeException( "Error while transforming document to relational values" );
                 }
 
-                normalizedTuple.add( 0, rexBuilder.makeLiteral( id.value ) );
-                String parsed = tuple.serialize();
-                normalizedTuple.add( 1, rexBuilder.makeLiteral( parsed ) );
+                normalizedTuple.add( 0, rexBuilder.makeLiteral( PolyBinary.of( id.serialize().getBytes() ), AlgDataTypeFactory.DEFAULT.createPolyType( PolyType.VARBINARY, 2024 ), PolyType.VARBINARY ) );
+                byte[] parsed = tuple.serialize().getBytes();
+                normalizedTuple.add( 1, rexBuilder.makeLiteral( PolyBinary.of( parsed ), AlgDataTypeFactory.DEFAULT.createPolyType( PolyType.VARBINARY, 2024 ), PolyType.VARBINARY ) );
                 normalized.add( ImmutableList.copyOf( normalizedTuple ) );
             }
         }

core/src/main/java/org/polypheny/db/algebra/enumerable/EnumerableTransformer.java

@@ -54,6 +54,7 @@
 import org.polypheny.db.plan.AlgTraitSet;
 import org.polypheny.db.schema.trait.ModelTrait;
 import org.polypheny.db.schema.trait.ModelTraitDef;
+import org.polypheny.db.type.entity.PolyBinary;
 import org.polypheny.db.type.entity.PolyString;
 import org.polypheny.db.type.entity.document.PolyDocument;
 import org.polypheny.db.util.BuiltInMethod;
@@ -324,7 +325,7 @@ private Result implementDocumentOnRelational( EnumerableAlgImplementor implement
         ParameterExpression target = Expressions.parameter( Object[].class );
 
         for ( AlgDataTypeField field : getInput( 0 ).getRowType().getFieldList() ) {
-            UnaryExpression element = Expressions.convert_( Expressions.arrayIndex( target, Expressions.constant( field.getIndex() ) ), PolyString.class );
+            UnaryExpression element = Expressions.convert_( Expressions.arrayIndex( target, Expressions.constant( field.getIndex() ) ), PolyBinary.class );
             Expression el = Expressions.call( RefactorFunctions.class, "toDocument", element );
             if ( field.getName().equals( DocumentType.DOCUMENT_DATA ) ) {
                 expressions.add( 0, el );

core/src/main/java/org/polypheny/db/algebra/logical/document/LogicalDocumentValues.java

@@ -16,28 +16,18 @@
 
 package org.polypheny.db.algebra.logical.document;
 
-import com.google.common.collect.ImmutableList;
-import java.util.ArrayList;
 import java.util.List;
-import org.bson.BsonDocument;
-import org.bson.BsonObjectId;
-import org.bson.BsonValue;
-import org.bson.types.ObjectId;
 import org.polypheny.db.algebra.AlgNode;
 import org.polypheny.db.algebra.AlgShuttle;
 import org.polypheny.db.algebra.core.document.DocumentValues;
 import org.polypheny.db.algebra.core.relational.RelationalTransformable;
 import org.polypheny.db.algebra.type.AlgDataType;
-import org.polypheny.db.algebra.type.AlgDataTypeField;
-import org.polypheny.db.algebra.type.DocumentType;
 import org.polypheny.db.catalog.logistic.NamespaceType;
 import org.polypheny.db.plan.AlgOptCluster;
 import org.polypheny.db.plan.AlgTraitSet;
 import org.polypheny.db.plan.Convention;
-import org.polypheny.db.rex.RexLiteral;
 import org.polypheny.db.type.PolyType;
 import org.polypheny.db.type.entity.document.PolyDocument;
-import org.polypheny.db.util.BsonUtil;
 
 
 public class LogicalDocumentValues extends DocumentValues implements RelationalTransformable {
@@ -81,47 +71,6 @@ public static AlgNode create( AlgOptCluster cluster, List<PolyDocument> document
     }
 
 
-    private static ImmutableList<BsonValue> bsonify( ImmutableList<ImmutableList<RexLiteral>> tuples, AlgDataType rowType ) {
-        List<BsonValue> docs = new ArrayList<>();
-
-        for ( ImmutableList<RexLiteral> values : tuples ) {
-            BsonDocument doc = new BsonDocument();
-            int pos = 0;
-            for ( RexLiteral value : values ) {
-                AlgDataTypeField field = rowType.getFieldList().get( pos );
-
-                if ( field.getName().equals( DocumentType.DOCUMENT_ID ) ) {
-                    String _id = value.value.asString().value;
-                    ObjectId objectId;
-                    if ( _id.matches( "ObjectId\\([0-9abcdef]{24}\\)" ) ) {
-                        objectId = new ObjectId( _id.substring( 9, 33 ) );
-                    } else {
-                        objectId = ObjectId.get();
-                    }
-                    doc.put( DocumentType.DOCUMENT_ID, new BsonObjectId( objectId ) );
-                } else if ( field.getName().equals( "_data" ) ) {
-                    BsonDocument docVal = new BsonDocument();
-                    if ( !value.isNull() && value.value.asString().value.length() != 0 ) {
-                        String data = BsonUtil.fixBson( value.value.asString().value );
-                        if ( data.matches( "[{].*[}]" ) ) {
-                            docVal = BsonDocument.parse( data );
-                        } else {
-                            throw new RuntimeException( "The inserted document is not valid." );
-                        }
-                    }
-                    doc.put( DocumentType.DOCUMENT_DATA, docVal );
-                } else {
-                    doc.put( field.getName(), BsonUtil.getAsBson( value, null ) );
-                }
-
-                pos++;
-            }
-            docs.add( doc );
-        }
-        return ImmutableList.copyOf( docs );
-    }
-
-
     public static LogicalDocumentValues createOneRow( AlgOptCluster cluster ) {
         final AlgDataType rowType =
                 cluster.getTypeFactory()

core/src/main/java/org/polypheny/db/algebra/type/DocumentType.java

@@ -70,8 +70,8 @@ public static DocumentType ofId() {
 
     public static AlgDataType asRelational() {
         return new AlgRecordType( List.of(
-                new AlgDataTypeFieldImpl( DOCUMENT_ID, 0, AlgDataTypeFactory.DEFAULT.createPolyType( PolyType.VARCHAR, 2024 ) ),
-                new AlgDataTypeFieldImpl( DOCUMENT_DATA, 1, AlgDataTypeFactory.DEFAULT.createPolyType( PolyType.VARCHAR, 2024 ) )
+                new AlgDataTypeFieldImpl( DOCUMENT_ID, 0, AlgDataTypeFactory.DEFAULT.createPolyType( PolyType.VARBINARY, 2024 ) ),
+                new AlgDataTypeFieldImpl( DOCUMENT_DATA, 1, AlgDataTypeFactory.DEFAULT.createPolyType( PolyType.VARBINARY, 2024 ) )
         ) );
     }
 

core/src/main/java/org/polypheny/db/functions/RefactorFunctions.java

@@ -19,6 +19,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import org.polypheny.db.type.entity.PolyBinary;
 import org.polypheny.db.type.entity.PolyInterval;
 import org.polypheny.db.type.entity.PolyLong;
 import org.polypheny.db.type.entity.PolyString;
@@ -49,14 +50,14 @@ public static PolyValue[] toObjectArray( PolyValue... obj ) {
 
 
     @SuppressWarnings("unused")
-    public static PolyString fromDocument( PolyValue doc ) {
-        return PolyString.of( doc.serialize() );
+    public static PolyBinary fromDocument( PolyValue doc ) {
+        return PolyBinary.of( doc.serialize().getBytes() );
     }
 
 
     @SuppressWarnings("unused")
-    public static PolyValue toDocument( PolyString json ) {
-        return PolyValue.deserialize( json.value );
+    public static PolyValue toDocument( PolyBinary json ) {
+        return PolyValue.deserialize( new String( json.value.getBytes() ) );
     }
 
 

core/src/main/java/org/polypheny/db/type/entity/PolyBinary.java

@@ -57,6 +57,11 @@ public static PolyBinary of( byte[] value ) {
     }
 
 
+    public static PolyBinary ofNullable( byte[] value ) {
+        return value == null ? null : PolyBinary.of( value );
+    }
+
+
     @Override
     public int compareTo( @NotNull PolyValue o ) {
         return 0;

core/src/main/java/org/polypheny/db/type/entity/PolyString.java

@@ -163,6 +163,19 @@ public PolySerializable copy() {
     }
 
 
+    public String asCharset( String charset ) {
+        return asCharset( Charset.forName( charset ) );
+    }
+
+
+    public String asCharset( Charset charset ) {
+        if ( this.charset.equals( charset ) ) {
+            return value;
+        }
+        return new String( value.getBytes( this.charset ), charset );
+    }
+
+
     public static class PolyStringSerializerDef extends SimpleSerializerDef<PolyString> {
 
         @Override

core/src/main/java/org/polypheny/db/util/Triple.java

@@ -16,18 +16,16 @@
 
 package org.polypheny.db.util;
 
-import lombok.AllArgsConstructor;
 import lombok.EqualsAndHashCode;
 import lombok.Value;
 
-@Value
-@AllArgsConstructor
+@Value(staticConstructor = "of")
 @EqualsAndHashCode
 public class Triple<A, B, C> implements Comparable<Triple<A, B, C>> {
 
-    A left;
-    B middle;
-    C right;
+    public A left;
+    public B middle;
+    public C right;
 
 
     @Override

dbms/src/main/java/org/polypheny/db/PolyphenyDb.java

@@ -110,7 +110,7 @@ public class PolyphenyDb {
     public boolean daemonMode = false;
 
     @Option(name = { "-defaultStore" }, description = "Type of default store")
-    public String defaultStoreName = "postgresql";
+    public String defaultStoreName = "hsqldb";
 
     @Option(name = { "-defaultSource" }, description = "Type of default source")
     public String defaultSourceName = "csv";

plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/JdbcToEnumerableConverter.java

@@ -83,6 +83,7 @@
 import org.polypheny.db.type.ArrayType;
 import org.polypheny.db.type.PolyType;
 import org.polypheny.db.type.entity.PolyBigDecimal;
+import org.polypheny.db.type.entity.PolyBinary;
 import org.polypheny.db.type.entity.PolyBoolean;
 import org.polypheny.db.type.entity.PolyDate;
 import org.polypheny.db.type.entity.PolyDefaults;
@@ -454,6 +455,9 @@ private static Expression getOfPolyExpression( AlgDataType fieldType, Expression
             case ARRAY:
                 poly = Expressions.call( PolyList.class, fieldType.isNullable() ? "ofNullable" : "of", source ); // todo might change
                 break;
+            case VARBINARY:
+                poly = Expressions.call( PolyBinary.class, fieldType.isNullable() ? "ofNullable" : "of", Expressions.convert_( source, byte[].class ) );
+                break;
             case FILE:
                 poly = Expressions.call( PolyFile.class, fieldType.isNullable() ? "ofNullable" : "of", Expressions.convert_( source, byte[].class ) );
                 break;

plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/ResultSetEnumerable.java

@@ -293,7 +293,6 @@ private static void setDynamicParam( PreparedStatement preparedStatement, int i,
                 break;
             case ARRAY:
                 if ( connectionHandler.getDialect().supportsNestedArrays() ) {
-                    // apparently even postgres is able to support nested arrays now
                     Array array = getArray( value, type, connectionHandler );
                     preparedStatement.setArray( i, array );
                     array.free(); // according to documentation this is advised to not hog the memory

plugins/postgres-adapter/src/main/java/org/polypheny/db/adapter/postgres/store/PostgresqlStore.java

@@ -173,7 +173,7 @@ public void updateColumnType( Context context, long allocId, LogicalColumn newCo
         if ( column.collectionsType != null ) {
             builder.append( " " ).append( column.collectionsType );
         }
-        if ( column.length != null ) {
+        if ( column.length != null && column.type != PolyType.VARBINARY ) {
             builder.append( "(" );
             builder.append( column.length );
             if ( column.scale != null ) {
@@ -323,7 +323,7 @@ protected String getTypeString( PolyType type ) {
             case BOOLEAN:
                 return "BOOLEAN";
             case VARBINARY:
-                return "VARBINARY";
+                return "BYTEA";
             case TINYINT:
                 return "SMALLINT";
             case SMALLINT:

plugins/sql-language/src/main/java/org/polypheny/db/sql/language/dialect/PostgresqlSqlDialect.java

@@ -109,6 +109,7 @@ public SqlNode getCastSpec( AlgDataType type ) {
                 // Postgres has a double type but it is named differently
                 castSpec = "_double precision";
                 break;
+            case VARBINARY:
             case FILE:
             case IMAGE:
             case VIDEO: