Seal tensor buffer.

Signed-off-by: vegetableysm <[email protected]>

java/hive/src/main/java/io/v6d/hive/ql/io/VineyardOutputFormat.java

@@ -20,9 +20,12 @@
 import io.v6d.modules.basic.dataframe.DataFrameBuilder;
 import io.v6d.modules.basic.tensor.TensorBuilder;
 import io.v6d.modules.basic.tensor.ITensor;
+import io.v6d.modules.basic.arrow.BufferBuilder;
+import io.v6d.core.client.ds.ObjectMeta;
 
 import java.io.IOException;
 import java.util.Properties;
+import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.arrow.memory.BufferAllocator;
@@ -111,6 +114,7 @@ public SinkRecordWriter(
         } else {
             System.out.printf("connected to vineyard succeed!\n");
         }
+
     }
 
     @Override
@@ -133,6 +137,15 @@ public void write(Writable w) throws IOException {
         //     System.out.printf("field %d: %s\n", i, structVector.getChildrenFromFields().get(i).getName());
         // }
         System.out.println("==============");
+
+        try{
+            BufferBuilder builder = new BufferBuilder(client, 100);
+            ObjectMeta meta =  builder.seal(client);
+            System.out.printf("buffer id: %d\n", meta.getId());
+        } catch (Exception e) {
+            System.out.printf("failed to seal buffer: %s\n", e);
+        }
+
         for (int i = 0; i < root.getRowCount(); i++) {
             System.out.printf("row %d: ", i);
             for (int j = 0; j < root.getFieldVectors().size(); ++j) {
@@ -157,7 +170,11 @@ public void write(Writable w) throws IOException {
                 // TODO: other type
                 case Int:
                     System.out.printf("int\n");
-                    tensorBuilder = new TensorBuilder(client, root.getFieldVectors().get(i));
+                    List<Integer> shape = new ArrayList<Integer>(1);
+                    shape.add(root.getRowCount());
+                    tensorBuilder = new TensorBuilder(client, shape, root.getFieldVectors().get(i));
+                    // tensorBuilder.setValues(root.getFieldVectors().get(i));
+                    // tensorBuilder = new TensorBuilder(root.getFieldVectors().get(i));
                     // column
                     dataFrameBuilder.addColumn(schema.getFields().get(i).getName(), tensorBuilder);
                     break;

java/modules/basic/src/main/java/io/v6d/modules/basic/dataframe/DataFrameBuilder.java

@@ -46,6 +46,7 @@ public DataFrameBuilder(Client client) {
         mapper = new ObjectMapper();
         columns = new ArrayList<>();
         tensorBuilders = new ArrayList<>();
+        System.out.println("values size:" + values.size());
     }
 
     public void set_index(TensorBuilder index) {
@@ -66,6 +67,7 @@ public void addColumn(String column, TensorBuilder builder) {
         columnCount++;
         columns.add(getJsonNode(column));
         tensorBuilders.add(builder);
+        System.out.println("values size:" + values.size());
     }
 
     @Override
@@ -99,13 +101,13 @@ public ObjectMeta seal(Client client) {
 
         dataFrameMeta.setListValue("columns_", columns);
         dataFrameMeta.setValue("__values_-size", values.size());
-        for (int i = 0; i < values.size(); i++) {
-            // seal tensor
-            values.forEach((key,value)->{
-                dataFrameMeta.setValue("__values_-key-", key);
-                dataFrameMeta.addMember("__values_-value-", value.seal(client));
-            });
-        }
+        System.out.println("values size:" + values.size());
+        // seal tensor
+        values.forEach((key,value)->{
+            System.out.println("key:" + key.toString());
+            dataFrameMeta.setValue("__values_-key-", key);
+            dataFrameMeta.addMember("__values_-value-", value.seal(client));
+        });
 
         // DataFrame dataFrame = new DataFrame(dataFrameMeta, rowCount, columnCount, columns, tensors);
         // dataFrame.setColumns(columns);

java/modules/basic/src/main/java/io/v6d/modules/basic/tensor/TensorBuilder.java

@@ -5,27 +5,74 @@
 import io.v6d.core.client.ds.ObjectBase;
 import io.v6d.core.client.ds.ObjectBuilder;
 import io.v6d.core.client.ds.ObjectMeta;
+import io.v6d.modules.basic.arrow.Buffer;
+import io.v6d.modules.basic.arrow.BufferBuilder;
+import io.v6d.modules.basic.arrow.Int32ArrayBuilder;
 import io.v6d.core.common.util.VineyardException;
 import io.v6d.modules.basic.arrow.TableBuilder;
 
 import java.util.ArrayList;
+import java.util.Collection;
+import java.lang.Long;
 
 import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.types.Types;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class TensorBuilder implements ObjectBuilder{
-    private Logger logger = LoggerFactory.getLogger(TableBuilder.class);
     private FieldVector values;
-    private Client client;
+    private BufferBuilder bufferBuilder;
+    private Collection<Integer> shape;
+    private Types.MinorType dtype;
+    private int partition_index_ = 0;
+    private long size = 0;
 
-    public TensorBuilder(Client client, FieldVector vector) {
-        this.client = client;
+    public TensorBuilder(FieldVector vector) {
+        System.out.println("TensorBuilder with vector");
         values = vector;
     }
 
-    public void append(Object value) {
-        System.out.println("append " + value);
+    public TensorBuilder(IPCClient client, Collection<Integer> shape) {
+        System.out.println("TensorBuilder with shape");
+        this.shape = shape;
+        int count = 0;
+        for (Integer item : shape) {
+            count += item;
+        }
+        //TODO:Support other type.
+        size = count * Integer.SIZE;
+        //TODO: init intBuilder
+    }
+
+    public TensorBuilder(IPCClient client, Collection<Integer> shape, FieldVector values) {
+        System.out.println("TensorBuilder with shape");
+        this.shape = shape;
+        int count = 0;
+        for (Integer item : shape) {
+            count += item.intValue();
+        }
+        System.out.print("count:" + count);
+        //TODO:Support other type.
+        size = count * Integer.SIZE;
+        try {
+            bufferBuilder = new BufferBuilder(client, size);
+            for (int i = 0; i < values.getValueCount(); i++) {
+                bufferBuilder.getBuffer().setInt(i, ((IntVector)values).get(i));
+            }
+        } catch (VineyardException e) {
+            System.out.println("TensorBuilder error");
+            e.printStackTrace();
+        }
+        System.out.println("TensorBuilder end");
+    }
+
+    public void setValues(FieldVector values) {
+        System.out.println("setValues ");
+        this.values = values;
+        // other type
+        this.size = values.getBufferSize() * Integer.SIZE;
     }
 
     public FieldVector getBuffer() {
@@ -47,10 +94,27 @@ public ObjectMeta seal(Client client) {
         this.build(client);
         ObjectMeta tensorMeta = ObjectMeta.empty();
 
-        // TODO: set the typename
+        // TODO: set other typename
         tensorMeta.setTypename("vineyard::Tensor<int>");
+        tensorMeta.setValue("value_type_", "int");
+        // Int32ArrayBuilder intBuilder = new Int32ArrayBuilder(client, );
+
+        try {
+            tensorMeta.addMember("buffer_", bufferBuilder.seal(client));
+        } catch (VineyardException e) {
+            System.out.println("Seal buffer error");
+            e.printStackTrace();
+            return null;
+        }
+        // tensorMeta.addMember("buffer_", ????);
+        tensorMeta.setListValue("shape_", shape);
+        tensorMeta.setValue("partition_index_", partition_index_);
+        tensorMeta.setNBytes(size);
+
         try {
             client.createMetaData(tensorMeta);
+            System.out.println("Tensor id:" + tensorMeta.getId().value());
+            System.out.println("Tensor id:" + tensorMeta.getId());
         } catch (VineyardException e) {
             e.printStackTrace();
             return null;