From 08d2fa6c18930ed4888116c2c5ece00f2b95fbae Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Fri, 31 Jan 2025 14:57:09 +0000 Subject: [PATCH] wire hex encoding for tensors --- container-search/abi-spec.json | 3 + .../com/yahoo/search/query/Presentation.java | 28 +++- .../query/properties/QueryProperties.java | 2 +- .../yahoo/search/rendering/JsonRenderer.java | 82 ++++++------ .../com/yahoo/search/result/FeatureData.java | 43 +++--- .../rendering/JsonRendererTestCase.java | 126 +++++++++++++++--- 6 files changed, 204 insertions(+), 80 deletions(-) diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index c17d2819da6e..3bddbad2aedc 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -5443,6 +5443,8 @@ "public java.util.Set getSummaryFields()", "public void setSummaryFields(java.lang.String)", "public boolean getTensorShortForm()", + "public boolean getTensorHexDense()", + "public java.lang.String getTensorFormat()", "public void setTensorShortForm(java.lang.String)", "public void setTensorFormat(java.lang.String)", "public void setTensorShortForm(boolean)", @@ -8067,6 +8069,7 @@ "public java.lang.String toJson()", "public java.lang.String toJson(boolean)", "public java.lang.String toJson(boolean, boolean)", + "public java.lang.String toJson(com.yahoo.tensor.serialization.JsonFormat$EncodeOptions)", "public java.lang.StringBuilder writeJson(java.lang.StringBuilder)", "public java.lang.Double getDouble(java.lang.String)", "public com.yahoo.tensor.Tensor getTensor(java.lang.String)", diff --git a/container-search/src/main/java/com/yahoo/search/query/Presentation.java b/container-search/src/main/java/com/yahoo/search/query/Presentation.java index 27e0a4f26ea0..c0ee5f7678f0 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Presentation.java +++ b/container-search/src/main/java/com/yahoo/search/query/Presentation.java @@ -80,6 +80,9 @@ public class Presentation implements Cloneable { /** Whether to renders tensors in short form */ private boolean tensorDirectValues = false; // TODO: Flip default on Vespa 9 + /** Whether to dense (part of) tensors in hex string form */ + private boolean tensorHexDense = false; + /** Set of explicitly requested summary fields, instead of summary classes */ private Set summaryFields = LazySet.newHashSet(); @@ -186,6 +189,20 @@ public void setSummaryFields(String asString) { */ public boolean getTensorShortForm() { return tensorShortForm; } + /** whether dense part of tensors should be represented as a string of hex digits */ + public boolean getTensorHexDense() { return tensorHexDense; } + + /** the current tensor format, see setTensorFormat() */ + public String getTensorFormat() { + String format = "long"; + if (tensorShortForm) format = "short"; + if (tensorHexDense) format = "hex"; + if (tensorDirectValues) { + return (format + "-value"); + } + return format; + } + /** @deprecated use setTensorFormat(). */ @Deprecated // TODO: Remove on Vespa 9 public void setTensorShortForm(String value) { @@ -199,6 +216,16 @@ public void setTensorShortForm(String value) { */ public void setTensorFormat(String value) { switch (value) { + case "hex" : + tensorHexDense = true; + tensorShortForm = true; + tensorDirectValues = false; + break; + case "hex-value" : + tensorHexDense = true; + tensorShortForm = true; + tensorDirectValues = true; + break; case "short" : tensorShortForm = true; tensorDirectValues = false; @@ -254,4 +281,3 @@ public int hashCode() { } } - diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index 19d92662d335..d0d35bb5bbb8 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -133,7 +133,7 @@ private static Map createPropertySetterMap() { map.put(CompoundName.fromComponents(Presentation.PRESENTATION, Presentation.FORMAT), GetterSetter.of(query -> query.getPresentation().getFormat(), (query, value) -> query.getPresentation().setFormat(asString(value, "")))); map.put(CompoundName.fromComponents(Presentation.PRESENTATION, Presentation.TIMING), GetterSetter.of(query -> query.getPresentation().getTiming(), (query, value) -> query.getPresentation().setTiming(asBoolean(value, true)))); map.put(CompoundName.fromComponents(Presentation.PRESENTATION, Presentation.SUMMARY_FIELDS), GetterSetter.of(query -> query.getPresentation().getSummaryFields(), (query, value) -> query.getPresentation().setSummaryFields(asString(value, "")))); - map.put(CompoundName.fromComponents(Presentation.PRESENTATION, Presentation.FORMAT, Presentation.TENSORS), GetterSetter.of(query -> query.getPresentation().getTensorShortForm(), (query, value) -> query.getPresentation().setTensorFormat(asString(value, "short")))); // TODO: Switch default to short-value on Vespa 9); + map.put(CompoundName.fromComponents(Presentation.PRESENTATION, Presentation.FORMAT, Presentation.TENSORS), GetterSetter.of(query -> query.getPresentation().getTensorFormat(), (query, value) -> query.getPresentation().setTensorFormat(asString(value, "short")))); // TODO: Switch default to short-value on Vespa 9); map.put(Query.HITS, GetterSetter.of(Query::getHits, (query, value) -> query.setHits(asInteger(value,10)))); map.put(Query.OFFSET, GetterSetter.of(Query::getOffset, (query, value) -> query.setOffset(asInteger(value,0)))); map.put(Query.TIMEOUT, GetterSetter.of(Query::getTimeout, (query, value) -> query.setTimeout(value.toString()))); diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java index 20937d2e1614..6f8cee70e843 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java @@ -132,8 +132,7 @@ static class FieldConsumerSettings { volatile boolean jsonWsets = true; volatile boolean jsonMapsAll = true; volatile boolean jsonWsetsAll = false; - volatile boolean tensorShortForm = true; - volatile boolean tensorDirectValues = false; + volatile JsonFormat.EncodeOptions tensorOptions; boolean convertDeep() { return (jsonDeepMaps || jsonWsets); } void init() { this.debugRendering = false; @@ -141,8 +140,7 @@ void init() { this.jsonWsets = true; this.jsonMapsAll = true; this.jsonWsetsAll = true; - this.tensorShortForm = true; - this.tensorDirectValues = false; + this.tensorOptions = new JsonFormat.EncodeOptions(true, false, false); } void getSettings(Query q) { if (q == null) { @@ -156,9 +154,11 @@ void getSettings(Query q) { // we may need more fine tuning, but for now use the same query parameters here: this.jsonMapsAll = props.getBoolean(WRAP_DEEP_MAPS, true); this.jsonWsetsAll = props.getBoolean(WRAP_WSETS, true); - this.tensorShortForm = q.getPresentation().getTensorShortForm(); - this.tensorDirectValues = q.getPresentation().getTensorDirectValues(); - } + this.tensorOptions = new JsonFormat.EncodeOptions( + q.getPresentation().getTensorShortForm(), + q.getPresentation().getTensorDirectValues(), + q.getPresentation().getTensorHexDense()); + } } private volatile FieldConsumerSettings fieldConsumerSettings; @@ -560,14 +560,16 @@ public static class FieldConsumer implements Hit.RawUtf8Consumer, TraceRenderer. /** Invoke this from your constructor when sub-classing {@link FieldConsumer} */ protected FieldConsumer(boolean debugRendering, boolean tensorShortForm, boolean jsonMaps) { - this(null, debugRendering, tensorShortForm, jsonMaps); + this(null, debugRendering, new JsonFormat.EncodeOptions(tensorShortForm, false, false), jsonMaps); } - private FieldConsumer(JsonGenerator generator, boolean debugRendering, boolean tensorShortForm, boolean jsonMaps) { + private FieldConsumer(JsonGenerator generator, boolean debugRendering, + JsonFormat.EncodeOptions tensorOptions, + boolean jsonMaps) { this.generator = generator; this.settings = new FieldConsumerSettings(); this.settings.debugRendering = debugRendering; - this.settings.tensorShortForm = tensorShortForm; + this.settings.tensorOptions = tensorOptions; this.settings.jsonDeepMaps = jsonMaps; } @@ -768,27 +770,27 @@ protected void renderFieldContents(Object field) throws IOException { public void accept(Object field) throws IOException { if (field == null) { generator().writeNull(); - } else if (field instanceof Boolean) { - generator().writeBoolean((Boolean)field); - } else if (field instanceof Number) { - renderNumberField((Number) field); - } else if (field instanceof TreeNode) { - generator().writeTree((TreeNode) field); - } else if (field instanceof Tensor) { - renderTensor(Optional.of((Tensor)field)); - } else if (field instanceof FeatureData) { - generator().writeRawValue(((FeatureData)field).toJson(settings.tensorShortForm, settings.tensorDirectValues)); - } else if (field instanceof Inspectable) { - renderInspectorDirect(((Inspectable)field).inspect()); - } else if (field instanceof JsonProducer) { - generator().writeRawValue(((JsonProducer) field).toJson()); - } else if (field instanceof StringFieldValue) { - generator().writeString(((StringFieldValue)field).getString()); - } else if (field instanceof TensorFieldValue) { - renderTensor(((TensorFieldValue)field).getTensor()); - } else if (field instanceof FieldValue) { - // the null below is the field which has already been written - ((FieldValue) field).serialize(null, new JsonWriter(generator)); + } else if (field instanceof Boolean bool) { + generator().writeBoolean(bool); + } else if (field instanceof Number num) { + renderNumberField(num); + } else if (field instanceof TreeNode treenode) { + generator().writeTree(treenode); + } else if (field instanceof Tensor t) { + renderTensor(Optional.of(t)); + } else if (field instanceof FeatureData featureData) { + generator().writeRawValue(featureData.toJson(settings.tensorOptions)); + } else if (field instanceof Inspectable i) { + renderInspectorDirect(i.inspect()); + } else if (field instanceof JsonProducer jp) { + generator().writeRawValue(jp.toJson()); + } else if (field instanceof StringFieldValue sfv) { + generator().writeString(sfv.getString()); + } else if (field instanceof TensorFieldValue tfv) { + renderTensor(tfv.getTensor()); + } else if (field instanceof FieldValue fv) { + // the null below is the field name which has already been written + fv.serialize(null, new JsonWriter(generator)); } else { generator().writeString(field.toString()); } @@ -797,27 +799,27 @@ public void accept(Object field) throws IOException { private void renderNumberField(Number field) throws IOException { if (field instanceof Integer) { generator().writeNumber(field.intValue()); - } else if (field instanceof Float) { + } else if (field instanceof Float) { generator().writeNumber(field.floatValue()); - } else if (field instanceof Double) { + } else if (field instanceof Double) { generator().writeNumber(field.doubleValue()); } else if (field instanceof Long) { generator().writeNumber(field.longValue()); } else if (field instanceof Byte || field instanceof Short) { generator().writeNumber(field.intValue()); - } else if (field instanceof BigInteger) { - generator().writeNumber((BigInteger) field); - } else if (field instanceof BigDecimal) { - generator().writeNumber((BigDecimal) field); + } else if (field instanceof BigInteger bigint) { + generator().writeNumber(bigint); + } else if (field instanceof BigDecimal bigdec) { + generator().writeNumber(bigdec); } else { generator().writeNumber(field.doubleValue()); } } private void renderTensor(Optional tensor) throws IOException { - generator().writeRawValue(new String(JsonFormat.encode(tensor.orElse(Tensor.Builder.of(TensorType.empty).build()), - settings.tensorShortForm, settings.tensorDirectValues), - StandardCharsets.UTF_8)); + var t = tensor.orElse(Tensor.Builder.of(TensorType.empty).build()); + byte[] json = JsonFormat.encode(t, settings.tensorOptions); + generator().writeRawValue(new String(json, StandardCharsets.UTF_8)); } private JsonGenerator generator() { diff --git a/container-search/src/main/java/com/yahoo/search/result/FeatureData.java b/container-search/src/main/java/com/yahoo/search/result/FeatureData.java index 55c8c6c28eeb..09b98623d9a0 100644 --- a/container-search/src/main/java/com/yahoo/search/result/FeatureData.java +++ b/container-search/src/main/java/com/yahoo/search/result/FeatureData.java @@ -42,9 +42,6 @@ public class FeatureData implements Inspectable, JsonProducer { /** The lazily computed feature names of this */ private Set featureNames = null; - /** The lazily computed json form of this */ - private String jsonForm = null; - public FeatureData(Inspector encodedValues) { this.encodedValues = Objects.requireNonNull(encodedValues); } @@ -71,40 +68,43 @@ public Inspector inspect() { @Override public String toJson() { - return toJson(false, false); + return toJson(new JsonFormat.EncodeOptions(false, false, false)); } public String toJson(boolean tensorShortForm) { - return toJson(tensorShortForm, false); + return toJson(new JsonFormat.EncodeOptions(tensorShortForm, false, false)); } public String toJson(boolean tensorShortForm, boolean tensorDirectValues) { - return writeJson(tensorShortForm, tensorDirectValues, new StringBuilder()).toString(); + return toJson(new JsonFormat.EncodeOptions(tensorShortForm, tensorDirectValues, false)); + } + + public String toJson(JsonFormat.EncodeOptions tensorOptions) { + return writeJson(tensorOptions, new StringBuilder()).toString(); } @Override public StringBuilder writeJson(StringBuilder target) { - return JsonRender.render(encodedValues, new Encoder(target, true, false, false)); + return writeJson(new JsonFormat.EncodeOptions(false, false, false), target); } - private StringBuilder writeJson(boolean tensorShortForm, boolean tensorDirectValues, StringBuilder target) { + private StringBuilder writeJson(JsonFormat.EncodeOptions tensorOptions, StringBuilder target) { if (this == empty) return target.append("{}"); - if (jsonForm != null) return target.append(jsonForm); if (encodedValues != null) - return JsonRender.render(encodedValues, new Encoder(target, true, tensorShortForm, tensorDirectValues)); + return JsonRender.render(encodedValues, new Encoder(target, true, tensorOptions)); else - return writeJson(values, tensorShortForm, tensorDirectValues, target); + return writeJson(values, tensorOptions, target); } - private StringBuilder writeJson(Map values, boolean tensorShortForm, boolean tensorDirectValues, StringBuilder target) { + private StringBuilder writeJson(Map values, JsonFormat.EncodeOptions tensorOptions, StringBuilder target) { target.append("{"); for (Map.Entry entry : values.entrySet()) { target.append("\"").append(entry.getKey()).append("\":"); if (entry.getValue().type().rank() == 0) { target.append(entry.getValue().asDouble()); } else { - byte[] encodedTensor = JsonFormat.encode(entry.getValue(), tensorShortForm, tensorDirectValues); + byte[] encodedTensor = JsonFormat.encode(entry.getValue(), tensorOptions); target.append(new String(encodedTensor, StandardCharsets.UTF_8)); } target.append(","); @@ -149,7 +149,7 @@ private Tensor decodeTensor(String featureName) { return switch (featureValue.type()) { case DOUBLE -> Tensor.from(featureValue.asDouble()); - case DATA -> TypedBinaryFormat.decode(Optional.empty(), GrowableByteBuffer.wrap(featureValue.asData())); + case DATA -> tensorFromData(featureValue.asData()); default -> throw new IllegalStateException("Unexpected feature value type " + featureValue.type()); }; } @@ -192,23 +192,24 @@ public boolean equals(Object other) { /** A JSON encoder which encodes DATA as a tensor */ private static class Encoder extends JsonRender.StringEncoder { - private final boolean tensorShortForm; - private final boolean tensorDirectValues; + private final JsonFormat.EncodeOptions tensorOptions; - Encoder(StringBuilder out, boolean compact, boolean tensorShortForm, boolean tensorDirectValues) { + Encoder(StringBuilder out, boolean compact, JsonFormat.EncodeOptions tensorOptions) { super(out, compact); - this.tensorShortForm = tensorShortForm; - this.tensorDirectValues = tensorDirectValues; + this.tensorOptions = tensorOptions; } @Override public void encodeDATA(byte[] value) { // This could be done more efficiently ... - Tensor tensor = TypedBinaryFormat.decode(Optional.empty(), GrowableByteBuffer.wrap(value)); - byte[] encodedTensor = JsonFormat.encode(tensor, tensorShortForm, tensorDirectValues); + Tensor tensor = tensorFromData(value); + byte[] encodedTensor = JsonFormat.encode(tensor, tensorOptions); target().append(new String(encodedTensor, StandardCharsets.UTF_8)); } } + private static Tensor tensorFromData(byte[] value) { + return TypedBinaryFormat.decode(Optional.empty(), GrowableByteBuffer.wrap(value)); + } } diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java index 611df6ad284b..a6cb04d0f09e 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java @@ -175,14 +175,14 @@ void testTensorRendering() throws ExecutionException, InterruptedException, IOEx "relevance":1.0, "fields":{ "tensor_standard":{"type":"tensor(x{},y{})","cells":[{"address":{"x":"a","y":"0"},"value":1.0},{"address":{"x":"b","y":"1"},"value":2.0}]}, - "tensor_indexed":{"type":"tensor(x[2],y[3])","values":[[1.0,2.0,3.0],[4.0,5.0,6.0]]}, + "tensor_indexed":{"type":"tensor(x[2],y[3])","values":[[1.0,2.0,3.0],[4.0,5.0,6.0]]}, "tensor_single_mapped":{"type":"tensor(x{})","cells":{"a":1.0,"b":2.0}}, - "tensor_mixed":{"type":"tensor(x{},y[2])","blocks":{"a":[1.0,2.0],"b":[3.0,4.0]}}, + "tensor_mixed":{"type":"tensor(x{},y[2])","blocks":{"a":[1.0,2.0],"b":[3.0,4.0]}}, "summaryfeatures":{ "tensor_standard":{"type":"tensor(x{},y{})","cells":[{"address":{"x":"a","y":"0"},"value":1.0},{"address":{"x":"b","y":"1"},"value":2.0}]}, - "tensor_indexed":{"type":"tensor(x[2],y[3])","values":[[1.0,2.0,3.0],[4.0,5.0,6.0]]}, + "tensor_indexed":{"type":"tensor(x[2],y[3])","values":[[1.0,2.0,3.0],[4.0,5.0,6.0]]}, "tensor_single_mapped":{"type":"tensor(x{})","cells":{"a":1.0,"b":2.0}}, - "tensor_mixed":{"type":"tensor(x{},y[2])","blocks":{"a":[1.0,2.0],"b":[3.0,4.0]}} + "tensor_mixed":{"type":"tensor(x{},y[2])","blocks":{"a":[1.0,2.0],"b":[3.0,4.0]}} } } }] @@ -202,14 +202,14 @@ void testTensorRendering() throws ExecutionException, InterruptedException, IOEx "relevance":1.0, "fields":{ "tensor_standard":{"type":"tensor(x{},y{})","cells":[{"address":{"x":"a","y":"0"},"value":1.0},{"address":{"x":"b","y":"1"},"value":2.0}]}, - "tensor_indexed":{"type":"tensor(x[2],y[3])","cells":[{"address":{"x":"0","y":"0"},"value":1.0},{"address":{"x":"0","y":"1"},"value":2.0},{"address":{"x":"0","y":"2"},"value":3.0},{"address":{"x":"1","y":"0"},"value":4.0},{"address":{"x":"1","y":"1"},"value":5.0},{"address":{"x":"1","y":"2"},"value":6.0}]}, + "tensor_indexed":{"type":"tensor(x[2],y[3])","cells":[{"address":{"x":"0","y":"0"},"value":1.0},{"address":{"x":"0","y":"1"},"value":2.0},{"address":{"x":"0","y":"2"},"value":3.0},{"address":{"x":"1","y":"0"},"value":4.0},{"address":{"x":"1","y":"1"},"value":5.0},{"address":{"x":"1","y":"2"},"value":6.0}]}, "tensor_single_mapped":{"type":"tensor(x{})","cells":[{"address":{"x":"a"},"value":1.0},{"address":{"x":"b"},"value":2.0}]}, - "tensor_mixed":{"type":"tensor(x{},y[2])","cells":[{"address":{"x":"a","y":"0"},"value":1.0},{"address":{"x":"a","y":"1"},"value":2.0},{"address":{"x":"b","y":"0"},"value":3.0},{"address":{"x":"b","y":"1"},"value":4.0}]}, + "tensor_mixed":{"type":"tensor(x{},y[2])","cells":[{"address":{"x":"a","y":"0"},"value":1.0},{"address":{"x":"a","y":"1"},"value":2.0},{"address":{"x":"b","y":"0"},"value":3.0},{"address":{"x":"b","y":"1"},"value":4.0}]}, "summaryfeatures":{ "tensor_standard":{"type":"tensor(x{},y{})","cells":[{"address":{"x":"a","y":"0"},"value":1.0},{"address":{"x":"b","y":"1"},"value":2.0}]}, - "tensor_indexed":{"type":"tensor(x[2],y[3])","cells":[{"address":{"x":"0","y":"0"},"value":1.0},{"address":{"x":"0","y":"1"},"value":2.0},{"address":{"x":"0","y":"2"},"value":3.0},{"address":{"x":"1","y":"0"},"value":4.0},{"address":{"x":"1","y":"1"},"value":5.0},{"address":{"x":"1","y":"2"},"value":6.0}]}, + "tensor_indexed":{"type":"tensor(x[2],y[3])","cells":[{"address":{"x":"0","y":"0"},"value":1.0},{"address":{"x":"0","y":"1"},"value":2.0},{"address":{"x":"0","y":"2"},"value":3.0},{"address":{"x":"1","y":"0"},"value":4.0},{"address":{"x":"1","y":"1"},"value":5.0},{"address":{"x":"1","y":"2"},"value":6.0}]}, "tensor_single_mapped":{"type":"tensor(x{})","cells":[{"address":{"x":"a"},"value":1.0},{"address":{"x":"b"},"value":2.0}]}, - "tensor_mixed":{"type":"tensor(x{},y[2])","cells":[{"address":{"x":"a","y":"0"},"value":1.0},{"address":{"x":"a","y":"1"},"value":2.0},{"address":{"x":"b","y":"0"},"value":3.0},{"address":{"x":"b","y":"1"},"value":4.0}]} + "tensor_mixed":{"type":"tensor(x{},y[2])","cells":[{"address":{"x":"a","y":"0"},"value":1.0},{"address":{"x":"a","y":"1"},"value":2.0},{"address":{"x":"b","y":"0"},"value":3.0},{"address":{"x":"b","y":"1"},"value":4.0}]} } } }] @@ -275,6 +275,73 @@ void testTensorRendering() throws ExecutionException, InterruptedException, IOEx assertTensorRendering(shortDirectJson, "short-value"); assertTensorRendering(longDirectJson, "long-value"); + String hexJson = """ + { + "root": { + "id": "toplevel", + "relevance": 1.0, + "fields": { + "totalCount": 1 + }, + "children": [ + { + "id": "tensors", + "relevance": 1.0, + "fields": { + "tensor_standard": { "type": "tensor(x{},y{})", "cells": [ { "address": { "x": "a", "y": "0" }, "value": 1.0 }, { "address": { "x": "b", "y": "1" }, "value": 2.0 } ] }, + "tensor_indexed": { "type": "tensor(x[2],y[3])", "values": "3F8040004040408040A040C0" }, + "tensor_single_mapped": { "type": "tensor(x{})", "cells": { "a": 1.0, "b": 2.0 } }, + "tensor_mixed": { "type": "tensor(x{},y[2])", "blocks": { "a": "3F804000", "b": "40404080" } }, + "summaryfeatures": { + "tensor_standard": { "type": "tensor(x{},y{})", "cells": [ { "address": { "x": "a", "y": "0" }, "value": 1.0 }, { "address": { "x": "b", "y": "1" }, "value": 2.0 } ] }, + "tensor_indexed": { "type": "tensor(x[2],y[3])", "values": "3F8040004040408040A040C0" }, + "tensor_single_mapped": { "type": "tensor(x{})", "cells": { "a": 1.0, "b": 2.0 } }, + "tensor_mixed": { "type": "tensor(x{},y[2])", "blocks": { "a": "3F804000", "b": "40404080" } } + } + } + } + ] + } + }"""; + + String hexDirectJson = """ + { + "root": { + "id": "toplevel", + "relevance": 1.0, + "fields": { + "totalCount": 1 + }, + "children": [ + { + "id": "tensors", + "relevance": 1.0, + "fields": { + "tensor_standard": [ + { "address": { "x": "a", "y": "0" }, "value": 1.0 }, + { "address": { "x": "b", "y": "1" }, "value": 2.0 } + ], + "tensor_indexed": "3F8040004040408040A040C0", + "tensor_single_mapped": { "a": 1.0, "b": 2.0 }, + "tensor_mixed": { "a": "3F804000", "b": "40404080" }, + "summaryfeatures": { + "tensor_standard": [ + { "address": { "x": "a", "y": "0" }, "value": 1.0 }, + { "address": { "x": "b", "y": "1" }, "value": 2.0 } + ], + "tensor_indexed": "3F8040004040408040A040C0", + "tensor_single_mapped": { "a": 1.0, "b": 2.0 }, + "tensor_mixed": { "a": "3F804000", "b": "40404080" } + } + } + } + ] + } + }"""; + + assertTensorRendering(hexJson, "hex"); + assertTensorRendering(hexDirectJson, "hex-value"); + try { render(new Result(new Query("/?presentation.format.tensors=unknown"))); fail("Expected exception"); @@ -285,20 +352,32 @@ void testTensorRendering() throws ExecutionException, InterruptedException, IOEx } } + static Tensor mmTensor() { + return Tensor.from("tensor(x{},y{}):{ {x:a,y:0}:1.0, {x:b,y:1}:2.0 }"); + } + static Tensor iiTensor() { + return Tensor.from("tensor(x[2],y[3]):[[1,2,3],[4,5,6]]"); + } + static Tensor mTensor() { + return Tensor.from("tensor(x{}):{ a:1, b:2 }"); + } + static Tensor miTensor() { + return Tensor.from("tensor(x{},y[2]):{a:[1,2], b:[3,4]}"); + } private void assertTensorRendering(String expected, String format) throws ExecutionException, InterruptedException, IOException { Slime slime = new Slime(); Cursor features = slime.setObject(); - features.setData("tensor_standard", TypedBinaryFormat.encode(Tensor.from("tensor(x{},y{}):{ {x:a,y:0}:1.0, {x:b,y:1}:2.0 }"))); - features.setData("tensor_indexed", TypedBinaryFormat.encode(Tensor.from("tensor(x[2],y[3]):[[1,2,3],[4,5,6]]"))); - features.setData("tensor_single_mapped", TypedBinaryFormat.encode(Tensor.from("tensor(x{}):{ a:1, b:2 }"))); - features.setData("tensor_mixed", TypedBinaryFormat.encode(Tensor.from("tensor(x{},y[2]):{a:[1,2], b:[3,4]}"))); + features.setData("tensor_standard", TypedBinaryFormat.encode(mmTensor())); + features.setData("tensor_indexed", TypedBinaryFormat.encode(iiTensor())); + features.setData("tensor_single_mapped", TypedBinaryFormat.encode(mTensor())); + features.setData("tensor_mixed", TypedBinaryFormat.encode(miTensor())); FeatureData summaryFeatures = new FeatureData(new SlimeAdapter(slime.get())); Hit h = new Hit("tensors"); - h.setField("tensor_standard", new TensorFieldValue(Tensor.from("tensor(x{},y{}):{ {x:a,y:0}:1.0, {x:b,y:1}:2.0 }"))); - h.setField("tensor_indexed", new TensorFieldValue(Tensor.from("tensor(x[2],y[3]):[[1,2,3],[4,5,6]]"))); - h.setField("tensor_single_mapped", new TensorFieldValue(Tensor.from("tensor(x{}):{ a:1, b:2 }"))); - h.setField("tensor_mixed", new TensorFieldValue(Tensor.from("tensor(x{},y[2]):{a:[1,2], b:[3,4]}"))); + h.setField("tensor_standard", new TensorFieldValue(mmTensor())); + h.setField("tensor_indexed", new TensorFieldValue(iiTensor())); + h.setField("tensor_single_mapped", new TensorFieldValue(mTensor())); + h.setField("tensor_mixed", new TensorFieldValue(miTensor())); h.setField("summaryfeatures", summaryFeatures); Result result1 = new Result(new Query("/?presentation.format.tensors=" + format)); @@ -311,6 +390,17 @@ private void assertTensorRendering(String expected, String format) throws Execut result2.hits().add(h); result2.setTotalHitCount(1L); assertEqualJson(expected, render(result2)); + + summaryFeatures = new FeatureData( + Map.of("tensor_standard", mmTensor(), + "tensor_indexed", iiTensor(), + "tensor_single_mapped", mTensor(), + "tensor_mixed", miTensor())); + h.setField("summaryfeatures", summaryFeatures); + Result result3 = new Result(new Query("/?presentation.format.tensors=" + format)); + result3.hits().add(h); + result3.setTotalHitCount(1L); + assertEqualJson(expected, render(result3)); } @Test @@ -1643,7 +1733,9 @@ private String render(Execution execution, Result r) throws InterruptedException private void assertEqualJson(String expected, String generated) { assertEquals("", validateJSON(expected)); assertEquals("", validateJSON(generated)); - assertEquals(JSON.canonical(expected), JSON.canonical(generated)); + if (! JSON.equals(expected, generated)) { + assertEquals(JSON.canonical(expected), JSON.canonical(generated)); + } } @SuppressWarnings("unchecked")