Skip to content

Commit

Permalink
Changes to support IP
Browse files Browse the repository at this point in the history
Signed-off-by: bharath-techie <[email protected]>
  • Loading branch information
bharath-techie committed Nov 14, 2024
1 parent e9f77e3 commit 217db78
Show file tree
Hide file tree
Showing 11 changed files with 164 additions and 23 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ public class StarTreeMapperIT extends OpenSearchIntegTestCase {
.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB))
.build();

private static XContentBuilder createMinimalTestMapping(boolean invalidDim, boolean invalidMetric, boolean ipdim) {
private static XContentBuilder createMinimalTestMapping(boolean invalidDim, boolean invalidMetric, boolean wildcard) {
try {
return jsonBuilder().startObject()
.startObject("composite")
Expand All @@ -68,7 +68,7 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool
.endObject()
.startArray("ordered_dimensions")
.startObject()
.field("name", getDim(invalidDim, ipdim))
.field("name", getDim(invalidDim, wildcard))
.endObject()
.startObject()
.field("name", "keyword_dv")
Expand Down Expand Up @@ -104,6 +104,10 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool
.endObject()
.startObject("ip")
.field("type", "ip")
.field("doc_values", true)
.endObject()
.startObject("wildcard")
.field("type", "wildcard")
.field("doc_values", false)
.endObject()
.endObject()
Expand Down Expand Up @@ -362,11 +366,11 @@ private XContentBuilder getMappingWithDuplicateFields(boolean isDuplicateDim, bo
return mapping;
}

private static String getDim(boolean hasDocValues, boolean isKeyword) {
private static String getDim(boolean hasDocValues, boolean isWildCard) {
if (hasDocValues) {
return random().nextBoolean() ? "numeric" : "keyword";
} else if (isKeyword) {
return "ip";
return random().nextBoolean() ? "numeric" : random().nextBoolean() ? "keyword" : "ip";
} else if (isWildCard) {
return "wildcard";
}
return "numeric_dv";
}
Expand Down Expand Up @@ -748,7 +752,7 @@ public void testUnsupportedDim() {
() -> prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createMinimalTestMapping(false, false, true)).get()
);
assertEquals(
"Failed to parse mapping [_doc]: unsupported field type associated with dimension [ip] as part of star tree field [startree-1]",
"Failed to parse mapping [_doc]: unsupported field type associated with dimension [wildcard] as part of star tree field [startree-1]",
ex.getMessage()
);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues;
import org.opensearch.index.mapper.CompositeMappedFieldType;
import org.opensearch.index.mapper.DocCountFieldMapper;
import org.opensearch.index.mapper.IpFieldMapper;
import org.opensearch.index.mapper.KeywordFieldMapper;
import org.opensearch.index.mapper.MapperService;

Expand Down Expand Up @@ -277,7 +278,8 @@ public SortedNumericDocValues getSortedNumeric(FieldInfo field) {
}

private boolean isSortedSetField(String field) {
return mapperService.fieldType(field) instanceof KeywordFieldMapper.KeywordFieldType;
return mapperService.fieldType(field) instanceof KeywordFieldMapper.KeywordFieldType
|| mapperService.fieldType(field) instanceof IpFieldMapper.IpFieldType;
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import java.util.stream.Collectors;

import static org.opensearch.index.compositeindex.datacube.DateDimension.CALENDAR_INTERVALS;
import static org.opensearch.index.compositeindex.datacube.IpDimension.IP;
import static org.opensearch.index.compositeindex.datacube.KeywordDimension.KEYWORD;

/**
Expand All @@ -46,6 +47,8 @@ public static Dimension parseAndCreateDimension(
return new NumericDimension(name);
case KEYWORD:
return new KeywordDimension(name);
case IP:
return new IpDimension(name);
default:
throw new IllegalArgumentException(
String.format(Locale.ROOT, "unsupported field type associated with dimension [%s] as part of star tree field", name)
Expand All @@ -71,6 +74,8 @@ public static Dimension parseAndCreateDimension(
return new NumericDimension(name);
case KEYWORD:
return new KeywordDimension(name);
case IP:
return new IpDimension(name);
default:
throw new IllegalArgumentException(
String.format(Locale.ROOT, "unsupported field type associated with star tree dimension [%s]", name)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,5 +33,11 @@ public enum DimensionType {
* Represents a keyword dimension type.
* This is used for dimensions that contain keyword ordinals.
*/
KEYWORD
KEYWORD,

/**
* Represents an IP dimension type.
* This is used for dimensions that contain IP ordinals.
*/
IP
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.index.compositeindex.datacube;

import org.apache.lucene.index.DocValuesType;
import org.opensearch.common.annotation.ExperimentalApi;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.index.mapper.CompositeDataCubeFieldType;

import java.io.IOException;
import java.util.List;
import java.util.Objects;
import java.util.function.Consumer;

/**
* Composite index keyword dimension class
*
* @opensearch.experimental
*/
@ExperimentalApi
public class IpDimension implements Dimension {
public static final String IP = "ip";
private final String field;

public IpDimension(String field) {
this.field = field;
}

@Override
public String getField() {
return field;
}

@Override
public int getNumSubDimensions() {
return 1;
}

@Override
public void setDimensionValues(Long value, Consumer<Long> dimSetter) {
// This will set the keyword dimension value's ordinal
dimSetter.accept(value);
}

@Override
public List<String> getSubDimensionNames() {
return List.of(field);
}

@Override
public DocValuesType getDocValuesType() {
return DocValuesType.SORTED_SET;
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(CompositeDataCubeFieldType.NAME, field);
builder.field(CompositeDataCubeFieldType.TYPE, IP);
builder.endObject();
return builder;
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
IpDimension dimension = (IpDimension) o;
return Objects.equals(field, dimension.getField());

Check warning on line 75 in server/src/main/java/org/opensearch/index/compositeindex/datacube/IpDimension.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/compositeindex/datacube/IpDimension.java#L74-L75

Added lines #L74 - L75 were not covered by tests
}

@Override
public int hashCode() {
return Objects.hash(field);

Check warning on line 80 in server/src/main/java/org/opensearch/index/compositeindex/datacube/IpDimension.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/compositeindex/datacube/IpDimension.java#L80

Added line #L80 was not covered by tests
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
import org.opensearch.common.collect.Tuple;
import org.opensearch.common.logging.DeprecationLogger;
import org.opensearch.common.network.InetAddresses;
import org.opensearch.index.compositeindex.datacube.DimensionType;
import org.opensearch.index.fielddata.IndexFieldData;
import org.opensearch.index.fielddata.ScriptDocValues;
import org.opensearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData;
Expand All @@ -62,6 +63,7 @@
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.BiFunction;
import java.util.function.Supplier;

Expand Down Expand Up @@ -155,6 +157,11 @@ public IpFieldMapper build(BuilderContext context) {
);
}

@Override
public Optional<DimensionType> getSupportedDataCubeDimensionType() {
return Optional.of(DimensionType.IP);
}

}

public static final TypeParser PARSER = new TypeParser((n, c) -> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.InetAddressPoint;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
Expand All @@ -25,6 +26,7 @@
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.BytesRef;
import org.opensearch.common.lucene.Lucene;
import org.opensearch.common.network.InetAddresses;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.index.codec.composite.CompositeIndexFieldInfo;
import org.opensearch.index.codec.composite.CompositeIndexReader;
Expand All @@ -36,6 +38,8 @@
import org.opensearch.index.mapper.NumberFieldMapper;

import java.io.IOException;
import java.net.InetAddress;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
Expand Down Expand Up @@ -65,12 +69,15 @@ public void testStarTreeKeywordDocValues() throws IOException {
doc.add(new SortedNumericDocValuesField("sndv", 1));
doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text1")));
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text2")));
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.10")))));
iw.addDocument(doc);
doc = new Document();
doc.add(new StringField("_id", "2", Field.Store.NO));
doc.add(new SortedNumericDocValuesField("sndv", 1));
doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text11")));
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text22")));
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.11")))));

iw.addDocument(doc);
iw.flush();
iw.deleteDocuments(new Term("_id", "2"));
Expand All @@ -80,12 +87,14 @@ public void testStarTreeKeywordDocValues() throws IOException {
doc.add(new SortedNumericDocValuesField("sndv", 2));
doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text1")));
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text2")));
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.10")))));
iw.addDocument(doc);
doc = new Document();
doc.add(new StringField("_id", "4", Field.Store.NO));
doc.add(new SortedNumericDocValuesField("sndv", 2));
doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text11")));
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text22")));
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.11")))));
iw.addDocument(doc);
iw.flush();
iw.deleteDocuments(new Term("_id", "4"));
Expand Down Expand Up @@ -166,6 +175,9 @@ public void testStarTreeKeywordDocValuesWithDeletions() throws IOException {

doc.add(new SortedSetDocValuesField("keyword2", new BytesRef(keyword2Value)));
map.put(keyword1Value + "-" + keyword2Value, sndvValue + map.getOrDefault(keyword1Value + "-" + keyword2Value, 0));
doc.add(
new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10." + i))))
);
iw.addDocument(doc);
documents.put(id, doc);
}
Expand Down Expand Up @@ -221,9 +233,7 @@ public void testStarTreeKeywordDocValuesWithDeletions() throws IOException {
SortedSetStarTreeValuesIterator k1 = (SortedSetStarTreeValuesIterator) starTreeValues.getDimensionValuesIterator(
"keyword1"
);
SortedSetStarTreeValuesIterator k2 = (SortedSetStarTreeValuesIterator) starTreeValues.getDimensionValuesIterator(
"keyword2"
);
SortedSetStarTreeValuesIterator k2 = (SortedSetStarTreeValuesIterator) starTreeValues.getDimensionValuesIterator("ip1");
for (StarTreeDocument starDoc : actualStarTreeDocuments) {
String keyword1 = null;
if (starDoc.dimensions[0] != null) {
Expand All @@ -232,7 +242,11 @@ public void testStarTreeKeywordDocValuesWithDeletions() throws IOException {

String keyword2 = null;
if (starDoc.dimensions[1] != null) {
keyword2 = k2.lookupOrd(starDoc.dimensions[1]).utf8ToString();
BytesRef encoded = k2.lookupOrd(starDoc.dimensions[1]);
InetAddress address = InetAddressPoint.decode(
Arrays.copyOfRange(encoded.bytes, encoded.offset, encoded.offset + encoded.length)
);
keyword2 = InetAddresses.toAddrString(address);
}
double metric = (double) starDoc.metrics[0];
if (map.containsKey(keyword1 + "-" + keyword2)) {
Expand All @@ -254,21 +268,28 @@ public void testStarKeywordDocValuesWithMissingDocs() throws IOException {
Document doc = new Document();
doc.add(new SortedNumericDocValuesField("sndv", 1));
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text2")));
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.10")))));

iw.addDocument(doc);
doc = new Document();
doc.add(new SortedNumericDocValuesField("sndv", 1));
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text22")));
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.11")))));
iw.addDocument(doc);
iw.forceMerge(1);
doc = new Document();
doc.add(new SortedNumericDocValuesField("sndv", 2));
doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text1")));
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text2")));
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.10")))));

iw.addDocument(doc);
doc = new Document();
doc.add(new SortedNumericDocValuesField("sndv", 2));
doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text11")));
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text22")));
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.11")))));

iw.addDocument(doc);
iw.forceMerge(1);
iw.close();
Expand Down Expand Up @@ -340,11 +361,14 @@ public void testStarKeywordDocValuesWithMissingDocsInSegment() throws IOExceptio
doc.add(new SortedNumericDocValuesField("sndv", 2));
doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text1")));
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text2")));
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.10")))));
iw.addDocument(doc);
doc = new Document();
doc.add(new SortedNumericDocValuesField("sndv", 2));
doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text11")));
doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text22")));
doc.add(new SortedSetDocValuesField("ip1", new BytesRef(InetAddressPoint.encode(InetAddresses.forString("10.10.10.11")))));

iw.addDocument(doc);
iw.forceMerge(1);
iw.close();
Expand Down Expand Up @@ -538,7 +562,7 @@ protected XContentBuilder getMapping() throws IOException {
b.field("name", "keyword1");
b.endObject();
b.startObject();
b.field("name", "keyword2");
b.field("name", "ip1");
b.endObject();
b.endArray();
b.startArray("metrics");
Expand Down Expand Up @@ -566,6 +590,9 @@ protected XContentBuilder getMapping() throws IOException {
b.startObject("keyword2");
b.field("type", "keyword");
b.endObject();
b.startObject("ip1");
b.field("type", "ip");
b.endObject();
b.endObject();
});
}
Expand Down
Loading

0 comments on commit 217db78

Please sign in to comment.