diff --git a/.gitignore b/.gitignore index 96cbff8d90..b8638f2e72 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ gradle-app.setting .ci/output java-client/bin +samples/bin \ No newline at end of file diff --git a/USER_GUIDE.md b/USER_GUIDE.md index aba3e6f193..c0d55b07ff 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -1,89 +1,52 @@ -# User Guide - -- [User Guide](#user-guide) - - [Sample data](#sample-data) - - [IndexData class](#indexdata-class) - - [Create a client](#create-a-client) - - [Create a client using `RestClientTransport`](#create-a-client-using-restclienttransport) - - [Create a client using `ApacheHttpClient5Transport`](#create-a-client-using-apachehttpclient5transport) - - [Create an index](#create-an-index) - - [Create an index with default settings](#create-an-index-with-default-settings) - - [Create an index with custom settings and mappings](#create-an-index-with-custom-settings-and-mappings) - - [Index data](#index-data) - - [Search for the documents](#search-for-the-documents) - - [Get raw JSON results](#get-raw-json-results) - - [Search documents using a match query](#search-documents-using-a-match-query) - - [Search documents using k-NN](#search-documents-using-k-nn) - - [Exact k-NN with scoring script](#exact-k-nn-with-scoring-script) - - [Exact k-NN with painless scripting extension](#exact-k-nn-with-painless-scripting-extension) - - [Search documents using suggesters](#search-documents-using-suggesters) - - [App Data class](#app-data-class) - - [Using completion suggester](#using-completion-suggester) - - [Using term suggester](#using-term-suggester) - - [Using phrase suggester](#using-phrase-suggester) - - [Bulk requests](#bulk-requests) - - [Aggregations](#aggregations) - - [Delete the document](#delete-the-document) - - [Delete the index](#delete-the-index) - - [Data Stream API](#data-stream-api) - - [Create a data stream](#create-a-data-stream) - - [Get data stream](#get-data-stream) - - [Data stream stats](#data-stream-stats) - - [Delete data stream and backing indices](#delete-data-stream-and-backing-indices) - - [Point-In-Time API](#point-in-time-api) - - [Creating a point in time](#creating-a-point-in-time) - - [List all point in time](#list-all-point-in-time) - - [Delete point in time](#delete-point-in-time) - - [Cat API](#cat-api) - - [Cat Indices](#cat-indices) - - [Cat aliases](#cat-aliases) - - [Cat nodes](#cat-nodes) - - [Cat point in time segments](#cat-point-in-time-segments) -- [Using different transport options](#using-different-transport-options) - - [Amazon Managed OpenSearch](#amazon-managed-opensearch) - -## Sample data - -### IndexData class +- [OpenSearch Java Client User Guide](#opensearch-java-client-user-guide) + - [Setup](#setup) + - [Basic Features](#basic-features) + - [Creating a client](#creating-a-client) + - [Using `RestClientTransport`](#using-restclienttransport) + - [Using `ApacheHttpClient5Transport`](#using-apachehttpclient5transport) + - [Creating an index](#creating-an-index) + - [With default settings](#with-default-settings) + - [With custom settings and mappings](#with-custom-settings-and-mappings) + - [Indexing data](#indexing-data) + - [Searching for a document](#searching-for-a-document) + - [Deleting a document](#deleting-a-document) + - [Deleting an index](#deleting-an-index) + - [Advanced Features](#advanced-features) + - [Plugins](#plugins) -```java -static class IndexData { - private String firstName; - private String lastName; - - public IndexData(String firstName, String lastName) { - this.firstName = firstName; - this.lastName = lastName; - } - - public String getFirstName() { - return firstName; - } - - public void setFirstName(String firstName) { - this.firstName = firstName; - } - - public String getLastName() { - return lastName; - } - - public void setLastName(String lastName) { - this.lastName = lastName; - } - - @Override - public String toString() { - return String.format("IndexData{first name='%s', last name='%s'}", firstName, lastName); - } +# OpenSearch Java Client User Guide + +## Setup + +To start using the OpenSearch Java client, you need to provide a transport. The default `ApacheHttpClient5TransportBuilder` transport comes with the Java client. To use the OpenSearch Java client with the default transport, add it to your `pom.xml` file as a dependency: + +``` + + org.opensearch.client + opensearch-java + 2.6.0 + +``` + +If you’re using Gradle, add the following dependencies to your project: + +``` +dependencies { + implementation 'org.opensearch.client:opensearch-java:2.6.0' } ``` -## Create a client +## Basic Features + +In the example below, we create a client, create an index with default and non-default settings, insert a document into the index, search for the document, delete the document, and finally delete the index. + +You can find working versions of the code below that can be run with a local instance of OpenSearch in [samples](./samples/). + +### Creating a client There are multiple low level transports which `OpenSearchClient` could be configured with. -### Create a client using `RestClientTransport` +#### Using `RestClientTransport` ```java import org.apache.http.HttpHost; @@ -109,7 +72,7 @@ Transport transport = new RestClientTransport(restClient, OpenSearchClient client = new OpenSearchClient(transport); ``` -### Create a client using `ApacheHttpClient5Transport` +#### Using `ApacheHttpClient5Transport` ```java import org.apache.hc.core5.http.HttpHost; @@ -132,9 +95,13 @@ The Apache HttpClient 5 based transport has dependences on Apache HttpClient 5 a implementation("org.apache.httpcomponents.core5", "httpcore5", "5.2.2") ``` -## Create an index +Upcoming OpenSearch `3.0.0` release brings HTTP/2 support and as such, the `RestClientTransport` would switch to HTTP/2 if available (for both HTTPS and/or HTTP protocols). The desired protocol could be forced using `RestClientBuilder.HttpClientConfigCallback`. -### Create an index with default settings +See [SampleClient.java](./samples/src/main/java/org/opensearch/client/samples/SampleClient.java) for a working sample. + +### Creating an index + +#### With default settings ```java String index = "sample-index"; @@ -142,7 +109,7 @@ CreateIndexRequest createIndexRequest = new CreateIndexRequest.Builder().index(i client.indices().create(createIndexRequest); ``` -### Create an index with custom settings and mappings +#### With custom settings and mappings ```java String index = "sample-index"; @@ -161,19 +128,21 @@ CreateIndexRequest createIndexRequest = new CreateIndexRequest.Builder() client.indices().create(createIndexRequest); ``` -## Index data +### Indexing data + +[IndexData](./samples/src/main/java/org/opensearch/client/samples/util/IndexData.java) refers to sample data class. ```java -IndexData indexData = new IndexData("John", "Doe"); +IndexData indexData = new IndexData("Document 1", "Text for document 1"); IndexRequest indexRequest = new IndexRequest.Builder().index(index).id("1").document(indexData).build(); client.index(indexRequest); -indexData = new IndexData("John", "Joe"); +indexData = new IndexData("Document 2", "Text for document 2"); indexRequest = new IndexRequest.Builder().index(index).id("2").document(indexData).build(); client.index(indexRequest); ``` -## Search for the documents +### Searching for a document ```java SearchResponse searchResponse = client.search(s -> s.index(index), IndexData.class); @@ -182,444 +151,7 @@ for (int i = 0; i < searchResponse.hits().hits().size(); i++) { } ``` -### Get raw JSON results - -When the target class is not defined or if the response result is a semi-structured data not tied to an object definition, use a raw JSON data representation as the target class. For example, the below snippet uses `ObjectNode` from jackson. - -```java -SearchResponse searchResponse = client.search(b -> b.index(index), ObjectNode.class); -for (int i = 0; i < searchResponse.hits().hits().size(); i++) { - System.out.println(searchResponse.hits().hits().get(i).source()); -} -``` - -## Search documents using a match query - -```java -SearchRequest searchRequest = new SearchRequest.Builder().query(q -> q.match(m -> m.field("firstName") - .query(FieldValue.of("John")))) - .build(); - -SearchResponse searchResponse = client.search(searchRequest, IndexData.class); -for (int i = 0; i < searchResponse.hits().hits().size(); i++) { - System.out.println(searchResponse.hits().hits().get(i).source()); -} -``` - -## Search documents using k-NN - -### Exact k-NN with scoring script - -1. Create index with custom mapping - -```java -String index = "my-knn-index-1"; -TypeMapping mapping = new TypeMapping.Builder() - .properties("my_vector", new Property.Builder() - .knnVector(new KnnVectorProperty.Builder() - .dimension(4) - .build()) - .build()) - .build(); -CreateIndexRequest createIndexRequest = new CreateIndexRequest.Builder() - .index(index) - .mappings(mapping) - .build(); -client.indices().create(createIndexRequest); -``` - -2. Index documents - -```java -JsonObject doc1 = Json.createObjectBuilder() - .add("my_vector", Json.createArrayBuilder().add(1.5).add(5.5).add(4.5).add(6.4).build()) - .add("price", 10.3) - .build(); -JsonObject doc2 = Json.createObjectBuilder() - .add("my_vector", Json.createArrayBuilder().add(2.5).add(3.5).add(5.6).add(6.7).build()) - .add("price", 5.5) - .build(); -JsonObject doc3 = Json.createObjectBuilder() - .add("my_vector", Json.createArrayBuilder().add(4.5).add(5.5).add(6.7).add(3.7).build()) - .add("price", 4.4) - .build(); - -ArrayList operations = new ArrayList<>(); -operations.add(new BulkOperation.Builder().index( - IndexOperation.of(io -> io.index(index).id("1").document(doc1)) - ).build()); -operations.add(new BulkOperation.Builder().index( - IndexOperation.of(io -> io.index(index).id("2").document(doc2)) - ).build()); -operations.add(new BulkOperation.Builder().index( - IndexOperation.of(io -> io.index(index).id("3").document(doc3)) - ).build()); - -BulkRequest bulkRequest = new BulkRequest.Builder() - .index(index) - .operations(operations) - .build(); -client.bulk(bulkRequest); -``` - -3. Search documents using k-NN script score (_This implementation utilizes `com.fasterxml.jackson.databind.JsonNode` as the target document class, which is not part of the OpenSearch Java library. However, any document class that matches the searched data can be used instead._) - -```java -InlineScript inlineScript = new InlineScript.Builder() - .source("knn_score") - .lang("knn") - .params(Map.of( - "field", JsonData.of("my_vector"), - "query_value", JsonData.of(List.of(1.5, 5.5, 4.5, 6.4)), - "space_type", JsonData.of("cosinesimil") - )) - .build(); -Query query = new Query.Builder() - .scriptScore(new ScriptScoreQuery.Builder() - .query(new Query.Builder() - .matchAll(new MatchAllQuery.Builder().build()) - .build()) - .script(new Script.Builder() - .inline(inlineScript) - .build()) - .build()) - .build(); -SearchRequest searchRequest = new SearchRequest.Builder() - .index(index) - .query(query) - .build(); -SearchResponse searchResponse = client.search(searchRequest, JsonNode.class); -``` - -### Exact k-NN with painless scripting extension - -1. Create index with custom mapping - -```java -String index = "my-knn-index-1"; -TypeMapping mapping = new TypeMapping.Builder() - .properties("my_vector", new Property.Builder() - .knnVector(new KnnVectorProperty.Builder() - .dimension(4) - .build()) - .build()) - .build(); -CreateIndexRequest createIndexRequest = new CreateIndexRequest.Builder() - .index(index) - .mappings(mapping) - .build(); -client.indices().create(createIndexRequest); -``` - -2. Index documents - -```java -JsonObject doc1 = Json.createObjectBuilder() - .add("my_vector", Json.createArrayBuilder().add(1.5).add(5.5).add(4.5).add(6.4).build()) - .add("price", 10.3) - .build(); -JsonObject doc2 = Json.createObjectBuilder() - .add("my_vector", Json.createArrayBuilder().add(2.5).add(3.5).add(5.6).add(6.7).build()) - .add("price", 5.5) - .build(); -JsonObject doc3 = Json.createObjectBuilder() - .add("my_vector", Json.createArrayBuilder().add(4.5).add(5.5).add(6.7).add(3.7).build()) - .add("price", 4.4) - .build(); - -ArrayList operations = new ArrayList<>(); -operations.add(new BulkOperation.Builder().index( - IndexOperation.of(io -> io.index(index).id("1").document(doc1)) - ).build()); -operations.add(new BulkOperation.Builder().index( - IndexOperation.of(io -> io.index(index).id("2").document(doc2)) - ).build()); -operations.add(new BulkOperation.Builder().index( - IndexOperation.of(io -> io.index(index).id("3").document(doc3)) - ).build()); - -BulkRequest bulkRequest = new BulkRequest.Builder() - .index(index) - .operations(operations) - .build(); -client.bulk(bulkRequest); -``` - -3. Search documents using k-NN with painless scripting extension (_This implementation utilizes `com.fasterxml.jackson.databind.JsonNode` as the target document class, which is not part of the OpenSearch Java library. However, any document class that matches the searched data can be used instead._) - -```java -InlineScript inlineScript = new InlineScript.Builder() - .source("1.0 + cosineSimilarity(params.query_value, doc[params.field])") - .params(Map.of( - "field", JsonData.of("my_vector"), - "query_value", JsonData.of(List.of(1.5, 5.5, 4.5, 6.4)) - )) - .build(); -Query query = new Query.Builder() - .scriptScore(new ScriptScoreQuery.Builder() - .query(new Query.Builder() - .matchAll(new MatchAllQuery.Builder().build()) - .build()) - .script(new Script.Builder() - .inline(inlineScript) - .build()) - .build()) - .build(); -SearchRequest searchRequest = new SearchRequest.Builder() - .index(index) - .query(query) - .build(); -SearchResult searchResult = client.search(searchRequest, JsonNode.class); -``` - -## Search documents using suggesters - -### App Data class - -```java -public static class AppData { - - private int intValue; - private String msg; - - public int getIntValue() { - return intValue; - } - - public void setIntValue(int intValue) { - this.intValue = intValue; - } - - public String getMsg() { - return msg; - } - - public void setMsg(String msg) { - this.msg = msg; - } -} -``` - -### Using completion suggester - -```java -String index = "completion-suggester"; - -Property intValueProp = new Property.Builder() - .long_(v -> v) - .build(); -Property msgCompletionProp = new Property.Builder() - .completion(c -> c) - .build(); -client.indices().create(c -> c - .index(index) - .mappings(m -> m - .properties("intValue", intValueProp) - .properties("msg", msgCompletionProp))); - -AppData appData = new AppData(); -appData.setIntValue(1337); -appData.setMsg("foo"); - -client.index(b -> b - .index(index) - .id("1") - .document(appData) - .refresh(Refresh.True)); - -appData.setIntValue(1338); -appData.setMsg("foobar"); - -client.index(b -> b - .index(index) - .id("2") - .document(appData) - .refresh(Refresh.True)); - -String suggesterName = "msgSuggester"; - -CompletionSuggester completionSuggester = FieldSuggesterBuilders.completion() - .field("msg") - .size(1) - .build(); -FieldSuggester fieldSuggester = new FieldSuggester.Builder().prefix("foo") - .completion(completionSuggester) - .build(); -Suggester suggester = new Suggester.Builder() - .suggesters(Collections.singletonMap(suggesterName, fieldSuggester)) - .build(); -SearchRequest searchRequest = new SearchRequest.Builder() - .index(index) - .suggest(suggester) - .build(); - -SearchResponse response = client.search(searchRequest, AppData.class); -``` - -### Using term suggester - -```java - String index = "term-suggester"; - -// term suggester does not require a special mapping -client.indices().create(c -> c - .index(index)); - -AppData appData = new AppData(); -appData.setIntValue(1337); -appData.setMsg("foo"); - -client.index(b -> b - .index(index) - .id("1") - .document(appData) - .refresh(Refresh.True)); - -appData.setIntValue(1338); -appData.setMsg("foobar"); - -client.index(b -> b - .index(index) - .id("2") - .document(appData) - .refresh(Refresh.True)); - -String suggesterName = "msgSuggester"; - -TermSuggester termSuggester = FieldSuggesterBuilders.term() - .field("msg") - .size(1) - .build(); -FieldSuggester fieldSuggester = new FieldSuggester.Builder().text("fool") - .term(termSuggester) - .build(); -Suggester suggester = new Suggester.Builder() - .suggesters(Collections.singletonMap(suggesterName, fieldSuggester)) - .build(); -SearchRequest searchRequest = new SearchRequest.Builder() - .index(index) - .suggest(suggester) - .build(); - -SearchResponse response = client.search(searchRequest, AppData.class); -``` - -### Using phrase suggester - -```java -String index = "test-phrase-suggester"; - -ShingleTokenFilter shingleTokenFilter = new ShingleTokenFilter.Builder().minShingleSize("2") - .maxShingleSize("3") - .build(); - -Analyzer analyzer = new Analyzer.Builder() - .custom(new CustomAnalyzer.Builder().tokenizer("standard") - .filter(Arrays.asList("lowercase", "shingle")).build()) - .build(); - -TokenFilter tokenFilter = new TokenFilter.Builder() - .definition(new TokenFilterDefinition.Builder() - .shingle(shingleTokenFilter).build()) - .build(); - -IndexSettingsAnalysis indexSettingsAnalysis = new IndexSettingsAnalysis.Builder() - .analyzer("trigram", analyzer) - .filter("shingle", tokenFilter) - .build(); - -IndexSettings settings = new IndexSettings.Builder().analysis(indexSettingsAnalysis).build(); - -TypeMapping mapping = new TypeMapping.Builder().properties("msg", new Property.Builder() - .text(new TextProperty.Builder().fields("trigram", new Property.Builder() - .text(new TextProperty.Builder().analyzer("trigram").build()) - .build()).build()) - .build()).build(); - -client.indices().create(c -> c - .index(index) - .settings(settings) - .mappings(mapping)); - -AppData appData = new AppData(); -appData.setIntValue(1337); -appData.setMsg("Design Patterns"); - -client.index(b -> b - .index(index) - .id("1") - .document(appData) - .refresh(Refresh.True)); - -appData.setIntValue(1338); -appData.setMsg("Software Architecture Patterns Explained"); - -client.index(b -> b - .index(index) - .id("2") - .document(appData) - .refresh(Refresh.True)); - -String suggesterName = "msgSuggester"; - -PhraseSuggester phraseSuggester = FieldSuggesterBuilders.phrase() - .field("msg.trigram") - .build(); -FieldSuggester fieldSuggester = new FieldSuggester.Builder().text("design paterns") - .phrase(phraseSuggester) - .build(); -Suggester suggester = new Suggester.Builder() - .suggesters(Collections.singletonMap(suggesterName, fieldSuggester)) - .build(); -SearchRequest searchRequest = new SearchRequest.Builder() - .index(index) - .suggest(suggester) - .build(); - -SearchResponse response = client.search(searchRequest, AppData.class); -``` - -## Bulk requests - -```java -ArrayList ops = new ArrayList<>(); -SimplePojo doc1 = new SimplePojo("Document 1", "The text of document 1"); -ops.add(new BulkOperation.Builder().index( - IndexOperation.of(io -> io.index(TEST_INDEX).id("id1").document(doc1)) -).build()); -SimplePojo doc2 = new SimplePojo("Document 2", "The text of document 2"); -ops.add(new BulkOperation.Builder().index( - IndexOperation.of(io -> io.index(TEST_INDEX).id("id2").document(doc2)) -).build()); -SimplePojo doc3 = getLongDoc("Long Document 3", 100000); -ops.add(new BulkOperation.Builder().index( - IndexOperation.of(io -> io.index(TEST_INDEX).id("id3").document(doc3)) -).build()); - -BulkRequest.Builder bulkReq = new BulkRequest.Builder() - .index(index) - .operations(ops) - .refresh(Refresh.WaitFor); -BulkResponse bulkResponse = client.bulk(bulkReq.build()); -``` - -## Aggregations - -```java -SearchRequest searchRequest = new SearchRequest.Builder().query(q -> q.match(m -> m.field("firstName") - .query(FieldValue.of("John")))) - .aggregations("firstNames", new Aggregation.Builder().terms(t -> t.field("firstName.keyword")) - .build()) - .build(); - -SearchResponse searchResponse = client.search(searchRequest, IndexData.class); -for (Map.Entry entry : searchResponse.aggregations().entrySet()) { - System.out.println("Agg - " + entry.getKey()); - entry.getValue().sterms().buckets().array().forEach(b -> System.out.printf("%s : %d%n", b.key(), b.docCount())); -} -``` - -## Delete the document +### Deleting a document The following sample code deletes a document whose ID is 1. @@ -627,159 +159,24 @@ The following sample code deletes a document whose ID is 1. client.delete(d -> d.index(index).id("1")); ``` -## Delete the index +## Deleting an index ```java -DeleteIndexRequest deleteIndexRequest = new DeleteRequest.Builder().index(index).build(); +DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest.Builder().index(index).build(); DeleteIndexResponse deleteIndexResponse = client.indices().delete(deleteIndexRequest); ``` -## Data Stream API - -### Create a data stream +You can find a working sample of the above code in [IndexingBasics.java](./samples/src/main/java/org/opensearch/client/samples/IndexingBasics.java). -Before creating a data stream, you need to create an index template which configures a set of indices as a data stream. -A data stream must have a timestamp field. If not specified, OpenSearch uses `@timestamp` as the default timestamp field name. +## Advanced Features -The following sample code creates an index template for data stream with a custom timestamp field, and creates a data stream -which matches the name pattern specified in the index template. +- [Authentication (IAM, SigV4)](./guides/auth.md) +- [Bulk Indexing](./guides/bulk.md) +- [Cat APIs](./guides/cat.md) +- [Data Stream APIs](./guides/data_stream.md) +- [Point-in-Time APIs](./guides/point_in_time.md) +- [Search](./guides/search.md) -```java -String dataStreamIndexTemplateName = "sample-data-stream-template"; -String timestampFieldName = "my_timestamp_field"; -String namePattern = "sample-data-stream-*"; -String dataStreamName = "sample-data-stream-1"; - -// Create an index template which configures data stream -PutIndexTemplateRequest putIndexTemplateRequest = new PutIndexTemplateRequest.Builder() - .name(dataStreamIndexTemplateName) - .indexPatterns(namePattern) - .dataStream(new DataStream.Builder() - .timestampField(t -> t.name(timestampFieldName)) - .build()) - .build(); -PutIndexTemplateResponse putIndexTemplateResponse = javaClient().indices().putIndexTemplate(putIndexTemplateRequest); +## Plugins -// Create a data stream -CreateDataStreamRequest createDataStreamRequest = new CreateDataStreamRequest.Builder().name(dataStreamName).build(); -CreateDataStreamResponse createDataStreamResponse = javaClient().indices().createDataStream(createDataStreamRequest); -``` - -### Get data stream - -```java -GetDataStreamRequest getDataStreamRequest = new GetDataStreamRequest.Builder().name(dataStreamName).build(); -GetDataStreamResponse getDataStreamResponse = javaClient().indices().getDataStream(getDataStreamRequest); -``` - -### Data stream stats - -```java -DataStreamsStatsRequest dataStreamsStatsRequest = new DataStreamsStatsRequest.Builder().name(dataStreamName).build(); -DataStreamsStatsResponse dataStreamsStatsResponse = javaClient().indices().dataStreamsStats(dataStreamsStatsRequest); -``` - -### Delete data stream and backing indices - -```java -DeleteDataStreamRequest deleteDataStreamRequest = new DeleteDataStreamRequest.Builder().name(dataStreamName).build(); -DeleteDataStreamResponse deleteDataStreamResponse = javaClient().indices().deleteDataStream(deleteDataStreamRequest); -``` - -## Point-In-Time API - -### Creating a point in time - -Creates a PIT. The keep_alive query parameter is required; it specifies how long to keep a PIT. - -```java -CreatePitRequest createPitRequest = new CreatePitRequest.Builder() - .targetIndexes(Collections.singletonList(index)) - .keepAlive(new Time.Builder().time("100m").build()).build(); - -CreatePitResponse createPitResponse = javaClient() - .createPit(createPitRequest); -``` - -### List all point in time - -Returns all PITs in the OpenSearch cluster. - -```java -ListAllPitResponse listAllPitResponse = javaClient().listAllPit(); -``` - -### Delete point in time - -Deletes one, several, or all PITs. PITs are automatically deleted when the keep_alive time period elapses. However, to deallocate resources, you can delete a PIT using the Delete PIT API. The Delete PIT API supports deleting a list of PITs by ID or deleting all PITs at once. - -```java -DeletePitRequest deletePitRequest = new DeletePitRequest.Builder() - .pitId(Collections.singletonList("pit_id")).build(); - -DeletePitResponse deletePitResponse = javaClient() - .deletePit(deletePitRequest); -``` - -## Cat API - -### Cat Indices - -The following sample code cat indices with required headers and sorted by creation date - -```java -IndicesRequest indicesRequest = new IndicesRequest.Builder() - .headers("index,health,status,pri,rep,doc.count,creation.date,creation.date.string").sort("creation.date").build(); -IndicesResponse indicesResponse = javaClient().cat().indices(indicesRequest); -``` - -### Cat aliases - -The following sample code cat aliases with name "test-alias" and sorted by index - -```java -AliasesRequest aliasesRequest = new AliasesRequest.Builder().name("test-alias").sort("index").build(); -AliasesResponse aliasesResponse = javaClient().cat().aliases(aliasesRequest); -``` - -### Cat nodes - -The following sample code cat nodes sorted by cpu - -```java -NodesResponse nodesResponse = javaClient().cat().nodes(r -> r.sort("cpu")); -``` - -### Cat point in time segments - -Similarly to the CAT Segments API, the PIT Segments API provides low-level information about the disk utilization of a PIT by describing its Lucene segments. - -```java -SegmentsResponse pitSegmentsResponse = javaClient().cat() - .pitSegments(r -> r.headers("index,shard,id,segment,size")); -``` - -# Using different transport options - -## Amazon Managed OpenSearch - -Use `AwsSdk2Transport` to make requests to Amazon Managed OpenSearch and OpenSearch Serverless. - -```java -SdkHttpClient httpClient = ApacheHttpClient.builder().build(); - -OpenSearchClient client = new OpenSearchClient( - new AwsSdk2Transport( - httpClient, - "search-...us-west-2.es.amazonaws.com", // OpenSearch endpoint, without https:// - "es" // signing service name, use "aoss" for OpenSearch Serverless - Region.US_WEST_2, // signing service region - AwsSdk2TransportOptions.builder().build() - ) -); - -InfoResponse info = client.info(); -System.out.println(info.version().distribution() + ": " + info.version().number()); - -httpClient.close(); -``` +- [k-NN](guides/plugins/knn.md) \ No newline at end of file diff --git a/guides/auth.md b/guides/auth.md new file mode 100644 index 0000000000..658464642d --- /dev/null +++ b/guides/auth.md @@ -0,0 +1,27 @@ +- [Authentication](#authentication) + - [Amazon OpenSearch Service](#amazon-opensearch-service) + +# Authentication + +## Amazon OpenSearch Service + +Requests to [OpenSearch Service and OpenSearch Serverless](https://docs.aws.amazon.com/opensearch-service/index.html) must be signed using the AWS signing protocol. Use `AwsSdk2Transport` to send signed requests. + +```java +SdkHttpClient httpClient = ApacheHttpClient.builder().build(); + +OpenSearchClient client = new OpenSearchClient( + new AwsSdk2Transport( + httpClient, + "search-...us-west-2.es.amazonaws.com", // OpenSearch endpoint, without https:// + "es" // signing service name, use "aoss" for OpenSearch Serverless + Region.US_WEST_2, // signing service region + AwsSdk2TransportOptions.builder().build() + ) +); + +InfoResponse info = client.info(); +System.out.println(info.version().distribution() + ": " + info.version().number()); + +httpClient.close(); +``` \ No newline at end of file diff --git a/guides/bulk.md b/guides/bulk.md new file mode 100644 index 0000000000..3c12c2ddc7 --- /dev/null +++ b/guides/bulk.md @@ -0,0 +1,41 @@ +- [Bulk](#bulk) + - [Bulk Indexing](#bulk-indexing) + - [Bulk requests](#bulk-requests) + +# Bulk + +The [Bulk API](https://opensearch.org/docs/latest/api-reference/document-apis/bulk/) lets you add, update, or delete multiple documents in a single request. + +## Bulk Indexing + +## Bulk requests + +```java +String indexName = "sample-index"; +CreateIndexRequest createIndexRequest = new CreateIndexRequest.Builder().index(index).build(); +client.indices().create(createIndexRequest); + +ArrayList ops = new ArrayList<>(); +IndexData doc1 = new IndexData("Document 1", "The text of document 1"); +ops.add(new BulkOperation.Builder().index( + IndexOperation.of(io -> io.index(indexName).id("id1").document(doc1)) +).build()); +IndexData doc2 = new IndexData("Document 2", "The text of document 2"); +ops.add(new BulkOperation.Builder().index( + IndexOperation.of(io -> io.index(indexName).id("id2").document(doc2)) +).build()); +IndexData doc3 = new IndexData("Document 3", "The text of document 3"); +ops.add(new BulkOperation.Builder().index( + IndexOperation.of(io -> io.index(indexName).id("id3").document(doc3)) +).build()); + +BulkRequest.Builder bulkReq = new BulkRequest.Builder() + .index(indexName) + .operations(ops) + .refresh(Refresh.WaitFor); +BulkResponse bulkResponse = client.bulk(bulkReq.build()); +``` + +[IndexData](../samples/src/main/java/org/opensearch/client/samples/util/IndexData.java) refers to sample data class. + +You can find a working sample of the above code in [Bulk.java](../samples/src/main/java/org/opensearch/client/samples/Bulk.java). \ No newline at end of file diff --git a/guides/cat.md b/guides/cat.md new file mode 100644 index 0000000000..a5a3adcb5e --- /dev/null +++ b/guides/cat.md @@ -0,0 +1,42 @@ +- [Cat API](#cat-api) + - [Cat Indices](#cat-indices) + - [Cat aliases](#cat-aliases) + - [Cat nodes](#cat-nodes) + - [Cat point in time segments](#cat-point-in-time-segments) + +# Cat API + +The CAT API is a human-readable interface that returns plain text instead of traditional JSON. + +## Cat Indices +The following sample code cat indices with required headers and sorted by creation date + +```java +IndicesRequest indicesRequest = new IndicesRequest.Builder() + .headers("index,health,status,pri,rep,doc.count,creation.date,creation.date.string").sort("creation.date").build(); +IndicesResponse indicesResponse = javaClient().cat().indices(indicesRequest); +``` + + +## Cat aliases +The following sample code cat aliases with name "test-alias" and sorted by index + +```java +AliasesRequest aliasesRequest = new AliasesRequest.Builder().name("test-alias").sort("index").build(); +AliasesResponse aliasesResponse = javaClient().cat().aliases(aliasesRequest); +``` + +## Cat nodes +The following sample code cat nodes sorted by cpu + +```java +NodesResponse nodesResponse = javaClient().cat().nodes(r -> r.sort("cpu")); +``` + +## Cat point in time segments +Similarly to the CAT Segments API, the PIT Segments API provides low-level information about the disk utilization of a PIT by describing its Lucene segments. + +```java +SegmentsResponse pitSegmentsResponse = javaClient().cat() + .pitSegments(r -> r.headers("index,shard,id,segment,size")); +``` \ No newline at end of file diff --git a/guides/data_stream.md b/guides/data_stream.md new file mode 100644 index 0000000000..7c49807f8c --- /dev/null +++ b/guides/data_stream.md @@ -0,0 +1,54 @@ +- [Data Stream API](#data-stream-api) + - [Create a data stream](#create-a-data-stream) + - [Get data stream](#get-data-stream) + - [Data stream stats](#data-stream-stats) + - [Delete data stream and backing indices](#delete-data-stream-and-backing-indices) + +# Data Stream API + +## Create a data stream +Before creating a data stream, you need to create an index template which configures a set of indices as a data stream. +A data stream must have a timestamp field. If not specified, OpenSearch uses `@timestamp` as the default timestamp field name. + +The following sample code creates an index template for data stream with a custom timestamp field, and creates a data stream +which matches the name pattern specified in the index template. +```java +String dataStreamIndexTemplateName = "sample-data-stream-template"; +String timestampFieldName = "my_timestamp_field"; +String namePattern = "sample-data-stream-*"; +String dataStreamName = "sample-data-stream-1"; + +// Create an index template which configures data stream +PutIndexTemplateRequest putIndexTemplateRequest = new PutIndexTemplateRequest.Builder() + .name(dataStreamIndexTemplateName) + .indexPatterns(namePattern) + .dataStream(new DataStream.Builder() + .timestampField(t -> t.name(timestampFieldName)) + .build()) + .build(); +PutIndexTemplateResponse putIndexTemplateResponse = client.indices().putIndexTemplate(putIndexTemplateRequest); + +// Create a data stream +CreateDataStreamRequest createDataStreamRequest = new CreateDataStreamRequest.Builder().name(dataStreamName).build(); +CreateDataStreamResponse createDataStreamResponse = client.indices().createDataStream(createDataStreamRequest); +``` + +## Get data stream +```java +GetDataStreamRequest getDataStreamRequest = new GetDataStreamRequest.Builder().name(dataStreamName).build(); +GetDataStreamResponse getDataStreamResponse = client.indices().getDataStream(getDataStreamRequest); +``` + +## Data stream stats +```java +DataStreamsStatsRequest dataStreamsStatsRequest = new DataStreamsStatsRequest.Builder().name(dataStreamName).build(); +DataStreamsStatsResponse dataStreamsStatsResponse = client.indices().dataStreamsStats(dataStreamsStatsRequest); +``` + +## Delete data stream and backing indices +```java +DeleteDataStreamRequest deleteDataStreamRequest = new DeleteDataStreamRequest.Builder().name(dataStreamName).build(); +DeleteDataStreamResponse deleteDataStreamResponse = client.indices().deleteDataStream(deleteDataStreamRequest); +``` + +You can find a working sample of the above code in [DataStreamBasics.java](../samples/src/main/java/org/opensearch/client/samples/DataStreamBasics.java). \ No newline at end of file diff --git a/guides/plugins/knn.md b/guides/plugins/knn.md new file mode 100644 index 0000000000..a2c3db72b8 --- /dev/null +++ b/guides/plugins/knn.md @@ -0,0 +1,288 @@ +- [k-NN Plugin](#k-nn-plugin) + - [Basic Approximate k-NN](#basic-approximate-k-nn) + - [Create an Index](#create-an-index) + - [Index Vectors](#index-vectors) + - [Search for Nearest Neighbors](#search-for-nearest-neighbors) + - [Approximate k-NN with a Boolean Filter](#approximate-k-nn-with-a-boolean-filter) + - [Approximate k-NN with an Efficient Filter](#approximate-k-nn-with-an-efficient-filter) + - [Exact k-NN with a scoring script](#exact-k-nn-with-a-scoring-script) + - [Exact k-NN with the Painless scripting extensions](#exact-k-nn-with-the-painless-scripting-extensions) + +# k-NN Plugin + +Short for k-nearest neighbors, the k-NN plugin enables users to search for the k-nearest neighbors to a query point across an index of vectors. See the [plugin's documentation](https://opensearch.org/docs/latest/search-plugins/knn/index/) for more information. + +## Basic Approximate k-NN + +In the following example we create a 5-dimensional k-NN index with random data. You can find a synchronous version of this working sample in [samples/src/main/java/org/opensearch/client/samples/knn/KnnBasics.java](../../samples/src/main/java/org/opensearch/client/samples/knn/KnnBasics.java). + +```bash +$ ./gradlew :samples:run -Dsamples.mainClass=knn.KnnBasics + +[Main] INFO - Running main class: org.opensearch.client.samples.knn.KnnBasics +[KnnBasics] INFO - Server: opensearch@2.7.0 +[KnnBasics] INFO - Creating index my-index +[KnnBasics] INFO - Indexing 10 vectors +[KnnBasics] INFO - Waiting for indexing to finish +[KnnBasics] INFO - Searching for vector [0.67, 0.67, 0.37, 0.0, 0.72] +[KnnBasics] INFO - Found {values=[0.32, 0.96, 0.41, 0.04, 0.9]} with score 0.8050233 +[KnnBasics] INFO - Found {values=[0.04, 0.58, 0.13, 0.27, 0.37]} with score 0.6031363 +[KnnBasics] INFO - Found {values=[0.96, 0.88, 0.8, 0.41, 0.18]} with score 0.5640794 +[KnnBasics] INFO - Deleting index my-index +``` + +### Create an Index + +```java +final var indexName = "my-index"; +final var dimensions = 5; + +client.indices().create(r -> r + .index(indexName) + .settings(s -> s.knn(true)) + .mappings(m -> m + .properties("values", p -> p + .knnVector(k -> k.dimension(dimensions))))); +``` + +### Index Vectors + +Given the following document class definition: + +```java +public static class Doc { + private float[] values; + + public Doc() {} + + public Doc(float[] values) { + this.values = values; + } + + public static Doc rand(int dimensions) { + var values = new float[dimensions]; + for (var i = 0; i < dimensions; ++i) { + values[i] = Math.round(Math.random() * 100.0) / 100.0f; + } + return new Doc(values); + } + + // Getters/Setters & toString elided +} +``` + +Create 10 random vectors and insert them using the bulk API: + +```java +final var nVectors = 10; +var bulkRequest = new BulkRequest.Builder(); +for (var i = 0; i < nVectors; ++i) { + var id = Integer.toString(i); + var doc = Doc.rand(dimensions); + bulkRequest.operations(b -> b + .index(o -> o + .index(indexName) + .id(id) + .document(doc))); +} + +client.bulk(bulkRequest.build()); + +client.indices().refresh(i -> i.index(indexName)); +``` + +### Search for Nearest Neighbors + +Create a random vector of the same size and search for its nearest neighbors. + +```java +final var searchVector = new float[dimensions]; +for (var i = 0; i < dimensions; ++i) { + searchVector[i] = Math.round(Math.random() * 100.0) / 100.0f; +} + +var searchResponse = client.search(s -> s + .index(indexName) + .query(q -> q + .knn(k -> k + .field("values") + .vector(searchVector) + .k(3))), + Doc.class); + +for (var hit : searchResponse.hits().hits()) { + System.out.println(hit.source()); +} +``` + +## Approximate k-NN with a Boolean Filter + +In the [KnnBooleanFilter.java sample](../../samples/src/main/java/org/opensearch/client/samples/knn/KnnBooleanFilter.java) we create a 5-dimensional k-NN index with random data and a `metadata` field that contains a book genre (e.g. `fiction`). The search query is a k-NN search filtered by genre. The filter clause is outside the k-NN query clause and is applied after the k-NN search. + +```java +var searchResponse = client.search(s -> s + .index(indexName) + .query(q -> q + .bool(b -> b + .filter(f -> f + .bool(b2 -> b2 + .must(m -> m + .term(t -> t + .field("metadata.genre") + .value(v -> v.stringValue(searchGenre)))))) + .must(m -> m + .knn(k -> k + .field("values") + .vector(searchVector) + .k(5))))), + Doc.class); +``` + +```bash +$ ./gradlew :samples:run -Dsamples.mainClass=knn.KnnBooleanFilter + +[Main] INFO - Running main class: org.opensearch.client.samples.knn.KnnBooleanFilter +[KnnBooleanFilter] INFO - Server: opensearch@2.7.0 +[KnnBooleanFilter] INFO - Creating index my-index +[KnnBooleanFilter] INFO - Indexing 3000 vectors +[KnnBooleanFilter] INFO - Waiting for indexing to finish +[KnnBooleanFilter] INFO - Searching for vector [0.18, 0.71, 0.44, 0.03, 0.42] with the 'drama' genre +[KnnBooleanFilter] INFO - Found {values=[0.21, 0.58, 0.55, 0.09, 0.45], metadata={genre=drama}} with score 0.966744 +[KnnBooleanFilter] INFO - Deleting index my-index +``` + +## Approximate k-NN with an Efficient Filter + +In the [KnnEfficientFilter.java sample](../../samples/src/main/java/org/opensearch/client/samples/knn/KnnEfficientFilter.java) we implement the example in [the k-NN documentation](https://opensearch.org/docs/latest/search-plugins/knn/filter-search-knn/), which creates an index that uses the Lucene engine and HNSW as the method in the mapping, containing hotel location and parking data, then search for the top three hotels near the location with the coordinates `[5, 4]` that are rated between 8 and 10, inclusive, and provide parking. + +```java +var searchResponse = client.search(s -> s + .index(indexName) + .size(3) + .query(q -> q + .knn(k -> k + .field("location") + .vector(searchLocation) + .k(3) + .filter(Query.of(f -> f + .bool(b -> b + .must(m -> m + .range(r -> r + .field("rating") + .gte(JsonData.of(searchRatingMin)) + .lte(JsonData.of(searchRatingMax)))) + .must(m -> m + .term(t -> t + .field("parking") + .value(FieldValue.of(searchParking))))))))), + Hotel.class); +``` + +```bash +$ ./gradlew :samples:run -Dsamples.mainClass=knn.KnnEfficientFilter + +[Main] INFO - Running main class: org.opensearch.client.samples.knn.KnnEfficientFilter +[KnnEfficientFilter] INFO - Server: opensearch@2.7.0 +[KnnEfficientFilter] INFO - Creating index hotels-index +[KnnEfficientFilter] INFO - Indexing hotel {location=[5.2, 4.0], parking=true, rating=5} with id 1 +[KnnEfficientFilter] INFO - Indexing hotel {location=[5.2, 3.9], parking=false, rating=4} with id 2 +[KnnEfficientFilter] INFO - Indexing hotel {location=[4.9, 3.4], parking=true, rating=9} with id 3 +[KnnEfficientFilter] INFO - Indexing hotel {location=[4.2, 4.6], parking=false, rating=6} with id 4 +[KnnEfficientFilter] INFO - Indexing hotel {location=[3.3, 4.5], parking=true, rating=8} with id 5 +[KnnEfficientFilter] INFO - Indexing hotel {location=[6.4, 3.4], parking=true, rating=9} with id 6 +[KnnEfficientFilter] INFO - Indexing hotel {location=[4.2, 6.2], parking=true, rating=5} with id 7 +[KnnEfficientFilter] INFO - Indexing hotel {location=[2.4, 4.0], parking=true, rating=8} with id 8 +[KnnEfficientFilter] INFO - Indexing hotel {location=[1.4, 3.2], parking=false, rating=5} with id 9 +[KnnEfficientFilter] INFO - Indexing hotel {location=[7.0, 9.9], parking=true, rating=9} with id 10 +[KnnEfficientFilter] INFO - Indexing hotel {location=[3.0, 2.3], parking=false, rating=6} with id 11 +[KnnEfficientFilter] INFO - Indexing hotel {location=[5.0, 1.0], parking=true, rating=3} with id 12 +[KnnEfficientFilter] INFO - Indexing 12 documents +[KnnEfficientFilter] INFO - Waiting for indexing to finish +[KnnEfficientFilter] INFO - Searching for hotel near [5.0, 4.0] with rating >=8,<=10 and parking=true +[KnnEfficientFilter] INFO - Found {location=[4.9, 3.4], parking=true, rating=9} with score 0.72992706 +[KnnEfficientFilter] INFO - Found {location=[6.4, 3.4], parking=true, rating=9} with score 0.3012048 +[KnnEfficientFilter] INFO - Found {location=[3.3, 4.5], parking=true, rating=8} with score 0.24154587 +[KnnEfficientFilter] INFO - Deleting index hotels-index +``` + +## Exact k-NN with a scoring script + +In the [KnnScriptScore.java sample](../../samples/src/main/java/org/opensearch/client/samples/knn/KnnScriptScore.java) we create a 5-dimensional k-NN index with random data. The search query uses the [k-NN scoring script](https://opensearch.org/docs/latest/search-plugins/knn/knn-score-script/) to calculate exact nearest neighbors. + +```java +var searchResponse = client.search(s -> s + .index(indexName) + .query(q -> q + .scriptScore(ss -> ss + .query(qq -> qq.matchAll(m -> m)) + .script(sss -> sss + .inline(i -> i + .source("knn_score") + .lang("knn") + .params("field", JsonData.of("values")) + .params("query_value", JsonData.of(searchVector)) + .params("space_type", JsonData.of("cosinesimil")))))), + Doc.class); +``` + +```bash +$ ./gradlew :samples:run -Dsamples.mainClass=knn.KnnScriptScore + +[Main] INFO - Running main class: org.opensearch.client.samples.knn.KnnScriptScore +[KnnScriptScore] INFO - Server: opensearch@2.7.0 +[KnnScriptScore] INFO - Creating index my-index +[KnnScriptScore] INFO - Indexing 10 vectors +[KnnScriptScore] INFO - Waiting for indexing to finish +[KnnScriptScore] INFO - Searching for vector [0.94, 0.1, 0.39, 0.63, 0.42] +[KnnScriptScore] INFO - Found {values=[0.66, 0.23, 0.15, 0.44, 0.13]} with score 1.9564294 +[KnnScriptScore] INFO - Found {values=[0.94, 0.05, 0.86, 0.68, 0.05]} with score 1.90958 +[KnnScriptScore] INFO - Found {values=[0.88, 0.72, 0.29, 0.48, 0.56]} with score 1.8788767 +[KnnScriptScore] INFO - Found {values=[0.97, 0.99, 0.66, 0.61, 0.91]} with score 1.847905 +[KnnScriptScore] INFO - Found {values=[0.18, 0.29, 0.43, 0.63, 0.25]} with score 1.7819176 +[KnnScriptScore] INFO - Found {values=[0.35, 0.2, 0.62, 0.4, 0.96]} with score 1.7673628 +[KnnScriptScore] INFO - Found {values=[0.34, 0.59, 0.05, 0.47, 0.54]} with score 1.7316635 +[KnnScriptScore] INFO - Found {values=[0.55, 0.98, 0.07, 0.57, 0.06]} with score 1.6385877 +[KnnScriptScore] INFO - Found {values=[0.03, 0.72, 0.89, 0.83, 0.46]} with score 1.6147845 +[KnnScriptScore] INFO - Found {values=[0.17, 0.81, 0.09, 0.21, 0.3]} with score 1.4616101 +[KnnScriptScore] INFO - Deleting index my-index +``` + +## Exact k-NN with the Painless scripting extensions + +In the [KnnPainlessScript.java sample](../../samples/src/main/java/org/opensearch/client/samples/knn/KnnPainlessScript.java) we create a 5-dimensional k-NN index with random data. The search query uses the [k-NN Painless extensions](https://opensearch.org/docs/latest/search-plugins/knn/painless-functions/) to calculate exact nearest neighbors. + +```java +var searchResponse = client.search(s -> s + .index(indexName) + .query(q -> q + .scriptScore(ss -> ss + .query(qq -> qq.matchAll(m -> m)) + .script(sss -> sss + .inline(i -> i + .source("1.0 + cosineSimilarity(params.query_value, doc[params.field])") + .params("field", JsonData.of("values")) + .params("query_value", JsonData.of(searchVector)))))), + Doc.class); +``` + +```bash +$ ./gradlew :samples:run -Dsamples.mainClass=knn.KnnPainlessScript + +[Main] INFO - Running main class: org.opensearch.client.samples.knn.KnnPainlessScript +[KnnPainlessScript] INFO - Server: opensearch@2.7.0 +[KnnPainlessScript] INFO - Creating index my-index +[KnnPainlessScript] INFO - Indexing 10 vectors +[KnnPainlessScript] INFO - Waiting for indexing to finish +[KnnPainlessScript] INFO - Searching for vector [0.57, 0.86, 0.37, 0.07, 0.38] +[KnnPainlessScript] INFO - Found {values=[1.0, 0.6, 0.66, 0.03, 0.18]} with score 1.8911908 +[KnnPainlessScript] INFO - Found {values=[0.4, 0.39, 0.63, 0.09, 0.39]} with score 1.8776901 +[KnnPainlessScript] INFO - Found {values=[0.32, 0.98, 0.7, 0.7, 0.77]} with score 1.8616674 +[KnnPainlessScript] INFO - Found {values=[0.93, 0.35, 0.27, 0.45, 0.81]} with score 1.789043 +[KnnPainlessScript] INFO - Found {values=[0.81, 0.36, 0.87, 0.78, 0.56]} with score 1.7457235 +[KnnPainlessScript] INFO - Found {values=[0.55, 0.19, 0.61, 0.42, 0.4]} with score 1.743325 +[KnnPainlessScript] INFO - Found {values=[0.12, 0.54, 0.09, 0.83, 0.28]} with score 1.6045148 +[KnnPainlessScript] INFO - Found {values=[0.0, 0.04, 0.63, 0.07, 0.9]} with score 1.479921 +[KnnPainlessScript] INFO - Found {values=[0.41, 0.05, 0.52, 1.0, 0.18]} with score 1.4306322 +[KnnPainlessScript] INFO - Found {values=[0.22, 0.1, 0.59, 0.89, 0.15]} with score 1.4274814 +[KnnPainlessScript] INFO - Deleting index my-index +``` \ No newline at end of file diff --git a/guides/point_in_time.md b/guides/point_in_time.md new file mode 100644 index 0000000000..f2fa36118b --- /dev/null +++ b/guides/point_in_time.md @@ -0,0 +1,53 @@ +- [Point-in-Time](#point-in-time) + - [Point-In-Time API](#point-in-time-api) + - [Creating a point in time](#creating-a-point-in-time) + - [List all point in time](#list-all-point-in-time) + - [Delete point in time](#delete-point-in-time) + +# Point-in-Time + +[Point in Time (PIT)](https://opensearch.org/docs/latest/search-plugins/point-in-time/) lets you run different queries against a dataset that is fixed in time. + +## Point-In-Time API + +### Creating a point in time + +To create a PIT, first create an index. + +``` +java +String index = "sample-index"; +CreateIndexRequest createIndexRequest = new CreateIndexRequest.Builder().index(index).build(); +client.indices().create(createIndexRequest); +``` + +To create a PIT, the keep_alive query parameter is required; it specifies how long to keep a PIT. + +```java +CreatePitRequest createPitRequest = new CreatePitRequest.Builder() + .targetIndexes(Collections.singletonList(index)) + .keepAlive(new Time.Builder().time("100m").build()).build(); + +CreatePitResponse createPitResponse = client.createPit(createPitRequest); +``` + +### List all point in time + +Returns all PITs in the OpenSearch cluster. + +```java +ListAllPitResponse listAllPitResponse = client.listAllPit(); +``` + +### Delete point in time + +Deletes one, several, or all PITs. PITs are automatically deleted when the keep_alive time period elapses. However, to deallocate resources, you can delete a PIT using the Delete PIT API. The Delete PIT API supports deleting a list of PITs by ID or deleting all PITs at once. + +```java +DeletePitRequest deletePitRequest = new DeletePitRequest.Builder() + .pitId(Collections.singletonList(createPitResponse.pitId())).build(); + +DeletePitResponse deletePitResponse = client.deletePit(deletePitRequest); +``` + +You can find a working sample of the above code in [PointInTime.java](../samples/src/main/java/org/opensearch/client/samples/PointInTime.java). \ No newline at end of file diff --git a/guides/search.md b/guides/search.md new file mode 100644 index 0000000000..d343b30420 --- /dev/null +++ b/guides/search.md @@ -0,0 +1,283 @@ +- [Search](#search) + - [Setup](#setup) + - [Search API](#search-api) + - [Basic Search](#basic-search) + - [Get raw JSON results](#get-raw-json-results) + - [Search documents using a match query](#search-documents-using-a-match-query) + - [Search documents using suggesters](#search-documents-using-suggesters) + - [Using completion suggester](#using-completion-suggester) + - [Using term suggester](#using-term-suggester) + - [Using phrase suggester](#using-phrase-suggester) + - [Aggregations](#aggregations) + +# Search + +OpenSearch provides a powerful search API that allows you to search for documents in an index. The search API supports a number of parameters that allow you to customize the search operation. In this guide, we will explore the search API and its parameters. + +## Setup + +To get started, first create a client, create an index and index some documents: + +```java +import org.apache.hc.core5.http.HttpHost; + +final HttpHost[] hosts = new HttpHost[] { + new HttpHost("http", "localhost", 9200) + }; + +final OpenSearchTransport transport = ApacheHttpClient5TransportBuilder + .builder(hosts) + .setMapper(new JacksonJsonpMapper()) + .build(); +OpenSearchClient client = new OpenSearchClient(transport); + +String index = "sample-index"; +CreateIndexRequest createIndexRequest = new CreateIndexRequest.Builder().index(index).build(); +client.indices().create(createIndexRequest); + +IndexData indexData = new IndexData("Document 1", "Text for document 1"); +IndexRequest indexRequest = new IndexRequest.Builder().index(index).id("1").document(indexData).build(); +client.index(indexRequest); + +indexData = new IndexData("Document 2", "Text for document 2"); +indexRequest = new IndexRequest.Builder().index(index).id("2").document(indexData).build(); +client.index(indexRequest); +``` + +[IndexData](../samples/src/main/java/org/opensearch/client/samples/util/IndexData.java) refers to sample data class. + +## Search API + +### Basic Search + +```java +SearchResponse searchResponse = client.search(s -> s.index(index), IndexData.class); +for (int i = 0; i < searchResponse.hits().hits().size(); i++) { + System.out.println(searchResponse.hits().hits().get(i).source()); +} +``` + +#### Get raw JSON results + +When the target class is not defined or if the response result is a semi-structured data not tied to an object definition, use a raw JSON data representation as the target class. For example, the below snippet uses `ObjectNode` from jackson. + +```java +SearchResponse searchResponse = client.search(b -> b.index(index), ObjectNode.class); +for (int i = 0; i < searchResponse.hits().hits().size(); i++) { + System.out.println(searchResponse.hits().hits().get(i).source()); +} +``` + +### Search documents using a match query + +```java +SearchRequest searchRequest = new SearchRequest.Builder().query(q -> q.match(m -> m.field("text") + .query(FieldValue.of("Text for document 2")))) + .build(); + +SearchResponse searchResponse = client.search(searchRequest, IndexData.class); +for (int i = 0; i < searchResponse.hits().hits().size(); i++) { + System.out.println(searchResponse.hits().hits().get(i).source()); +} +``` + +### Search documents using suggesters + +[AppData](../samples/src/main/java/org/opensearch/client/samples/util/AppData.java) refers to the sample data class used in the below samples. + +#### Using completion suggester + +```java +String index = "completion-suggester"; + +Property intValueProp = new Property.Builder() + .long_(v -> v) + .build(); +Property msgCompletionProp = new Property.Builder() + .completion(c -> c) + .build(); +client.indices().create(c -> c + .index(index) + .mappings(m -> m + .properties("intValue", intValueProp) + .properties("msg", msgCompletionProp))); + +AppData appData = new AppData(); +appData.setIntValue(1337); +appData.setMsg("foo"); + +client.index(b -> b + .index(index) + .id("1") + .document(appData) + .refresh(Refresh.True)); + +appData.setIntValue(1338); +appData.setMsg("foobar"); + +client.index(b -> b + .index(index) + .id("2") + .document(appData) + .refresh(Refresh.True)); + +String suggesterName = "msgSuggester"; + +CompletionSuggester completionSuggester = FieldSuggesterBuilders.completion() + .field("msg") + .size(1) + .build(); +FieldSuggester fieldSuggester = new FieldSuggester.Builder().prefix("foo") + .completion(completionSuggester) + .build(); +Suggester suggester = new Suggester.Builder() + .suggesters(Collections.singletonMap(suggesterName, fieldSuggester)) + .build(); +SearchRequest searchRequest = new SearchRequest.Builder() + .index(index) + .suggest(suggester) + .build(); + +SearchResponse response = client.search(searchRequest, AppData.class); +``` + +#### Using term suggester + +```java +String index = "term-suggester"; + +// term suggester does not require a special mapping +client.indices().create(c -> c + .index(index)); + +AppData appData = new AppData(); +appData.setIntValue(1337); +appData.setMsg("foo"); + +client.index(b -> b + .index(index) + .id("1") + .document(appData) + .refresh(Refresh.True)); + +appData.setIntValue(1338); +appData.setMsg("foobar"); + +client.index(b -> b + .index(index) + .id("2") + .document(appData) + .refresh(Refresh.True)); + +String suggesterName = "msgSuggester"; + +TermSuggester termSuggester = FieldSuggesterBuilders.term() + .field("msg") + .size(1) + .build(); +FieldSuggester fieldSuggester = new FieldSuggester.Builder().text("fool") + .term(termSuggester) + .build(); +Suggester suggester = new Suggester.Builder() + .suggesters(Collections.singletonMap(suggesterName, fieldSuggester)) + .build(); +SearchRequest searchRequest = new SearchRequest.Builder() + .index(index) + .suggest(suggester) + .build(); + +SearchResponse response = client.search(searchRequest, AppData.class); +``` + +#### Using phrase suggester + +```java +String index = "test-phrase-suggester"; + +ShingleTokenFilter shingleTokenFilter = new ShingleTokenFilter.Builder().minShingleSize("2") + .maxShingleSize("3") + .build(); + +Analyzer analyzer = new Analyzer.Builder() + .custom(new CustomAnalyzer.Builder().tokenizer("standard") + .filter(Arrays.asList("lowercase", "shingle")).build()) + .build(); + +TokenFilter tokenFilter = new TokenFilter.Builder() + .definition(new TokenFilterDefinition.Builder() + .shingle(shingleTokenFilter).build()) + .build(); + +IndexSettingsAnalysis indexSettingsAnalysis = new IndexSettingsAnalysis.Builder() + .analyzer("trigram", analyzer) + .filter("shingle", tokenFilter) + .build(); + +IndexSettings settings = new IndexSettings.Builder().analysis(indexSettingsAnalysis).build(); + +TypeMapping mapping = new TypeMapping.Builder().properties("msg", new Property.Builder() + .text(new TextProperty.Builder().fields("trigram", new Property.Builder() + .text(new TextProperty.Builder().analyzer("trigram").build()) + .build()).build()) + .build()).build(); + +client.indices().create(c -> c + .index(index) + .settings(settings) + .mappings(mapping)); + +AppData appData = new AppData(); +appData.setIntValue(1337); +appData.setMsg("Design Patterns"); + +client.index(b -> b + .index(index) + .id("1") + .document(appData) + .refresh(Refresh.True)); + +appData.setIntValue(1338); +appData.setMsg("Software Architecture Patterns Explained"); + +client.index(b -> b + .index(index) + .id("2") + .document(appData) + .refresh(Refresh.True)); + +String suggesterName = "msgSuggester"; + +PhraseSuggester phraseSuggester = FieldSuggesterBuilders.phrase() + .field("msg.trigram") + .build(); +FieldSuggester fieldSuggester = new FieldSuggester.Builder().text("design paterns") + .phrase(phraseSuggester) + .build(); +Suggester suggester = new Suggester.Builder() + .suggesters(Collections.singletonMap(suggesterName, fieldSuggester)) + .build(); +SearchRequest searchRequest = new SearchRequest.Builder() + .index(index) + .suggest(suggester) + .build(); + +SearchResponse response = client.search(searchRequest, AppData.class); +``` + +### Aggregations + +```java +SearchRequest searchRequest = new SearchRequest.Builder().query(q -> q.match(m -> m.field("title") + .query(FieldValue.of("Document 1")))) + .aggregations("titles", new Aggregation.Builder().terms(t -> t.field("title.keyword")) + .build()) + .build(); + +SearchResponse searchResponse = client.search(searchRequest, IndexData.class); +for (Map.Entry entry : searchResponse.aggregations().entrySet()) { + System.out.println("Agg - " + entry.getKey()); + entry.getValue().sterms().buckets().array().forEach(b -> System.out.printf("%s : %d%n", b.key(), b.docCount())); +} +``` + +You can find a working sample of the above code in [Search.java](../samples/src/main/java/org/opensearch/client/samples/Search.java). \ No newline at end of file diff --git a/samples/build.gradle.kts b/samples/build.gradle.kts new file mode 100644 index 0000000000..8c635d83d9 --- /dev/null +++ b/samples/build.gradle.kts @@ -0,0 +1,41 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +plugins { + java + application + checkstyle +} + +checkstyle { + toolVersion = "10.0" +} + +java { + targetCompatibility = JavaVersion.VERSION_11 + sourceCompatibility = JavaVersion.VERSION_11 +} + +dependencies { + implementation(project(":java-client")) + implementation("org.apache.logging.log4j", "log4j-api","[2.17.1,3.0)") + implementation("org.apache.logging.log4j", "log4j-core","[2.17.1,3.0)") + implementation("org.apache.logging.log4j", "log4j-slf4j2-impl","[2.17.1,3.0)") + implementation("commons-logging", "commons-logging", "1.2") + implementation("com.fasterxml.jackson.core", "jackson-databind", "2.15.2") + implementation("org.apache.httpcomponents.client5", "httpclient5", "5.2.1") + implementation("org.apache.httpcomponents.core5", "httpcore5", "5.2.2") +} + +application { + mainClass.set("org.opensearch.client.samples.Main") +} + +tasks.named("run") { + systemProperty("samples.mainClass", System.getProperty("samples.mainClass")) +} diff --git a/samples/src/main/java/org/opensearch/client/samples/Bulk.java b/samples/src/main/java/org/opensearch/client/samples/Bulk.java new file mode 100644 index 0000000000..b3f84f04e6 --- /dev/null +++ b/samples/src/main/java/org/opensearch/client/samples/Bulk.java @@ -0,0 +1,104 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.client.samples; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.opensearch._types.Refresh; +import org.opensearch.client.opensearch._types.mapping.IntegerNumberProperty; +import org.opensearch.client.opensearch._types.mapping.Property; +import org.opensearch.client.opensearch._types.mapping.TypeMapping; +import org.opensearch.client.opensearch._types.query_dsl.Query; +import org.opensearch.client.opensearch.core.BulkRequest; +import org.opensearch.client.opensearch.core.BulkResponse; +import org.opensearch.client.opensearch.core.SearchRequest; +import org.opensearch.client.opensearch.core.SearchResponse; +import org.opensearch.client.opensearch.core.bulk.BulkOperation; +import org.opensearch.client.opensearch.core.bulk.IndexOperation; +import org.opensearch.client.opensearch.indices.CreateIndexRequest; +import org.opensearch.client.opensearch.indices.DeleteIndexRequest; +import org.opensearch.client.opensearch.indices.IndexSettings; +import org.opensearch.client.samples.util.IndexData; + +/** + * Run with: ./gradlew :samples:run -Dsamples.mainClass=Bulk + */ +public class Bulk { + private static final Logger LOGGER = LogManager.getLogger(Bulk.class); + + public static void main(String[] args) { + try { + var client = SampleClient.create(); + + var version = client.info().version(); + LOGGER.info("Server: {}@{}", version.distribution(), version.number()); + + final var indexName = "my-index"; + + if (!client.indices().exists(r -> r.index(indexName)).value()) { + LOGGER.info("Creating index {}", indexName); + IndexSettings settings = new IndexSettings.Builder() + .numberOfShards("2") + .numberOfReplicas("1") + .build(); + TypeMapping mapping = new TypeMapping.Builder() + .properties("age", new Property.Builder().integer(new IntegerNumberProperty.Builder().build()).build()) + .build(); + CreateIndexRequest createIndexRequest = new CreateIndexRequest.Builder() + .index(indexName) + .settings(settings) + .mappings(mapping) + .build(); + client.indices().create(createIndexRequest); + } + + LOGGER.info("Bulk indexing documents"); + ArrayList ops = new ArrayList<>(); + IndexData doc1 = new IndexData("Document 1", "The text of document 1"); + ops.add(new BulkOperation.Builder().index( + IndexOperation.of(io -> io.index(indexName).id("id1").document(doc1)) + ).build()); + IndexData doc2 = new IndexData("Document 2", "The text of document 2"); + ops.add(new BulkOperation.Builder().index( + IndexOperation.of(io -> io.index(indexName).id("id2").document(doc2)) + ).build()); + IndexData doc3 = new IndexData("Document 3", "The text of document 3"); + ops.add(new BulkOperation.Builder().index( + IndexOperation.of(io -> io.index(indexName).id("id3").document(doc3)) + ).build()); + + BulkRequest.Builder bulkReq = new BulkRequest.Builder() + .index(indexName) + .operations(ops) + .refresh(Refresh.WaitFor); + BulkResponse bulkResponse = client.bulk(bulkReq.build()); + LOGGER.info("Bulk response items: {}", bulkResponse.items().size()); + + Query query = Query.of(qb -> qb.match(mb -> mb.field("title").query(fv -> fv.stringValue("Document")))); + final SearchRequest.Builder searchReq = new SearchRequest.Builder() + .allowPartialSearchResults(false) + .index(List.of(indexName)) + .size(10) + .source(sc -> sc.fetch(false)) + .ignoreThrottled(false) + .query(query); + SearchResponse searchResponse = client.search(searchReq.build(), IndexData.class); + LOGGER.info("Found {} documents", searchResponse.hits().hits().size()); + + LOGGER.info("Deleting index {}", indexName); + DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest.Builder().index(indexName).build(); + client.indices().delete(deleteIndexRequest); + } catch (Exception e) { + LOGGER.error("Unexpected exception", e); + } + } +} diff --git a/samples/src/main/java/org/opensearch/client/samples/DataStreamBasics.java b/samples/src/main/java/org/opensearch/client/samples/DataStreamBasics.java new file mode 100644 index 0000000000..e6d8bf5875 --- /dev/null +++ b/samples/src/main/java/org/opensearch/client/samples/DataStreamBasics.java @@ -0,0 +1,75 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.client.samples; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.opensearch.indices.CreateDataStreamRequest; +import org.opensearch.client.opensearch.indices.DataStream; +import org.opensearch.client.opensearch.indices.DataStreamsStatsRequest; +import org.opensearch.client.opensearch.indices.DataStreamsStatsResponse; +import org.opensearch.client.opensearch.indices.DeleteDataStreamRequest; +import org.opensearch.client.opensearch.indices.GetDataStreamRequest; +import org.opensearch.client.opensearch.indices.GetDataStreamResponse; +import org.opensearch.client.opensearch.indices.PutIndexTemplateRequest; + +/** + * Run with: ./gradlew :samples:run -Dsamples.mainClass=DataStreamBasics + */ +public class DataStreamBasics { + private static final Logger LOGGER = LogManager.getLogger(DataStreamBasics.class); + + public static void main(String[] args) { + try { + var client = SampleClient.create(); + + var version = client.info().version(); + LOGGER.info("Server: {}@{}", version.distribution(), version.number()); + + String dataStreamIndexTemplateName = "sample-data-stream-template"; + String timestampFieldName = "my_timestamp_field"; + String namePattern = "sample-data-stream-*"; + String dataStreamName = "sample-data-stream-1"; + + // Create an index template which configures data stream + PutIndexTemplateRequest putIndexTemplateRequest = new PutIndexTemplateRequest.Builder() + .name(dataStreamIndexTemplateName) + .indexPatterns(namePattern) + .dataStream(new DataStream.Builder() + .timestampField(t -> t.name(timestampFieldName)) + .build()) + .build(); + client.indices().putIndexTemplate(putIndexTemplateRequest); + + // Create a data stream + CreateDataStreamRequest createDataStreamRequest = new CreateDataStreamRequest.Builder().name(dataStreamName).build(); + client.indices().createDataStream(createDataStreamRequest); + LOGGER.info("Created a data stream"); + + GetDataStreamRequest getDataStreamRequest = new GetDataStreamRequest.Builder().name(dataStreamName).build(); + GetDataStreamResponse getDataStreamResponse = client.indices().getDataStream(getDataStreamRequest); + LOGGER.info("Found {} data streams", getDataStreamResponse.dataStreams().size()); + + DataStreamsStatsRequest dataStreamsStatsRequest = new DataStreamsStatsRequest.Builder().name(dataStreamName).build(); + DataStreamsStatsResponse dataStreamsStatsResponse = client.indices().dataStreamsStats(dataStreamsStatsRequest); + LOGGER.info("Data stream stats:"); + LOGGER.info("Shards: {}", dataStreamsStatsResponse.shards()); + LOGGER.info("Data stream count: {}", dataStreamsStatsResponse.dataStreamCount()); + LOGGER.info("Backing indices: {}", dataStreamsStatsResponse.backingIndices()); + LOGGER.info("Total store size: {}", dataStreamsStatsResponse.totalStoreSize()); + LOGGER.info("Data stream name: {}", dataStreamsStatsResponse.dataStreams().get(0).dataStream()); + + LOGGER.info("Deleting data stream"); + DeleteDataStreamRequest deleteDataStreamRequest = new DeleteDataStreamRequest.Builder().name(dataStreamName).build(); + client.indices().deleteDataStream(deleteDataStreamRequest); + } catch (Exception e) { + LOGGER.error("Unexpected exception", e); + } + } +} diff --git a/samples/src/main/java/org/opensearch/client/samples/IndexingBasics.java b/samples/src/main/java/org/opensearch/client/samples/IndexingBasics.java new file mode 100644 index 0000000000..bba03de80b --- /dev/null +++ b/samples/src/main/java/org/opensearch/client/samples/IndexingBasics.java @@ -0,0 +1,118 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.client.samples; + +import java.util.ArrayList; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.opensearch._types.mapping.DynamicMapping; +import org.opensearch.client.opensearch._types.mapping.FieldNamesField; +import org.opensearch.client.opensearch._types.mapping.IntegerNumberProperty; +import org.opensearch.client.opensearch._types.mapping.Property; +import org.opensearch.client.opensearch._types.mapping.RoutingField; +import org.opensearch.client.opensearch._types.mapping.SourceField; +import org.opensearch.client.opensearch._types.mapping.TypeMapping; +import org.opensearch.client.opensearch.core.IndexRequest; +import org.opensearch.client.opensearch.core.SearchResponse; +import org.opensearch.client.opensearch.indices.CreateIndexRequest; +import org.opensearch.client.opensearch.indices.DeleteIndexRequest; +import org.opensearch.client.opensearch.indices.GetMappingRequest; +import org.opensearch.client.opensearch.indices.GetMappingResponse; +import org.opensearch.client.opensearch.indices.IndexSettings; +import org.opensearch.client.opensearch.indices.PutMappingRequest; +import org.opensearch.client.samples.util.IndexData; + +/** + * Run with: ./gradlew :samples:run -Dsamples.mainClass=IndexingBasics + */ +public class IndexingBasics { + private static final Logger LOGGER = LogManager.getLogger(IndexingBasics.class); + + public static void main(String[] args) { + try { + var client = SampleClient.create(); + + var version = client.info().version(); + LOGGER.info("Server: {}@{}", version.distribution(), version.number()); + + final var indexName = "my-index"; + + if (!client.indices().exists(r -> r.index(indexName)).value()) { + LOGGER.info("Creating index {}", indexName); + IndexSettings settings = new IndexSettings.Builder() + .numberOfShards("2") + .numberOfReplicas("1") + .build(); + TypeMapping mapping = new TypeMapping.Builder() + .properties("age", new Property.Builder().integer(new IntegerNumberProperty.Builder().build()).build()) + .build(); + CreateIndexRequest createIndexRequest = new CreateIndexRequest.Builder() + .index(indexName) + .settings(settings) + .mappings(mapping) + .build(); + client.indices().create(createIndexRequest); + } + + LOGGER.info("Indexing documents"); + IndexData indexData = new IndexData("Document 1", "Text for document 1"); + IndexRequest indexRequest = new IndexRequest.Builder().index(indexName) + .id("1") + .document(indexData) + .build(); + client.index(indexRequest); + + indexData = new IndexData("Document 2", "Text for document 2"); + indexRequest = new IndexRequest.Builder().index(indexName).id("2").document(indexData).build(); + client.index(indexRequest); + + // wait for the document to index + Thread.sleep(3000); + + SearchResponse searchResponse = client.search(s -> s.index(indexName), IndexData.class); + for (var hit : searchResponse.hits().hits()) { + LOGGER.info("Found {} with score {}", hit.source(), hit.score()); + } + + LOGGER.info("Adding a new mapping to index {}", indexName); + PutMappingRequest.Builder mappingsRequestBuilder = new PutMappingRequest.Builder().index(indexName) + .source( + new SourceField.Builder() + .enabled(true) + .build()) + .routing( + new RoutingField.Builder() + .required(false) + .build()) + .dynamic(DynamicMapping.Strict) + .meta("key", JsonData.of("key value")) + .fieldNames(new FieldNamesField.Builder().enabled(false).build()) + .dateDetection(false) + .dynamicDateFormats(new ArrayList<>()) + .dynamicTemplates(new ArrayList<>()) + .numericDetection(false); + client.indices().putMapping(mappingsRequestBuilder.build()); + + GetMappingRequest mappingsRequest = new GetMappingRequest.Builder().index(indexName).build(); + GetMappingResponse getMappingResponse = client.indices().getMapping(mappingsRequest); + LOGGER.info("Mappings {} found for index {}", getMappingResponse.result().get(indexName).mappings(), indexName); + + LOGGER.info("Deleting document with id 1"); + client.delete(d -> d.index(indexName).id("1")); + + LOGGER.info("Deleting index {}", indexName); + DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest.Builder().index(indexName).build(); + client.indices().delete(deleteIndexRequest); + } catch (Exception e) { + LOGGER.error("Unexpected exception", e); + } + } +} diff --git a/samples/src/main/java/org/opensearch/client/samples/Main.java b/samples/src/main/java/org/opensearch/client/samples/Main.java new file mode 100644 index 0000000000..f6f9e4ac01 --- /dev/null +++ b/samples/src/main/java/org/opensearch/client/samples/Main.java @@ -0,0 +1,39 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.client.samples; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class Main { + private static final Logger LOGGER = LogManager.getLogger(Main.class); + + public static void main(String[] args) { + var mainClass = System.getProperty("samples.mainClass"); + if (mainClass == null || mainClass.isEmpty()) { + LOGGER.error("Please specify the main class to run with -Dsamples.mainClass=
"); + System.exit(1); + } + + if (!mainClass.startsWith("org.")) { + mainClass = Main.class.getPackageName() + "." + mainClass; + } + + LOGGER.info("Running main class: {}", mainClass); + + try { + final var clazz = Class.forName(mainClass); + final var mainMethod = clazz.getMethod("main", String[].class); + mainMethod.invoke(null, (Object) args); + } catch (Exception e) { + LOGGER.error("Failed to run main class", e); + System.exit(1); + } + } +} diff --git a/samples/src/main/java/org/opensearch/client/samples/PointInTime.java b/samples/src/main/java/org/opensearch/client/samples/PointInTime.java new file mode 100644 index 0000000000..55a348ef8c --- /dev/null +++ b/samples/src/main/java/org/opensearch/client/samples/PointInTime.java @@ -0,0 +1,81 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.client.samples; + +import java.util.Collections; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.opensearch._types.Time; +import org.opensearch.client.opensearch._types.mapping.IntegerNumberProperty; +import org.opensearch.client.opensearch._types.mapping.Property; +import org.opensearch.client.opensearch._types.mapping.TypeMapping; +import org.opensearch.client.opensearch.core.pit.CreatePitRequest; +import org.opensearch.client.opensearch.core.pit.CreatePitResponse; +import org.opensearch.client.opensearch.core.pit.DeletePitRequest; +import org.opensearch.client.opensearch.core.pit.DeletePitResponse; +import org.opensearch.client.opensearch.core.pit.ListAllPitResponse; +import org.opensearch.client.opensearch.indices.CreateIndexRequest; +import org.opensearch.client.opensearch.indices.DeleteIndexRequest; +import org.opensearch.client.opensearch.indices.IndexSettings; + +/** + * Run with: ./gradlew :samples:run -Dsamples.mainClass=PointInTime + */ +public class PointInTime { + private static final Logger LOGGER = LogManager.getLogger(PointInTime.class); + + public static void main(String[] args) { + try { + var client = SampleClient.create(); + + var version = client.info().version(); + LOGGER.info("Server: {}@{}", version.distribution(), version.number()); + + final var indexName = "my-index"; + + if (!client.indices().exists(r -> r.index(indexName)).value()) { + LOGGER.info("Creating index {}", indexName); + IndexSettings settings = new IndexSettings.Builder() + .numberOfShards("2") + .numberOfReplicas("1") + .build(); + TypeMapping mapping = new TypeMapping.Builder() + .properties("age", new Property.Builder().integer(new IntegerNumberProperty.Builder().build()).build()) + .build(); + CreateIndexRequest createIndexRequest = new CreateIndexRequest.Builder() + .index(indexName) + .settings(settings) + .mappings(mapping) + .build(); + client.indices().create(createIndexRequest); + } + + CreatePitRequest createPitRequest = new CreatePitRequest.Builder() + .targetIndexes(Collections.singletonList(indexName)) + .keepAlive(new Time.Builder().time("100m").build()).build(); + + CreatePitResponse createPitResponse = client.createPit(createPitRequest); + LOGGER.info("PIT created with id: {}", createPitResponse.pitId()); + + ListAllPitResponse listAllPitResponse = client.listAllPit(); + LOGGER.info("Found {} PITs", listAllPitResponse.pits().size()); + + DeletePitRequest deletePitRequest = new DeletePitRequest.Builder() + .pitId(Collections.singletonList(createPitResponse.pitId())).build(); + DeletePitResponse deletePitResponse = client.deletePit(deletePitRequest); + LOGGER.info("Deleting PIT: {}", deletePitResponse.pits()); + + LOGGER.info("Deleting index {}", indexName); + DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest.Builder().index(indexName).build(); + client.indices().delete(deleteIndexRequest); + } catch (Exception e) { + LOGGER.error("Unexpected exception", e); + } + } +} diff --git a/samples/src/main/java/org/opensearch/client/samples/SampleClient.java b/samples/src/main/java/org/opensearch/client/samples/SampleClient.java new file mode 100644 index 0000000000..9b73d9504f --- /dev/null +++ b/samples/src/main/java/org/opensearch/client/samples/SampleClient.java @@ -0,0 +1,71 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.client.samples; + +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import org.apache.hc.client5.http.auth.AuthScope; +import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; +import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; +import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; +import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.ssl.SSLContextBuilder; +import org.opensearch.client.json.jackson.JacksonJsonpMapper; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder; + +public class SampleClient { + public static OpenSearchClient create() throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { + var env = System.getenv(); + var https = Boolean.parseBoolean(env.getOrDefault("HTTPS", "true")); + var hostname = env.getOrDefault("HOST", "localhost"); + var port = Integer.parseInt(env.getOrDefault("PORT", "9200")); + var user = env.getOrDefault("USERNAME", "admin"); + var pass = env.getOrDefault("PASSWORD", "admin"); + + final var hosts = new HttpHost[]{ + new HttpHost(https ? "https" : "http", hostname, port) + }; + + final var sslContext = SSLContextBuilder.create() + .loadTrustMaterial(null, (chains, authType) -> true) + .build(); + + final var transport = ApacheHttpClient5TransportBuilder + .builder(hosts) + .setMapper(new JacksonJsonpMapper()) + .setHttpClientConfigCallback(httpClientBuilder -> { + final var credentialsProvider = new BasicCredentialsProvider(); + for (final var host : hosts) { + credentialsProvider.setCredentials( + new AuthScope(host), + new UsernamePasswordCredentials(user, pass.toCharArray())); + } + + // Disable SSL/TLS verification as our local testing clusters use self-signed certificates + final var tlsStrategy = ClientTlsStrategyBuilder.create() + .setSslContext(sslContext) + .setHostnameVerifier(NoopHostnameVerifier.INSTANCE) + .build(); + + final var connectionManager = PoolingAsyncClientConnectionManagerBuilder.create() + .setTlsStrategy(tlsStrategy) + .build(); + + return httpClientBuilder + .setDefaultCredentialsProvider(credentialsProvider) + .setConnectionManager(connectionManager); + }) + .build(); + return new OpenSearchClient(transport); + } +} diff --git a/samples/src/main/java/org/opensearch/client/samples/Search.java b/samples/src/main/java/org/opensearch/client/samples/Search.java new file mode 100644 index 0000000000..ccc1acb679 --- /dev/null +++ b/samples/src/main/java/org/opensearch/client/samples/Search.java @@ -0,0 +1,315 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.client.samples; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch._types.FieldValue; +import org.opensearch.client.opensearch._types.Refresh; +import org.opensearch.client.opensearch._types.aggregations.Aggregate; +import org.opensearch.client.opensearch._types.aggregations.Aggregation; +import org.opensearch.client.opensearch._types.analysis.Analyzer; +import org.opensearch.client.opensearch._types.analysis.CustomAnalyzer; +import org.opensearch.client.opensearch._types.analysis.ShingleTokenFilter; +import org.opensearch.client.opensearch._types.analysis.TokenFilter; +import org.opensearch.client.opensearch._types.analysis.TokenFilterDefinition; +import org.opensearch.client.opensearch._types.mapping.Property; +import org.opensearch.client.opensearch._types.mapping.TextProperty; +import org.opensearch.client.opensearch._types.mapping.TypeMapping; +import org.opensearch.client.opensearch.core.IndexRequest; +import org.opensearch.client.opensearch.core.SearchRequest; +import org.opensearch.client.opensearch.core.SearchResponse; +import org.opensearch.client.opensearch.core.search.CompletionSuggester; +import org.opensearch.client.opensearch.core.search.FieldSuggester; +import org.opensearch.client.opensearch.core.search.FieldSuggesterBuilders; +import org.opensearch.client.opensearch.core.search.PhraseSuggester; +import org.opensearch.client.opensearch.core.search.Suggester; +import org.opensearch.client.opensearch.core.search.TermSuggester; +import org.opensearch.client.opensearch.indices.CreateIndexRequest; +import org.opensearch.client.opensearch.indices.DeleteIndexRequest; +import org.opensearch.client.opensearch.indices.IndexSettings; +import org.opensearch.client.opensearch.indices.IndexSettingsAnalysis; +import org.opensearch.client.samples.util.AppData; +import org.opensearch.client.samples.util.IndexData; + +/** + * Run with: ./gradlew :samples:run -Dsamples.mainClass=Search + */ +public class Search { + private static final Logger LOGGER = LogManager.getLogger(IndexingBasics.class); + + private static OpenSearchClient client; + + public static void main(String[] args) { + try { + client = SampleClient.create(); + + var version = client.info().version(); + LOGGER.info("Server: {}@{}", version.distribution(), version.number()); + + final var indexName = "my-index"; + + if (!client.indices().exists(r -> r.index(indexName)).value()) { + LOGGER.info("Creating index {}", indexName); + CreateIndexRequest createIndexRequest = new CreateIndexRequest.Builder().index(indexName).build(); + client.indices().create(createIndexRequest); + } + + LOGGER.info("Indexing documents"); + IndexData indexData = new IndexData("Document 1", "Text for document 1"); + IndexRequest indexRequest = new IndexRequest.Builder().index(indexName) + .id("1") + .document(indexData) + .build(); + client.index(indexRequest); + + indexData = new IndexData("Document 2", "Text for document 2"); + indexRequest = new IndexRequest.Builder().index(indexName).id("2").document(indexData).build(); + client.index(indexRequest); + + // wait for the document to index + Thread.sleep(3000); + + SearchResponse searchResponse = client.search(s -> s.index(indexName), IndexData.class); + for (var hit : searchResponse.hits().hits()) { + LOGGER.info("Found {} with score {}", hit.source(), hit.score()); + } + + SearchRequest searchRequest = new SearchRequest.Builder().query(q -> q.match(m -> m.field("text") + .query(FieldValue.of("Text for document 2")))) + .build(); + + searchResponse = client.search(searchRequest, IndexData.class); + for (var hit : searchResponse.hits().hits()) { + LOGGER.info("Found {} with score {}", hit.source(), hit.score()); + } + + searchRequest = new SearchRequest.Builder().query(q -> q.match(m -> m.field("title") + .query(FieldValue.of("Document 1")))) + .aggregations("titles", + new Aggregation.Builder().terms(t -> t.field("title.keyword")) + .build()) + .build(); + + searchResponse = client.search(searchRequest, IndexData.class); + for (Map.Entry entry : searchResponse.aggregations().entrySet()) { + LOGGER.info("Agg - {}", entry.getKey()); + entry.getValue().sterms().buckets().array().forEach(b -> LOGGER.info("{} : {}", b.key(), b.docCount())); + } + + LOGGER.info("Deleting index {}", indexName); + DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest.Builder().index(indexName).build(); + client.indices().delete(deleteIndexRequest); + + searchWithCompletionSuggester(); + searchWithTermSuggester(); + searchWithPhraseSuggester(); + } catch (Exception e) { + LOGGER.error("Unexpected exception", e); + } + } + + public static void searchWithCompletionSuggester() { + try { + var index = "completion-suggester"; + Property intValueProp = new Property.Builder() + .long_(v -> v) + .build(); + Property msgCompletionProp = new Property.Builder() + .completion(c -> c) + .build(); + client.indices().create(c -> c + .index(index) + .mappings(m -> m + .properties("intValue", intValueProp) + .properties("msg", msgCompletionProp))); + + AppData appData = new AppData(); + appData.setIntValue(1337); + appData.setMsg("foo"); + + client.index(b -> b + .index(index) + .id("1") + .document(appData) + .refresh(Refresh.True)); + + appData.setIntValue(1338); + appData.setMsg("foobar"); + + client.index(b -> b + .index(index) + .id("2") + .document(appData) + .refresh(Refresh.True)); + + String suggesterName = "msgSuggester"; + + CompletionSuggester completionSuggester = FieldSuggesterBuilders.completion() + .field("msg") + .size(1) + .build(); + FieldSuggester fieldSuggester = new FieldSuggester.Builder().prefix("foo") + .completion(completionSuggester) + .build(); + Suggester suggester = new Suggester.Builder() + .suggesters(Collections.singletonMap(suggesterName, fieldSuggester)) + .build(); + SearchRequest searchRequest = new SearchRequest.Builder() + .index(index) + .suggest(suggester) + .build(); + + SearchResponse searchResponse = client.search(searchRequest, AppData.class); + LOGGER.info("Suggester response size {}", searchResponse.suggest().get(suggesterName).size()); + + client.indices().delete(new DeleteIndexRequest.Builder().index(index).build()); + } catch (Exception e) { + LOGGER.error("Unexpected exception", e); + } + } + + public static void searchWithTermSuggester() { + try { + String index = "term-suggester"; + + // term suggester does not require a special mapping + client.indices().create(c -> c + .index(index)); + + AppData appData = new AppData(); + appData.setIntValue(1337); + appData.setMsg("foo"); + + client.index(b -> b + .index(index) + .id("1") + .document(appData) + .refresh(Refresh.True)); + + appData.setIntValue(1338); + appData.setMsg("foobar"); + + client.index(b -> b + .index(index) + .id("2") + .document(appData) + .refresh(Refresh.True)); + + String suggesterName = "msgSuggester"; + + TermSuggester termSuggester = FieldSuggesterBuilders.term() + .field("msg") + .size(1) + .build(); + FieldSuggester fieldSuggester = new FieldSuggester.Builder().text("fool") + .term(termSuggester) + .build(); + Suggester suggester = new Suggester.Builder() + .suggesters(Collections.singletonMap(suggesterName, fieldSuggester)) + .build(); + SearchRequest searchRequest = new SearchRequest.Builder() + .index(index) + .suggest(suggester) + .build(); + + SearchResponse searchResponse = client.search(searchRequest, AppData.class); + LOGGER.info("Suggester response size {}", searchResponse.suggest().get(suggesterName).size()); + + client.indices().delete(new DeleteIndexRequest.Builder().index(index).build()); + } catch (Exception e) { + LOGGER.error("Unexpected exception", e); + } + } + + public static void searchWithPhraseSuggester() { + try { + String index = "test-phrase-suggester"; + + ShingleTokenFilter shingleTokenFilter = new ShingleTokenFilter.Builder().minShingleSize("2") + .maxShingleSize("3") + .build(); + + Analyzer analyzer = new Analyzer.Builder() + .custom(new CustomAnalyzer.Builder().tokenizer("standard") + .filter(Arrays.asList("lowercase", "shingle")).build()) + .build(); + + TokenFilter tokenFilter = new TokenFilter.Builder() + .definition(new TokenFilterDefinition.Builder() + .shingle(shingleTokenFilter).build()) + .build(); + + IndexSettingsAnalysis indexSettingsAnalysis = new IndexSettingsAnalysis.Builder() + .analyzer("trigram", analyzer) + .filter("shingle", tokenFilter) + .build(); + + IndexSettings settings = new IndexSettings.Builder().analysis(indexSettingsAnalysis).build(); + + TypeMapping mapping = new TypeMapping.Builder().properties("msg", new Property.Builder() + .text(new TextProperty.Builder().fields("trigram", new Property.Builder() + .text(new TextProperty.Builder().analyzer("trigram").build()) + .build()).build()) + .build()).build(); + + client.indices().create(c -> c + .index(index) + .settings(settings) + .mappings(mapping)); + + AppData appData = new AppData(); + appData.setIntValue(1337); + appData.setMsg("Design Patterns"); + + client.index(b -> b + .index(index) + .id("1") + .document(appData) + .refresh(Refresh.True)); + + appData.setIntValue(1338); + appData.setMsg("Software Architecture Patterns Explained"); + + client.index(b -> b + .index(index) + .id("2") + .document(appData) + .refresh(Refresh.True)); + + String suggesterName = "msgSuggester"; + + PhraseSuggester phraseSuggester = FieldSuggesterBuilders.phrase() + .field("msg.trigram") + .build(); + FieldSuggester fieldSuggester = new FieldSuggester.Builder().text("design paterns") + .phrase(phraseSuggester) + .build(); + Suggester suggester = new Suggester.Builder() + .suggesters(Collections.singletonMap(suggesterName, fieldSuggester)) + .build(); + SearchRequest searchRequest = new SearchRequest.Builder() + .index(index) + .suggest(suggester) + .build(); + + SearchResponse searchResponse = client.search(searchRequest, AppData.class); + LOGGER.info("Suggester response size {}", searchResponse.suggest().get(suggesterName).size()); + + client.indices().delete(new DeleteIndexRequest.Builder().index(index).build()); + } catch (Exception e) { + LOGGER.error("Unexpected exception", e); + } + } +} diff --git a/samples/src/main/java/org/opensearch/client/samples/knn/KnnBasics.java b/samples/src/main/java/org/opensearch/client/samples/knn/KnnBasics.java new file mode 100644 index 0000000000..1ac93a2723 --- /dev/null +++ b/samples/src/main/java/org/opensearch/client/samples/knn/KnnBasics.java @@ -0,0 +1,113 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.client.samples.knn; + +import java.util.Arrays; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.opensearch.core.BulkRequest; +import org.opensearch.client.samples.SampleClient; +import org.opensearch.client.samples.util.RandUtil; + +/** + * Run with: ./gradlew :samples:run -Dsamples.mainClass=knn.KnnBasics + */ +public class KnnBasics { + private static final Logger LOGGER = LogManager.getLogger(KnnBasics.class); + + public static void main(String[] args) { + try { + var client = SampleClient.create(); + + var version = client.info().version(); + LOGGER.info("Server: {}@{}", version.distribution(), version.number()); + + final var indexName = "my-index"; + final var dimensions = 5; + + if (!client.indices().exists(r -> r.index(indexName)).value()) { + LOGGER.info("Creating index {}", indexName); + client.indices().create(r -> r + .index(indexName) + .settings(s -> s.knn(true)) + .mappings(m -> m + .properties("values", p -> p + .knnVector(k -> k.dimension(dimensions))))); + } + + final var nVectors = 10; + var bulkRequest = new BulkRequest.Builder(); + for (var i = 0; i < nVectors; ++i) { + var id = Integer.toString(i); + var doc = Doc.rand(dimensions); + bulkRequest.operations(b -> b + .index(o -> o + .index(indexName) + .id(id) + .document(doc))); + } + + LOGGER.info("Indexing {} vectors", nVectors); + client.bulk(bulkRequest.build()); + + LOGGER.info("Waiting for indexing to finish"); + client.indices().refresh(i -> i.index(indexName)); + + final var searchVector = RandUtil.rand2SfArray(dimensions); + LOGGER.info("Searching for vector {}", searchVector); + + var searchResponse = client.search(s -> s + .index(indexName) + .query(q -> q + .knn(k -> k + .field("values") + .vector(searchVector) + .k(3))), + Doc.class); + + for (var hit : searchResponse.hits().hits()) { + LOGGER.info("Found {} with score {}", hit.source(), hit.score()); + } + + LOGGER.info("Deleting index {}", indexName); + client.indices().delete(r -> r.index(indexName)); + } catch (Exception e) { + LOGGER.error("Unexpected exception", e); + } + } + + public static class Doc { + private float[] values; + + public Doc() {} + + public Doc(float[] values) { + this.values = values; + } + + public static Doc rand(int dimensions) { + return new Doc(RandUtil.rand2SfArray(dimensions)); + } + + public float[] getValues() { + return values; + } + + public void setValues(float[] values) { + this.values = values; + } + + @Override + public String toString() { + return "{" + + "values=" + Arrays.toString(values) + + '}'; + } + } +} diff --git a/samples/src/main/java/org/opensearch/client/samples/knn/KnnBooleanFilter.java b/samples/src/main/java/org/opensearch/client/samples/knn/KnnBooleanFilter.java new file mode 100644 index 0000000000..47f637a8d0 --- /dev/null +++ b/samples/src/main/java/org/opensearch/client/samples/knn/KnnBooleanFilter.java @@ -0,0 +1,159 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.client.samples.knn; + +import java.util.Arrays; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.opensearch.core.BulkRequest; +import org.opensearch.client.samples.SampleClient; +import org.opensearch.client.samples.util.RandUtil; + +/** + * Run with: ./gradlew :samples:run -Dsamples.mainClass=knn.KnnBooleanFilter + */ +public class KnnBooleanFilter { + private static final Logger LOGGER = LogManager.getLogger(KnnBooleanFilter.class); + + public static void main(String[] args) { + try { + var client = SampleClient.create(); + + var version = client.info().version(); + LOGGER.info("Server: {}@{}", version.distribution(), version.number()); + + final var indexName = "my-index"; + final var dimensions = 5; + + if (!client.indices().exists(r -> r.index(indexName)).value()) { + LOGGER.info("Creating index {}", indexName); + client.indices().create(r -> r + .index(indexName) + .settings(s -> s.knn(true)) + .mappings(m -> m + .properties("values", p -> p + .knnVector(k -> k.dimension(dimensions))))); + } + + final var nVectors = 3000; + final var genres = new String[] {"fiction", "drama", "romance"}; + var bulkRequest = new BulkRequest.Builder(); + for (var i = 0; i < nVectors; ++i) { + var id = Integer.toString(i); + var doc = Doc.rand(dimensions, genres); + bulkRequest.operations(b -> b + .index(o -> o + .index(indexName) + .id(id) + .document(doc))); + } + + LOGGER.info("Indexing {} vectors", nVectors); + client.bulk(bulkRequest.build()); + + LOGGER.info("Waiting for indexing to finish"); + client.indices().refresh(i -> i.index(indexName)); + + final var searchGenre = RandUtil.choice(genres); + final var searchVector = RandUtil.rand2SfArray(dimensions); + LOGGER.info("Searching for vector {} with the '{}' genre", searchVector, searchGenre); + + var searchResponse = client.search(s -> s + .index(indexName) + .query(q -> q + .bool(b -> b + .filter(f -> f + .bool(b2 -> b2 + .must(m -> m + .term(t -> t + .field("metadata.genre") + .value(v -> v.stringValue(searchGenre)))))) + .must(m -> m + .knn(k -> k + .field("values") + .vector(searchVector) + .k(5))))), + Doc.class); + + for (var hit : searchResponse.hits().hits()) { + LOGGER.info("Found {} with score {}", hit.source(), hit.score()); + } + + LOGGER.info("Deleting index {}", indexName); + client.indices().delete(r -> r.index(indexName)); + } catch (Exception e) { + LOGGER.error("Unexpected exception", e); + } + } + + public static class Doc { + private float[] values; + private Metadata metadata; + + public Doc() {} + + public Doc(float[] values, Metadata metadata) { + this.values = values; + this.metadata = metadata; + } + + public static Doc rand(int dimensions, String[] genres) { + return new Doc(RandUtil.rand2SfArray(dimensions), new Metadata(RandUtil.choice(genres))); + } + + public float[] getValues() { + return values; + } + + public void setValues(float[] values) { + this.values = values; + } + + public Metadata getMetadata() { + return metadata; + } + + public void setMetadata(Metadata metadata) { + this.metadata = metadata; + } + + @Override + public String toString() { + return "{" + + "values=" + Arrays.toString(values) + + ", metadata=" + metadata + + '}'; + } + } + + private static class Metadata { + private String genre; + + private Metadata() {} + + private Metadata(String genre) { + this.genre = genre; + } + + public String getGenre() { + return genre; + } + + public void setGenre(String genre) { + this.genre = genre; + } + + @Override + public String toString() { + return "{" + + "genre=" + genre + + '}'; + } + } +} diff --git a/samples/src/main/java/org/opensearch/client/samples/knn/KnnEfficientFilter.java b/samples/src/main/java/org/opensearch/client/samples/knn/KnnEfficientFilter.java new file mode 100644 index 0000000000..6f512a0f1d --- /dev/null +++ b/samples/src/main/java/org/opensearch/client/samples/knn/KnnEfficientFilter.java @@ -0,0 +1,174 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.client.samples.knn; + +import java.util.Arrays; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.opensearch._types.FieldValue; +import org.opensearch.client.opensearch._types.query_dsl.Query; +import org.opensearch.client.opensearch.core.BulkRequest; +import org.opensearch.client.samples.SampleClient; + +/** + * Run with: ./gradlew :samples:run -Dsamples.mainClass=knn.KnnEfficientFilter + */ +public class KnnEfficientFilter { + private static final Logger LOGGER = LogManager.getLogger(KnnEfficientFilter.class); + + public static void main(String[] args) { + try { + var client = SampleClient.create(); + + var version = client.info().version(); + LOGGER.info("Server: {}@{}", version.distribution(), version.number()); + + final var indexName = "hotels-index"; + + if (!client.indices().exists(r -> r.index(indexName)).value()) { + LOGGER.info("Creating index {}", indexName); + client.indices().create(r -> r + .index(indexName) + .settings(s -> s + .knn(true) + .knnAlgoParamEfSearch(100) + .numberOfShards("1") + .numberOfReplicas("0")) + .mappings(m -> m + .properties("location", p -> p + .knnVector(k -> k + .dimension(2) + .method(v -> v + .name("hnsw") + .spaceType("l2") + .engine("lucene") + .parameters("ef_construction", JsonData.of(100)) + .parameters("m", JsonData.of(16))))))); + } + + final var hotels = new Hotel[] { + new Hotel(5.2f, 4.f, true, 5), + new Hotel(5.2f, 3.9f, false, 4), + new Hotel(4.9f, 3.4f, true, 9), + new Hotel(4.2f, 4.6f, false, 6), + new Hotel(3.3f, 4.5f, true, 8), + new Hotel(6.4f, 3.4f, true, 9), + new Hotel(4.2f, 6.2f, true, 5), + new Hotel(2.4f, 4.0f, true, 8), + new Hotel(1.4f, 3.2f, false, 5), + new Hotel(7.0f, 9.9f, true, 9), + new Hotel(3.0f, 2.3f, false, 6), + new Hotel(5.0f, 1.0f, true, 3), + }; + var bulkRequest = new BulkRequest.Builder(); + for (var i = 0; i < hotels.length; ++i) { + final var id = Integer.toString(i + 1); + final var hotel = hotels[i]; + LOGGER.info("Indexing hotel {} with id {}", hotel, id); + bulkRequest.operations(b -> b + .index(o -> o + .index(indexName) + .id(id) + .document(hotel))); + } + + LOGGER.info("Indexing {} documents", hotels.length); + client.bulk(bulkRequest.build()); + + LOGGER.info("Waiting for indexing to finish"); + client.indices().refresh(i -> i.index(indexName)); + + final var searchLocation = new float[]{ 5.0f, 4.0f }; + final var searchRatingMin = 8; + final var searchRatingMax = 10; + final var searchParking = true; + LOGGER.info( + "Searching for hotel near {} with rating >={},<={} and parking={}", + searchLocation, searchRatingMin, searchRatingMax, searchParking); + + var searchResponse = client.search(s -> s + .index(indexName) + .size(3) + .query(q -> q + .knn(k -> k + .field("location") + .vector(searchLocation) + .k(3) + .filter(Query.of(f -> f + .bool(b -> b + .must(m -> m + .range(r -> r + .field("rating") + .gte(JsonData.of(searchRatingMin)) + .lte(JsonData.of(searchRatingMax)))) + .must(m -> m + .term(t -> t + .field("parking") + .value(FieldValue.of(searchParking))))))))), + Hotel.class); + + for (var hit : searchResponse.hits().hits()) { + LOGGER.info("Found {} with score {}", hit.source(), hit.score()); + } + + LOGGER.info("Deleting index {}", indexName); + client.indices().delete(r -> r.index(indexName)); + } catch (Exception e) { + LOGGER.error("Unexpected exception", e); + } + } + + public static class Hotel { + private float[] location; + private boolean parking; + private int rating; + + public Hotel() {} + + public Hotel(float locX, float locY, boolean parking, int rating) { + this.location = new float[] {locX, locY}; + this.parking = parking; + this.rating = rating; + } + + public float[] getLocation() { + return location; + } + + public void setLocation(float[] location) { + this.location = location; + } + + public boolean isParking() { + return parking; + } + + public void setParking(boolean parking) { + this.parking = parking; + } + + public int getRating() { + return rating; + } + + public void setRating(int rating) { + this.rating = rating; + } + + @Override + public String toString() { + return "{" + + "location=" + Arrays.toString(location) + + ", parking=" + parking + + ", rating=" + rating + + '}'; + } + } +} diff --git a/samples/src/main/java/org/opensearch/client/samples/knn/KnnPainlessScript.java b/samples/src/main/java/org/opensearch/client/samples/knn/KnnPainlessScript.java new file mode 100644 index 0000000000..3ff803138a --- /dev/null +++ b/samples/src/main/java/org/opensearch/client/samples/knn/KnnPainlessScript.java @@ -0,0 +1,117 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.client.samples.knn; + +import java.util.Arrays; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.opensearch.core.BulkRequest; +import org.opensearch.client.samples.SampleClient; +import org.opensearch.client.samples.util.RandUtil; + +/** + * Run with: ./gradlew :samples:run -Dsamples.mainClass=knn.KnnPainlessScript + */ +public class KnnPainlessScript { + private static final Logger LOGGER = LogManager.getLogger(KnnPainlessScript.class); + + public static void main(String[] args) { + try { + var client = SampleClient.create(); + + var version = client.info().version(); + LOGGER.info("Server: {}@{}", version.distribution(), version.number()); + + final var indexName = "my-index"; + final var dimensions = 5; + + if (!client.indices().exists(r -> r.index(indexName)).value()) { + LOGGER.info("Creating index {}", indexName); + client.indices().create(r -> r + .index(indexName) + .settings(s -> s.knn(true)) + .mappings(m -> m + .properties("values", p -> p + .knnVector(k -> k.dimension(dimensions))))); + } + + final var nVectors = 10; + var bulkRequest = new BulkRequest.Builder(); + for (var i = 0; i < nVectors; ++i) { + var id = Integer.toString(i); + var doc = Doc.rand(dimensions); + bulkRequest.operations(b -> b + .index(o -> o + .index(indexName) + .id(id) + .document(doc))); + } + + LOGGER.info("Indexing {} vectors", nVectors); + client.bulk(bulkRequest.build()); + + LOGGER.info("Waiting for indexing to finish"); + client.indices().refresh(i -> i.index(indexName)); + + final var searchVector = RandUtil.rand2SfArray(dimensions); + LOGGER.info("Searching for vector {}", searchVector); + + var searchResponse = client.search(s -> s + .index(indexName) + .query(q -> q + .scriptScore(ss -> ss + .query(qq -> qq.matchAll(m -> m)) + .script(sss -> sss + .inline(i -> i + .source("1.0 + cosineSimilarity(params.query_value, doc[params.field])") + .params("field", JsonData.of("values")) + .params("query_value", JsonData.of(searchVector)))))), + Doc.class); + + for (var hit : searchResponse.hits().hits()) { + LOGGER.info("Found {} with score {}", hit.source(), hit.score()); + } + + LOGGER.info("Deleting index {}", indexName); + client.indices().delete(r -> r.index(indexName)); + } catch (Exception e) { + LOGGER.error("Unexpected exception", e); + } + } + + public static class Doc { + private float[] values; + + public Doc() {} + + public Doc(float[] values) { + this.values = values; + } + + public static Doc rand(int dimensions) { + return new Doc(RandUtil.rand2SfArray(dimensions)); + } + + public float[] getValues() { + return values; + } + + public void setValues(float[] values) { + this.values = values; + } + + @Override + public String toString() { + return "{" + + "values=" + Arrays.toString(values) + + '}'; + } + } +} diff --git a/samples/src/main/java/org/opensearch/client/samples/knn/KnnScriptScore.java b/samples/src/main/java/org/opensearch/client/samples/knn/KnnScriptScore.java new file mode 100644 index 0000000000..0ee46e4fc0 --- /dev/null +++ b/samples/src/main/java/org/opensearch/client/samples/knn/KnnScriptScore.java @@ -0,0 +1,119 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.client.samples.knn; + +import java.util.Arrays; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.opensearch.core.BulkRequest; +import org.opensearch.client.samples.SampleClient; +import org.opensearch.client.samples.util.RandUtil; + +/** + * Run with: ./gradlew :samples:run -Dsamples.mainClass=knn.KnnScriptScore + */ +public class KnnScriptScore { + private static final Logger LOGGER = LogManager.getLogger(KnnScriptScore.class); + + public static void main(String[] args) { + try { + var client = SampleClient.create(); + + var version = client.info().version(); + LOGGER.info("Server: {}@{}", version.distribution(), version.number()); + + final var indexName = "my-index"; + final var dimensions = 5; + + if (!client.indices().exists(r -> r.index(indexName)).value()) { + LOGGER.info("Creating index {}", indexName); + client.indices().create(r -> r + .index(indexName) + .settings(s -> s.knn(true)) + .mappings(m -> m + .properties("values", p -> p + .knnVector(k -> k.dimension(dimensions))))); + } + + final var nVectors = 10; + var bulkRequest = new BulkRequest.Builder(); + for (var i = 0; i < nVectors; ++i) { + var id = Integer.toString(i); + var doc = Doc.rand(dimensions); + bulkRequest.operations(b -> b + .index(o -> o + .index(indexName) + .id(id) + .document(doc))); + } + + LOGGER.info("Indexing {} vectors", nVectors); + client.bulk(bulkRequest.build()); + + LOGGER.info("Waiting for indexing to finish"); + client.indices().refresh(i -> i.index(indexName)); + + final var searchVector = RandUtil.rand2SfArray(dimensions); + LOGGER.info("Searching for vector {}", searchVector); + + var searchResponse = client.search(s -> s + .index(indexName) + .query(q -> q + .scriptScore(ss -> ss + .query(qq -> qq.matchAll(m -> m)) + .script(sss -> sss + .inline(i -> i + .source("knn_score") + .lang("knn") + .params("field", JsonData.of("values")) + .params("query_value", JsonData.of(searchVector)) + .params("space_type", JsonData.of("cosinesimil")))))), + Doc.class); + + for (var hit : searchResponse.hits().hits()) { + LOGGER.info("Found {} with score {}", hit.source(), hit.score()); + } + + LOGGER.info("Deleting index {}", indexName); + client.indices().delete(r -> r.index(indexName)); + } catch (Exception e) { + LOGGER.error("Unexpected exception", e); + } + } + + public static class Doc { + private float[] values; + + public Doc() {} + + public Doc(float[] values) { + this.values = values; + } + + public static Doc rand(int dimensions) { + return new Doc(RandUtil.rand2SfArray(dimensions)); + } + + public float[] getValues() { + return values; + } + + public void setValues(float[] values) { + this.values = values; + } + + @Override + public String toString() { + return "{" + + "values=" + Arrays.toString(values) + + '}'; + } + } +} diff --git a/samples/src/main/java/org/opensearch/client/samples/util/AppData.java b/samples/src/main/java/org/opensearch/client/samples/util/AppData.java new file mode 100644 index 0000000000..3b6968722a --- /dev/null +++ b/samples/src/main/java/org/opensearch/client/samples/util/AppData.java @@ -0,0 +1,30 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.client.samples.util; + +public class AppData { + private int intValue; + private String msg; + + public int getIntValue() { + return intValue; + } + + public void setIntValue(int intValue) { + this.intValue = intValue; + } + + public String getMsg() { + return msg; + } + + public void setMsg(String msg) { + this.msg = msg; + } +} diff --git a/samples/src/main/java/org/opensearch/client/samples/util/IndexData.java b/samples/src/main/java/org/opensearch/client/samples/util/IndexData.java new file mode 100644 index 0000000000..759f03be8f --- /dev/null +++ b/samples/src/main/java/org/opensearch/client/samples/util/IndexData.java @@ -0,0 +1,43 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.client.samples.util; + +public class IndexData { + private String title; + private String text; + + public IndexData() { + } + + public IndexData(String title, String text) { + this.title = title; + this.text = text; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getText() { + return text; + } + + public void setText(String text) { + this.text = text; + } + + @Override + public String toString() { + return String.format("IndexData{title='%s', text='%s'}", title, text); + } +} diff --git a/samples/src/main/java/org/opensearch/client/samples/util/RandUtil.java b/samples/src/main/java/org/opensearch/client/samples/util/RandUtil.java new file mode 100644 index 0000000000..fbc2249621 --- /dev/null +++ b/samples/src/main/java/org/opensearch/client/samples/util/RandUtil.java @@ -0,0 +1,23 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.client.samples.util; + +public class RandUtil { + public static float[] rand2SfArray(int n) { + var arr = new float[n]; + for (var i = 0; i < n; ++i) { + arr[i] = Math.round(Math.random() * 100.0) / 100.0f; + } + return arr; + } + + public static T choice(T[] arr) { + return arr[(int)Math.floor(Math.random() * arr.length)]; + } +} diff --git a/samples/src/main/resources/log4j2.xml b/samples/src/main/resources/log4j2.xml new file mode 100644 index 0000000000..a388d16da0 --- /dev/null +++ b/samples/src/main/resources/log4j2.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index 2f1f60b93b..6d462291d4 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -32,3 +32,4 @@ rootProject.name = "opensearch-java" include("java-client") +include("samples") \ No newline at end of file