diff --git a/.buildkite/pipelines/pull-request/packaging-tests-unix.yml b/.buildkite/pipelines/pull-request/packaging-tests-unix.yml index e94baac8d9448..04ccc41891b3b 100644 --- a/.buildkite/pipelines/pull-request/packaging-tests-unix.yml +++ b/.buildkite/pipelines/pull-request/packaging-tests-unix.yml @@ -5,7 +5,7 @@ steps: steps: - label: "{{matrix.image}} / docker / packaging-tests-unix" key: "packaging-tests-unix-docker" - command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.docker + command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.docker-cloud-ess timeout_in_minutes: 300 matrix: setup: diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 235a4b2dbb4ad..ea4f34bcbe11e 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -8,6 +8,7 @@ "admin", "write" ], + "allowed_list": ["elastic-renovate-prod[bot]"], "set_commit_status": false, "build_on_commit": true, "build_on_comment": true, diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java index 27f4d68b0bc3f..652defa7b39cd 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java @@ -30,10 +30,13 @@ import org.elasticsearch.compute.data.BooleanBlock; import org.elasticsearch.compute.data.BooleanVector; import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; import org.elasticsearch.compute.data.DoubleBlock; import org.elasticsearch.compute.data.ElementType; import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.OrdinalBytesRefVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.AggregationOperator; import org.elasticsearch.compute.operator.DriverContext; @@ -78,7 +81,10 @@ public class AggregatorBenchmark { private static final String DOUBLES = "doubles"; private static final String BOOLEANS = "booleans"; private static final String BYTES_REFS = "bytes_refs"; + private static final String ORDINALS = "ordinals"; private static final String TWO_LONGS = "two_" + LONGS; + private static final String TWO_BYTES_REFS = "two_" + BYTES_REFS; + private static final String TWO_ORDINALS = "two_" + ORDINALS; private static final String LONGS_AND_BYTES_REFS = LONGS + "_and_" + BYTES_REFS; private static final String TWO_LONGS_AND_BYTES_REFS = "two_" + LONGS + "_and_" + BYTES_REFS; @@ -119,7 +125,21 @@ public class AggregatorBenchmark { } } - @Param({ NONE, LONGS, INTS, DOUBLES, BOOLEANS, BYTES_REFS, TWO_LONGS, LONGS_AND_BYTES_REFS, TWO_LONGS_AND_BYTES_REFS }) + @Param( + { + NONE, + LONGS, + INTS, + DOUBLES, + BOOLEANS, + BYTES_REFS, + ORDINALS, + TWO_LONGS, + TWO_BYTES_REFS, + TWO_ORDINALS, + LONGS_AND_BYTES_REFS, + TWO_LONGS_AND_BYTES_REFS } + ) public String grouping; @Param({ COUNT, COUNT_DISTINCT, MIN, MAX, SUM }) @@ -144,8 +164,12 @@ private static Operator operator(DriverContext driverContext, String grouping, S case INTS -> List.of(new BlockHash.GroupSpec(0, ElementType.INT)); case DOUBLES -> List.of(new BlockHash.GroupSpec(0, ElementType.DOUBLE)); case BOOLEANS -> List.of(new BlockHash.GroupSpec(0, ElementType.BOOLEAN)); - case BYTES_REFS -> List.of(new BlockHash.GroupSpec(0, ElementType.BYTES_REF)); + case BYTES_REFS, ORDINALS -> List.of(new BlockHash.GroupSpec(0, ElementType.BYTES_REF)); case TWO_LONGS -> List.of(new BlockHash.GroupSpec(0, ElementType.LONG), new BlockHash.GroupSpec(1, ElementType.LONG)); + case TWO_BYTES_REFS, TWO_ORDINALS -> List.of( + new BlockHash.GroupSpec(0, ElementType.BYTES_REF), + new BlockHash.GroupSpec(1, ElementType.BYTES_REF) + ); case LONGS_AND_BYTES_REFS -> List.of( new BlockHash.GroupSpec(0, ElementType.LONG), new BlockHash.GroupSpec(1, ElementType.BYTES_REF) @@ -218,6 +242,10 @@ private static void checkGrouped(String prefix, String grouping, String op, Stri checkGroupingBlock(prefix, LONGS, page.getBlock(0)); checkGroupingBlock(prefix, LONGS, page.getBlock(1)); } + case TWO_BYTES_REFS, TWO_ORDINALS -> { + checkGroupingBlock(prefix, BYTES_REFS, page.getBlock(0)); + checkGroupingBlock(prefix, BYTES_REFS, page.getBlock(1)); + } case LONGS_AND_BYTES_REFS -> { checkGroupingBlock(prefix, LONGS, page.getBlock(0)); checkGroupingBlock(prefix, BYTES_REFS, page.getBlock(1)); @@ -379,7 +407,7 @@ private static void checkGroupingBlock(String prefix, String grouping, Block blo throw new AssertionError(prefix + "bad group expected [true] but was [" + groups.getBoolean(1) + "]"); } } - case BYTES_REFS -> { + case BYTES_REFS, ORDINALS -> { BytesRefBlock groups = (BytesRefBlock) block; for (int g = 0; g < GROUPS; g++) { if (false == groups.getBytesRef(g, new BytesRef()).equals(bytesGroup(g))) { @@ -508,6 +536,8 @@ private static Block dataBlock(BlockFactory blockFactory, String blockType) { private static List groupingBlocks(String grouping, String blockType) { return switch (grouping) { case TWO_LONGS -> List.of(groupingBlock(LONGS, blockType), groupingBlock(LONGS, blockType)); + case TWO_BYTES_REFS -> List.of(groupingBlock(BYTES_REFS, blockType), groupingBlock(BYTES_REFS, blockType)); + case TWO_ORDINALS -> List.of(groupingBlock(ORDINALS, blockType), groupingBlock(ORDINALS, blockType)); case LONGS_AND_BYTES_REFS -> List.of(groupingBlock(LONGS, blockType), groupingBlock(BYTES_REFS, blockType)); case TWO_LONGS_AND_BYTES_REFS -> List.of( groupingBlock(LONGS, blockType), @@ -570,6 +600,19 @@ private static Block groupingBlock(String grouping, String blockType) { } yield builder.build(); } + case ORDINALS -> { + IntVector.Builder ordinals = blockFactory.newIntVectorBuilder(BLOCK_LENGTH * valuesPerGroup); + for (int i = 0; i < BLOCK_LENGTH; i++) { + for (int v = 0; v < valuesPerGroup; v++) { + ordinals.appendInt(i % GROUPS); + } + } + BytesRefVector.Builder bytes = blockFactory.newBytesRefVectorBuilder(BLOCK_LENGTH * valuesPerGroup); + for (int i = 0; i < GROUPS; i++) { + bytes.appendBytesRef(bytesGroup(i)); + } + yield new OrdinalBytesRefVector(ordinals.build(), bytes.build()).asBlock(); + } default -> throw new UnsupportedOperationException("unsupported grouping [" + grouping + "]"); }; } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java index 95f279bfa5162..d80256ee36a17 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java @@ -24,22 +24,17 @@ public enum DockerBase { // Base image with extras for Cloud CLOUD("ubuntu:20.04", "-cloud", "apt-get"), - // Based on CLOUD above, with more extras. We don't set a base image because - // we programmatically extend from the Cloud image. - CLOUD_ESS(null, "-cloud-ess", "apt-get"), - // Chainguard based wolfi image with latest jdk // This is usually updated via renovatebot // spotless:off - WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:90888b190da54062f67f3fef1372eb0ae7d81ea55f5a1f56d748b13e4853d984", + WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:277ebb42c458ef39cb4028f9204f0b3d51d8cd628ea737a65696a1143c3e42fe", "-wolfi", "apk" ), // spotless:on - // Based on WOLFI above, with more extras. We don't set a base image because - // we programmatically extend from the Wolfi image. - WOLFI_ESS(null, "-wolfi-ess", "apk"); + // we programmatically extend from the wolfi image. + CLOUD_ESS(null, "-cloud-ess", "apk"); private final String image; private final String suffix; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java index 6b93ea10283ae..19309fe2da8a3 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java @@ -181,9 +181,6 @@ private static String distributionProjectName(ElasticsearchDistribution distribu if (distribution.getType() == InternalElasticsearchDistributionTypes.DOCKER_WOLFI) { return projectName + "wolfi-docker" + archString + "-export"; } - if (distribution.getType() == InternalElasticsearchDistributionTypes.DOCKER_WOLFI_ESS) { - return projectName + "wolfi-ess-docker" + archString + "-export"; - } return projectName + distribution.getType().getName(); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerWolfiEssElasticsearchDistributionType.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerWolfiEssElasticsearchDistributionType.java deleted file mode 100644 index 550c43d43a536..0000000000000 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerWolfiEssElasticsearchDistributionType.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.gradle.internal.distribution; - -import org.elasticsearch.gradle.ElasticsearchDistributionType; - -public class DockerWolfiEssElasticsearchDistributionType implements ElasticsearchDistributionType { - - DockerWolfiEssElasticsearchDistributionType() {} - - @Override - public String getName() { - return "dockerWolfiEss"; - } - - @Override - public boolean isDocker() { - return true; - } -} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java index 077a47041861f..ba0e76b3f5b99 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java @@ -22,7 +22,6 @@ public class InternalElasticsearchDistributionTypes { public static ElasticsearchDistributionType DOCKER_CLOUD = new DockerCloudElasticsearchDistributionType(); public static ElasticsearchDistributionType DOCKER_CLOUD_ESS = new DockerCloudEssElasticsearchDistributionType(); public static ElasticsearchDistributionType DOCKER_WOLFI = new DockerWolfiElasticsearchDistributionType(); - public static ElasticsearchDistributionType DOCKER_WOLFI_ESS = new DockerWolfiEssElasticsearchDistributionType(); public static List ALL_INTERNAL = List.of( DEB, @@ -32,7 +31,6 @@ public class InternalElasticsearchDistributionTypes { DOCKER_IRONBANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, - DOCKER_WOLFI, - DOCKER_WOLFI_ESS + DOCKER_WOLFI ); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java index cc852e615726a..77ab9557eac33 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java @@ -54,7 +54,6 @@ import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_IRONBANK; import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_UBI; import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_WOLFI; -import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_WOLFI_ESS; import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.RPM; /** @@ -153,7 +152,6 @@ private static Map> lifecycleTask lifecyleTasks.put(DOCKER_CLOUD, project.getTasks().register(taskPrefix + ".docker-cloud")); lifecyleTasks.put(DOCKER_CLOUD_ESS, project.getTasks().register(taskPrefix + ".docker-cloud-ess")); lifecyleTasks.put(DOCKER_WOLFI, project.getTasks().register(taskPrefix + ".docker-wolfi")); - lifecyleTasks.put(DOCKER_WOLFI_ESS, project.getTasks().register(taskPrefix + ".docker-wolfi-ess")); lifecyleTasks.put(ARCHIVE, project.getTasks().register(taskPrefix + ".archives")); lifecyleTasks.put(DEB, project.getTasks().register(taskPrefix + ".packages")); lifecyleTasks.put(RPM, lifecyleTasks.get(DEB)); diff --git a/distribution/docker/README.md b/distribution/docker/README.md index 28e6ff314d91a..49facab461edc 100644 --- a/distribution/docker/README.md +++ b/distribution/docker/README.md @@ -7,7 +7,7 @@ the [DockerBase] enum. * UBI - the same as the default image, but based upon [RedHat's UBI images][ubi], specifically their minimal flavour. * Wolfi - the same as the default image, but based upon [Wolfi](https://github.com/wolfi-dev) - * Wolfi ESS - this directly extends the Wolfi image, and adds all ES plugins + * Cloud ESS - this directly extends the Wolfi image, and adds all ES plugins that the ES build generates in an archive directory. It also sets an environment variable that points at this directory. This allows plugins to be installed from the archive instead of the internet, speeding up @@ -23,7 +23,6 @@ the [DockerBase] enum. software (FOSS) and Commercial off-the-shelf (COTS). In practice, this is another UBI build, this time on the regular UBI image, with extra hardening. See below for more details. - * Cloud - this is mostly the same as the default image, with some notable differences: * `filebeat` and `metricbeat` are included * `wget` is included @@ -31,12 +30,6 @@ the [DockerBase] enum. `/app/elasticsearch.sh`. In normal use this file would be bind-mounted in, but the image ships a stub version of this file so that the image can still be tested. - * Cloud ESS - this directly extends the Cloud image, and adds all ES plugins - that the ES build generates in an archive directory. It also sets an - environment variable that points at this directory. This allows plugins to - be installed from the archive instead of the internet, speeding up - deployment times. - The long-term goal is for both Cloud images to be retired in favour of the default image. diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 99c482d91085a..e40ac68bbacf4 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -381,7 +381,7 @@ private static List generateTags(DockerBase base, Architecture architect String image = "elasticsearch${base.suffix}" String namespace = 'elasticsearch' - if (base == DockerBase.CLOUD || base == DockerBase.CLOUD_ESS || base == DockerBase.WOLFI_ESS) { + if (base == DockerBase.CLOUD || base == DockerBase.CLOUD_ESS) { namespace += '-ci' } @@ -446,7 +446,8 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) { } } -void addBuildEssDockerImageTask(Architecture architecture, DockerBase dockerBase) { +void addBuildEssDockerImageTask(Architecture architecture) { + DockerBase dockerBase = DockerBase.CLOUD_ESS String arch = architecture == Architecture.AARCH64 ? '-aarch64' : '' String contextDir = "${project.buildDir}/docker-context/elasticsearch${dockerBase.suffix}-${VersionProperties.elasticsearch}-docker-build-context${arch}" @@ -460,22 +461,20 @@ void addBuildEssDockerImageTask(Architecture architecture, DockerBase dockerBase from configurations.allPlugins } - if (dockerBase == DockerBase.WOLFI_ESS) { - // If we're performing a release build, but `build.id` hasn't been set, we can - // infer that we're not at the Docker building stage of the build, and therefore - // we should skip the beats part of the build. - String buildId = providers.systemProperty('build.id').getOrNull() - boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null || useDra + // If we're performing a release build, but `build.id` hasn't been set, we can + // infer that we're not at the Docker building stage of the build, and therefore + // we should skip the beats part of the build. + String buildId = providers.systemProperty('build.id').getOrNull() + boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null || useDra - if (includeBeats) { - from configurations.getByName("filebeat_${architecture.classifier}") - from configurations.getByName("metricbeat_${architecture.classifier}") - } - // For some reason, the artifact name can differ depending on what repository we used. - rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz" + if (includeBeats) { + from configurations.getByName("filebeat_${architecture.classifier}") + from configurations.getByName("metricbeat_${architecture.classifier}") } + // For some reason, the artifact name can differ depending on what repository we used. + rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz" - String baseSuffix = dockerBase == DockerBase.CLOUD_ESS ? DockerBase.CLOUD.suffix : DockerBase.WOLFI.suffix + String baseSuffix = DockerBase.WOLFI.suffix from(projectDir.resolve("src/docker/Dockerfile.ess")) { expand( [ @@ -493,7 +492,7 @@ void addBuildEssDockerImageTask(Architecture architecture, DockerBase dockerBase final TaskProvider buildDockerImageTask = tasks.register(taskName("build", architecture, dockerBase, "DockerImage"), DockerBuildTask) { - DockerBase base = dockerBase == DockerBase.CLOUD_ESS ? DockerBase.CLOUD : DockerBase.WOLFI + DockerBase base = DockerBase.WOLFI TaskProvider buildBaseTask = tasks.named(taskName("build", architecture, base, "DockerImage")) inputs.files(buildBaseTask) @@ -519,7 +518,7 @@ void addBuildEssDockerImageTask(Architecture architecture, DockerBase dockerBase for (final Architecture architecture : Architecture.values()) { for (final DockerBase base : DockerBase.values()) { - if (base == DockerBase.CLOUD_ESS || base == DockerBase.WOLFI_ESS) { + if (base == DockerBase.CLOUD_ESS) { continue } addBuildDockerContextTask(architecture, base) @@ -527,8 +526,7 @@ for (final Architecture architecture : Architecture.values()) { addBuildDockerImageTask(architecture, base) } - addBuildEssDockerImageTask(architecture, DockerBase.CLOUD_ESS) - addBuildEssDockerImageTask(architecture, DockerBase.WOLFI_ESS) + addBuildEssDockerImageTask(architecture) } def exportDockerImages = tasks.register("exportDockerImages") @@ -564,8 +562,7 @@ subprojects { Project subProject -> (base == DockerBase.CLOUD ? 'cloud.tar' : (base == DockerBase.CLOUD_ESS ? 'cloud-ess.tar' : (base == DockerBase.WOLFI ? 'wolfi.tar' : - (base == DockerBase.WOLFI_ESS ? 'wolfi-ess.tar' : - 'docker.tar'))))) + 'docker.tar')))) final String artifactName = "elasticsearch${arch}${base.suffix}_test" final String exportTaskName = taskName("export", architecture, base, 'DockerImage') diff --git a/distribution/docker/src/docker/Dockerfile.ess b/distribution/docker/src/docker/Dockerfile.ess index 3ca5e8f2b42a3..197af28b93455 100644 --- a/distribution/docker/src/docker/Dockerfile.ess +++ b/distribution/docker/src/docker/Dockerfile.ess @@ -2,26 +2,24 @@ FROM ${base_image} AS builder USER root -<% if (docker_base == "wolfi_ess") { %> - # Add plugins infrastructure - RUN mkdir -p /opt/plugins/archive - RUN chmod -R 0555 /opt/plugins - - COPY filebeat-${version}.tar.gz metricbeat-${version}.tar.gz /tmp/ - RUN set -eux ; \\ - for beat in filebeat metricbeat ; do \\ - if [ ! -s /tmp/\$beat-${version}.tar.gz ]; then \\ - echo "/tmp/\$beat-${version}.tar.gz is empty - cannot uncompress" 2>&1 ; \\ - exit 1 ; \\ - fi ; \\ - if ! tar tf /tmp/\$beat-${version}.tar.gz >/dev/null; then \\ - echo "/tmp/\$beat-${version}.tar.gz is corrupt - cannot uncompress" 2>&1 ; \\ - exit 1 ; \\ - fi ; \\ - mkdir -p /opt/\$beat ; \\ - tar xf /tmp/\$beat-${version}.tar.gz -C /opt/\$beat --strip-components=1 ; \\ - done -<% } %> +# Add plugins infrastructure +RUN mkdir -p /opt/plugins/archive +RUN chmod -R 0555 /opt/plugins + +COPY filebeat-${version}.tar.gz metricbeat-${version}.tar.gz /tmp/ +RUN set -eux ; \\ + for beat in filebeat metricbeat ; do \\ + if [ ! -s /tmp/\$beat-${version}.tar.gz ]; then \\ + echo "/tmp/\$beat-${version}.tar.gz is empty - cannot uncompress" 2>&1 ; \\ + exit 1 ; \\ + fi ; \\ + if ! tar tf /tmp/\$beat-${version}.tar.gz >/dev/null; then \\ + echo "/tmp/\$beat-${version}.tar.gz is corrupt - cannot uncompress" 2>&1 ; \\ + exit 1 ; \\ + fi ; \\ + mkdir -p /opt/\$beat ; \\ + tar xf /tmp/\$beat-${version}.tar.gz -C /opt/\$beat --strip-components=1 ; \\ + done COPY plugins/*.zip /opt/plugins/archive/ @@ -29,7 +27,6 @@ RUN chown 1000:1000 /opt/plugins/archive/* RUN chmod 0444 /opt/plugins/archive/* FROM ${base_image} -<% if (docker_base == "wolfi_ess") { %> USER root RUN <%= retry.loop("apk", "export DEBIAN_FRONTEND=noninteractive && apk update && apk update && apk add --no-cache wget") %> @@ -44,8 +41,4 @@ RUN mkdir /app && \\ COPY --from=builder --chown=0:0 /opt /opt USER 1000:0 -<% } else { %> -COPY --from=builder /opt/plugins /opt/plugins -<% } %> - ENV ES_PLUGIN_ARCHIVE_DIR /opt/plugins/archive diff --git a/distribution/tools/entitlement-agent/impl/build.gradle b/distribution/tools/entitlement-agent/impl/build.gradle index f73e21505d483..16f134bf0e693 100644 --- a/distribution/tools/entitlement-agent/impl/build.gradle +++ b/distribution/tools/entitlement-agent/impl/build.gradle @@ -12,6 +12,13 @@ apply plugin: 'elasticsearch.build' dependencies { compileOnly project(':distribution:tools:entitlement-agent') implementation 'org.ow2.asm:asm:9.7' + testImplementation project(":test:framework") + testImplementation project(":distribution:tools:entitlement-bridge") + testImplementation 'org.ow2.asm:asm-util:9.7' +} + +tasks.named('test').configure { + systemProperty "tests.security.manager", "false" } tasks.named('forbiddenApisMain').configure { diff --git a/distribution/tools/entitlement-agent/impl/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/ASMUtils.java b/distribution/tools/entitlement-agent/impl/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/ASMUtils.java new file mode 100644 index 0000000000000..d7aaa6d854e9c --- /dev/null +++ b/distribution/tools/entitlement-agent/impl/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/ASMUtils.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.instrumentation.impl; + +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.util.Printer; +import org.objectweb.asm.util.Textifier; +import org.objectweb.asm.util.TraceClassVisitor; + +import java.io.PrintWriter; +import java.io.StringWriter; + +public class ASMUtils { + public static String bytecode2text(byte[] classBytes) { + ClassReader classReader = new ClassReader(classBytes); + StringWriter stringWriter = new StringWriter(); + try (PrintWriter printWriter = new PrintWriter(stringWriter)) { + Printer printer = new Textifier(); // For a textual representation + TraceClassVisitor traceClassVisitor = new TraceClassVisitor(null, printer, printWriter); + classReader.accept(traceClassVisitor, 0); + return stringWriter.toString(); + } + } +} diff --git a/distribution/tools/entitlement-agent/impl/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterTests.java b/distribution/tools/entitlement-agent/impl/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterTests.java new file mode 100644 index 0000000000000..e807ecee4f103 --- /dev/null +++ b/distribution/tools/entitlement-agent/impl/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterTests.java @@ -0,0 +1,153 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.instrumentation.impl; + +import org.elasticsearch.entitlement.api.EntitlementChecks; +import org.elasticsearch.entitlement.api.EntitlementProvider; +import org.elasticsearch.entitlement.instrumentation.InstrumentationService; +import org.elasticsearch.entitlement.instrumentation.MethodKey; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Map; + +import static org.elasticsearch.entitlement.instrumentation.impl.ASMUtils.bytecode2text; + +/** + * This tests {@link InstrumenterImpl} in isolation, without a java agent. + * It causes the methods to be instrumented, and verifies that the instrumentation is called as expected. + * Problems with bytecode generation are easier to debug this way than in the context of an agent. + */ +@ESTestCase.WithoutSecurityManager +public class InstrumenterTests extends ESTestCase { + final InstrumentationService instrumentationService = new InstrumentationServiceImpl(); + + private static TestEntitlementManager getTestChecks() { + return (TestEntitlementManager) EntitlementProvider.checks(); + } + + @Before + public void initialize() { + getTestChecks().isActive = false; + } + + /** + * Contains all the virtual methods from {@link ClassToInstrument}, + * allowing this test to call them on the dynamically loaded instrumented class. + */ + public interface Testable {} + + /** + * This is a placeholder for real class library methods. + * Without the java agent, we can't instrument the real methods, so we instrument this instead. + *

+ * Methods of this class must have the same signature and the same static/virtual condition as the corresponding real method. + * They should assert that the arguments came through correctly. + * They must not throw {@link TestException}. + */ + public static class ClassToInstrument implements Testable { + public static void systemExit(int status) { + assertEquals(123, status); + } + } + + static final class TestException extends RuntimeException {} + + /** + * We're not testing the permission checking logic here. + * This is a trivial implementation of {@link EntitlementChecks} that just always throws, + * just to demonstrate that the injected bytecodes succeed in calling these methods. + */ + public static class TestEntitlementManager implements EntitlementChecks { + /** + * This allows us to test that the instrumentation is correct in both cases: + * if the check throws, and if it doesn't. + */ + volatile boolean isActive; + + @Override + public void checkSystemExit(Class callerClass, int status) { + assertSame(InstrumenterTests.class, callerClass); + assertEquals(123, status); + throwIfActive(); + } + + private void throwIfActive() { + if (isActive) { + throw new TestException(); + } + } + } + + public void test() throws Exception { + // This test doesn't replace ClassToInstrument in-place but instead loads a separate + // class ClassToInstrument_NEW that contains the instrumentation. Because of this, + // we need to configure the Transformer to use a MethodKey and instrumentationMethod + // with slightly different signatures (using the common interface Testable) which + // is not what would happen when it's run by the agent. + + MethodKey k1 = instrumentationService.methodKeyForTarget(ClassToInstrument.class.getMethod("systemExit", int.class)); + Method v1 = EntitlementChecks.class.getMethod("checkSystemExit", Class.class, int.class); + var instrumenter = new InstrumenterImpl("_NEW", Map.of(k1, v1)); + + byte[] newBytecode = instrumenter.instrumentClassFile(ClassToInstrument.class).bytecodes(); + + if (logger.isTraceEnabled()) { + logger.trace("Bytecode after instrumentation:\n{}", bytecode2text(newBytecode)); + } + + Class newClass = new TestLoader(Testable.class.getClassLoader()).defineClassFromBytes( + ClassToInstrument.class.getName() + "_NEW", + newBytecode + ); + + // Before checking is active, nothing should throw + callStaticSystemExit(newClass, 123); + + getTestChecks().isActive = true; + + // After checking is activated, everything should throw + assertThrows(TestException.class, () -> callStaticSystemExit(newClass, 123)); + } + + /** + * Calling a static method of a dynamically loaded class is significantly more cumbersome + * than calling a virtual method. + */ + private static void callStaticSystemExit(Class c, int status) throws NoSuchMethodException, IllegalAccessException { + try { + c.getMethod("systemExit", int.class).invoke(null, status); + } catch (InvocationTargetException e) { + Throwable cause = e.getCause(); + if (cause instanceof TestException n) { + // Sometimes we're expecting this one! + throw n; + } else { + throw new AssertionError(cause); + } + } + } + + static class TestLoader extends ClassLoader { + TestLoader(ClassLoader parent) { + super(parent); + } + + public Class defineClassFromBytes(String name, byte[] bytes) { + return defineClass(name, bytes, 0, bytes.length); + } + } + + private static final Logger logger = LogManager.getLogger(InstrumenterTests.class); +} diff --git a/distribution/tools/entitlement-agent/impl/src/test/resources/META-INF/services/org.elasticsearch.entitlement.api.EntitlementChecks b/distribution/tools/entitlement-agent/impl/src/test/resources/META-INF/services/org.elasticsearch.entitlement.api.EntitlementChecks new file mode 100644 index 0000000000000..983585190b35a --- /dev/null +++ b/distribution/tools/entitlement-agent/impl/src/test/resources/META-INF/services/org.elasticsearch.entitlement.api.EntitlementChecks @@ -0,0 +1,10 @@ +# + # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + # or more contributor license agreements. Licensed under the "Elastic License + # 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + # Public License v 1"; you may not use this file except in compliance with, at + # your election, the "Elastic License 2.0", the "GNU Affero General Public + # License v3.0 only", or the "Server Side Public License, v 1". +# + +org.elasticsearch.entitlement.instrumentation.impl.InstrumenterTests$TestEntitlementManager diff --git a/distribution/tools/entitlement-agent/src/test/java/org/elasticsearch/entitlement/agent/EntitlementAgentTests.java b/distribution/tools/entitlement-agent/src/test/java/org/elasticsearch/entitlement/agent/EntitlementAgentTests.java index bb775d302c1d0..cf7991626029a 100644 --- a/distribution/tools/entitlement-agent/src/test/java/org/elasticsearch/entitlement/agent/EntitlementAgentTests.java +++ b/distribution/tools/entitlement-agent/src/test/java/org/elasticsearch/entitlement/agent/EntitlementAgentTests.java @@ -24,6 +24,10 @@ * to make sure it works with the entitlement granted and throws without it. * The only exception is {@link System#exit}, where we can't that it works without * terminating the JVM. + *

+ * If you're trying to debug the instrumentation code, take a look at {@code InstrumenterTests}. + * That tests the bytecode portion without firing up an agent, which makes everything easier to troubleshoot. + *

* See {@code build.gradle} for how we set the command line arguments for this test. */ @WithoutSecurityManager diff --git a/docs/changelog/112938.yaml b/docs/changelog/112938.yaml new file mode 100644 index 0000000000000..82b98871c3352 --- /dev/null +++ b/docs/changelog/112938.yaml @@ -0,0 +1,35 @@ +pr: 112938 +summary: Enhance SORT push-down to Lucene to cover references to fields and ST_DISTANCE function +area: ES|QL +type: enhancement +issues: + - 109973 +highlight: + title: Enhance SORT push-down to Lucene to cover references to fields and ST_DISTANCE function + body: |- + The most used and likely most valuable geospatial search query in Elasticsearch is the sorted proximity search, + finding items within a certain distance of a point of interest and sorting the results by distance. + This has been possible in ES|QL since 8.15.0, but the sorting was done in-memory, not pushed down to Lucene. + Now the sorting is pushed down to Lucene, which results in a significant performance improvement. + + Queries that perform both filtering and sorting on distance are supported. For example: + + [source,esql] + ---- + FROM test + | EVAL distance = ST_DISTANCE(location, TO_GEOPOINT("POINT(37.7749, -122.4194)")) + | WHERE distance < 1000000 + | SORT distance ASC, name DESC + | LIMIT 10 + ---- + + In addition, the support for sorting on EVAL expressions has been extended to cover references to fields: + + [source,esql] + ---- + FROM test + | EVAL ref = field + | SORT ref ASC + | LIMIT 10 + ---- + notable: false diff --git a/docs/changelog/113374.yaml b/docs/changelog/113374.yaml new file mode 100644 index 0000000000000..f1d5750de0f60 --- /dev/null +++ b/docs/changelog/113374.yaml @@ -0,0 +1,5 @@ +pr: 113374 +summary: Add ESQL match function +area: ES|QL +type: feature +issues: [] diff --git a/docs/changelog/114021.yaml b/docs/changelog/114021.yaml new file mode 100644 index 0000000000000..e9dab5dce5685 --- /dev/null +++ b/docs/changelog/114021.yaml @@ -0,0 +1,5 @@ +pr: 114021 +summary: "ESQL: Speed up grouping by bytes" +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/114382.yaml b/docs/changelog/114382.yaml new file mode 100644 index 0000000000000..9f572e14f4737 --- /dev/null +++ b/docs/changelog/114382.yaml @@ -0,0 +1,5 @@ +pr: 114382 +summary: "[ES|QL] Add hypot function" +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/114411.yaml b/docs/changelog/114411.yaml new file mode 100644 index 0000000000000..23bff3c8e25ba --- /dev/null +++ b/docs/changelog/114411.yaml @@ -0,0 +1,5 @@ +pr: 114411 +summary: "ESQL: Push down filters even in case of renames in Evals" +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/114457.yaml b/docs/changelog/114457.yaml new file mode 100644 index 0000000000000..9558c41852f69 --- /dev/null +++ b/docs/changelog/114457.yaml @@ -0,0 +1,6 @@ +pr: 114457 +summary: "[Inference API] Introduce Update API to change some aspects of existing\ + \ inference endpoints" +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/114533.yaml b/docs/changelog/114533.yaml new file mode 100644 index 0000000000000..f45589e8de921 --- /dev/null +++ b/docs/changelog/114533.yaml @@ -0,0 +1,5 @@ +pr: 114533 +summary: Fix dim validation for bit `element_type` +area: Vector Search +type: bug +issues: [] diff --git a/docs/changelog/114623.yaml b/docs/changelog/114623.yaml new file mode 100644 index 0000000000000..817a8e874bcc0 --- /dev/null +++ b/docs/changelog/114623.yaml @@ -0,0 +1,5 @@ +pr: 114623 +summary: Preserve thread context when waiting for segment generation in RTG +area: CRUD +type: bug +issues: [] diff --git a/docs/changelog/114638.yaml b/docs/changelog/114638.yaml new file mode 100644 index 0000000000000..0386aacfe3e18 --- /dev/null +++ b/docs/changelog/114638.yaml @@ -0,0 +1,7 @@ +pr: 114638 +summary: "ES|QL: Restrict sorting for `_source` and counter field types" +area: ES|QL +type: bug +issues: + - 114423 + - 111976 diff --git a/docs/reference/esql/esql-across-clusters.asciidoc b/docs/reference/esql/esql-across-clusters.asciidoc index cfcb5de73602c..db266fafde9d6 100644 --- a/docs/reference/esql/esql-across-clusters.asciidoc +++ b/docs/reference/esql/esql-across-clusters.asciidoc @@ -188,9 +188,10 @@ FROM *:my-index-000001 [[ccq-cluster-details]] ==== Cross-cluster metadata -ES|QL {ccs} responses include metadata about the search on each cluster when the response format is JSON. +Using the `"include_ccs_metadata": true` option, users can request that +ES|QL {ccs} responses include metadata about the search on each cluster (when the response format is JSON). Here we show an example using the async search endpoint. {ccs-cap} metadata is also present in the synchronous -search endpoint. +search endpoint response when requested. [source,console] ---- @@ -200,7 +201,8 @@ POST /_query/async?format=json FROM my-index-000001,cluster_one:my-index-000001,cluster_two:my-index* | STATS COUNT(http.response.status_code) BY user.id | LIMIT 2 - """ + """, + "include_ccs_metadata": true } ---- // TEST[setup:my_index] @@ -238,7 +240,7 @@ Which returns: "(local)": { <4> "status": "successful", "indices": "blogs", - "took": 36, <5> + "took": 41, <5> "_shards": { <6> "total": 13, "successful": 13, @@ -260,7 +262,7 @@ Which returns: "cluster_two": { "status": "successful", "indices": "cluster_two:my-index*", - "took": 41, + "took": 40, "_shards": { "total": 18, "successful": 18, @@ -286,7 +288,7 @@ it is identified as "(local)". <5> How long (in milliseconds) the search took on each cluster. This can be useful to determine which clusters have slower response times than others. <6> The shard details for the search on that cluster, including a count of shards that were -skipped due to the can-match phase. Shards are skipped when they cannot have any matching data +skipped due to the can-match phase results. Shards are skipped when they cannot have any matching data and therefore are not included in the full ES|QL query. @@ -294,9 +296,6 @@ The cross-cluster metadata can be used to determine whether any data came back f For instance, in the query below, the wildcard expression for `cluster-two` did not resolve to a concrete index (or indices). The cluster is, therefore, marked as 'skipped' and the total number of shards searched is set to zero. -Since the other cluster did have a matching index, the search did not return an error, but -instead returned all the matching data it could find. - [source,console] ---- @@ -306,7 +305,8 @@ POST /_query/async?format=json FROM cluster_one:my-index*,cluster_two:logs* | STATS COUNT(http.response.status_code) BY user.id | LIMIT 2 - """ + """, + "include_ccs_metadata": true } ---- // TEST[continued] diff --git a/docs/reference/esql/esql-query-api.asciidoc b/docs/reference/esql/esql-query-api.asciidoc index d1db21043a5b5..b1582721ad0e0 100644 --- a/docs/reference/esql/esql-query-api.asciidoc +++ b/docs/reference/esql/esql-query-api.asciidoc @@ -67,6 +67,11 @@ precedence. `false`. The API only supports this parameter for CBOR, JSON, SMILE, and YAML responses. See <>. +`include_ccs_metadata`:: +(Optional, boolean) If `true`, cross-cluster searches will include metadata about the query +on each cluster. Defaults to `false`. The API only supports this parameter for CBOR, JSON, SMILE, +and YAML responses. See <>. + `locale`:: (Optional, string) Returns results (especially dates) formatted per the conventions of the locale. For syntax, refer to <>. @@ -85,6 +90,7 @@ https://en.wikipedia.org/wiki/Query_plan[EXPLAIN PLAN]. `query`:: (Required, string) {esql} query to run. For syntax, refer to <>. + ifeval::["{release-state}"=="unreleased"] `table`:: (Optional, object) Named "table" parameters that can be referenced by the <> command. @@ -108,6 +114,13 @@ returned if `drop_null_columns` is sent with the request. (array of arrays) Values for the search results. +`_clusters`:: +(object) +Metadata about clusters involved in the execution of a cross-cluster query. Only returned (1) for +cross-cluster searches and (2) when `include_ccs_metadata` is sent in the body and set to `true` +and (3) when `format` of the response is set to JSON (the default), CBOR, SMILE, or YAML. +See <> for more information. + `profile`:: (object) Profile describing the execution of the query. Only returned if `profile` was sent in the body. diff --git a/docs/reference/esql/functions/description/hypot.asciidoc b/docs/reference/esql/functions/description/hypot.asciidoc new file mode 100644 index 0000000000000..5162f0d9ef98f --- /dev/null +++ b/docs/reference/esql/functions/description/hypot.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Returns the hypotenuse of two numbers. The input can be any numeric values, the return value is always a double. Hypotenuses of infinities are null. diff --git a/docs/reference/esql/functions/description/match.asciidoc b/docs/reference/esql/functions/description/match.asciidoc new file mode 100644 index 0000000000000..2a27fe4814395 --- /dev/null +++ b/docs/reference/esql/functions/description/match.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Performs a match query on the specified field. Returns true if the provided query matches the row. diff --git a/docs/reference/esql/functions/examples/hypot.asciidoc b/docs/reference/esql/functions/examples/hypot.asciidoc new file mode 100644 index 0000000000000..6dbcc62e8755e --- /dev/null +++ b/docs/reference/esql/functions/examples/hypot.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/math.csv-spec[tag=hypot] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/math.csv-spec[tag=hypot-result] +|=== + diff --git a/docs/reference/esql/functions/examples/match.asciidoc b/docs/reference/esql/functions/examples/match.asciidoc new file mode 100644 index 0000000000000..3f31d68ea9abb --- /dev/null +++ b/docs/reference/esql/functions/examples/match.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/match-function.csv-spec[tag=match-with-field] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/match-function.csv-spec[tag=match-with-field-result] +|=== + diff --git a/docs/reference/esql/functions/kibana/definition/hypot.json b/docs/reference/esql/functions/kibana/definition/hypot.json new file mode 100644 index 0000000000000..06971f07a3585 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/hypot.json @@ -0,0 +1,301 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "hypot", + "description" : "Returns the hypotenuse of two numbers. The input can be any numeric values, the return value is always a double.\nHypotenuses of infinities are null.", + "signatures" : [ + { + "params" : [ + { + "name" : "number1", + "type" : "double", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "double", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "double", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "integer", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "double", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "double", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "unsigned_long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "integer", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "double", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "integer", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "integer", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "integer", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "integer", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "unsigned_long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "double", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "integer", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "unsigned_long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "unsigned_long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "double", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "unsigned_long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "integer", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "unsigned_long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number1", + "type" : "unsigned_long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + }, + { + "name" : "number2", + "type" : "unsigned_long", + "optional" : false, + "description" : "Numeric expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "double" + } + ], + "examples" : [ + "ROW a = 3.0, b = 4.0\n| EVAL c = HYPOT(a, b)" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/match.json b/docs/reference/esql/functions/kibana/definition/match.json new file mode 100644 index 0000000000000..d2fe0bba53866 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/match.json @@ -0,0 +1,85 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "match", + "description" : "Performs a match query on the specified field. Returns true if the provided query matches the row.", + "signatures" : [ + { + "params" : [ + { + "name" : "field", + "type" : "keyword", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Text you wish to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "keyword", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "text", + "optional" : false, + "description" : "Text you wish to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "text", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Text you wish to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "text", + "optional" : false, + "description" : "Field that the query will target." + }, + { + "name" : "query", + "type" : "text", + "optional" : false, + "description" : "Text you wish to find in the provided field." + } + ], + "variadic" : false, + "returnType" : "boolean" + } + ], + "examples" : [ + "from books \n| where match(author, \"Faulkner\")\n| keep book_no, author \n| sort book_no \n| limit 5;" + ], + "preview" : true, + "snapshot_only" : true +} diff --git a/docs/reference/esql/functions/kibana/docs/hypot.md b/docs/reference/esql/functions/kibana/docs/hypot.md new file mode 100644 index 0000000000000..f0cbea6b88e55 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/hypot.md @@ -0,0 +1,12 @@ + + +### HYPOT +Returns the hypotenuse of two numbers. The input can be any numeric values, the return value is always a double. +Hypotenuses of infinities are null. + +``` +ROW a = 3.0, b = 4.0 +| EVAL c = HYPOT(a, b) +``` diff --git a/docs/reference/esql/functions/kibana/docs/match.md b/docs/reference/esql/functions/kibana/docs/match.md new file mode 100644 index 0000000000000..3c06662982bbf --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/match.md @@ -0,0 +1,14 @@ + + +### MATCH +Performs a match query on the specified field. Returns true if the provided query matches the row. + +``` +from books +| where match(author, "Faulkner") +| keep book_no, author +| sort book_no +| limit 5; +``` diff --git a/docs/reference/esql/functions/layout/hypot.asciidoc b/docs/reference/esql/functions/layout/hypot.asciidoc new file mode 100644 index 0000000000000..84376a9f15908 --- /dev/null +++ b/docs/reference/esql/functions/layout/hypot.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-hypot]] +=== `HYPOT` + +*Syntax* + +[.text-center] +image::esql/functions/signature/hypot.svg[Embedded,opts=inline] + +include::../parameters/hypot.asciidoc[] +include::../description/hypot.asciidoc[] +include::../types/hypot.asciidoc[] +include::../examples/hypot.asciidoc[] diff --git a/docs/reference/esql/functions/layout/match.asciidoc b/docs/reference/esql/functions/layout/match.asciidoc new file mode 100644 index 0000000000000..e62c81548c2b1 --- /dev/null +++ b/docs/reference/esql/functions/layout/match.asciidoc @@ -0,0 +1,17 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-match]] +=== `MATCH` + +preview::["Do not use on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] + +*Syntax* + +[.text-center] +image::esql/functions/signature/match.svg[Embedded,opts=inline] + +include::../parameters/match.asciidoc[] +include::../description/match.asciidoc[] +include::../types/match.asciidoc[] +include::../examples/match.asciidoc[] diff --git a/docs/reference/esql/functions/math-functions.asciidoc b/docs/reference/esql/functions/math-functions.asciidoc index e311208795533..9fedfa57f50c5 100644 --- a/docs/reference/esql/functions/math-functions.asciidoc +++ b/docs/reference/esql/functions/math-functions.asciidoc @@ -20,6 +20,7 @@ * <> * <> * <> +* <> * <> * <> * <> @@ -46,6 +47,7 @@ include::layout/cosh.asciidoc[] include::layout/e.asciidoc[] include::layout/exp.asciidoc[] include::layout/floor.asciidoc[] +include::layout/hypot.asciidoc[] include::layout/log.asciidoc[] include::layout/log10.asciidoc[] include::layout/pi.asciidoc[] diff --git a/docs/reference/esql/functions/parameters/hypot.asciidoc b/docs/reference/esql/functions/parameters/hypot.asciidoc new file mode 100644 index 0000000000000..9d6c7d50c7bec --- /dev/null +++ b/docs/reference/esql/functions/parameters/hypot.asciidoc @@ -0,0 +1,9 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`number1`:: +Numeric expression. If `null`, the function returns `null`. + +`number2`:: +Numeric expression. If `null`, the function returns `null`. diff --git a/docs/reference/esql/functions/parameters/match.asciidoc b/docs/reference/esql/functions/parameters/match.asciidoc new file mode 100644 index 0000000000000..f18adb28cd20c --- /dev/null +++ b/docs/reference/esql/functions/parameters/match.asciidoc @@ -0,0 +1,9 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`field`:: +Field that the query will target. + +`query`:: +Text you wish to find in the provided field. diff --git a/docs/reference/esql/functions/signature/hypot.svg b/docs/reference/esql/functions/signature/hypot.svg new file mode 100644 index 0000000000000..b849ea42cfd9e --- /dev/null +++ b/docs/reference/esql/functions/signature/hypot.svg @@ -0,0 +1 @@ +HYPOT(number1,number2) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/match.svg b/docs/reference/esql/functions/signature/match.svg new file mode 100644 index 0000000000000..e7bb001247a9d --- /dev/null +++ b/docs/reference/esql/functions/signature/match.svg @@ -0,0 +1 @@ +MATCH(field,query) diff --git a/docs/reference/esql/functions/types/hypot.asciidoc b/docs/reference/esql/functions/types/hypot.asciidoc new file mode 100644 index 0000000000000..dd06ba96d7f34 --- /dev/null +++ b/docs/reference/esql/functions/types/hypot.asciidoc @@ -0,0 +1,24 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +number1 | number2 | result +double | double | double +double | integer | double +double | long | double +double | unsigned_long | double +integer | double | double +integer | integer | double +integer | long | double +integer | unsigned_long | double +long | double | double +long | integer | double +long | long | double +long | unsigned_long | double +unsigned_long | double | double +unsigned_long | integer | double +unsigned_long | long | double +unsigned_long | unsigned_long | double +|=== diff --git a/docs/reference/esql/functions/types/match.asciidoc b/docs/reference/esql/functions/types/match.asciidoc new file mode 100644 index 0000000000000..7523b29c62b1d --- /dev/null +++ b/docs/reference/esql/functions/types/match.asciidoc @@ -0,0 +1,12 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +field | query | result +keyword | keyword | boolean +keyword | text | boolean +text | keyword | boolean +text | text | boolean +|=== diff --git a/docs/reference/index-modules.asciidoc b/docs/reference/index-modules.asciidoc index ed8cf6c1494e4..1c8f1db216b75 100644 --- a/docs/reference/index-modules.asciidoc +++ b/docs/reference/index-modules.asciidoc @@ -122,7 +122,7 @@ preview:[] The number of shards a custom <> value can go to. Defaults to 1 and can only be set at index creation time. This value must be less - than the `index.number_of_shards` unless the `index.number_of_shards` value is also 1. + than the `index.number_of_routing_shards` unless the `index.number_of_routing_shards` value is also 1. See <> for more details about how this setting is used. [[ccr-index-soft-deletes]] diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 443417e6a5b92..0b5c1ae6528f9 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -4222,6 +4222,11 @@ + + + + + diff --git a/modules/aggregations/build.gradle b/modules/aggregations/build.gradle index 1b3aac13b3608..f558ce8b9cfdb 100644 --- a/modules/aggregations/build.gradle +++ b/modules/aggregations/build.gradle @@ -48,4 +48,5 @@ dependencies { tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("aggregations/date_agg_per_day_of_week/Date aggregartion per day of week", "week-date behaviour has changed") + task.skipTest("aggregations/time_series/Configure with no synthetic source", "temporary until backport") }) diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml index 1703d4908a753..acab855e17df6 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml @@ -291,23 +291,6 @@ setup: sum: sum: field: val ---- -"Configure with no synthetic source": - - requires: - cluster_features: ["gte_v8.15.0"] - reason: "Error message changed in 8.15.0" - - - do: - catch: '/Indices with with index mode \[time_series\] only support synthetic source/' - indices.create: - index: tsdb_error - body: - settings: - mode: time_series - routing_path: [key] - mappings: - _source: - enabled: false --- "Number for keyword routing field": diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamOptionsIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamOptionsIT.java new file mode 100644 index 0000000000000..980cc32a12c68 --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamOptionsIT.java @@ -0,0 +1,144 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.datastreams; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.junit.Before; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; + +/** + * This should be a yaml test, but in order to write one we would need to expose the new APIs in the rest-api-spec. + * We do not want to do that until the feature flag is removed. For this reason, we temporarily, test the new APIs here. + * Please convert this to a yaml test when the feature flag is removed. + */ +public class DataStreamOptionsIT extends DisabledSecurityDataStreamTestCase { + + private static final String DATA_STREAM_NAME = "failure-data-stream"; + + @SuppressWarnings("unchecked") + @Before + public void setup() throws IOException { + Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/ds-template"); + putComposableIndexTemplateRequest.setJsonEntity(""" + { + "index_patterns": ["failure-data-stream"], + "template": { + "settings": { + "number_of_replicas": 0 + } + }, + "data_stream": { + "failure_store": true + } + } + """); + assertOK(client().performRequest(putComposableIndexTemplateRequest)); + + assertOK(client().performRequest(new Request("PUT", "/_data_stream/" + DATA_STREAM_NAME))); + // Initialize the failure store. + assertOK(client().performRequest(new Request("POST", DATA_STREAM_NAME + "/_rollover?target_failure_store"))); + ensureGreen(DATA_STREAM_NAME); + + final Response dataStreamResponse = client().performRequest(new Request("GET", "/_data_stream/" + DATA_STREAM_NAME)); + List dataStreams = (List) entityAsMap(dataStreamResponse).get("data_streams"); + assertThat(dataStreams.size(), is(1)); + Map dataStream = (Map) dataStreams.get(0); + assertThat(dataStream.get("name"), equalTo(DATA_STREAM_NAME)); + List backingIndices = getIndices(dataStream); + assertThat(backingIndices.size(), is(1)); + List failureStore = getFailureStore(dataStream); + assertThat(failureStore.size(), is(1)); + } + + public void testEnableDisableFailureStore() throws IOException { + { + assertAcknowledged(client().performRequest(new Request("DELETE", "/_data_stream/" + DATA_STREAM_NAME + "/_options"))); + assertFailureStore(false, 1); + assertDataStreamOptions(null); + } + { + Request enableRequest = new Request("PUT", "/_data_stream/" + DATA_STREAM_NAME + "/_options"); + enableRequest.setJsonEntity(""" + { + "failure_store": { + "enabled": true + } + }"""); + assertAcknowledged(client().performRequest(enableRequest)); + assertFailureStore(true, 1); + assertDataStreamOptions(true); + } + + { + Request disableRequest = new Request("PUT", "/_data_stream/" + DATA_STREAM_NAME + "/_options"); + disableRequest.setJsonEntity(""" + { + "failure_store": { + "enabled": false + } + }"""); + assertAcknowledged(client().performRequest(disableRequest)); + assertFailureStore(false, 1); + assertDataStreamOptions(false); + } + } + + @SuppressWarnings("unchecked") + private void assertFailureStore(boolean failureStoreEnabled, int failureStoreSize) throws IOException { + final Response dataStreamResponse = client().performRequest(new Request("GET", "/_data_stream/" + DATA_STREAM_NAME)); + List dataStreams = (List) entityAsMap(dataStreamResponse).get("data_streams"); + assertThat(dataStreams.size(), is(1)); + Map dataStream = (Map) dataStreams.get(0); + assertThat(dataStream.get("name"), equalTo(DATA_STREAM_NAME)); + assertThat(dataStream.containsKey("failure_store"), is(true)); + // Ensure the failure store is set to the provided value + assertThat(((Map) dataStream.get("failure_store")).get("enabled"), equalTo(failureStoreEnabled)); + // And the failure indices preserved + List failureStore = getFailureStore(dataStream); + assertThat(failureStore.size(), is(failureStoreSize)); + } + + @SuppressWarnings("unchecked") + private void assertDataStreamOptions(Boolean failureStoreEnabled) throws IOException { + final Response dataStreamResponse = client().performRequest(new Request("GET", "/_data_stream/" + DATA_STREAM_NAME + "/_options")); + List dataStreams = (List) entityAsMap(dataStreamResponse).get("data_streams"); + assertThat(dataStreams.size(), is(1)); + Map dataStream = (Map) dataStreams.get(0); + assertThat(dataStream.get("name"), equalTo(DATA_STREAM_NAME)); + Map> options = (Map>) dataStream.get("options"); + if (failureStoreEnabled == null) { + assertThat(options, nullValue()); + } else { + assertThat(options.containsKey("failure_store"), is(true)); + assertThat(options.get("failure_store").get("enabled"), equalTo(failureStoreEnabled)); + } + } + + @SuppressWarnings("unchecked") + private List getFailureStore(Map response) { + var failureStore = (Map) response.get("failure_store"); + return getIndices(failureStore); + + } + + @SuppressWarnings("unchecked") + private List getIndices(Map response) { + List> indices = (List>) response.get("indices"); + return indices.stream().map(index -> index.get("index_name")).toList(); + } +} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java index db6c12c8bc565..ab78f48b6cddf 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java @@ -15,7 +15,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.distribution.DistributionType; -import org.hamcrest.Matchers; import org.junit.Before; import org.junit.ClassRule; @@ -115,18 +114,62 @@ public void testConfigureStoredSourceBeforeIndexCreation() throws IOException { } }"""; + assertOK(putComponentTemplate(client, "logs@custom", storedSourceMapping)); + assertOK(createDataStream(client, "logs-custom-dev")); + + var mapping = getMapping(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0)); + String sourceMode = (String) subObject("_source").apply(mapping).get("mode"); + assertThat(sourceMode, equalTo("stored")); + } + + public void testConfigureDisabledSourceBeforeIndexCreation() { + var storedSourceMapping = """ + { + "template": { + "settings": { + "index": { + "mode": "logsdb" + } + }, + "mappings": { + "_source": { + "enabled": false + } + } + } + }"""; + Exception e = assertThrows(ResponseException.class, () -> putComponentTemplate(client, "logs@custom", storedSourceMapping)); assertThat( e.getMessage(), - containsString("Failed to parse mapping: Indices with with index mode [logsdb] only support synthetic source") + containsString("Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode") ); assertThat(e.getMessage(), containsString("mapper_parsing_exception")); + } - assertOK(createDataStream(client, "logs-custom-dev")); + public void testConfigureDisabledSourceModeBeforeIndexCreation() { + var storedSourceMapping = """ + { + "template": { + "settings": { + "index": { + "mode": "logsdb" + } + }, + "mappings": { + "_source": { + "mode": "disabled" + } + } + } + }"""; - var mapping = getMapping(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0)); - String sourceMode = (String) subObject("_source").apply(mapping).get("mode"); - assertThat(sourceMode, equalTo("synthetic")); + Exception e = assertThrows(ResponseException.class, () -> putComponentTemplate(client, "logs@custom", storedSourceMapping)); + assertThat( + e.getMessage(), + containsString("Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode") + ); + assertThat(e.getMessage(), containsString("mapper_parsing_exception")); } public void testConfigureStoredSourceWhenIndexIsCreated() throws IOException { @@ -142,8 +185,45 @@ public void testConfigureStoredSourceWhenIndexIsCreated() throws IOException { }"""; assertOK(putComponentTemplate(client, "logs@custom", storedSourceMapping)); + assertOK(createDataStream(client, "logs-custom-dev")); + + var mapping = getMapping(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0)); + String sourceMode = (String) subObject("_source").apply(mapping).get("mode"); + assertThat(sourceMode, equalTo("stored")); + } + + public void testConfigureDisabledSourceWhenIndexIsCreated() throws IOException { + var disabledModeMapping = """ + { + "template": { + "mappings": { + "_source": { + "enabled": false + } + } + } + }"""; + + assertOK(putComponentTemplate(client, "logs@custom", disabledModeMapping)); + ResponseException e = expectThrows(ResponseException.class, () -> createDataStream(client, "logs-custom-dev")); + assertThat(e.getMessage(), containsString("_source can not be disabled in index using [logsdb] index mode")); + } + + public void testConfigureDisabledSourceModeWhenIndexIsCreated() throws IOException { + var disabledModeMapping = """ + { + "template": { + "mappings": { + "_source": { + "mode": "disabled" + } + } + } + }"""; + + assertOK(putComponentTemplate(client, "logs@custom", disabledModeMapping)); ResponseException e = expectThrows(ResponseException.class, () -> createDataStream(client, "logs-custom-dev")); - assertThat(e.getMessage(), containsString("Indices with with index mode [logsdb] only support synthetic source")); + assertThat(e.getMessage(), containsString("_source can not be disabled in index using [logsdb] index mode")); } public void testOverrideIndexCodec() throws IOException { @@ -377,7 +457,7 @@ public void testIgnoreAboveSetting() throws IOException { ); assertThat( ex.getMessage(), - Matchers.containsString("Failed to parse value [" + newValue + "] for setting [index.mapping.ignore_above]") + containsString("Failed to parse value [" + newValue + "] for setting [index.mapping.ignore_above]") ); } } diff --git a/modules/data-streams/src/main/java/module-info.java b/modules/data-streams/src/main/java/module-info.java index 16229f9eb2394..2d49029c1023c 100644 --- a/modules/data-streams/src/main/java/module-info.java +++ b/modules/data-streams/src/main/java/module-info.java @@ -17,6 +17,7 @@ exports org.elasticsearch.datastreams.action to org.elasticsearch.server; exports org.elasticsearch.datastreams.lifecycle.action to org.elasticsearch.server; exports org.elasticsearch.datastreams.lifecycle; + exports org.elasticsearch.datastreams.options.action to org.elasticsearch.server; provides org.elasticsearch.features.FeatureSpecification with org.elasticsearch.datastreams.DataStreamFeatures; } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java index 1a6465a251021..cb7445705537a 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.datastreams.lifecycle.GetDataStreamLifecycleAction; import org.elasticsearch.action.datastreams.lifecycle.PutDataStreamLifecycleAction; import org.elasticsearch.client.internal.OriginSettingClient; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -56,6 +57,15 @@ import org.elasticsearch.datastreams.lifecycle.rest.RestExplainDataStreamLifecycleAction; import org.elasticsearch.datastreams.lifecycle.rest.RestGetDataStreamLifecycleAction; import org.elasticsearch.datastreams.lifecycle.rest.RestPutDataStreamLifecycleAction; +import org.elasticsearch.datastreams.options.action.DeleteDataStreamOptionsAction; +import org.elasticsearch.datastreams.options.action.GetDataStreamOptionsAction; +import org.elasticsearch.datastreams.options.action.PutDataStreamOptionsAction; +import org.elasticsearch.datastreams.options.action.TransportDeleteDataStreamOptionsAction; +import org.elasticsearch.datastreams.options.action.TransportGetDataStreamOptionsAction; +import org.elasticsearch.datastreams.options.action.TransportPutDataStreamOptionsAction; +import org.elasticsearch.datastreams.options.rest.RestDeleteDataStreamOptionsAction; +import org.elasticsearch.datastreams.options.rest.RestGetDataStreamOptionsAction; +import org.elasticsearch.datastreams.options.rest.RestPutDataStreamOptionsAction; import org.elasticsearch.datastreams.rest.RestCreateDataStreamAction; import org.elasticsearch.datastreams.rest.RestDataStreamsStatsAction; import org.elasticsearch.datastreams.rest.RestDeleteDataStreamAction; @@ -229,6 +239,11 @@ public Collection createComponents(PluginServices services) { actions.add(new ActionHandler<>(DeleteDataStreamLifecycleAction.INSTANCE, TransportDeleteDataStreamLifecycleAction.class)); actions.add(new ActionHandler<>(ExplainDataStreamLifecycleAction.INSTANCE, TransportExplainDataStreamLifecycleAction.class)); actions.add(new ActionHandler<>(GetDataStreamLifecycleStatsAction.INSTANCE, TransportGetDataStreamLifecycleStatsAction.class)); + if (DataStream.isFailureStoreFeatureFlagEnabled()) { + actions.add(new ActionHandler<>(GetDataStreamOptionsAction.INSTANCE, TransportGetDataStreamOptionsAction.class)); + actions.add(new ActionHandler<>(PutDataStreamOptionsAction.INSTANCE, TransportPutDataStreamOptionsAction.class)); + actions.add(new ActionHandler<>(DeleteDataStreamOptionsAction.INSTANCE, TransportDeleteDataStreamOptionsAction.class)); + } return actions; } @@ -261,6 +276,11 @@ public List getRestHandlers( handlers.add(new RestDeleteDataStreamLifecycleAction()); handlers.add(new RestExplainDataStreamLifecycleAction()); handlers.add(new RestDataStreamLifecycleStatsAction()); + if (DataStream.isFailureStoreFeatureFlagEnabled()) { + handlers.add(new RestGetDataStreamOptionsAction()); + handlers.add(new RestPutDataStreamOptionsAction()); + handlers.add(new RestDeleteDataStreamOptionsAction()); + } return handlers; } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamLifecycleAction.java index 1c4659efc2f8b..1595348649528 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamLifecycleAction.java @@ -34,7 +34,26 @@ private DeleteDataStreamLifecycleAction() {/* no instances */} public static final class Request extends AcknowledgedRequest implements IndicesRequest.Replaceable { private String[] names; - private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true, false); + private IndicesOptions indicesOptions = IndicesOptions.builder() + .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) + .wildcardOptions( + IndicesOptions.WildcardOptions.builder() + .matchOpen(true) + .matchClosed(true) + .includeHidden(false) + .resolveAliases(false) + .allowEmptyExpressions(true) + .build() + ) + .gatekeeperOptions( + IndicesOptions.GatekeeperOptions.builder() + .allowAliasToMultipleIndices(false) + .allowClosedIndices(true) + .ignoreThrottled(false) + .allowFailureIndices(false) + .build() + ) + .build(); public Request(StreamInput in) throws IOException { super(in); diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/DeleteDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/DeleteDataStreamOptionsAction.java new file mode 100644 index 0000000000000..98a29dd636ddf --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/DeleteDataStreamOptionsAction.java @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.datastreams.options.action; + +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; + +/** + * Removes the data stream options configuration from the requested data streams. + */ +public class DeleteDataStreamOptionsAction { + + public static final ActionType INSTANCE = new ActionType<>("indices:admin/data_stream/options/delete"); + + private DeleteDataStreamOptionsAction() {/* no instances */} + + public static final class Request extends AcknowledgedRequest implements IndicesRequest.Replaceable { + + private String[] names; + private IndicesOptions indicesOptions = IndicesOptions.builder() + .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) + .wildcardOptions( + IndicesOptions.WildcardOptions.builder().matchOpen(true).matchClosed(true).allowEmptyExpressions(true).resolveAliases(false) + ) + .gatekeeperOptions(IndicesOptions.GatekeeperOptions.builder().allowAliasToMultipleIndices(false).allowClosedIndices(true)) + .build(); + + public Request(StreamInput in) throws IOException { + super(in); + this.names = in.readOptionalStringArray(); + this.indicesOptions = IndicesOptions.readIndicesOptions(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalStringArray(names); + indicesOptions.writeIndicesOptions(out); + } + + public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String[] names) { + super(masterNodeTimeout, ackTimeout); + this.names = names; + } + + public String[] getNames() { + return names; + } + + @Override + public String[] indices() { + return names; + } + + @Override + public IndicesOptions indicesOptions() { + return indicesOptions; + } + + public Request indicesOptions(IndicesOptions indicesOptions) { + this.indicesOptions = indicesOptions; + return this; + } + + @Override + public boolean includeDataStreams() { + return true; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Arrays.equals(names, request.names) && Objects.equals(indicesOptions, request.indicesOptions); + } + + @Override + public int hashCode() { + int result = Objects.hash(indicesOptions); + result = 31 * result + Arrays.hashCode(names); + return result; + } + + @Override + public IndicesRequest indices(String... indices) { + this.names = indices; + return this; + } + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/GetDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/GetDataStreamOptionsAction.java new file mode 100644 index 0000000000000..c1354da1129ca --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/GetDataStreamOptionsAction.java @@ -0,0 +1,223 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.datastreams.options.action; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.cluster.metadata.DataStreamOptions; +import org.elasticsearch.common.collect.Iterators; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ChunkedToXContentObject; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; + +/** + * This action retrieves the data stream options from every data stream. Currently, data stream options only support + * failure store. + */ +public class GetDataStreamOptionsAction { + + public static final ActionType INSTANCE = new ActionType<>("indices:admin/data_stream/options/get"); + + private GetDataStreamOptionsAction() {/* no instances */} + + public static class Request extends MasterNodeReadRequest implements IndicesRequest.Replaceable { + + private String[] names; + private IndicesOptions indicesOptions = IndicesOptions.builder() + .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) + .wildcardOptions( + IndicesOptions.WildcardOptions.builder().matchOpen(true).matchClosed(true).allowEmptyExpressions(true).resolveAliases(false) + ) + .gatekeeperOptions(IndicesOptions.GatekeeperOptions.builder().allowAliasToMultipleIndices(false).allowClosedIndices(true)) + .build(); + private boolean includeDefaults = false; + + public Request(TimeValue masterNodeTimeout, String[] names) { + super(masterNodeTimeout); + this.names = names; + } + + public Request(TimeValue masterNodeTimeout, String[] names, boolean includeDefaults) { + super(masterNodeTimeout); + this.names = names; + this.includeDefaults = includeDefaults; + } + + public String[] getNames() { + return names; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + public Request(StreamInput in) throws IOException { + super(in); + this.names = in.readOptionalStringArray(); + this.indicesOptions = IndicesOptions.readIndicesOptions(in); + this.includeDefaults = in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalStringArray(names); + indicesOptions.writeIndicesOptions(out); + out.writeBoolean(includeDefaults); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Arrays.equals(names, request.names) + && indicesOptions.equals(request.indicesOptions) + && includeDefaults == request.includeDefaults; + } + + @Override + public int hashCode() { + int result = Objects.hash(indicesOptions, includeDefaults); + result = 31 * result + Arrays.hashCode(names); + return result; + } + + @Override + public String[] indices() { + return names; + } + + @Override + public IndicesOptions indicesOptions() { + return indicesOptions; + } + + public boolean includeDefaults() { + return includeDefaults; + } + + public Request indicesOptions(IndicesOptions indicesOptions) { + this.indicesOptions = indicesOptions; + return this; + } + + @Override + public boolean includeDataStreams() { + return true; + } + + @Override + public IndicesRequest indices(String... indices) { + this.names = indices; + return this; + } + + public Request includeDefaults(boolean includeDefaults) { + this.includeDefaults = includeDefaults; + return this; + } + } + + public static class Response extends ActionResponse implements ChunkedToXContentObject { + public static final ParseField DATA_STREAMS_FIELD = new ParseField("data_streams"); + + public record DataStreamEntry(String dataStreamName, DataStreamOptions dataStreamOptions) implements Writeable, ToXContentObject { + + public static final ParseField NAME_FIELD = new ParseField("name"); + public static final ParseField OPTIONS_FIELD = new ParseField("options"); + + DataStreamEntry(StreamInput in) throws IOException { + this(in.readString(), in.readOptionalWriteable(DataStreamOptions::read)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(dataStreamName); + out.writeOptionalWriteable(dataStreamOptions); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(NAME_FIELD.getPreferredName(), dataStreamName); + if (dataStreamOptions != null && dataStreamOptions.isEmpty() == false) { + builder.field(OPTIONS_FIELD.getPreferredName(), dataStreamOptions); + } + builder.endObject(); + return builder; + } + } + + private final List dataStreams; + + public Response(List dataStreams) { + this.dataStreams = dataStreams; + } + + public Response(StreamInput in) throws IOException { + this(in.readCollectionAsList(DataStreamEntry::new)); + } + + public List getDataStreams() { + return dataStreams; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeCollection(dataStreams); + } + + @Override + public Iterator toXContentChunked(ToXContent.Params outerParams) { + return Iterators.concat(Iterators.single((builder, params) -> { + builder.startObject(); + builder.startArray(DATA_STREAMS_FIELD.getPreferredName()); + return builder; + }), + Iterators.map(dataStreams.iterator(), entry -> (builder, params) -> entry.toXContent(builder, outerParams)), + Iterators.single((builder, params) -> { + builder.endArray(); + builder.endObject(); + return builder; + }) + ); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Response response = (Response) o; + return dataStreams.equals(response.dataStreams); + } + + @Override + public int hashCode() { + return Objects.hash(dataStreams); + } + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/PutDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/PutDataStreamOptionsAction.java new file mode 100644 index 0000000000000..d055a6972312a --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/PutDataStreamOptionsAction.java @@ -0,0 +1,165 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.datastreams.options.action; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.cluster.metadata.DataStreamFailureStore; +import org.elasticsearch.cluster.metadata.DataStreamOptions; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +/** + * Sets the data stream options that was provided in the request to the requested data streams. + */ +public class PutDataStreamOptionsAction { + + public static final ActionType INSTANCE = new ActionType<>("indices:admin/data_stream/options/put"); + + private PutDataStreamOptionsAction() {/* no instances */} + + public static final class Request extends AcknowledgedRequest implements IndicesRequest.Replaceable { + + public interface Factory { + Request create(@Nullable DataStreamFailureStore dataStreamFailureStore); + } + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "put_data_stream_options_request", + false, + (args, factory) -> factory.create((DataStreamFailureStore) args[0]) + ); + + static { + PARSER.declareObjectOrNull( + ConstructingObjectParser.optionalConstructorArg(), + (p, c) -> DataStreamFailureStore.PARSER.parse(p, null), + null, + new ParseField("failure_store") + ); + } + + public static Request parseRequest(XContentParser parser, Factory factory) { + return PARSER.apply(parser, factory); + } + + private String[] names; + private IndicesOptions indicesOptions = IndicesOptions.builder() + .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) + .wildcardOptions( + IndicesOptions.WildcardOptions.builder().matchOpen(true).matchClosed(true).allowEmptyExpressions(true).resolveAliases(false) + ) + .gatekeeperOptions(IndicesOptions.GatekeeperOptions.builder().allowAliasToMultipleIndices(false).allowClosedIndices(true)) + .build(); + private final DataStreamOptions options; + + public Request(StreamInput in) throws IOException { + super(in); + this.names = in.readStringArray(); + this.indicesOptions = IndicesOptions.readIndicesOptions(in); + options = DataStreamOptions.read(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeStringArray(names); + indicesOptions.writeIndicesOptions(out); + out.writeWriteable(options); + } + + public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String[] names, DataStreamOptions options) { + super(masterNodeTimeout, ackTimeout); + this.names = names; + this.options = options; + } + + public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String[] names, @Nullable DataStreamFailureStore failureStore) { + super(masterNodeTimeout, ackTimeout); + this.names = names; + this.options = new DataStreamOptions(failureStore); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (options.failureStore() == null) { + validationException = addValidationError("At least one option needs to be provided", validationException); + } + return validationException; + } + + public String[] getNames() { + return names; + } + + public DataStreamOptions getOptions() { + return options; + } + + @Override + public String[] indices() { + return names; + } + + @Override + public IndicesOptions indicesOptions() { + return indicesOptions; + } + + public Request indicesOptions(IndicesOptions indicesOptions) { + this.indicesOptions = indicesOptions; + return this; + } + + @Override + public boolean includeDataStreams() { + return true; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Arrays.equals(names, request.names) + && Objects.equals(indicesOptions, request.indicesOptions) + && options.equals(request.options); + } + + @Override + public int hashCode() { + int result = Objects.hash(indicesOptions, options); + result = 31 * result + Arrays.hashCode(names); + return result; + } + + @Override + public IndicesRequest indices(String... names) { + this.names = names; + return this; + } + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportDeleteDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportDeleteDataStreamOptionsAction.java new file mode 100644 index 0000000000000..ead23ed78222b --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportDeleteDataStreamOptionsAction.java @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.datastreams.options.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.datastreams.DataStreamsActionUtil; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetadataDataStreamsService; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.indices.SystemIndices; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.List; + +/** + * Transport action that resolves the data stream names from the request and removes any configured data stream options from them. + */ +public class TransportDeleteDataStreamOptionsAction extends AcknowledgedTransportMasterNodeAction { + + private final MetadataDataStreamsService metadataDataStreamsService; + private final SystemIndices systemIndices; + + @Inject + public TransportDeleteDataStreamOptionsAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + MetadataDataStreamsService metadataDataStreamsService, + SystemIndices systemIndices + ) { + super( + DeleteDataStreamOptionsAction.INSTANCE.name(), + transportService, + clusterService, + threadPool, + actionFilters, + DeleteDataStreamOptionsAction.Request::new, + indexNameExpressionResolver, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.metadataDataStreamsService = metadataDataStreamsService; + this.systemIndices = systemIndices; + } + + @Override + protected void masterOperation( + Task task, + DeleteDataStreamOptionsAction.Request request, + ClusterState state, + ActionListener listener + ) { + List dataStreamNames = DataStreamsActionUtil.getDataStreamNames( + indexNameExpressionResolver, + state, + request.getNames(), + request.indicesOptions() + ); + for (String name : dataStreamNames) { + systemIndices.validateDataStreamAccess(name, threadPool.getThreadContext()); + } + metadataDataStreamsService.removeDataStreamOptions(dataStreamNames, request.ackTimeout(), request.masterNodeTimeout(), listener); + } + + @Override + protected ClusterBlockException checkBlock(DeleteDataStreamOptionsAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportGetDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportGetDataStreamOptionsAction.java new file mode 100644 index 0000000000000..b032b35c943c0 --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportGetDataStreamOptionsAction.java @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.datastreams.options.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.datastreams.DataStreamsActionUtil; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.indices.SystemIndices; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Collects the data streams from the cluster state and then returns for each data stream its name and its + * data stream options. Currently, data stream options include only the failure store configuration. + */ +public class TransportGetDataStreamOptionsAction extends TransportMasterNodeReadAction< + GetDataStreamOptionsAction.Request, + GetDataStreamOptionsAction.Response> { + + private final SystemIndices systemIndices; + + @Inject + public TransportGetDataStreamOptionsAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + SystemIndices systemIndices + ) { + super( + GetDataStreamOptionsAction.INSTANCE.name(), + transportService, + clusterService, + threadPool, + actionFilters, + GetDataStreamOptionsAction.Request::new, + indexNameExpressionResolver, + GetDataStreamOptionsAction.Response::new, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.systemIndices = systemIndices; + } + + @Override + protected void masterOperation( + Task task, + GetDataStreamOptionsAction.Request request, + ClusterState state, + ActionListener listener + ) { + List requestedDataStreams = DataStreamsActionUtil.getDataStreamNames( + indexNameExpressionResolver, + state, + request.getNames(), + request.indicesOptions() + ); + Map dataStreams = state.metadata().dataStreams(); + for (String name : requestedDataStreams) { + systemIndices.validateDataStreamAccess(name, threadPool.getThreadContext()); + } + listener.onResponse( + new GetDataStreamOptionsAction.Response( + requestedDataStreams.stream() + .map(dataStreams::get) + .filter(Objects::nonNull) + .map( + dataStream -> new GetDataStreamOptionsAction.Response.DataStreamEntry( + dataStream.getName(), + dataStream.getDataStreamOptions() + ) + ) + .sorted(Comparator.comparing(GetDataStreamOptionsAction.Response.DataStreamEntry::dataStreamName)) + .toList() + ) + ); + } + + @Override + protected ClusterBlockException checkBlock(GetDataStreamOptionsAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportPutDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportPutDataStreamOptionsAction.java new file mode 100644 index 0000000000000..b1386232c44f9 --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/TransportPutDataStreamOptionsAction.java @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.datastreams.options.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.datastreams.DataStreamsActionUtil; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetadataDataStreamsService; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.indices.SystemIndices; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.List; + +/** + * Transport action that resolves the data stream names from the request and sets the data stream lifecycle provided in the request. + */ +public class TransportPutDataStreamOptionsAction extends AcknowledgedTransportMasterNodeAction { + + private final MetadataDataStreamsService metadataDataStreamsService; + private final SystemIndices systemIndices; + + @Inject + public TransportPutDataStreamOptionsAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + MetadataDataStreamsService metadataDataStreamsService, + SystemIndices systemIndices + ) { + super( + PutDataStreamOptionsAction.INSTANCE.name(), + transportService, + clusterService, + threadPool, + actionFilters, + PutDataStreamOptionsAction.Request::new, + indexNameExpressionResolver, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.metadataDataStreamsService = metadataDataStreamsService; + this.systemIndices = systemIndices; + } + + @Override + protected void masterOperation( + Task task, + PutDataStreamOptionsAction.Request request, + ClusterState state, + ActionListener listener + ) { + List dataStreamNames = DataStreamsActionUtil.getDataStreamNames( + indexNameExpressionResolver, + state, + request.getNames(), + request.indicesOptions() + ); + for (String name : dataStreamNames) { + systemIndices.validateDataStreamAccess(name, threadPool.getThreadContext()); + } + metadataDataStreamsService.setDataStreamOptions( + dataStreamNames, + request.getOptions(), + request.ackTimeout(), + request.masterNodeTimeout(), + listener + ); + } + + @Override + protected ClusterBlockException checkBlock(PutDataStreamOptionsAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestDeleteDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestDeleteDataStreamOptionsAction.java new file mode 100644 index 0000000000000..96460632ff443 --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestDeleteDataStreamOptionsAction.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.datastreams.options.rest; + +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.datastreams.options.action.DeleteDataStreamOptionsAction; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestToXContentListener; + +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.DELETE; +import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout; + +@ServerlessScope(Scope.INTERNAL) +public class RestDeleteDataStreamOptionsAction extends BaseRestHandler { + + @Override + public String getName() { + return "delete_data_stream_options_action"; + } + + @Override + public List routes() { + return List.of(new Route(DELETE, "/_data_stream/{name}/_options")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { + final var deleteDataOptionsRequest = new DeleteDataStreamOptionsAction.Request( + getMasterNodeTimeout(request), + request.paramAsTime("timeout", AcknowledgedRequest.DEFAULT_ACK_TIMEOUT), + Strings.splitStringByCommaToArray(request.param("name")) + ); + deleteDataOptionsRequest.indicesOptions(IndicesOptions.fromRequest(request, deleteDataOptionsRequest.indicesOptions())); + return channel -> client.execute( + DeleteDataStreamOptionsAction.INSTANCE, + deleteDataOptionsRequest, + new RestToXContentListener<>(channel) + ); + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestGetDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestGetDataStreamOptionsAction.java new file mode 100644 index 0000000000000..6d6530efce1b9 --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestGetDataStreamOptionsAction.java @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.datastreams.options.rest; + +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.datastreams.options.action.GetDataStreamOptionsAction; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestUtils; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestRefCountedChunkedToXContentListener; + +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.GET; + +@ServerlessScope(Scope.PUBLIC) +public class RestGetDataStreamOptionsAction extends BaseRestHandler { + + @Override + public String getName() { + return "get_data_stream_options_action"; + } + + @Override + public List routes() { + return List.of(new Route(GET, "/_data_stream/{name}/_options")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { + GetDataStreamOptionsAction.Request getDataStreamOptionsRequest = new GetDataStreamOptionsAction.Request( + RestUtils.getMasterNodeTimeout(request), + Strings.splitStringByCommaToArray(request.param("name")) + ); + getDataStreamOptionsRequest.includeDefaults(request.paramAsBoolean("include_defaults", false)); + getDataStreamOptionsRequest.indicesOptions(IndicesOptions.fromRequest(request, getDataStreamOptionsRequest.indicesOptions())); + return channel -> client.execute( + GetDataStreamOptionsAction.INSTANCE, + getDataStreamOptionsRequest, + new RestRefCountedChunkedToXContentListener<>(channel) + ); + } + + @Override + public boolean allowSystemIndexAccessByDefault() { + return true; + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestPutDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestPutDataStreamOptionsAction.java new file mode 100644 index 0000000000000..9191b96b6039e --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/rest/RestPutDataStreamOptionsAction.java @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.datastreams.options.rest; + +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.datastreams.options.action.PutDataStreamOptionsAction; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.PUT; +import static org.elasticsearch.rest.RestUtils.getAckTimeout; +import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout; + +@ServerlessScope(Scope.PUBLIC) +public class RestPutDataStreamOptionsAction extends BaseRestHandler { + + @Override + public String getName() { + return "put_data_stream_options_action"; + } + + @Override + public List routes() { + return List.of(new Route(PUT, "/_data_stream/{name}/_options")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + try (XContentParser parser = request.contentParser()) { + PutDataStreamOptionsAction.Request putOptionsRequest = PutDataStreamOptionsAction.Request.parseRequest( + parser, + (failureStore) -> new PutDataStreamOptionsAction.Request( + getMasterNodeTimeout(request), + getAckTimeout(request), + Strings.splitStringByCommaToArray(request.param("name")), + failureStore + ) + ); + putOptionsRequest.indicesOptions(IndicesOptions.fromRequest(request, putOptionsRequest.indicesOptions())); + return channel -> client.execute(PutDataStreamOptionsAction.INSTANCE, putOptionsRequest, new RestToXContentListener<>(channel)); + } + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloader.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloader.java index fa46540e29f7a..3bbb0539f193a 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloader.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloader.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.Strings; import org.elasticsearch.common.hash.MessageDigests; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.query.BoolQueryBuilder; @@ -236,7 +237,7 @@ boolean processDatabase(String id, DatabaseConfiguration database) throws IOExce logger.debug("Processing database [{}] for configuration [{}]", name, database.id()); try (ProviderDownload downloader = downloaderFor(database)) { - if (downloader.validCredentials()) { + if (downloader != null && downloader.validCredentials()) { // the name that comes from the enterprise downloader cluster state doesn't include the .mmdb extension, // but the downloading and indexing of database code expects it to be there, so we add it on here before continuing final String fileName = name + ".mmdb"; @@ -443,10 +444,17 @@ private void scheduleNextRun(TimeValue time) { } } + @Nullable private ProviderDownload downloaderFor(DatabaseConfiguration database) { - assert database.provider() instanceof DatabaseConfiguration.Maxmind - : "Attempt to use maxmind downloader with a provider of type" + database.provider().getClass(); - return new MaxmindDownload(database.name(), (DatabaseConfiguration.Maxmind) database.provider()); + if (database.provider() instanceof DatabaseConfiguration.Maxmind) { + return new MaxmindDownload(database.name(), (DatabaseConfiguration.Maxmind) database.provider()); + } else if (database.provider() instanceof DatabaseConfiguration.Ipinfo) { + // as a temporary implementation detail, null here means 'not actually supported *just yet*' + return null; + } else { + assert false : "Attempted to use database downloader with unsupported provider type [" + database.provider().getClass() + "]"; + return null; + } } class MaxmindDownload implements ProviderDownload { diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java index a7828a9f3a0b7..eacf2e5a2ee57 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java @@ -55,6 +55,7 @@ import static org.elasticsearch.ingest.geoip.GeoIpDownloader.GEOIP_DOWNLOADER; import static org.elasticsearch.ingest.geoip.GeoIpProcessor.Factory.downloadDatabaseOnPipelineCreation; import static org.elasticsearch.ingest.geoip.GeoIpProcessor.GEOIP_TYPE; +import static org.elasticsearch.ingest.geoip.GeoIpProcessor.IP_LOCATION_TYPE; /** * Persistent task executor that is responsible for starting {@link GeoIpDownloader} after task is allocated by master node. @@ -297,9 +298,18 @@ private static boolean hasAtLeastOneGeoipProcessor(Map processor return false; } - final Map processorConfig = (Map) processor.get(GEOIP_TYPE); - if (processorConfig != null) { - return downloadDatabaseOnPipelineCreation(GEOIP_TYPE, processorConfig, null) == downloadDatabaseOnPipelineCreation; + { + final Map processorConfig = (Map) processor.get(GEOIP_TYPE); + if (processorConfig != null) { + return downloadDatabaseOnPipelineCreation(GEOIP_TYPE, processorConfig, null) == downloadDatabaseOnPipelineCreation; + } + } + + { + final Map processorConfig = (Map) processor.get(IP_LOCATION_TYPE); + if (processorConfig != null) { + return downloadDatabaseOnPipelineCreation(IP_LOCATION_TYPE, processorConfig, null) == downloadDatabaseOnPipelineCreation; + } } return isProcessorWithOnFailureGeoIpProcessor(processor, downloadDatabaseOnPipelineCreation) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java index f8ca6d87924a4..6c64cb755bb32 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java @@ -42,6 +42,7 @@ public final class GeoIpProcessor extends AbstractProcessor { + "in a future version of Elasticsearch"; // TODO add a message about migration? public static final String GEOIP_TYPE = "geoip"; + public static final String IP_LOCATION_TYPE = "ip_location"; private final String type; private final String field; @@ -225,7 +226,7 @@ public Processor create( final Map config ) throws IOException { String ipField = readStringProperty(type, processorTag, config, "field"); - String targetField = readStringProperty(type, processorTag, config, "target_field", "geoip"); + String targetField = readStringProperty(type, processorTag, config, "target_field", type); String databaseFile = readStringProperty(type, processorTag, config, "database_file", "GeoLite2-City.mmdb"); List propertyNames = readOptionalList(type, processorTag, config, "properties"); boolean ignoreMissing = readBooleanProperty(type, processorTag, config, "ignore_missing", false); diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java index 49932f342086e..cc0bec583483e 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java @@ -71,6 +71,7 @@ import java.util.function.Predicate; import java.util.function.Supplier; +import static java.util.Map.entry; import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; import static org.elasticsearch.ingest.EnterpriseGeoIpTask.ENTERPRISE_GEOIP_DOWNLOADER; import static org.elasticsearch.ingest.IngestService.INGEST_ORIGIN; @@ -129,7 +130,10 @@ public Map getProcessors(Processor.Parameters paramet parameters.ingestService.getClusterService() ); databaseRegistry.set(registry); - return Map.of(GeoIpProcessor.GEOIP_TYPE, new GeoIpProcessor.Factory(GeoIpProcessor.GEOIP_TYPE, registry)); + return Map.ofEntries( + entry(GeoIpProcessor.GEOIP_TYPE, new GeoIpProcessor.Factory(GeoIpProcessor.GEOIP_TYPE, registry)), + entry(GeoIpProcessor.IP_LOCATION_TYPE, new GeoIpProcessor.Factory(GeoIpProcessor.IP_LOCATION_TYPE, registry)) + ); } @Override @@ -239,6 +243,11 @@ public List getNamedWriteables() { DatabaseConfiguration.Maxmind.NAME, DatabaseConfiguration.Maxmind::new ), + new NamedWriteableRegistry.Entry( + DatabaseConfiguration.Provider.class, + DatabaseConfiguration.Ipinfo.NAME, + DatabaseConfiguration.Ipinfo::new + ), new NamedWriteableRegistry.Entry( DatabaseConfiguration.Provider.class, DatabaseConfiguration.Local.NAME, diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java index 3399b71879e26..a26364f9305e1 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.Objects; import java.util.Set; import java.util.regex.Pattern; @@ -78,8 +79,19 @@ public record DatabaseConfiguration(String id, String name, Provider provider) i // "GeoLite2-Country" ); + public static final Set IPINFO_NAMES = Set.of( + // these file names are from https://ipinfo.io/developers/database-filename-reference + "asn", // "Free IP to ASN" + "country", // "Free IP to Country" + // "country_asn" // "Free IP to Country + IP to ASN", not supported at present + "standard_asn", // commercial "ASN" + "standard_location", // commercial "IP Geolocation" + "standard_privacy" // commercial "Privacy Detection" (sometimes "Anonymous IP") + ); + private static final ParseField NAME = new ParseField("name"); private static final ParseField MAXMIND = new ParseField(Maxmind.NAME); + private static final ParseField IPINFO = new ParseField(Ipinfo.NAME); private static final ParseField WEB = new ParseField(Web.NAME); private static final ParseField LOCAL = new ParseField(Local.NAME); @@ -89,12 +101,21 @@ public record DatabaseConfiguration(String id, String name, Provider provider) i (a, id) -> { String name = (String) a[0]; Provider provider; + + // one and only one provider object must be present + final long numNonNulls = Arrays.stream(a, 1, a.length).filter(Objects::nonNull).count(); + if (numNonNulls != 1) { + throw new IllegalArgumentException("Exactly one provider object must be specified, but [" + numNonNulls + "] were found"); + } + if (a[1] != null) { provider = (Maxmind) a[1]; } else if (a[2] != null) { - provider = (Web) a[2]; + provider = (Ipinfo) a[2]; + } else if (a[3] != null) { + provider = (Web) a[3]; } else { - provider = (Local) a[3]; + provider = (Local) a[4]; } return new DatabaseConfiguration(id, name, provider); } @@ -107,6 +128,7 @@ public record DatabaseConfiguration(String id, String name, Provider provider) i (parser, id) -> Maxmind.PARSER.apply(parser, null), MAXMIND ); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (parser, id) -> Ipinfo.PARSER.apply(parser, null), IPINFO); PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (parser, id) -> Web.PARSER.apply(parser, null), WEB); PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (parser, id) -> Local.PARSER.apply(parser, null), LOCAL); } @@ -194,8 +216,16 @@ public ActionRequestValidationException validate() { err.addValidationError("invalid name [" + name + "]: cannot be empty"); } - if (MAXMIND_NAMES.contains(name) == false) { - err.addValidationError("invalid name [" + name + "]: must be a supported name ([" + MAXMIND_NAMES + "])"); + // provider-specific name validation + if (provider instanceof Maxmind) { + if (MAXMIND_NAMES.contains(name) == false) { + err.addValidationError("invalid name [" + name + "]: must be a supported name ([" + MAXMIND_NAMES + "])"); + } + } + if (provider instanceof Ipinfo) { + if (IPINFO_NAMES.contains(name) == false) { + err.addValidationError("invalid name [" + name + "]: must be a supported name ([" + IPINFO_NAMES + "])"); + } } // important: the name must be unique across all configurations of this same type, @@ -234,7 +264,7 @@ public String getWriteableName() { private static final ParseField ACCOUNT_ID = new ParseField("account_id"); - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("database", false, (a, id) -> { + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("maxmind", false, (a, id) -> { String accountId = (String) a[0]; return new Maxmind(accountId); }); @@ -247,10 +277,6 @@ public Maxmind(StreamInput in) throws IOException { this(in.readString()); } - public static Maxmind parse(XContentParser parser) { - return PARSER.apply(parser, null); - } - @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(accountId); @@ -270,6 +296,37 @@ public boolean isReadOnly() { } } + public record Ipinfo() implements Provider { + public static final String NAME = "ipinfo"; + + // this'll become a ConstructingObjectParser once we accept the token (securely) in the json definition + private static final ObjectParser PARSER = new ObjectParser<>("ipinfo", Ipinfo::new); + + public Ipinfo(StreamInput in) throws IOException { + this(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException {} + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.endObject(); + return builder; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public boolean isReadOnly() { + return false; + } + } + public record Local(String type) implements Provider { public static final String NAME = "local"; @@ -288,10 +345,6 @@ public Local(StreamInput in) throws IOException { this(in.readString()); } - public static Local parse(XContentParser parser) { - return PARSER.apply(parser, null); - } - @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(type); @@ -325,10 +378,6 @@ public Web(StreamInput in) throws IOException { this(); } - public static Web parse(XContentParser parser) { - return PARSER.apply(parser, null); - } - @Override public void writeTo(StreamOutput out) throws IOException {} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestDeleteDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestDeleteDatabaseConfigurationAction.java index e836821a3b2f2..78ea73250d632 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestDeleteDatabaseConfigurationAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestDeleteDatabaseConfigurationAction.java @@ -27,7 +27,7 @@ public class RestDeleteDatabaseConfigurationAction extends BaseRestHandler { @Override public List routes() { - return List.of(new Route(DELETE, "/_ingest/geoip/database/{id}")); + return List.of(new Route(DELETE, "/_ingest/ip_location/database/{id}"), new Route(DELETE, "/_ingest/geoip/database/{id}")); } @Override diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestGetDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestGetDatabaseConfigurationAction.java index f34f388f22965..af446ee8d2bd9 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestGetDatabaseConfigurationAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestGetDatabaseConfigurationAction.java @@ -26,7 +26,12 @@ public class RestGetDatabaseConfigurationAction extends BaseRestHandler { @Override public List routes() { - return List.of(new Route(GET, "/_ingest/geoip/database"), new Route(GET, "/_ingest/geoip/database/{id}")); + return List.of( + new Route(GET, "/_ingest/ip_location/database"), + new Route(GET, "/_ingest/ip_location/database/{id}"), + new Route(GET, "/_ingest/geoip/database"), + new Route(GET, "/_ingest/geoip/database/{id}") + ); } @Override diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestPutDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestPutDatabaseConfigurationAction.java index c0b7a3f59f3aa..95b40df12fd1f 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestPutDatabaseConfigurationAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/RestPutDatabaseConfigurationAction.java @@ -29,7 +29,7 @@ public class RestPutDatabaseConfigurationAction extends BaseRestHandler { @Override public List routes() { - return List.of(new Route(PUT, "/_ingest/geoip/database/{id}")); + return List.of(new Route(PUT, "/_ingest/ip_location/database/{id}"), new Route(PUT, "/_ingest/geoip/database/{id}")); } @Override diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java index fda0e12bb1b76..dfb8fa78089d2 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java @@ -29,6 +29,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Strings; import org.elasticsearch.core.Tuple; +import org.elasticsearch.features.FeatureService; import org.elasticsearch.ingest.geoip.IngestGeoIpMetadata; import org.elasticsearch.ingest.geoip.direct.PutDatabaseConfigurationAction.Request; import org.elasticsearch.injection.guice.Inject; @@ -41,6 +42,8 @@ import java.util.Map; import java.util.Optional; +import static org.elasticsearch.ingest.IngestGeoIpFeatures.PUT_DATABASE_CONFIGURATION_ACTION_IPINFO; + public class TransportPutDatabaseConfigurationAction extends TransportMasterNodeAction { private static final Logger logger = LogManager.getLogger(TransportPutDatabaseConfigurationAction.class); @@ -58,6 +61,7 @@ public void taskSucceeded(UpdateDatabaseConfigurationTask task, Void unused) { } }; + private final FeatureService featureService; private final MasterServiceTaskQueue updateDatabaseConfigurationTaskQueue; @Inject @@ -66,7 +70,8 @@ public TransportPutDatabaseConfigurationAction( ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver + IndexNameExpressionResolver indexNameExpressionResolver, + FeatureService featureService ) { super( PutDatabaseConfigurationAction.NAME, @@ -79,6 +84,7 @@ public TransportPutDatabaseConfigurationAction( AcknowledgedResponse::readFrom, EsExecutors.DIRECT_EXECUTOR_SERVICE ); + this.featureService = featureService; this.updateDatabaseConfigurationTaskQueue = clusterService.createTaskQueue( "update-geoip-database-configuration-state-update", Priority.NORMAL, @@ -89,6 +95,19 @@ public TransportPutDatabaseConfigurationAction( @Override protected void masterOperation(Task task, Request request, ClusterState state, ActionListener listener) { final String id = request.getDatabase().id(); + + // if this is an ipinfo configuration, then make sure the whole cluster supports that feature + if (request.getDatabase().provider() instanceof DatabaseConfiguration.Ipinfo + && featureService.clusterHasFeature(clusterService.state(), PUT_DATABASE_CONFIGURATION_ACTION_IPINFO) == false) { + listener.onFailure( + new IllegalArgumentException( + "Unable to use ipinfo database configurations in mixed-clusters with nodes that do not support feature " + + PUT_DATABASE_CONFIGURATION_ACTION_IPINFO.id() + ) + ); + return; + } + updateDatabaseConfigurationTaskQueue.submitTask( Strings.format("update-geoip-database-configuration-[%s]", id), new UpdateDatabaseConfigurationTask(listener, request.getDatabase()), diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseTests.java new file mode 100644 index 0000000000000..39ecf4e70383b --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseTests.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.test.ESTestCase; + +import java.util.Set; + +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +public class DatabaseTests extends ESTestCase { + + public void testDatabasePropertyInvariants() { + // the city database is like a specialization of the country database + assertThat(Sets.difference(Database.Country.properties(), Database.City.properties()), is(empty())); + assertThat(Sets.difference(Database.Country.defaultProperties(), Database.City.defaultProperties()), is(empty())); + + // the isp database is like a specialization of the asn database + assertThat(Sets.difference(Database.Asn.properties(), Database.Isp.properties()), is(empty())); + assertThat(Sets.difference(Database.Asn.defaultProperties(), Database.Isp.defaultProperties()), is(empty())); + + // the enterprise database is like these other databases joined together + for (Database type : Set.of( + Database.City, + Database.Country, + Database.Asn, + Database.AnonymousIp, + Database.ConnectionType, + Database.Domain, + Database.Isp + )) { + assertThat(Sets.difference(type.properties(), Database.Enterprise.properties()), is(empty())); + } + // but in terms of the default fields, it's like a drop-in replacement for the city database + // n.b. this is just a choice we decided to make here at Elastic + assertThat(Database.Enterprise.defaultProperties(), equalTo(Database.City.defaultProperties())); + } + + public void testDatabaseVariantPropertyInvariants() { + // the second ASN variant database is like a specialization of the ASN database + assertThat(Sets.difference(Database.Asn.properties(), Database.AsnV2.properties()), is(empty())); + assertThat(Database.Asn.defaultProperties(), equalTo(Database.AsnV2.defaultProperties())); + + // the second City variant database is like a version of the ordinary City database but lacking many fields + assertThat(Sets.difference(Database.CityV2.properties(), Database.City.properties()), is(empty())); + assertThat(Sets.difference(Database.CityV2.defaultProperties(), Database.City.defaultProperties()), is(empty())); + + // the second Country variant database is like a version of the ordinary Country database but lacking come fields + assertThat(Sets.difference(Database.CountryV2.properties(), Database.CountryV2.properties()), is(empty())); + assertThat(Database.CountryV2.defaultProperties(), equalTo(Database.Country.defaultProperties())); + } +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java index 50b59c26749fc..e96bdbd6314b2 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java @@ -10,7 +10,6 @@ package org.elasticsearch.ingest.geoip; import org.elasticsearch.common.CheckedSupplier; -import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.IOUtils; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.RandomDocumentPicks; @@ -24,14 +23,12 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument; import static org.elasticsearch.ingest.geoip.GeoIpProcessor.GEOIP_TYPE; import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.is; @@ -57,71 +54,6 @@ public void cleanup() throws IOException { IOUtils.rm(tmpDir); } - public void testDatabasePropertyInvariants() { - // the city database is like a specialization of the country database - assertThat(Sets.difference(Database.Country.properties(), Database.City.properties()), is(empty())); - assertThat(Sets.difference(Database.Country.defaultProperties(), Database.City.defaultProperties()), is(empty())); - - // the isp database is like a specialization of the asn database - assertThat(Sets.difference(Database.Asn.properties(), Database.Isp.properties()), is(empty())); - assertThat(Sets.difference(Database.Asn.defaultProperties(), Database.Isp.defaultProperties()), is(empty())); - - // the enterprise database is like these other databases joined together - for (Database type : Set.of( - Database.City, - Database.Country, - Database.Asn, - Database.AnonymousIp, - Database.ConnectionType, - Database.Domain, - Database.Isp - )) { - assertThat(Sets.difference(type.properties(), Database.Enterprise.properties()), is(empty())); - } - // but in terms of the default fields, it's like a drop-in replacement for the city database - // n.b. this is just a choice we decided to make here at Elastic - assertThat(Database.Enterprise.defaultProperties(), equalTo(Database.City.defaultProperties())); - } - - public void testCity() throws Exception { - String ip = "8.8.8.8"; - GeoIpProcessor processor = new GeoIpProcessor( - GEOIP_TYPE, - randomAlphaOfLength(10), - null, - "source_field", - loader("GeoLite2-City.mmdb"), - () -> true, - "target_field", - ipDataLookupAll(Database.City), - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", ip); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(12)); - assertThat(geoData.get("ip"), equalTo(ip)); - assertThat(geoData.get("country_in_european_union"), equalTo(false)); - assertThat(geoData.get("country_iso_code"), equalTo("US")); - assertThat(geoData.get("country_name"), equalTo("United States")); - assertThat(geoData.get("continent_code"), equalTo("NA")); - assertThat(geoData.get("continent_name"), equalTo("North America")); - assertThat(geoData.get("timezone"), equalTo("America/Chicago")); - assertThat(geoData.get("location"), equalTo(Map.of("lat", 37.751d, "lon", -97.822d))); - assertThat(geoData.get("registered_country_in_european_union"), equalTo(false)); - assertThat(geoData.get("registered_country_iso_code"), equalTo("US")); - assertThat(geoData.get("registered_country_name"), equalTo("United States")); - } - public void testNullValueWithIgnoreMissing() throws Exception { GeoIpProcessor processor = new GeoIpProcessor( GEOIP_TYPE, @@ -208,369 +140,6 @@ public void testNonExistentWithoutIgnoreMissing() { assertThat(exception.getMessage(), equalTo("field [source_field] not present as part of path [source_field]")); } - public void testCity_withIpV6() throws Exception { - String ip = "2602:306:33d3:8000::3257:9652"; - GeoIpProcessor processor = new GeoIpProcessor( - GEOIP_TYPE, - randomAlphaOfLength(10), - null, - "source_field", - loader("GeoLite2-City.mmdb"), - () -> true, - "target_field", - ipDataLookupAll(Database.City), - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", ip); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(16)); - assertThat(geoData.get("ip"), equalTo(ip)); - assertThat(geoData.get("country_in_european_union"), equalTo(false)); - assertThat(geoData.get("country_iso_code"), equalTo("US")); - assertThat(geoData.get("country_name"), equalTo("United States")); - assertThat(geoData.get("continent_code"), equalTo("NA")); - assertThat(geoData.get("continent_name"), equalTo("North America")); - assertThat(geoData.get("region_iso_code"), equalTo("US-FL")); - assertThat(geoData.get("region_name"), equalTo("Florida")); - assertThat(geoData.get("city_name"), equalTo("Homestead")); - assertThat(geoData.get("timezone"), equalTo("America/New_York")); - assertThat(geoData.get("location"), equalTo(Map.of("lat", 25.4573d, "lon", -80.4572d))); - assertThat(geoData.get("accuracy_radius"), equalTo(50)); - assertThat(geoData.get("postal_code"), equalTo("33035")); - assertThat(geoData.get("registered_country_in_european_union"), equalTo(false)); - assertThat(geoData.get("registered_country_iso_code"), equalTo("US")); - assertThat(geoData.get("registered_country_name"), equalTo("United States")); - } - - public void testCityWithMissingLocation() throws Exception { - String ip = "80.231.5.0"; - GeoIpProcessor processor = new GeoIpProcessor( - GEOIP_TYPE, - randomAlphaOfLength(10), - null, - "source_field", - loader("GeoLite2-City.mmdb"), - () -> true, - "target_field", - ipDataLookupAll(Database.City), - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", ip); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(1)); - assertThat(geoData.get("ip"), equalTo(ip)); - } - - public void testCountry() throws Exception { - String ip = "82.170.213.79"; - GeoIpProcessor processor = new GeoIpProcessor( - GEOIP_TYPE, - randomAlphaOfLength(10), - null, - "source_field", - loader("GeoLite2-Country.mmdb"), - () -> true, - "target_field", - ipDataLookupAll(Database.Country), - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", ip); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(9)); - assertThat(geoData.get("ip"), equalTo(ip)); - assertThat(geoData.get("country_in_european_union"), equalTo(true)); - assertThat(geoData.get("country_iso_code"), equalTo("NL")); - assertThat(geoData.get("country_name"), equalTo("Netherlands")); - assertThat(geoData.get("continent_code"), equalTo("EU")); - assertThat(geoData.get("continent_name"), equalTo("Europe")); - assertThat(geoData.get("registered_country_in_european_union"), equalTo(true)); - assertThat(geoData.get("registered_country_iso_code"), equalTo("NL")); - assertThat(geoData.get("registered_country_name"), equalTo("Netherlands")); - } - - public void testCountryWithMissingLocation() throws Exception { - String ip = "80.231.5.0"; - GeoIpProcessor processor = new GeoIpProcessor( - GEOIP_TYPE, - randomAlphaOfLength(10), - null, - "source_field", - loader("GeoLite2-Country.mmdb"), - () -> true, - "target_field", - ipDataLookupAll(Database.Country), - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", ip); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(1)); - assertThat(geoData.get("ip"), equalTo(ip)); - } - - public void testAsn() throws Exception { - String ip = "82.171.64.0"; - GeoIpProcessor processor = new GeoIpProcessor( - GEOIP_TYPE, - randomAlphaOfLength(10), - null, - "source_field", - loader("GeoLite2-ASN.mmdb"), - () -> true, - "target_field", - ipDataLookupAll(Database.Asn), - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", ip); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(4)); - assertThat(geoData.get("ip"), equalTo(ip)); - assertThat(geoData.get("asn"), equalTo(1136L)); - assertThat(geoData.get("organization_name"), equalTo("KPN B.V.")); - assertThat(geoData.get("network"), equalTo("82.168.0.0/14")); - } - - public void testAnonymmousIp() throws Exception { - String ip = "81.2.69.1"; - GeoIpProcessor processor = new GeoIpProcessor( - GEOIP_TYPE, - randomAlphaOfLength(10), - null, - "source_field", - loader("GeoIP2-Anonymous-IP-Test.mmdb"), - () -> true, - "target_field", - ipDataLookupAll(Database.AnonymousIp), - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", ip); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(7)); - assertThat(geoData.get("ip"), equalTo(ip)); - assertThat(geoData.get("hosting_provider"), equalTo(true)); - assertThat(geoData.get("tor_exit_node"), equalTo(true)); - assertThat(geoData.get("anonymous_vpn"), equalTo(true)); - assertThat(geoData.get("anonymous"), equalTo(true)); - assertThat(geoData.get("public_proxy"), equalTo(true)); - assertThat(geoData.get("residential_proxy"), equalTo(true)); - } - - public void testConnectionType() throws Exception { - String ip = "214.78.120.5"; - GeoIpProcessor processor = new GeoIpProcessor( - GEOIP_TYPE, - randomAlphaOfLength(10), - null, - "source_field", - loader("GeoIP2-Connection-Type-Test.mmdb"), - () -> true, - "target_field", - ipDataLookupAll(Database.ConnectionType), - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", ip); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(2)); - assertThat(geoData.get("ip"), equalTo(ip)); - assertThat(geoData.get("connection_type"), equalTo("Satellite")); - } - - public void testDomain() throws Exception { - String ip = "69.219.64.2"; - GeoIpProcessor processor = new GeoIpProcessor( - GEOIP_TYPE, - randomAlphaOfLength(10), - null, - "source_field", - loader("GeoIP2-Domain-Test.mmdb"), - () -> true, - "target_field", - ipDataLookupAll(Database.Domain), - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", ip); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(2)); - assertThat(geoData.get("ip"), equalTo(ip)); - assertThat(geoData.get("domain"), equalTo("ameritech.net")); - } - - public void testEnterprise() throws Exception { - String ip = "74.209.24.4"; - GeoIpProcessor processor = new GeoIpProcessor( - GEOIP_TYPE, - randomAlphaOfLength(10), - null, - "source_field", - loader("GeoIP2-Enterprise-Test.mmdb"), - () -> true, - "target_field", - ipDataLookupAll(Database.Enterprise), - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", ip); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(33)); - assertThat(geoData.get("ip"), equalTo(ip)); - assertThat(geoData.get("country_confidence"), equalTo(99)); - assertThat(geoData.get("country_in_european_union"), equalTo(false)); - assertThat(geoData.get("country_iso_code"), equalTo("US")); - assertThat(geoData.get("country_name"), equalTo("United States")); - assertThat(geoData.get("continent_code"), equalTo("NA")); - assertThat(geoData.get("continent_name"), equalTo("North America")); - assertThat(geoData.get("region_iso_code"), equalTo("US-NY")); - assertThat(geoData.get("region_name"), equalTo("New York")); - assertThat(geoData.get("city_confidence"), equalTo(11)); - assertThat(geoData.get("city_name"), equalTo("Chatham")); - assertThat(geoData.get("timezone"), equalTo("America/New_York")); - assertThat(geoData.get("location"), equalTo(Map.of("lat", 42.3478, "lon", -73.5549))); - assertThat(geoData.get("accuracy_radius"), equalTo(27)); - assertThat(geoData.get("postal_code"), equalTo("12037")); - assertThat(geoData.get("city_confidence"), equalTo(11)); - assertThat(geoData.get("asn"), equalTo(14671L)); - assertThat(geoData.get("organization_name"), equalTo("FairPoint Communications")); - assertThat(geoData.get("network"), equalTo("74.209.16.0/20")); - assertThat(geoData.get("hosting_provider"), equalTo(false)); - assertThat(geoData.get("tor_exit_node"), equalTo(false)); - assertThat(geoData.get("anonymous_vpn"), equalTo(false)); - assertThat(geoData.get("anonymous"), equalTo(false)); - assertThat(geoData.get("public_proxy"), equalTo(false)); - assertThat(geoData.get("residential_proxy"), equalTo(false)); - assertThat(geoData.get("domain"), equalTo("frpt.net")); - assertThat(geoData.get("isp"), equalTo("Fairpoint Communications")); - assertThat(geoData.get("isp_organization_name"), equalTo("Fairpoint Communications")); - assertThat(geoData.get("user_type"), equalTo("residential")); - assertThat(geoData.get("connection_type"), equalTo("Cable/DSL")); - assertThat(geoData.get("registered_country_in_european_union"), equalTo(false)); - assertThat(geoData.get("registered_country_iso_code"), equalTo("US")); - assertThat(geoData.get("registered_country_name"), equalTo("United States")); - } - - public void testIsp() throws Exception { - String ip = "149.101.100.1"; - GeoIpProcessor processor = new GeoIpProcessor( - GEOIP_TYPE, - randomAlphaOfLength(10), - null, - "source_field", - loader("GeoIP2-ISP-Test.mmdb"), - () -> true, - "target_field", - ipDataLookupAll(Database.Isp), - false, - false, - "filename" - ); - - Map document = new HashMap<>(); - document.put("source_field", ip); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - - assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); - @SuppressWarnings("unchecked") - Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(8)); - assertThat(geoData.get("ip"), equalTo(ip)); - assertThat(geoData.get("asn"), equalTo(6167L)); - assertThat(geoData.get("organization_name"), equalTo("CELLCO-PART")); - assertThat(geoData.get("network"), equalTo("149.101.100.0/28")); - assertThat(geoData.get("isp"), equalTo("Verizon Wireless")); - assertThat(geoData.get("isp_organization_name"), equalTo("Verizon Wireless")); - assertThat(geoData.get("mobile_network_code"), equalTo("004")); - assertThat(geoData.get("mobile_country_code"), equalTo("310")); - } - public void testAddressIsNotInTheDatabase() throws Exception { GeoIpProcessor processor = new GeoIpProcessor( GEOIP_TYPE, @@ -594,9 +163,9 @@ public void testAddressIsNotInTheDatabase() throws Exception { } /** - * Don't silently do DNS lookups or anything trappy on bogus data + * Tests that an exception in the IpDataLookup is propagated out of the GeoIpProcessor's execute method */ - public void testInvalid() { + public void testExceptionPropagates() { GeoIpProcessor processor = new GeoIpProcessor( GEOIP_TYPE, randomAlphaOfLength(10), diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java index 4ecf3056db738..4167170567f52 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java @@ -15,15 +15,9 @@ import org.apache.lucene.util.Constants; import org.elasticsearch.common.network.NetworkAddress; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.SuppressForbidden; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.threadpool.TestThreadPool; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; import org.junit.After; import org.junit.Before; @@ -41,50 +35,27 @@ import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.parseBoolean; import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.parseLocationDouble; import static org.hamcrest.Matchers.anyOf; -import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.startsWith; public class IpinfoIpDataLookupsTests extends ESTestCase { - private ThreadPool threadPool; - private ResourceWatcherService resourceWatcherService; - // a temporary directory that mmdb files can be copied to and read from private Path tmpDir; @Before public void setup() { - threadPool = new TestThreadPool(ConfigDatabases.class.getSimpleName()); - Settings settings = Settings.builder().put("resource.reload.interval.high", TimeValue.timeValueMillis(100)).build(); - resourceWatcherService = new ResourceWatcherService(settings, threadPool); tmpDir = createTempDir(); } @After public void cleanup() throws IOException { - resourceWatcherService.close(); - threadPool.shutdownNow(); IOUtils.rm(tmpDir); } - public void testDatabasePropertyInvariants() { - // the second ASN variant database is like a specialization of the ASN database - assertThat(Sets.difference(Database.Asn.properties(), Database.AsnV2.properties()), is(empty())); - assertThat(Database.Asn.defaultProperties(), equalTo(Database.AsnV2.defaultProperties())); - - // the second City variant database is like a version of the ordinary City database but lacking many fields - assertThat(Sets.difference(Database.CityV2.properties(), Database.City.properties()), is(empty())); - assertThat(Sets.difference(Database.CityV2.defaultProperties(), Database.City.defaultProperties()), is(empty())); - - // the second Country variant database is like a version of the ordinary Country database but lacking come fields - assertThat(Sets.difference(Database.CountryV2.properties(), Database.CountryV2.properties()), is(empty())); - assertThat(Database.CountryV2.defaultProperties(), equalTo(Database.Country.defaultProperties())); - } - public void testParseAsn() { // expected case: "AS123" is 123 assertThat(parseAsn("AS123"), equalTo(123L)); @@ -126,53 +97,42 @@ public void testParseLocationDouble() { assertThat(parseLocationDouble("anythingelse"), nullValue()); } - public void testAsn() throws IOException { + public void testAsnFree() { assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); - Path configDir = tmpDir; - copyDatabase("ipinfo/ip_asn_sample.mmdb", configDir.resolve("ip_asn_sample.mmdb")); - copyDatabase("ipinfo/asn_sample.mmdb", configDir.resolve("asn_sample.mmdb")); - - GeoIpCache cache = new GeoIpCache(1000); // real cache to test purging of entries upon a reload - ConfigDatabases configDatabases = new ConfigDatabases(configDir, cache); - configDatabases.initialize(resourceWatcherService); - - // this is the 'free' ASN database (sample) - try (DatabaseReaderLazyLoader loader = configDatabases.getDatabase("ip_asn_sample.mmdb")) { - IpDataLookup lookup = new IpinfoIpDataLookups.Asn(Database.AsnV2.properties()); - Map data = lookup.getData(loader, "5.182.109.0"); - assertThat( - data, - equalTo( - Map.ofEntries( - entry("ip", "5.182.109.0"), - entry("organization_name", "M247 Europe SRL"), - entry("asn", 9009L), - entry("network", "5.182.109.0/24"), - entry("domain", "m247.com") - ) - ) - ); - } + String databaseName = "ip_asn_sample.mmdb"; + String ip = "5.182.109.0"; + assertExpectedLookupResults( + databaseName, + ip, + new IpinfoIpDataLookups.Asn(Database.AsnV2.properties()), + Map.ofEntries( + entry("ip", ip), + entry("organization_name", "M247 Europe SRL"), + entry("asn", 9009L), + entry("network", "5.182.109.0/24"), + entry("domain", "m247.com") + ) + ); + } - // this is the non-free or 'standard' ASN database (sample) - try (DatabaseReaderLazyLoader loader = configDatabases.getDatabase("asn_sample.mmdb")) { - IpDataLookup lookup = new IpinfoIpDataLookups.Asn(Database.AsnV2.properties()); - Map data = lookup.getData(loader, "23.53.116.0"); - assertThat( - data, - equalTo( - Map.ofEntries( - entry("ip", "23.53.116.0"), - entry("organization_name", "Akamai Technologies, Inc."), - entry("asn", 32787L), - entry("network", "23.53.116.0/24"), - entry("domain", "akamai.com"), - entry("type", "hosting"), - entry("country_iso_code", "US") - ) - ) - ); - } + public void testAsnStandard() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "asn_sample.mmdb"; + String ip = "23.53.116.0"; + assertExpectedLookupResults( + databaseName, + ip, + new IpinfoIpDataLookups.Asn(Database.AsnV2.properties()), + Map.ofEntries( + entry("ip", ip), + entry("organization_name", "Akamai Technologies, Inc."), + entry("asn", 32787L), + entry("network", "23.53.116.0/24"), + entry("domain", "akamai.com"), + entry("type", "hosting"), + entry("country_iso_code", "US") + ) + ); } public void testAsnInvariants() { @@ -212,62 +172,42 @@ public void testAsnInvariants() { } } - public void testCountry() throws IOException { + public void testCountryFree() { assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); - Path configDir = tmpDir; - copyDatabase("ipinfo/ip_country_sample.mmdb", configDir.resolve("ip_country_sample.mmdb")); - - GeoIpCache cache = new GeoIpCache(1000); // real cache to test purging of entries upon a reload - ConfigDatabases configDatabases = new ConfigDatabases(configDir, cache); - configDatabases.initialize(resourceWatcherService); - - // this is the 'free' Country database (sample) - try (DatabaseReaderLazyLoader loader = configDatabases.getDatabase("ip_country_sample.mmdb")) { - IpDataLookup lookup = new IpinfoIpDataLookups.Country(Database.CountryV2.properties()); - Map data = lookup.getData(loader, "4.221.143.168"); - assertThat( - data, - equalTo( - Map.ofEntries( - entry("ip", "4.221.143.168"), - entry("country_name", "South Africa"), - entry("country_iso_code", "ZA"), - entry("continent_name", "Africa"), - entry("continent_code", "AF") - ) - ) - ); - } + String databaseName = "ip_country_sample.mmdb"; + String ip = "4.221.143.168"; + assertExpectedLookupResults( + databaseName, + ip, + new IpinfoIpDataLookups.Country(Database.CountryV2.properties()), + Map.ofEntries( + entry("ip", ip), + entry("country_name", "South Africa"), + entry("country_iso_code", "ZA"), + entry("continent_name", "Africa"), + entry("continent_code", "AF") + ) + ); } - public void testGeolocation() throws IOException { + public void testGeolocationStandard() { assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); - Path configDir = tmpDir; - copyDatabase("ipinfo/ip_geolocation_sample.mmdb", configDir.resolve("ip_geolocation_sample.mmdb")); - - GeoIpCache cache = new GeoIpCache(1000); // real cache to test purging of entries upon a reload - ConfigDatabases configDatabases = new ConfigDatabases(configDir, cache); - configDatabases.initialize(resourceWatcherService); - - // this is the non-free or 'standard' Geolocation database (sample) - try (DatabaseReaderLazyLoader loader = configDatabases.getDatabase("ip_geolocation_sample.mmdb")) { - IpDataLookup lookup = new IpinfoIpDataLookups.Geolocation(Database.CityV2.properties()); - Map data = lookup.getData(loader, "2.124.90.182"); - assertThat( - data, - equalTo( - Map.ofEntries( - entry("ip", "2.124.90.182"), - entry("country_iso_code", "GB"), - entry("region_name", "England"), - entry("city_name", "London"), - entry("timezone", "Europe/London"), - entry("postal_code", "E1W"), - entry("location", Map.of("lat", 51.50853, "lon", -0.12574)) - ) - ) - ); - } + String databaseName = "ip_geolocation_sample.mmdb"; + String ip = "2.124.90.182"; + assertExpectedLookupResults( + databaseName, + ip, + new IpinfoIpDataLookups.Geolocation(Database.CityV2.properties()), + Map.ofEntries( + entry("ip", ip), + entry("country_iso_code", "GB"), + entry("region_name", "England"), + entry("city_name", "London"), + entry("timezone", "Europe/London"), + entry("postal_code", "E1W"), + entry("location", Map.of("lat", 51.50853, "lon", -0.12574)) + ) + ); } public void testGeolocationInvariants() { @@ -308,53 +248,43 @@ public void testGeolocationInvariants() { } } - public void testPrivacyDetection() throws IOException { + public void testPrivacyDetectionStandard() { assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); - Path configDir = tmpDir; - copyDatabase("ipinfo/privacy_detection_sample.mmdb", configDir.resolve("privacy_detection_sample.mmdb")); - - GeoIpCache cache = new GeoIpCache(1000); // real cache to test purging of entries upon a reload - ConfigDatabases configDatabases = new ConfigDatabases(configDir, cache); - configDatabases.initialize(resourceWatcherService); - - // testing the first row in the sample database - try (DatabaseReaderLazyLoader loader = configDatabases.getDatabase("privacy_detection_sample.mmdb")) { - IpDataLookup lookup = new IpinfoIpDataLookups.PrivacyDetection(Database.PrivacyDetection.properties()); - Map data = lookup.getData(loader, "1.53.59.33"); - assertThat( - data, - equalTo( - Map.ofEntries( - entry("ip", "1.53.59.33"), - entry("hosting", false), - entry("proxy", false), - entry("relay", false), - entry("tor", false), - entry("vpn", true) - ) - ) - ); - } + String databaseName = "privacy_detection_sample.mmdb"; + String ip = "1.53.59.33"; + assertExpectedLookupResults( + databaseName, + ip, + new IpinfoIpDataLookups.PrivacyDetection(Database.PrivacyDetection.properties()), + Map.ofEntries( + entry("ip", ip), + entry("hosting", false), + entry("proxy", false), + entry("relay", false), + entry("tor", false), + entry("vpn", true) + ) + ); + } - // testing a row with a non-empty service in the sample database - try (DatabaseReaderLazyLoader loader = configDatabases.getDatabase("privacy_detection_sample.mmdb")) { - IpDataLookup lookup = new IpinfoIpDataLookups.PrivacyDetection(Database.PrivacyDetection.properties()); - Map data = lookup.getData(loader, "216.131.74.65"); - assertThat( - data, - equalTo( - Map.ofEntries( - entry("ip", "216.131.74.65"), - entry("hosting", true), - entry("proxy", false), - entry("service", "FastVPN"), - entry("relay", false), - entry("tor", false), - entry("vpn", true) - ) - ) - ); - } + public void testPrivacyDetectionStandardNonEmptyService() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "privacy_detection_sample.mmdb"; + String ip = "216.131.74.65"; + assertExpectedLookupResults( + databaseName, + ip, + new IpinfoIpDataLookups.PrivacyDetection(Database.PrivacyDetection.properties()), + Map.ofEntries( + entry("ip", ip), + entry("hosting", true), + entry("proxy", false), + entry("service", "FastVPN"), + entry("relay", false), + entry("tor", false), + entry("vpn", true) + ) + ); } public void testPrivacyDetectionInvariants() { @@ -403,4 +333,29 @@ private static void assertDatabaseInvariants(final Path databasePath, final BiCo private static File pathToFile(Path databasePath) { return databasePath.toFile(); } + + private void assertExpectedLookupResults(String databaseName, String ip, IpDataLookup lookup, Map expected) { + try (DatabaseReaderLazyLoader loader = loader(databaseName)) { + Map actual = lookup.getData(loader, ip); + assertThat( + "The set of keys in the result are not the same as the set of expected keys", + actual.keySet(), + containsInAnyOrder(expected.keySet().toArray(new String[0])) + ); + for (Map.Entry entry : expected.entrySet()) { + assertThat("Unexpected value for key [" + entry.getKey() + "]", actual.get(entry.getKey()), equalTo(entry.getValue())); + } + } catch (AssertionError e) { + fail(e, "Assert failed for database [%s] with address [%s]", databaseName, ip); + } catch (Exception e) { + fail(e, "Exception for database [%s] with address [%s]", databaseName, ip); + } + } + + private DatabaseReaderLazyLoader loader(final String databaseName) { + Path path = tmpDir.resolve(databaseName); + copyDatabase("ipinfo/" + databaseName, path); // the ipinfo databases are prefixed on the test classpath + final GeoIpCache cache = new GeoIpCache(1000); + return new DatabaseReaderLazyLoader(cache, path, null); + } } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MMDBUtilTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MMDBUtilTests.java index 46a34c2cdad56..083e9b5bc32da 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MMDBUtilTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MMDBUtilTests.java @@ -83,39 +83,4 @@ public void testIsGzip() throws IOException { assertThat(MMDBUtil.isGzip(database), is(false)); assertThat(MMDBUtil.isGzip(gzipDatabase), is(true)); } - - public void testDatabaseTypeParsing() throws IOException { - // this test is a little bit overloaded -- it's testing that we're getting the expected sorts of - // database_type strings from these files, *and* it's also testing that we dispatch on those strings - // correctly and associated those files with the correct high-level Elasticsearch Database type. - // down the road it would probably make sense to split these out and find a better home for some of the - // logic, but for now it's probably more valuable to have the test *somewhere* than to get especially - // pedantic about where precisely it should be. - - copyDatabase("GeoLite2-City-Test.mmdb", tmpDir); - copyDatabase("GeoLite2-Country-Test.mmdb", tmpDir); - copyDatabase("GeoLite2-ASN-Test.mmdb", tmpDir); - copyDatabase("GeoIP2-Anonymous-IP-Test.mmdb", tmpDir); - copyDatabase("GeoIP2-City-Test.mmdb", tmpDir); - copyDatabase("GeoIP2-Country-Test.mmdb", tmpDir); - copyDatabase("GeoIP2-Connection-Type-Test.mmdb", tmpDir); - copyDatabase("GeoIP2-Domain-Test.mmdb", tmpDir); - copyDatabase("GeoIP2-Enterprise-Test.mmdb", tmpDir); - copyDatabase("GeoIP2-ISP-Test.mmdb", tmpDir); - - assertThat(parseDatabaseFromType("GeoLite2-City-Test.mmdb"), is(Database.City)); - assertThat(parseDatabaseFromType("GeoLite2-Country-Test.mmdb"), is(Database.Country)); - assertThat(parseDatabaseFromType("GeoLite2-ASN-Test.mmdb"), is(Database.Asn)); - assertThat(parseDatabaseFromType("GeoIP2-Anonymous-IP-Test.mmdb"), is(Database.AnonymousIp)); - assertThat(parseDatabaseFromType("GeoIP2-City-Test.mmdb"), is(Database.City)); - assertThat(parseDatabaseFromType("GeoIP2-Country-Test.mmdb"), is(Database.Country)); - assertThat(parseDatabaseFromType("GeoIP2-Connection-Type-Test.mmdb"), is(Database.ConnectionType)); - assertThat(parseDatabaseFromType("GeoIP2-Domain-Test.mmdb"), is(Database.Domain)); - assertThat(parseDatabaseFromType("GeoIP2-Enterprise-Test.mmdb"), is(Database.Enterprise)); - assertThat(parseDatabaseFromType("GeoIP2-ISP-Test.mmdb"), is(Database.Isp)); - } - - private Database parseDatabaseFromType(String databaseFile) throws IOException { - return IpDataLookupFactories.getDatabase(MMDBUtil.getDatabaseType(tmpDir.resolve(databaseFile))); - } } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookupsTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookupsTests.java new file mode 100644 index 0000000000000..57ee2191a590d --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookupsTests.java @@ -0,0 +1,339 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +import org.apache.lucene.util.Constants; +import org.elasticsearch.core.IOUtils; +import org.elasticsearch.test.ESTestCase; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Map; + +import static java.util.Map.entry; +import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +public class MaxmindIpDataLookupsTests extends ESTestCase { + + // a temporary directory that mmdb files can be copied to and read from + private Path tmpDir; + + @Before + public void setup() { + tmpDir = createTempDir(); + } + + @After + public void cleanup() throws IOException { + IOUtils.rm(tmpDir); + } + + public void testCity() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoLite2-City.mmdb"; + String ip = "8.8.8.8"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.City(Database.City.properties()), + Map.ofEntries( + entry("ip", ip), + entry("country_in_european_union", false), + entry("country_iso_code", "US"), + entry("country_name", "United States"), + entry("continent_code", "NA"), + entry("continent_name", "North America"), + entry("timezone", "America/Chicago"), + entry("location", Map.of("lat", 37.751d, "lon", -97.822d)), + entry("accuracy_radius", 1000), + entry("registered_country_in_european_union", false), + entry("registered_country_iso_code", "US"), + entry("registered_country_name", "United States") + ) + ); + } + + public void testCity_withIpV6() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoLite2-City.mmdb"; + String ip = "2602:306:33d3:8000::3257:9652"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.City(Database.City.properties()), + Map.ofEntries( + entry("ip", ip), + entry("country_in_european_union", false), + entry("country_iso_code", "US"), + entry("country_name", "United States"), + entry("continent_code", "NA"), + entry("continent_name", "North America"), + entry("region_iso_code", "US-FL"), + entry("region_name", "Florida"), + entry("city_name", "Homestead"), + entry("postal_code", "33035"), + entry("timezone", "America/New_York"), + entry("location", Map.of("lat", 25.4573d, "lon", -80.4572d)), + entry("accuracy_radius", 50), + entry("registered_country_in_european_union", false), + entry("registered_country_iso_code", "US"), + entry("registered_country_name", "United States") + ) + ); + } + + public void testCityWithMissingLocation() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoLite2-City.mmdb"; + String ip = "80.231.5.0"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.City(Database.City.properties()), + Map.ofEntries(entry("ip", ip)) + ); + } + + public void testCountry() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoLite2-Country.mmdb"; + String ip = "82.170.213.79"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.Country(Database.Country.properties()), + Map.ofEntries( + entry("ip", ip), + entry("country_in_european_union", true), + entry("country_iso_code", "NL"), + entry("country_name", "Netherlands"), + entry("continent_code", "EU"), + entry("continent_name", "Europe"), + entry("registered_country_in_european_union", true), + entry("registered_country_iso_code", "NL"), + entry("registered_country_name", "Netherlands") + ) + ); + } + + /** + * Don't silently do DNS lookups or anything trappy on bogus data + */ + public void testInvalid() throws IOException { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoLite2-Country.mmdb"; + String ip = "www.google.com"; + try (DatabaseReaderLazyLoader loader = loader(databaseName)) { + IpDataLookup lookup = new MaxmindIpDataLookups.Country(Database.Country.properties()); + IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> lookup.getData(loader, ip)); + assertThat(e.getMessage(), containsString("not an IP string literal")); + } + } + + public void testCountryWithMissingLocation() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoLite2-Country.mmdb"; + String ip = "80.231.5.0"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.Country(Database.Country.properties()), + Map.ofEntries(entry("ip", ip)) + ); + } + + public void testAsn() throws IOException { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoLite2-ASN.mmdb"; + String ip = "82.171.64.0"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.Asn(Database.Asn.properties()), + Map.ofEntries(entry("ip", ip), entry("organization_name", "KPN B.V."), entry("asn", 1136L), entry("network", "82.168.0.0/14")) + ); + } + + public void testAnonymousIp() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoIP2-Anonymous-IP-Test.mmdb"; + String ip = "81.2.69.1"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.AnonymousIp(Database.AnonymousIp.properties()), + Map.ofEntries( + entry("ip", ip), + entry("hosting_provider", true), + entry("tor_exit_node", true), + entry("anonymous_vpn", true), + entry("anonymous", true), + entry("public_proxy", true), + entry("residential_proxy", true) + ) + ); + } + + public void testConnectionType() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoIP2-Connection-Type-Test.mmdb"; + String ip = "214.78.120.5"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.ConnectionType(Database.ConnectionType.properties()), + Map.ofEntries(entry("ip", ip), entry("connection_type", "Satellite")) + ); + } + + public void testDomain() { + String databaseName = "GeoIP2-Domain-Test.mmdb"; + String ip = "69.219.64.2"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.Domain(Database.Domain.properties()), + Map.ofEntries(entry("ip", ip), entry("domain", "ameritech.net")) + ); + } + + public void testEnterprise() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoIP2-Enterprise-Test.mmdb"; + String ip = "74.209.24.4"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.Enterprise(Database.Enterprise.properties()), + Map.ofEntries( + entry("ip", ip), + entry("country_confidence", 99), + entry("country_in_european_union", false), + entry("country_iso_code", "US"), + entry("country_name", "United States"), + entry("continent_code", "NA"), + entry("continent_name", "North America"), + entry("region_iso_code", "US-NY"), + entry("region_name", "New York"), + entry("city_confidence", 11), + entry("city_name", "Chatham"), + entry("timezone", "America/New_York"), + entry("location", Map.of("lat", 42.3478, "lon", -73.5549)), + entry("accuracy_radius", 27), + entry("postal_code", "12037"), + entry("postal_confidence", 11), + entry("asn", 14671L), + entry("organization_name", "FairPoint Communications"), + entry("network", "74.209.16.0/20"), + entry("hosting_provider", false), + entry("tor_exit_node", false), + entry("anonymous_vpn", false), + entry("anonymous", false), + entry("public_proxy", false), + entry("residential_proxy", false), + entry("domain", "frpt.net"), + entry("isp", "Fairpoint Communications"), + entry("isp_organization_name", "Fairpoint Communications"), + entry("user_type", "residential"), + entry("connection_type", "Cable/DSL"), + entry("registered_country_in_european_union", false), + entry("registered_country_iso_code", "US"), + entry("registered_country_name", "United States") + ) + ); + } + + public void testIsp() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + String databaseName = "GeoIP2-ISP-Test.mmdb"; + String ip = "149.101.100.1"; + assertExpectedLookupResults( + databaseName, + ip, + new MaxmindIpDataLookups.Isp(Database.Isp.properties()), + Map.ofEntries( + entry("ip", ip), + entry("asn", 6167L), + entry("organization_name", "CELLCO-PART"), + entry("network", "149.101.100.0/28"), + entry("isp", "Verizon Wireless"), + entry("isp_organization_name", "Verizon Wireless"), + entry("mobile_network_code", "004"), + entry("mobile_country_code", "310") + ) + ); + } + + public void testDatabaseTypeParsing() throws IOException { + // this test is a little bit overloaded -- it's testing that we're getting the expected sorts of + // database_type strings from these files, *and* it's also testing that we dispatch on those strings + // correctly and associated those files with the correct high-level Elasticsearch Database type. + // down the road it would probably make sense to split these out and find a better home for some of the + // logic, but for now it's probably more valuable to have the test *somewhere* than to get especially + // pedantic about where precisely it should be. + + copyDatabase("GeoLite2-City-Test.mmdb", tmpDir); + copyDatabase("GeoLite2-Country-Test.mmdb", tmpDir); + copyDatabase("GeoLite2-ASN-Test.mmdb", tmpDir); + copyDatabase("GeoIP2-Anonymous-IP-Test.mmdb", tmpDir); + copyDatabase("GeoIP2-City-Test.mmdb", tmpDir); + copyDatabase("GeoIP2-Country-Test.mmdb", tmpDir); + copyDatabase("GeoIP2-Connection-Type-Test.mmdb", tmpDir); + copyDatabase("GeoIP2-Domain-Test.mmdb", tmpDir); + copyDatabase("GeoIP2-Enterprise-Test.mmdb", tmpDir); + copyDatabase("GeoIP2-ISP-Test.mmdb", tmpDir); + + assertThat(parseDatabaseFromType("GeoLite2-City-Test.mmdb"), is(Database.City)); + assertThat(parseDatabaseFromType("GeoLite2-Country-Test.mmdb"), is(Database.Country)); + assertThat(parseDatabaseFromType("GeoLite2-ASN-Test.mmdb"), is(Database.Asn)); + assertThat(parseDatabaseFromType("GeoIP2-Anonymous-IP-Test.mmdb"), is(Database.AnonymousIp)); + assertThat(parseDatabaseFromType("GeoIP2-City-Test.mmdb"), is(Database.City)); + assertThat(parseDatabaseFromType("GeoIP2-Country-Test.mmdb"), is(Database.Country)); + assertThat(parseDatabaseFromType("GeoIP2-Connection-Type-Test.mmdb"), is(Database.ConnectionType)); + assertThat(parseDatabaseFromType("GeoIP2-Domain-Test.mmdb"), is(Database.Domain)); + assertThat(parseDatabaseFromType("GeoIP2-Enterprise-Test.mmdb"), is(Database.Enterprise)); + assertThat(parseDatabaseFromType("GeoIP2-ISP-Test.mmdb"), is(Database.Isp)); + } + + private Database parseDatabaseFromType(String databaseFile) throws IOException { + return IpDataLookupFactories.getDatabase(MMDBUtil.getDatabaseType(tmpDir.resolve(databaseFile))); + } + + private void assertExpectedLookupResults(String databaseName, String ip, IpDataLookup lookup, Map expected) { + try (DatabaseReaderLazyLoader loader = loader(databaseName)) { + Map actual = lookup.getData(loader, ip); + assertThat( + "The set of keys in the result are not the same as the set of expected keys", + actual.keySet(), + containsInAnyOrder(expected.keySet().toArray(new String[0])) + ); + for (Map.Entry entry : expected.entrySet()) { + assertThat("Unexpected value for key [" + entry.getKey() + "]", actual.get(entry.getKey()), equalTo(entry.getValue())); + } + } catch (AssertionError e) { + fail(e, "Assert failed for database [%s] with address [%s]", databaseName, ip); + } catch (Exception e) { + fail(e, "Exception for database [%s] with address [%s]", databaseName, ip); + } + } + + private DatabaseReaderLazyLoader loader(final String databaseName) { + Path path = tmpDir.resolve(databaseName); + copyDatabase(databaseName, path); + final GeoIpCache cache = new GeoIpCache(1000); + return new DatabaseReaderLazyLoader(cache, path, null); + } +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationTests.java index 33356ad4235dc..76b2896afe7a0 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.ingest.geoip.IngestGeoIpPlugin; +import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.Ipinfo; import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.Local; import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.Maxmind; import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.Web; @@ -21,6 +22,7 @@ import java.io.IOException; import java.util.Set; +import static org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.IPINFO_NAMES; import static org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.MAXMIND_NAMES; public class DatabaseConfigurationTests extends AbstractXContentSerializingTestCase { @@ -44,13 +46,14 @@ protected DatabaseConfiguration createTestInstance() { } public static DatabaseConfiguration randomDatabaseConfiguration(String id) { + boolean useIpinfo = randomBoolean(); DatabaseConfiguration.Provider provider = switch (between(0, 2)) { - case 0 -> new Maxmind(randomAlphaOfLength(5)); + case 0 -> useIpinfo ? new Ipinfo() : new Maxmind(randomAlphaOfLength(5)); case 1 -> new Web(); case 2 -> new Local(randomAlphaOfLength(10)); default -> throw new AssertionError("failure, got illegal switch case"); }; - return new DatabaseConfiguration(id, randomFrom(MAXMIND_NAMES), provider); + return new DatabaseConfiguration(id, useIpinfo ? randomFrom(IPINFO_NAMES) : randomFrom(MAXMIND_NAMES), provider); } @Override @@ -61,21 +64,21 @@ protected DatabaseConfiguration mutateInstance(DatabaseConfiguration instance) { case 1: return new DatabaseConfiguration( instance.id(), - randomValueOtherThan(instance.name(), () -> randomFrom(MAXMIND_NAMES)), + randomValueOtherThan( + instance.name(), + () -> instance.provider() instanceof Ipinfo ? randomFrom(IPINFO_NAMES) : randomFrom(MAXMIND_NAMES) + ), instance.provider() ); case 2: DatabaseConfiguration.Provider provider = instance.provider(); - DatabaseConfiguration.Provider modifiedProvider; - if (provider instanceof Maxmind maxmind) { - modifiedProvider = new Maxmind(((Maxmind) instance.provider()).accountId() + randomAlphaOfLength(2)); - } else if (provider instanceof Web) { - modifiedProvider = new Maxmind(randomAlphaOfLength(20)); // can't modify a Web - } else if (provider instanceof Local local) { - modifiedProvider = new Local(local.type() + randomAlphaOfLength(2)); - } else { - throw new AssertionError("Unexpected provider type: " + provider.getClass()); - } + DatabaseConfiguration.Provider modifiedProvider = switch (provider) { + case Maxmind maxmind -> new Maxmind(maxmind.accountId() + randomAlphaOfLength(2)); + case Ipinfo ignored -> new Local(randomAlphaOfLength(20)); // can't modify Ipinfo + case Web ignored -> new Local(randomAlphaOfLength(20)); // can't modify a Web + case Local local -> new Local(local.type() + randomAlphaOfLength(2)); + default -> throw new AssertionError("Unexpected provider type: " + provider.getClass()); + }; return new DatabaseConfiguration(instance.id(), instance.name(), modifiedProvider); default: throw new AssertionError("failure, got illegal switch case"); diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml index 04fd2ac6a8189..a1104505bc240 100644 --- a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml +++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml @@ -1,8 +1,20 @@ +--- setup: - requires: cluster_features: ["geoip.downloader.database.configuration", "get_database_configuration_action.multi_node"] reason: "geoip downloader database configuration APIs added in 8.15, and updated in 8.16 to return more results" +--- +teardown: + - do: + ingest.delete_ip_location_database: + id: "my_database_1" + ignore: 404 + - do: + ingest.delete_ip_location_database: + id: "my_database_2" + ignore: 404 + --- "Test adding, getting, and removing geoip databases": - do: diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/50_ip_lookup_processor.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/50_ip_lookup_processor.yml new file mode 100644 index 0000000000000..fd73c715a5ac5 --- /dev/null +++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/50_ip_lookup_processor.yml @@ -0,0 +1,45 @@ +setup: + - requires: + cluster_features: + - "put_database_configuration_action.ipinfo" + reason: "ipinfo support added in 8.16" + +--- +"Test ip_location processor with defaults": + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "ip_location" : { + "field" : "field1" + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + id: "1" + pipeline: "my_pipeline" + body: {field1: "89.160.20.128"} + + - do: + get: + index: test + id: "1" + - match: { _source.field1: "89.160.20.128" } + - length: { _source.ip_location: 7 } + - match: { _source.ip_location.city_name: "Linköping" } + - match: { _source.ip_location.country_iso_code: "SE" } + - match: { _source.ip_location.location.lon: 15.6167 } + - match: { _source.ip_location.location.lat: 58.4167 } + - match: { _source.ip_location.region_iso_code: "SE-E" } + - match: { _source.ip_location.country_name: "Sweden" } + - match: { _source.ip_location.region_name: "Östergötland County" } + - match: { _source.ip_location.continent_name: "Europe" } diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/60_ip_location_databases.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/60_ip_location_databases.yml new file mode 100644 index 0000000000000..e2e9a1fdb5e28 --- /dev/null +++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/60_ip_location_databases.yml @@ -0,0 +1,137 @@ +--- +setup: + - requires: + cluster_features: + - "put_database_configuration_action.ipinfo" + reason: "ip location downloader database configuration APIs added in 8.16 to support more types" + +--- +teardown: + - do: + ingest.delete_ip_location_database: + id: "my_database_1" + ignore: 404 + - do: + ingest.delete_ip_location_database: + id: "my_database_2" + ignore: 404 + - do: + ingest.delete_ip_location_database: + id: "my_database_3" + ignore: 404 + +--- +"Test adding, getting, and removing ip location databases": + - do: + ingest.put_ip_location_database: + id: "my_database_1" + body: > + { + "name": "GeoIP2-City", + "maxmind": { + "account_id": "1234" + } + } + - match: { acknowledged: true } + + - do: + ingest.put_ip_location_database: + id: "my_database_1" + body: > + { + "name": "GeoIP2-Country", + "maxmind": { + "account_id": "4321" + } + } + - match: { acknowledged: true } + + - do: + ingest.put_ip_location_database: + id: "my_database_2" + body: > + { + "name": "GeoIP2-City", + "maxmind": { + "account_id": "1234" + } + } + - match: { acknowledged: true } + + - do: + catch: /illegal_argument_exception/ + ingest.put_ip_location_database: + id: "_web_TXlDdXN0b21HZW9MaXRlMi1DaXR5Lm1tZGI=" + body: > + { + "name": "GeoIP2-City", + "web": { + } + } + + - do: + ingest.put_ip_location_database: + id: "my_database_3" + body: > + { + "name": "standard_privacy", + "ipinfo": { + } + } + - match: { acknowledged: true } + + - do: + ingest.get_ip_location_database: + id: "my_database_1" + - length: { databases: 1 } + - match: { databases.0.id: "my_database_1" } + - gte: { databases.0.modified_date_millis: 0 } + - match: { databases.0.database.name: "GeoIP2-Country" } + - match: { databases.0.database.maxmind.account_id: "4321" } + + - do: + ingest.get_ip_location_database: {} + - length: { databases: 7 } + + - do: + ingest.get_ip_location_database: + id: "my_database_1,my_database_2" + - length: { databases: 2 } + + - do: + ingest.get_ip_location_database: + id: "_web_TXlDdXN0b21HZW9MaXRlMi1DaXR5Lm1tZGI=" + - length: { databases: 1 } + - match: { databases.0.id: "_web_TXlDdXN0b21HZW9MaXRlMi1DaXR5Lm1tZGI=" } + - gte: { databases.0.modified_date_millis: -1 } + - match: { databases.0.database.name: "MyCustomGeoLite2-City" } + + - do: + ingest.delete_ip_location_database: + id: "my_database_1" + + - do: + catch: /resource_not_found_exception/ + ingest.delete_ip_location_database: + id: "_web_TXlDdXN0b21HZW9MaXRlMi1DaXR5Lm1tZGI=" + + - do: + ingest.get_ip_location_database: {} + - length: { databases: 6 } + + - do: + ingest.get_ip_location_database: + id: "my_database_2" + - length: { databases: 1 } + - match: { databases.0.id: "my_database_2" } + - gte: { databases.0.modified_date_millis: 0 } + - match: { databases.0.database.name: "GeoIP2-City" } + - match: { databases.0.database.maxmind.account_id: "1234" } + + - do: + ingest.get_ip_location_database: + id: "my_database_3" + - length: { databases: 1 } + - match: { databases.0.id: "my_database_3" } + - gte: { databases.0.modified_date_millis: 0 } + - match: { databases.0.database.name: "standard_privacy" } diff --git a/muted-tests.yml b/muted-tests.yml index 4fdcb91cf37dc..2e6fd10c6ef65 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -68,9 +68,6 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/111448 - class: org.elasticsearch.search.SearchServiceTests issue: https://github.com/elastic/elasticsearch/issues/111529 -- class: org.elasticsearch.xpack.test.rest.XPackRestIT - method: test {p0=rollup/security_tests/Index-based access} - issue: https://github.com/elastic/elasticsearch/issues/111631 - class: org.elasticsearch.upgrades.FullClusterRestartIT method: testSnapshotRestore {cluster=OLD} issue: https://github.com/elastic/elasticsearch/issues/111777 @@ -286,12 +283,6 @@ tests: - class: org.elasticsearch.ingest.geoip.DatabaseNodeServiceIT method: testGzippedDatabase issue: https://github.com/elastic/elasticsearch/issues/113752 -- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT - method: test {p0=indices.split/40_routing_partition_size/more than 1} - issue: https://github.com/elastic/elasticsearch/issues/113841 -- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT - method: test {p0=indices.split/40_routing_partition_size/nested} - issue: https://github.com/elastic/elasticsearch/issues/113842 - class: org.elasticsearch.threadpool.SimpleThreadPoolIT method: testThreadPoolMetrics issue: https://github.com/elastic/elasticsearch/issues/108320 @@ -340,9 +331,6 @@ tests: - class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT method: test {yaml=cluster.stats/30_ccs_stats/cross-cluster search stats search} issue: https://github.com/elastic/elasticsearch/issues/114371 -- class: org.elasticsearch.xpack.esql.qa.single_node.RestEsqlIT - method: testProfileOrdinalsGroupingOperator {SYNC} - issue: https://github.com/elastic/elasticsearch/issues/114380 - class: org.elasticsearch.xpack.inference.services.cohere.CohereServiceTests method: testInfer_StreamRequest issue: https://github.com/elastic/elasticsearch/issues/114385 @@ -375,12 +363,12 @@ tests: - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT method: test {p0=synonyms/60_synonym_rule_get/Synonym rule not found} issue: https://github.com/elastic/elasticsearch/issues/114444 -- class: org.elasticsearch.datastreams.logsdb.qa.LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT - method: testTermsAggregation - issue: https://github.com/elastic/elasticsearch/issues/114554 -- class: org.elasticsearch.datastreams.logsdb.qa.LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT - method: testTermsQuery - issue: https://github.com/elastic/elasticsearch/issues/114563 +- class: org.elasticsearch.xpack.inference.integration.ModelRegistryIT + method: testGetModel + issue: https://github.com/elastic/elasticsearch/issues/114657 +- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT + method: test {yaml=reference/rest-api/usage/line_38} + issue: https://github.com/elastic/elasticsearch/issues/113694 # Examples: # diff --git a/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java b/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java index 8ce1bfdc61f6b..3a24427df24a3 100644 --- a/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java +++ b/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java @@ -89,7 +89,8 @@ public class CcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase { .setting("xpack.security.enabled", "false") // geohex_grid requires gold license .setting("xpack.license.self_generated.type", "trial") - .feature(FeatureFlag.TIME_SERIES_MODE); + .feature(FeatureFlag.TIME_SERIES_MODE) + .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED); private static ElasticsearchCluster remoteCluster = ElasticsearchCluster.local() .name(REMOTE_CLUSTER_NAME) diff --git a/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java b/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java index acdd540ca7b9d..5ada1e941266a 100644 --- a/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java +++ b/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java @@ -91,6 +91,7 @@ public class RcsCcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase { .setting("xpack.security.remote_cluster_server.ssl.enabled", "false") .setting("xpack.security.remote_cluster_client.ssl.enabled", "false") .feature(FeatureFlag.TIME_SERIES_MODE) + .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED) .user("test_admin", "x-pack-test-password"); private static ElasticsearchCluster fulfillingCluster = ElasticsearchCluster.local() diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java index 2a3e0c16fdc2f..4ca97bff42333 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java @@ -99,6 +99,7 @@ *
  • The default image with a custom, small base image
    • *
  • A UBI-based image
    • *
  • Another UBI image for Iron Bank
    • + *
  • A WOLFI-based image
    • *
  • Images for Cloud
    • * */ @@ -170,9 +171,7 @@ public void test012SecurityCanBeDisabled() throws Exception { public void test020PluginsListWithNoPlugins() { assumeTrue( "Only applies to non-Cloud images", - distribution.packaging != Packaging.DOCKER_CLOUD - && distribution().packaging != Packaging.DOCKER_CLOUD_ESS - && distribution().packaging != Packaging.DOCKER_WOLFI_ESS + distribution.packaging != Packaging.DOCKER_CLOUD && distribution().packaging != Packaging.DOCKER_CLOUD_ESS ); final Installation.Executables bin = installation.executables(); @@ -203,15 +202,14 @@ public void test021InstallPlugin() { * Checks that ESS images can install plugins from the local archive. */ public void test022InstallPluginsFromLocalArchive() { - assumeTrue( - "Only ESS images have a local archive", - distribution().packaging == Packaging.DOCKER_CLOUD_ESS || distribution().packaging == Packaging.DOCKER_WOLFI_ESS - ); + assumeTrue("Only ESS images have a local archive", distribution().packaging == Packaging.DOCKER_CLOUD_ESS); final String plugin = "analysis-icu"; final Installation.Executables bin = installation.executables(); + listPluginArchive().forEach(System.out::println); assertThat("Expected " + plugin + " to not be installed", listPlugins(), not(hasItems(plugin))); + assertThat("Expected " + plugin + " available in archive", listPluginArchive(), hasSize(16)); // Stuff the proxy settings with garbage, so any attempt to go out to the internet would fail sh.getEnv() @@ -259,10 +257,7 @@ public void test023InstallPluginUsingConfigFile() { * Checks that ESS images can manage plugins from the local archive by deploying a plugins config file. */ public void test024InstallPluginFromArchiveUsingConfigFile() { - assumeTrue( - "Only ESS image has a plugin archive", - distribution().packaging == Packaging.DOCKER_CLOUD_ESS || distribution().packaging == Packaging.DOCKER_WOLFI_ESS - ); + assumeTrue("Only ESS image has a plugin archive", distribution().packaging == Packaging.DOCKER_CLOUD_ESS); final String filename = "elasticsearch-plugins.yml"; append(tempDir.resolve(filename), """ @@ -394,7 +389,7 @@ public void test040JavaUsesTheOsProvidedKeystore() { if (distribution.packaging == Packaging.DOCKER_UBI || distribution.packaging == Packaging.DOCKER_IRON_BANK) { // In these images, the `cacerts` file ought to be a symlink here assertThat(path, equalTo("/etc/pki/ca-trust/extracted/java/cacerts")); - } else if (distribution.packaging == Packaging.DOCKER_WOLFI || distribution.packaging == Packaging.DOCKER_WOLFI_ESS) { + } else if (distribution.packaging == Packaging.DOCKER_WOLFI || distribution.packaging == Packaging.DOCKER_CLOUD_ESS) { // In these images, the `cacerts` file ought to be a symlink here assertThat(path, equalTo("/etc/ssl/certs/java/cacerts")); } else { @@ -1121,10 +1116,8 @@ public void test170DefaultShellIsBash() { */ public void test171AdditionalCliOptionsAreForwarded() throws Exception { assumeTrue( - "Does not apply to Cloud and wolfi ess images, because they don't use the default entrypoint", - distribution.packaging != Packaging.DOCKER_CLOUD - && distribution().packaging != Packaging.DOCKER_CLOUD_ESS - && distribution().packaging != Packaging.DOCKER_WOLFI_ESS + "Does not apply to Cloud and Cloud ESS images, because they don't use the default entrypoint", + distribution.packaging != Packaging.DOCKER_CLOUD && distribution().packaging != Packaging.DOCKER_CLOUD_ESS ); runContainer(distribution(), builder().runArgs("bin/elasticsearch", "-Ecluster.name=kimchy").envVar("ELASTIC_PASSWORD", PASSWORD)); @@ -1211,11 +1204,7 @@ public void test310IronBankImageHasNoAdditionalLabels() throws Exception { * Check that the Cloud image contains the required Beats */ public void test400CloudImageBundlesBeats() { - assumeTrue( - distribution.packaging == Packaging.DOCKER_CLOUD - || distribution.packaging == Packaging.DOCKER_CLOUD_ESS - || distribution.packaging == Packaging.DOCKER_WOLFI_ESS - ); + assumeTrue(distribution.packaging == Packaging.DOCKER_CLOUD || distribution.packaging == Packaging.DOCKER_CLOUD_ESS); final List contents = listContents("/opt"); assertThat("Expected beats in /opt", contents, hasItems("filebeat", "metricbeat")); @@ -1233,6 +1222,10 @@ private List listPlugins() { return sh.run(bin.pluginTool + " list").stdout().lines().collect(Collectors.toList()); } + private List listPluginArchive() { + return sh.run("ls -lh /opt/plugins/archive").stdout().lines().collect(Collectors.toList()); + } + /** * Check that readiness listener works */ diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java index 2aff1f258ed65..a988a446f561f 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java @@ -436,7 +436,7 @@ private void verifyKeystorePermissions() { switch (distribution.packaging) { case TAR, ZIP -> assertThat(keystore, file(File, ARCHIVE_OWNER, ARCHIVE_OWNER, p660)); case DEB, RPM -> assertThat(keystore, file(File, "root", "elasticsearch", p660)); - case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI, DOCKER_WOLFI_ESS -> assertThat( + case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> assertThat( keystore, DockerFileMatcher.file(p660) ); diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java index 487a00bdecac9..644990105f60f 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java @@ -245,7 +245,7 @@ protected static void install() throws Exception { installation = Packages.installPackage(sh, distribution); Packages.verifyPackageInstallation(installation, distribution, sh); } - case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI, DOCKER_WOLFI_ESS -> { + case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> { installation = Docker.runContainer(distribution); Docker.verifyContainerInstallation(installation); } @@ -338,7 +338,6 @@ public Shell.Result runElasticsearchStartCommand(String password, boolean daemon case DOCKER_CLOUD: case DOCKER_CLOUD_ESS: case DOCKER_WOLFI: - case DOCKER_WOLFI_ESS: // nothing, "installing" docker image is running it return Shell.NO_OP; default: @@ -362,7 +361,6 @@ public void stopElasticsearch() throws Exception { case DOCKER_CLOUD: case DOCKER_CLOUD_ESS: case DOCKER_WOLFI: - case DOCKER_WOLFI_ESS: // nothing, "installing" docker image is running it break; default: @@ -375,8 +373,7 @@ public void awaitElasticsearchStartup(Shell.Result result) throws Exception { switch (distribution.packaging) { case TAR, ZIP -> Archives.assertElasticsearchStarted(installation); case DEB, RPM -> Packages.assertElasticsearchStarted(sh, installation); - case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI, DOCKER_WOLFI_ESS -> Docker - .waitForElasticsearchToStart(); + case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> Docker.waitForElasticsearchToStart(); default -> throw new IllegalStateException("Unknown Elasticsearch packaging type."); } } diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java index d63d956dc5199..05cef4a0818ba 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java @@ -39,8 +39,6 @@ public Distribution(Path path) { this.packaging = Packaging.DOCKER_CLOUD_ESS; } else if (filename.endsWith(".wolfi.tar")) { this.packaging = Packaging.DOCKER_WOLFI; - } else if (filename.endsWith(".wolfi-ess.tar")) { - this.packaging = Packaging.DOCKER_WOLFI_ESS; } else { int lastDot = filename.lastIndexOf('.'); this.packaging = Packaging.valueOf(filename.substring(lastDot + 1).toUpperCase(Locale.ROOT)); @@ -65,7 +63,7 @@ public boolean isPackage() { */ public boolean isDocker() { return switch (packaging) { - case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI, DOCKER_WOLFI_ESS -> true; + case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> true; default -> false; }; } @@ -81,8 +79,7 @@ public enum Packaging { DOCKER_IRON_BANK(".ironbank.tar", Platforms.isDocker()), DOCKER_CLOUD(".cloud.tar", Platforms.isDocker()), DOCKER_CLOUD_ESS(".cloud-ess.tar", Platforms.isDocker()), - DOCKER_WOLFI(".wolfi.tar", Platforms.isDocker()), - DOCKER_WOLFI_ESS(".wolfi-ess.tar", Platforms.isDocker()); + DOCKER_WOLFI(".wolfi.tar", Platforms.isDocker()); /** The extension of this distribution's file */ public final String extension; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java index 6f7827663d46c..c38eaa58f0552 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java @@ -532,9 +532,7 @@ public static void verifyContainerInstallation(Installation es) { ) ); - if (es.distribution.packaging == Packaging.DOCKER_CLOUD - || es.distribution.packaging == Packaging.DOCKER_CLOUD_ESS - || es.distribution.packaging == Packaging.DOCKER_WOLFI_ESS) { + if (es.distribution.packaging == Packaging.DOCKER_CLOUD || es.distribution.packaging == Packaging.DOCKER_CLOUD_ESS) { verifyCloudContainerInstallation(es); } } @@ -543,7 +541,7 @@ private static void verifyCloudContainerInstallation(Installation es) { final String pluginArchive = "/opt/plugins/archive"; final List plugins = listContents(pluginArchive); - if (es.distribution.packaging == Packaging.DOCKER_CLOUD_ESS || es.distribution.packaging == Packaging.DOCKER_WOLFI_ESS) { + if (es.distribution.packaging == Packaging.DOCKER_CLOUD_ESS) { assertThat("ESS image should come with plugins in " + pluginArchive, plugins, not(empty())); final List repositoryPlugins = plugins.stream() diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java index a1529de825804..e562e7591564e 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java @@ -168,7 +168,6 @@ public static String getImageName(Distribution distribution) { case DOCKER_CLOUD -> "-cloud"; case DOCKER_CLOUD_ESS -> "-cloud-ess"; case DOCKER_WOLFI -> "-wolfi"; - case DOCKER_WOLFI_ESS -> "-wolfi-ess"; default -> throw new IllegalStateException("Unexpected distribution packaging type: " + distribution.packaging); }; diff --git a/qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java b/qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java index c68d27b883c53..e53c0564be297 100644 --- a/qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java +++ b/qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java @@ -35,6 +35,7 @@ public class SmokeTestMultiNodeClientYamlTestSuiteIT extends ESClientYamlSuiteTe // The first node does not have the ingest role so we're sure ingest requests are forwarded: .node(0, n -> n.setting("node.roles", "[master,data,ml,remote_cluster_client,transform]")) .feature(FeatureFlag.TIME_SERIES_MODE) + .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED) .build(); public SmokeTestMultiNodeClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { diff --git a/renovate.json b/renovate.json index 0a1d588e6332c..293a2bb262375 100644 --- a/renovate.json +++ b/renovate.json @@ -30,7 +30,7 @@ "\\s*\"?(?[^\\s:@\"]+)(?::(?[-a-zA-Z0-9.]+))?(?:@(?sha256:[a-zA-Z0-9]+))?\"?" ], "currentValueTemplate": "{{#if currentValue}}{{{currentValue}}}{{else}}latest{{/if}}", - "autoReplaceStringTemplate": "\"{{{depName}}}{{#if newValue}}:{{{newValue}}}{{/if}}{{#if newDigest}}@{{{newDigest}}}{{/if}}\"", + "autoReplaceStringTemplate": "{{{depName}}}{{#if newValue}}:{{{newValue}}}{{/if}}{{#if newDigest}}@{{{newDigest}}}{{/if}}\"", "datasourceTemplate": "docker" } ] diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index a742e83255bbb..27ae0c7f99db1 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -57,4 +57,8 @@ tasks.named("precommit").configure { tasks.named("yamlRestCompatTestTransform").configure({task -> task.skipTest("indices.sort/10_basic/Index Sort", "warning does not exist for compatibility") task.skipTest("search/330_fetch_fields/Test search rewrite", "warning does not exist for compatibility") + task.skipTest("tsdb/20_mapping/disabled source", "temporary until backported") + task.skipTest("logsdb/20_source_mapping/disabled _source is not supported", "temporary until backported") + task.skipTest("tsdb/20_mapping/regular source", "temporary until backported") + task.skipTest("logsdb/20_source_mapping/stored _source mode is not supported", "temporary until backported") }) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_ip_location_database.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_ip_location_database.json new file mode 100644 index 0000000000000..e97d1da276906 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_ip_location_database.json @@ -0,0 +1,31 @@ +{ + "ingest.delete_ip_location_database":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/delete-ip-location-database-api.html", + "description":"Deletes an ip location database configuration" + }, + "stability":"stable", + "visibility":"public", + "headers":{ + "accept": [ "application/json"] + }, + "url":{ + "paths":[ + { + "path":"/_ingest/ip_location/database/{id}", + "methods":[ + "DELETE" + ], + "parts":{ + "id":{ + "type":"list", + "description":"A comma-separated list of ip location database configurations to delete" + } + } + } + ] + }, + "params":{ + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_ip_location_database.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_ip_location_database.json new file mode 100644 index 0000000000000..a2e42fe6c8e59 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_ip_location_database.json @@ -0,0 +1,37 @@ +{ + "ingest.get_ip_location_database":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/get-ip-location-database-api.html", + "description":"Returns the specified ip location database configuration" + }, + "stability":"stable", + "visibility":"public", + "headers":{ + "accept": [ "application/json"] + }, + "url":{ + "paths":[ + { + "path":"/_ingest/ip_location/database", + "methods":[ + "GET" + ] + }, + { + "path":"/_ingest/ip_location/database/{id}", + "methods":[ + "GET" + ], + "parts":{ + "id":{ + "type":"list", + "description":"A comma-separated list of ip location database configurations to get; use `*` to get all ip location database configurations" + } + } + } + ] + }, + "params":{ + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_ip_location_database.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_ip_location_database.json new file mode 100644 index 0000000000000..18487969b1a90 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_ip_location_database.json @@ -0,0 +1,36 @@ +{ + "ingest.put_ip_location_database":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/put-ip-location-database-api.html", + "description":"Puts the configuration for a ip location database to be downloaded" + }, + "stability":"stable", + "visibility":"public", + "headers":{ + "accept": [ "application/json"], + "content_type": ["application/json"] + }, + "url":{ + "paths":[ + { + "path":"/_ingest/ip_location/database/{id}", + "methods":[ + "PUT" + ], + "parts":{ + "id":{ + "type":"string", + "description":"The id of the database configuration" + } + } + } + ] + }, + "params":{ + }, + "body":{ + "description":"The database configuration definition", + "required":true + } + } +} diff --git a/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java b/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java index 2b20e35019424..084e212a913b2 100644 --- a/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java +++ b/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java @@ -36,6 +36,7 @@ public class ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { .module("health-shards-availability") .module("data-streams") .feature(FeatureFlag.TIME_SERIES_MODE) + .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED) .build(); public ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml index dfe6c9820a16a..eab51427876aa 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml @@ -411,6 +411,55 @@ index param - nested array within array: - match: { hits.hits.0._source.path.to.some.3.id: [ 1000, 2000 ] } +--- +index param - nested array within array - disabled second pass: + - requires: + cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + settings: + index: + synthetic_source: + enable_second_doc_parsing_pass: false + mappings: + _source: + mode: synthetic + properties: + name: + type: keyword + path: + properties: + to: + properties: + some: + synthetic_source_keep: arrays + properties: + id: + type: integer + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "name": "A", "path": [ { "to": [ { "some" : [ { "id": 10 }, { "id": [1, 3, 2] } ] }, { "some": { "id": 100 } } ] }, { "to": { "some": { "id": [1000, 2000] } } } ] }' + - match: { errors: false } + + - do: + search: + index: test + sort: name + - match: { hits.hits.0._source.name: A } + - length: { hits.hits.0._source.path.to.some: 2} + - match: { hits.hits.0._source.path.to.some.0.id: 10 } + - match: { hits.hits.0._source.path.to.some.1.id: [ 1, 3, 2] } + + --- # 112156 stored field under object with store_array_source: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml index d209c839d904b..03c8def9f558c 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml @@ -1,21 +1,10 @@ --- -stored _source mode is not supported: +stored _source mode is supported: - requires: - test_runner_features: [capabilities] - capabilities: - - method: PUT - path: /{index} - capabilities: [logsdb_index_mode] - reason: "Support for 'logsdb' index mode capability required" - - - skip: - known_issues: - - cluster_feature: "gte_v8.15.0" - fixed_by: "gte_v8.16.0" - reason: "Development of logs index mode spans 8.15 and 8.16" + cluster_features: ["mapper.source.remove_synthetic_source_only_validation"] + reason: requires new validation logic - do: - catch: bad_request indices.create: index: test-stored-source body: @@ -25,31 +14,17 @@ stored _source mode is not supported: mappings: _source: mode: stored - properties: - "@timestamp": - type: date - host.name: - type: keyword + - do: + indices.get: + index: test-stored-source - - match: { error.type: "mapper_parsing_exception" } - - match: { error.root_cause.0.type: "mapper_parsing_exception" } - - match: { error.reason: "Failed to parse mapping: Indices with with index mode [logsdb] only support synthetic source" } + - match: { test-stored-source.mappings._source.mode: "stored" } --- disabled _source is not supported: - requires: - test_runner_features: [capabilities] - capabilities: - - method: PUT - path: /{index} - capabilities: [logsdb_index_mode] - reason: "Support for 'logsdb' index mode capability required" - - - skip: - known_issues: - - cluster_feature: "gte_v8.15.0" - fixed_by: "gte_v8.16.0" - reason: "Development of logs index mode spans 8.15 and 8.16" + cluster_features: ["mapper.source.remove_synthetic_source_only_validation"] + reason: requires new error message - do: catch: bad_request @@ -62,20 +37,15 @@ disabled _source is not supported: mappings: _source: enabled: false - properties: - "@timestamp": - type: date - host.name: - type: keyword - match: { error.type: "mapper_parsing_exception" } - match: { error.root_cause.0.type: "mapper_parsing_exception" } - - match: { error.reason: "Failed to parse mapping: Indices with with index mode [logsdb] only support synthetic source" } + - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode" } - do: catch: bad_request indices.create: - index: test-disabled-source + index: test-disabled-mode-source body: settings: index: @@ -83,12 +53,7 @@ disabled _source is not supported: mappings: _source: mode: disabled - properties: - "@timestamp": - type: date - host.name: - type: keyword - match: { error.type: "mapper_parsing_exception" } - match: { error.root_cause.0.type: "mapper_parsing_exception" } - - match: { error.reason: "Failed to parse mapping: Indices with with index mode [logsdb] only support synthetic source" } + - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode" } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/70_dense_vector_telemetry.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/70_dense_vector_telemetry.yml index 66b05e4d0d156..16574ceb587b4 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/70_dense_vector_telemetry.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/70_dense_vector_telemetry.yml @@ -21,13 +21,13 @@ setup: element_type: byte index_options: type: hnsw + m: 16 + ef_construction: 100 vector2: type: dense_vector dims: 1024 index: true similarity: dot_product - index_options: - type: int8_hnsw vector3: type: dense_vector dims: 100 diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml index ade153d284548..6a59c7bf75cbf 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml @@ -456,13 +456,12 @@ nested fields: - match: {tsdb-synthetic.mappings._source.mode: synthetic} --- -regular source: +stored source is supported: - requires: - cluster_features: ["gte_v8.7.0"] - reason: synthetic source + cluster_features: ["mapper.source.remove_synthetic_source_only_validation"] + reason: requires new validation logic - do: - catch: '/time series indices only support synthetic source/' indices.create: index: tsdb_index body: @@ -486,14 +485,21 @@ regular source: uid: type: keyword time_series_dimension: true + + - do: + indices.get: + index: tsdb_index + + - match: { tsdb_index.mappings._source.mode: "stored" } + --- -disabled source: +disabled source is not supported: - requires: - cluster_features: ["gte_v8.7.0"] - reason: synthetic source + cluster_features: ["mapper.source.remove_synthetic_source_only_validation"] + reason: requires new error message - do: - catch: '/time series indices only support synthetic source/' + catch: bad_request indices.create: index: tsdb_index body: @@ -518,6 +524,10 @@ disabled source: type: keyword time_series_dimension: true + - match: { error.type: "mapper_parsing_exception" } + - match: { error.root_cause.0.type: "mapper_parsing_exception" } + - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [time_series] index mode" } + --- source include/exclude: - requires: diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java index 3ca3f20917009..0647a24aa39c8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java @@ -881,7 +881,7 @@ public void testPartitionedTemplate() throws Exception { ); assertThat( eBadSettings.getMessage(), - containsString("partition size [6] should be a positive number less than the number of shards [5]") + containsString("partition size [6] should be a positive number less than the number of routing shards [5]") ); // provide an invalid mapping for a partitioned index @@ -913,7 +913,7 @@ public void testPartitionedTemplate() throws Exception { assertThat( eBadIndex.getMessage(), - containsString("partition size [6] should be a positive number less than the number of shards [5]") + containsString("partition size [6] should be a positive number less than the number of routing shards [5]") ); // finally, create a valid index diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 721a0e8314927..03186e63240e5 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -240,6 +240,8 @@ static TransportVersion def(int id) { public static final TransportVersion SIMULATE_INDEX_TEMPLATES_SUBSTITUTIONS = def(8_764_00_0); public static final TransportVersion RETRIEVERS_TELEMETRY_ADDED = def(8_765_00_0); public static final TransportVersion ESQL_CACHED_STRING_SERIALIZATION = def(8_766_00_0); + public static final TransportVersion CHUNK_SENTENCE_OVERLAP_SETTING_ADDED = def(8_767_00_0); + public static final TransportVersion OPT_IN_ESQL_CCS_EXECUTION_INFO = def(8_768_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java index cf06dd34fd5ca..d66cab1d2d717 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java @@ -58,7 +58,26 @@ public class IndicesAliasesRequest extends AcknowledgedRequest implements IndicesRequest.Replaceable { - public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.fromOptions( - false, - true, - true, - true, - false, - false, - true, - false - ); + public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.builder() + .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) + .wildcardOptions( + IndicesOptions.WildcardOptions.builder() + .matchOpen(true) + .matchClosed(true) + .allowEmptyExpressions(true) + .resolveAliases(false) + .build() + ) + .gatekeeperOptions( + IndicesOptions.GatekeeperOptions.builder() + .allowAliasToMultipleIndices(false) + .allowClosedIndices(true) + .ignoreThrottled(false) + .allowFailureIndices(true) + .build() + ) + .build(); private String[] indices; // Delete index should work by default on both open and closed indices. diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsStatsAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsStatsAction.java index 2bd4d223bc4ae..fbb084e8cd121 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsStatsAction.java @@ -40,7 +40,32 @@ public static class Request extends BroadcastRequest { public Request() { // this doesn't really matter since data stream name resolution isn't affected by IndicesOptions and // a data stream's backing indices are retrieved from its metadata - super(null, IndicesOptions.fromOptions(false, true, true, true, true, false, true, false)); + super( + null, + IndicesOptions.builder() + .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) + .wildcardOptions( + IndicesOptions.WildcardOptions.builder() + .matchOpen(true) + .matchClosed(true) + .includeHidden(false) + .resolveAliases(false) + .allowEmptyExpressions(true) + .build() + ) + .gatekeeperOptions( + IndicesOptions.GatekeeperOptions.builder() + .allowAliasToMultipleIndices(true) + .allowClosedIndices(true) + .ignoreThrottled(false) + .allowFailureIndices(true) + .build() + ) + .failureStoreOptions( + IndicesOptions.FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true).build() + ) + .build() + ); } public Request(StreamInput in) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/DeleteDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/DeleteDataStreamAction.java index 4f3e238796ed6..4f647d4f02884 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/DeleteDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/DeleteDataStreamAction.java @@ -46,7 +46,25 @@ public static class Request extends MasterNodeRequest implements Indice // empty response can be returned in case wildcards were used or // 404 status code returned in case no wildcard were used. private final boolean wildcardExpressionsOriginallySpecified; - private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true, false); + private IndicesOptions indicesOptions = IndicesOptions.builder() + .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) + .wildcardOptions( + IndicesOptions.WildcardOptions.builder() + .matchOpen(true) + .matchClosed(true) + .resolveAliases(false) + .allowEmptyExpressions(true) + .build() + ) + .gatekeeperOptions( + IndicesOptions.GatekeeperOptions.builder() + .allowAliasToMultipleIndices(false) + .allowClosedIndices(true) + .ignoreThrottled(false) + .allowFailureIndices(true) + .build() + ) + .build(); public Request(TimeValue masterNodeTimeout, String... names) { super(masterNodeTimeout); diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java index 8d7f440ab20e4..c1cf0fa7aab42 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java @@ -56,7 +56,26 @@ private GetDataStreamAction() { public static class Request extends MasterNodeReadRequest implements IndicesRequest.Replaceable { private String[] names; - private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true, false); + private IndicesOptions indicesOptions = IndicesOptions.builder() + .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) + .wildcardOptions( + IndicesOptions.WildcardOptions.builder() + .matchOpen(true) + .matchClosed(true) + .includeHidden(false) + .resolveAliases(false) + .allowEmptyExpressions(true) + .build() + ) + .gatekeeperOptions( + IndicesOptions.GatekeeperOptions.builder() + .allowAliasToMultipleIndices(false) + .allowClosedIndices(true) + .ignoreThrottled(false) + .allowFailureIndices(true) + .build() + ) + .build(); private boolean includeDefaults = false; private boolean verbose = false; diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java index 6314f47ab9516..bd628c88a1b1e 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java @@ -47,7 +47,26 @@ private GetDataStreamLifecycleAction() {/* no instances */} public static class Request extends MasterNodeReadRequest implements IndicesRequest.Replaceable { private String[] names; - private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true, false); + private IndicesOptions indicesOptions = IndicesOptions.builder() + .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) + .wildcardOptions( + IndicesOptions.WildcardOptions.builder() + .matchOpen(true) + .matchClosed(true) + .includeHidden(false) + .resolveAliases(false) + .allowEmptyExpressions(true) + .build() + ) + .gatekeeperOptions( + IndicesOptions.GatekeeperOptions.builder() + .allowAliasToMultipleIndices(false) + .allowClosedIndices(true) + .ignoreThrottled(false) + .allowFailureIndices(true) + .build() + ) + .build(); private boolean includeDefaults = false; public Request(TimeValue masterNodeTimeout, String[] names) { diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java index 77f723a46f168..b054d12890366 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java @@ -78,7 +78,26 @@ public static Request parseRequest(XContentParser parser, Factory factory) { } private String[] names; - private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true, false); + private IndicesOptions indicesOptions = IndicesOptions.builder() + .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) + .wildcardOptions( + IndicesOptions.WildcardOptions.builder() + .matchOpen(true) + .matchClosed(true) + .includeHidden(false) + .resolveAliases(false) + .allowEmptyExpressions(true) + .build() + ) + .gatekeeperOptions( + IndicesOptions.GatekeeperOptions.builder() + .allowAliasToMultipleIndices(false) + .allowClosedIndices(true) + .ignoreThrottled(false) + .allowFailureIndices(false) + .build() + ) + .build(); private final DataStreamLifecycle lifecycle; public Request(StreamInput in) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java index 99eac250641ae..fb4b3907d2bfd 100644 --- a/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java +++ b/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java @@ -20,6 +20,7 @@ import org.elasticsearch.action.NoShardAvailableActionException; import org.elasticsearch.action.admin.indices.refresh.TransportShardRefreshAction; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.action.support.replication.BasicReplicationRequest; import org.elasticsearch.action.support.single.shard.TransportSingleShardAction; import org.elasticsearch.client.internal.node.NodeClient; @@ -284,11 +285,11 @@ private void tryGetFromTranslog(GetRequest request, IndexShard indexShard, Disco } else { assert r.segmentGeneration() > -1L; assert r.primaryTerm() > Engine.UNKNOWN_PRIMARY_TERM; - indexShard.waitForPrimaryTermAndGeneration( - r.primaryTerm(), - r.segmentGeneration(), - listener.delegateFailureAndWrap((ll, aLong) -> super.asyncShardOperation(request, shardId, ll)) + final ActionListener termAndGenerationListener = ContextPreservingActionListener.wrapPreservingContext( + listener.delegateFailureAndWrap((ll, aLong) -> super.asyncShardOperation(request, shardId, ll)), + threadPool.getThreadContext() ); + indexShard.waitForPrimaryTermAndGeneration(r.primaryTerm(), r.segmentGeneration(), termAndGenerationListener); } } }), TransportGetFromTranslogAction.Response::new, getExecutor(request, shardId)) diff --git a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java index 772f36898202b..7ad61f60c0088 100644 --- a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java @@ -116,7 +116,8 @@ private void innerRun() throws Exception { // still use DFS_QUERY_THEN_FETCH, which does not perform the "query and fetch" optimization during the query phase. final boolean queryAndFetchOptimization = searchPhaseShardResults.length() == 1 && context.getRequest().hasKnnSearch() == false - && reducedQueryPhase.queryPhaseRankCoordinatorContext() == null; + && reducedQueryPhase.queryPhaseRankCoordinatorContext() == null + && (context.getRequest().source() == null || context.getRequest().source().rankBuilder() == null); if (queryAndFetchOptimization) { assert assertConsistentWithQueryAndFetchOptimization(); // query AND fetch optimization diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index 1c4eb1c191370..74786dff1648d 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -462,8 +462,7 @@ private static SearchHits getHits( : "not enough hits fetched. index [" + index + "] length: " + fetchResult.hits().getHits().length; SearchHit searchHit = fetchResult.hits().getHits()[index]; searchHit.shard(fetchResult.getSearchShardTarget()); - if (reducedQueryPhase.queryPhaseRankCoordinatorContext != null) { - assert shardDoc instanceof RankDoc; + if (shardDoc instanceof RankDoc) { searchHit.setRank(((RankDoc) shardDoc).rank); searchHit.score(shardDoc.score); long shardAndDoc = ShardDocSortField.encodeShardAndDoc(shardDoc.shardIndex, shardDoc.doc); @@ -735,6 +734,12 @@ static int getTopDocsSize(SearchRequest request) { return DEFAULT_SIZE; } SearchSourceBuilder source = request.source(); + if (source.rankBuilder() != null) { + // if we have a RankBuilder defined, it needs to have access to all the documents in order to rerank them + // so we override size here and keep all `rank_window_size` docs. + // Pagination is taking place later through RankFeaturePhaseRankCoordinatorContext#rankAndPaginate + return source.rankBuilder().rankWindowSize(); + } return (source.size() == -1 ? DEFAULT_SIZE : source.size()) + (source.from() == -1 ? SearchService.DEFAULT_FROM : source.from()); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 9760d84c67c5b..23e8e49aa16db 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -2265,7 +2265,7 @@ IndexMetadata build(boolean repair) { "routing partition size [" + routingPartitionSize + "] should be a positive number" - + " less than the number of shards [" + + " less than the number of routing shards [" + getRoutingNumShards() + "] for [" + index diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java index 8a46550f8a689..db3973c1a15a8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java @@ -45,6 +45,7 @@ public class MetadataDataStreamsService { private final DataStreamGlobalRetentionSettings globalRetentionSettings; private final MasterServiceTaskQueue updateLifecycleTaskQueue; private final MasterServiceTaskQueue setRolloverOnWriteTaskQueue; + private final MasterServiceTaskQueue updateOptionsTaskQueue; public MetadataDataStreamsService( ClusterService clusterService, @@ -93,6 +94,20 @@ public Tuple executeTask( Priority.NORMAL, rolloverOnWriteExecutor ); + ClusterStateTaskExecutor updateOptionsExecutor = new SimpleBatchedAckListenerTaskExecutor<>() { + + @Override + public Tuple executeTask( + UpdateOptionsTask modifyOptionsTask, + ClusterState clusterState + ) { + return new Tuple<>( + updateDataStreamOptions(clusterState, modifyOptionsTask.getDataStreamNames(), modifyOptionsTask.getOptions()), + modifyOptionsTask + ); + } + }; + this.updateOptionsTaskQueue = clusterService.createTaskQueue("modify-data-stream-options", Priority.NORMAL, updateOptionsExecutor); } public void modifyDataStream(final ModifyDataStreamsAction.Request request, final ActionListener listener) { @@ -147,6 +162,39 @@ public void removeLifecycle( ); } + /** + * Submits the task to set the provided data stream options to the requested data streams. + */ + public void setDataStreamOptions( + final List dataStreamNames, + DataStreamOptions options, + TimeValue ackTimeout, + TimeValue masterTimeout, + final ActionListener listener + ) { + updateOptionsTaskQueue.submitTask( + "set-data-stream-options", + new UpdateOptionsTask(dataStreamNames, options, ackTimeout, listener), + masterTimeout + ); + } + + /** + * Submits the task to remove the data stream options from the requested data streams. + */ + public void removeDataStreamOptions( + List dataStreamNames, + TimeValue ackTimeout, + TimeValue masterTimeout, + ActionListener listener + ) { + updateOptionsTaskQueue.submitTask( + "delete-data-stream-options", + new UpdateOptionsTask(dataStreamNames, null, ackTimeout, listener), + masterTimeout + ); + } + @SuppressForbidden(reason = "legacy usage of unbatched task") // TODO add support for batching here private void submitUnbatchedTask(@SuppressWarnings("SameParameterValue") String source, ClusterStateUpdateTask task) { clusterService.submitUnbatchedStateUpdateTask(source, task); @@ -228,6 +276,24 @@ ClusterState updateDataLifecycle(ClusterState currentState, List dataStr return ClusterState.builder(currentState).metadata(builder.build()).build(); } + /** + * Creates an updated cluster state in which the requested data streams have the data stream options provided. + * Visible for testing. + */ + ClusterState updateDataStreamOptions( + ClusterState currentState, + List dataStreamNames, + @Nullable DataStreamOptions dataStreamOptions + ) { + Metadata metadata = currentState.metadata(); + Metadata.Builder builder = Metadata.builder(metadata); + for (var dataStreamName : dataStreamNames) { + var dataStream = validateDataStream(metadata, dataStreamName); + builder.put(dataStream.copy().setDataStreamOptions(dataStreamOptions).build()); + } + return ClusterState.builder(currentState).metadata(builder.build()).build(); + } + /** * Creates an updated cluster state in which the requested data stream has the flag {@link DataStream#rolloverOnWrite()} * set to the value of the parameter rolloverOnWrite @@ -372,6 +438,34 @@ public DataStreamLifecycle getDataLifecycle() { } } + /** + * A cluster state update task that consists of the cluster state request and the listeners that need to be notified upon completion. + */ + static class UpdateOptionsTask extends AckedBatchedClusterStateUpdateTask { + + private final List dataStreamNames; + private final DataStreamOptions options; + + UpdateOptionsTask( + List dataStreamNames, + @Nullable DataStreamOptions options, + TimeValue ackTimeout, + ActionListener listener + ) { + super(ackTimeout, listener); + this.dataStreamNames = dataStreamNames; + this.options = options; + } + + public List getDataStreamNames() { + return dataStreamNames; + } + + public DataStreamOptions getOptions() { + return options; + } + } + /** * A cluster state update task that consists of the cluster state request and the listeners that need to be notified upon completion. */ diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index ad3d7d7f1c2ec..f5276bbe49b63 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -35,6 +35,7 @@ import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.FsDirectoryFactory; import org.elasticsearch.index.store.Store; @@ -186,6 +187,8 @@ public final class IndexScopedSettings extends AbstractScopedSettings { FieldMapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING, IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING, IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING, + IndexSettings.SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING, + SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING, // validate that built-in similarities don't get redefined Setting.groupSetting("index.similarity.", (s) -> { diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 5dfd698b2bb20..2d9e89223d7a6 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -217,8 +217,8 @@ public boolean shouldValidateTimestamp() { @Override public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) { - if (sourceFieldMapper.isSynthetic() == false) { - throw new IllegalArgumentException("time series indices only support synthetic source"); + if (sourceFieldMapper.enabled() == false) { + throw new IllegalArgumentException("_source can not be disabled in index using [" + IndexMode.TIME_SERIES + "] index mode"); } } @@ -292,8 +292,8 @@ public boolean shouldValidateTimestamp() { @Override public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) { - if (sourceFieldMapper.isSynthetic() == false) { - throw new IllegalArgumentException("Indices with with index mode [" + IndexMode.LOGSDB + "] only support synthetic source"); + if (sourceFieldMapper.enabled() == false) { + throw new IllegalArgumentException("_source can not be disabled in index using [" + IndexMode.LOGSDB + "] index mode"); } } diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index e82f9eff7d5e0..347b44a22e7c0 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -28,6 +28,7 @@ import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.ingest.IngestService; import org.elasticsearch.node.Node; @@ -651,6 +652,13 @@ public Iterator> settings() { Property.Final ); + public static final Setting SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING = Setting.boolSetting( + "index.synthetic_source.enable_second_doc_parsing_pass", + true, + Property.IndexScope, + Property.Dynamic + ); + /** * Returns true if TSDB encoding is enabled. The default is true */ @@ -820,6 +828,8 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) { private volatile long mappingDimensionFieldsLimit; private volatile boolean skipIgnoredSourceWrite; private volatile boolean skipIgnoredSourceRead; + private volatile boolean syntheticSourceSecondDocParsingPassEnabled; + private final SourceFieldMapper.Mode indexMappingSourceMode; /** * The maximum number of refresh listeners allows on this shard. @@ -980,6 +990,8 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti es87TSDBCodecEnabled = scopedSettings.get(TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING); skipIgnoredSourceWrite = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING); skipIgnoredSourceRead = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING); + syntheticSourceSecondDocParsingPassEnabled = scopedSettings.get(SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING); + indexMappingSourceMode = scopedSettings.get(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING); scopedSettings.addSettingsUpdateConsumer( MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING, @@ -1067,6 +1079,10 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti this::setSkipIgnoredSourceWrite ); scopedSettings.addSettingsUpdateConsumer(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING, this::setSkipIgnoredSourceRead); + scopedSettings.addSettingsUpdateConsumer( + SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING, + this::setSyntheticSourceSecondDocParsingPassEnabled + ); } private void setSearchIdleAfter(TimeValue searchIdleAfter) { @@ -1659,6 +1675,18 @@ private void setSkipIgnoredSourceRead(boolean value) { this.skipIgnoredSourceRead = value; } + private void setSyntheticSourceSecondDocParsingPassEnabled(boolean syntheticSourceSecondDocParsingPassEnabled) { + this.syntheticSourceSecondDocParsingPassEnabled = syntheticSourceSecondDocParsingPassEnabled; + } + + public boolean isSyntheticSourceSecondDocParsingPassEnabled() { + return syntheticSourceSecondDocParsingPassEnabled; + } + + public SourceFieldMapper.Mode getIndexMappingSourceMode() { + return indexMappingSourceMode; + } + /** * The bounds for {@code @timestamp} on this index or * {@code null} if there are no bounds. diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java index b1e91ad75e9a2..cc5454ee074e6 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java @@ -32,6 +32,8 @@ import java.io.IOException; +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; + public class ES813FlatVectorFormat extends KnnVectorsFormat { static final String NAME = "ES813FlatVectorFormat"; @@ -55,6 +57,11 @@ public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException return new ES813FlatVectorReader(format.fieldsReader(state)); } + @Override + public int getMaxDimensions(String fieldName) { + return MAX_DIMS_COUNT; + } + static class ES813FlatVectorWriter extends KnnVectorsWriter { private final FlatVectorsWriter writer; diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java index 248421fb99d1c..9491598653c44 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java @@ -30,6 +30,8 @@ import java.io.IOException; +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; + public class ES813Int8FlatVectorFormat extends KnnVectorsFormat { static final String NAME = "ES813Int8FlatVectorFormat"; @@ -58,6 +60,11 @@ public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException return new ES813FlatVectorReader(format.fieldsReader(state)); } + @Override + public int getMaxDimensions(String fieldName) { + return MAX_DIMS_COUNT; + } + @Override public String toString() { return NAME + "(name=" + NAME + ", innerFormat=" + format + ")"; diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormat.java index d6ce73dd4149a..6bb32d8e1ef52 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormat.java @@ -22,6 +22,7 @@ import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; public final class ES814HnswScalarQuantizedVectorsFormat extends KnnVectorsFormat { @@ -70,7 +71,7 @@ public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException @Override public int getMaxDimensions(String fieldName) { - return 1024; + return MAX_DIMS_COUNT; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorFormat.java index af771b6a27f19..5cd5872e10421 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorFormat.java @@ -18,6 +18,8 @@ import java.io.IOException; +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; + public class ES815BitFlatVectorFormat extends KnnVectorsFormat { static final String NAME = "ES815BitFlatVectorFormat"; @@ -45,4 +47,9 @@ public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException public String toString() { return NAME; } + + @Override + public int getMaxDimensions(String fieldName) { + return MAX_DIMS_COUNT; + } } diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815HnswBitVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815HnswBitVectorsFormat.java index 5e4656ea94c5b..186dfcbeb5d52 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815HnswBitVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815HnswBitVectorsFormat.java @@ -20,6 +20,8 @@ import java.io.IOException; +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; + public class ES815HnswBitVectorsFormat extends KnnVectorsFormat { static final String NAME = "ES815HnswBitVectorsFormat"; @@ -72,4 +74,9 @@ public String toString() { + flatVectorsFormat + ")"; } + + @Override + public int getMaxDimensions(String fieldName) { + return MAX_DIMS_COUNT; + } } diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsFormat.java index 523d5f6c4a91f..e32aea0fb04ae 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsFormat.java @@ -29,6 +29,8 @@ import java.io.IOException; +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; + /** * Copied from Lucene, replace with Lucene's implementation sometime after Lucene 10 */ @@ -68,6 +70,11 @@ public FlatVectorsReader fieldsReader(SegmentReadState state) throws IOException return new ES816BinaryQuantizedVectorsReader(state, rawVectorFormat.fieldsReader(state), scorer); } + @Override + public int getMaxDimensions(String fieldName) { + return MAX_DIMS_COUNT; + } + @Override public String toString() { return "ES816BinaryQuantizedVectorsFormat(name=" + NAME + ", flatVectorScorer=" + scorer + ")"; diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816HnswBinaryQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816HnswBinaryQuantizedVectorsFormat.java index 989f88e0a7857..097cdffff6ae4 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816HnswBinaryQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816HnswBinaryQuantizedVectorsFormat.java @@ -39,6 +39,7 @@ import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_NUM_MERGE_WORKER; import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.MAXIMUM_BEAM_WIDTH; import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.MAXIMUM_MAX_CONN; +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; /** * Copied from Lucene, replace with Lucene's implementation sometime after Lucene 10 @@ -128,7 +129,7 @@ public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException @Override public int getMaxDimensions(String fieldName) { - return 1024; + return MAX_DIMS_COUNT; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 0ff754d953934..bac987a3df96d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -208,6 +208,7 @@ private static List parseDocForMissingValues XContentParser parser = context.parser(); XContentParser.Token currentToken = parser.nextToken(); List path = new ArrayList<>(); + List isObjectInPath = new ArrayList<>(); // Tracks if path components correspond to an object or an array. String fieldName = null; while (currentToken != null) { while (currentToken != XContentParser.Token.FIELD_NAME) { @@ -218,11 +219,16 @@ private static List parseDocForMissingValues parser.skipChildren(); } else { path.add(fieldName); + isObjectInPath.add(currentToken == XContentParser.Token.START_OBJECT); } fieldName = null; } else if (currentToken == XContentParser.Token.END_OBJECT || currentToken == XContentParser.Token.END_ARRAY) { - if (currentToken == XContentParser.Token.END_OBJECT && path.isEmpty() == false) { + // Remove the path, if the scope type matches the one when the path was added. + if (isObjectInPath.isEmpty() == false + && (isObjectInPath.getLast() && currentToken == XContentParser.Token.END_OBJECT + || isObjectInPath.getLast() == false && currentToken == XContentParser.Token.END_ARRAY)) { path.removeLast(); + isObjectInPath.removeLast(); } fieldName = null; } @@ -237,7 +243,6 @@ private static List parseDocForMissingValues if (leaf != null) { parser.nextToken(); // Advance the parser to the value to be read. result.add(leaf.cloneWithValue(context.encodeFlattenedToken())); - parser.nextToken(); // Skip the token ending the value. fieldName = null; } currentToken = parser.nextToken(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index ac236e5a7e5fd..2eec14bd1a8d6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -111,6 +111,7 @@ public int get() { private final Set ignoredFields; private final List ignoredFieldValues; private final List ignoredFieldsMissingValues; + private final boolean inArrayScopeEnabled; private boolean inArrayScope; private final Map> dynamicMappers; @@ -143,6 +144,7 @@ private DocumentParserContext( Set ignoreFields, List ignoredFieldValues, List ignoredFieldsWithNoSource, + boolean inArrayScopeEnabled, boolean inArrayScope, Map> dynamicMappers, Map dynamicObjectMappers, @@ -164,6 +166,7 @@ private DocumentParserContext( this.ignoredFields = ignoreFields; this.ignoredFieldValues = ignoredFieldValues; this.ignoredFieldsMissingValues = ignoredFieldsWithNoSource; + this.inArrayScopeEnabled = inArrayScopeEnabled; this.inArrayScope = inArrayScope; this.dynamicMappers = dynamicMappers; this.dynamicObjectMappers = dynamicObjectMappers; @@ -188,6 +191,7 @@ private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic, in.ignoredFields, in.ignoredFieldValues, in.ignoredFieldsMissingValues, + in.inArrayScopeEnabled, in.inArrayScope, in.dynamicMappers, in.dynamicObjectMappers, @@ -219,6 +223,7 @@ protected DocumentParserContext( new HashSet<>(), new ArrayList<>(), new ArrayList<>(), + mappingParserContext.getIndexSettings().isSyntheticSourceSecondDocParsingPassEnabled(), false, new HashMap<>(), new HashMap<>(), @@ -371,7 +376,7 @@ public final Collection getIgnoredFieldsMiss * Applies to synthetic source only. */ public final DocumentParserContext maybeCloneForArray(Mapper mapper) throws IOException { - if (canAddIgnoredField() && mapper instanceof ObjectMapper) { + if (canAddIgnoredField() && mapper instanceof ObjectMapper && inArrayScopeEnabled) { boolean isNested = mapper instanceof NestedObjectMapper; if ((inArrayScope == false && isNested == false) || (inArrayScope && isNested)) { DocumentParserContext subcontext = switchParser(parser()); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index 4f90bd6e6f2c9..f3744c974e9e3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -58,6 +58,10 @@ public Set getFeatures() { @Override public Set getTestFeatures() { - return Set.of(RangeFieldMapper.DATE_RANGE_INDEXING_FIX, IgnoredSourceFieldMapper.DONT_EXPAND_DOTS_IN_IGNORED_SOURCE); + return Set.of( + RangeFieldMapper.DATE_RANGE_INDEXING_FIX, + IgnoredSourceFieldMapper.DONT_EXPAND_DOTS_IN_IGNORED_SOURCE, + SourceFieldMapper.REMOVE_SYNTHETIC_SOURCE_ONLY_VALIDATION + ); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java index 6737f08b1ac5b..9afa77161bef1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java @@ -12,7 +12,6 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.xcontent.XContentType; @@ -147,10 +146,6 @@ Mapping parse(@Nullable String type, MergeReason reason, Map map assert fieldNodeMap.isEmpty(); if (metadataFieldMapper instanceof SourceFieldMapper sfm) { - // Validation in other places should have failed first - assert sfm.isSynthetic() - || (sfm.isSynthetic() == false && mappingParserContext.getIndexSettings().getMode() != IndexMode.TIME_SERIES) - : "synthetic source can't be disabled in a time series index"; isSourceSynthetic = sfm.isSynthetic(); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index 5e63fee8c5adc..70c4a3ac213a2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Nullable; import org.elasticsearch.features.NodeFeature; @@ -41,6 +42,7 @@ public class ObjectMapper extends Mapper { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(ObjectMapper.class); + public static final FeatureFlag SUB_OBJECTS_AUTO_FEATURE_FLAG = new FeatureFlag("sub_objects_auto"); public static final String CONTENT_TYPE = "object"; static final String STORE_ARRAY_SOURCE_PARAM = "store_array_source"; @@ -74,7 +76,7 @@ public static Subobjects from(Object node) { if (value.equalsIgnoreCase("false")) { return DISABLED; } - if (value.equalsIgnoreCase("auto")) { + if (SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled() && value.equalsIgnoreCase("auto")) { return AUTO; } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 118cdbffc5db9..f9b9de97715ed 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -18,11 +18,13 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.core.Nullable; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.SearchExecutionContext; @@ -51,6 +53,9 @@ public class SourceFieldMapper extends MetadataFieldMapper { public static final NodeFeature SYNTHETIC_SOURCE_COPY_TO_INSIDE_OBJECTS_FIX = new NodeFeature( "mapper.source.synthetic_source_copy_to_inside_objects_fix" ); + public static final NodeFeature REMOVE_SYNTHETIC_SOURCE_ONLY_VALIDATION = new NodeFeature( + "mapper.source.remove_synthetic_source_only_validation" + ); public static final String NAME = "_source"; public static final String RECOVERY_SOURCE_NAME = "_recovery_source"; @@ -59,8 +64,16 @@ public class SourceFieldMapper extends MetadataFieldMapper { public static final String LOSSY_PARAMETERS_ALLOWED_SETTING_NAME = "index.lossy.source-mapping-parameters"; + public static final Setting INDEX_MAPPER_SOURCE_MODE_SETTING = Setting.enumSetting(SourceFieldMapper.Mode.class, settings -> { + final IndexMode indexMode = IndexSettings.MODE.get(settings); + return switch (indexMode) { + case IndexMode.LOGSDB, IndexMode.TIME_SERIES -> Mode.SYNTHETIC.name(); + default -> Mode.STORED.name(); + }; + }, "index.mapping.source.mode", value -> {}, Setting.Property.Final, Setting.Property.IndexScope); + /** The source mode */ - private enum Mode { + public enum Mode { DISABLED, STORED, SYNTHETIC @@ -93,6 +106,15 @@ private enum Mode { true ); + private static final SourceFieldMapper TSDB_DEFAULT_STORED = new SourceFieldMapper( + Mode.STORED, + Explicit.IMPLICIT_TRUE, + Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, + IndexMode.TIME_SERIES, + true + ); + private static final SourceFieldMapper TSDB_DEFAULT_NO_RECOVERY_SOURCE = new SourceFieldMapper( Mode.SYNTHETIC, Explicit.IMPLICIT_TRUE, @@ -102,6 +124,15 @@ private enum Mode { false ); + private static final SourceFieldMapper TSDB_DEFAULT_NO_RECOVERY_SOURCE_STORED = new SourceFieldMapper( + Mode.STORED, + Explicit.IMPLICIT_TRUE, + Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, + IndexMode.TIME_SERIES, + false + ); + private static final SourceFieldMapper LOGSDB_DEFAULT = new SourceFieldMapper( Mode.SYNTHETIC, Explicit.IMPLICIT_TRUE, @@ -111,6 +142,15 @@ private enum Mode { true ); + private static final SourceFieldMapper LOGSDB_DEFAULT_STORED = new SourceFieldMapper( + Mode.STORED, + Explicit.IMPLICIT_TRUE, + Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, + IndexMode.LOGSDB, + true + ); + private static final SourceFieldMapper LOGSDB_DEFAULT_NO_RECOVERY_SOURCE = new SourceFieldMapper( Mode.SYNTHETIC, Explicit.IMPLICIT_TRUE, @@ -120,6 +160,15 @@ private enum Mode { false ); + private static final SourceFieldMapper LOGSDB_DEFAULT_NO_RECOVERY_SOURCE_STORED = new SourceFieldMapper( + Mode.STORED, + Explicit.IMPLICIT_TRUE, + Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, + IndexMode.LOGSDB, + false + ); + /* * Synthetic source was added as the default for TSDB in v.8.7. The legacy field mapper below * is used in bwc tests and mixed clusters containing time series indexes created in an earlier version. @@ -194,6 +243,8 @@ public static class Builder extends MetadataFieldMapper.Builder { m -> Arrays.asList(toType(m).excludes) ); + private final Settings settings; + private final IndexMode indexMode; private final boolean supportsNonDefaultParameterValues; @@ -207,6 +258,7 @@ public Builder( boolean enableRecoverySource ) { super(Defaults.NAME); + this.settings = settings; this.indexMode = indexMode; this.supportsNonDefaultParameterValues = supportsCheckForNonDefaultParams == false || settings.getAsBoolean(LOSSY_PARAMETERS_ALLOWED_SETTING_NAME, true); @@ -223,10 +275,10 @@ protected Parameter[] getParameters() { return new Parameter[] { enabled, mode, includes, excludes }; } - private boolean isDefault() { - Mode m = mode.get(); - if (m != null - && (((indexMode != null && indexMode.isSyntheticSourceEnabled() && m == Mode.SYNTHETIC) == false) || m == Mode.DISABLED)) { + private boolean isDefault(final Mode sourceMode) { + if (sourceMode != null + && (((indexMode != null && indexMode.isSyntheticSourceEnabled() && sourceMode == Mode.SYNTHETIC) == false) + || sourceMode == Mode.DISABLED)) { return false; } return enabled.get().value() && includes.getValue().isEmpty() && excludes.getValue().isEmpty(); @@ -235,19 +287,18 @@ private boolean isDefault() { @Override public SourceFieldMapper build() { if (enabled.getValue().explicit()) { - if (indexMode != null && indexMode.isSyntheticSourceEnabled()) { - throw new MapperParsingException("Indices with with index mode [" + indexMode + "] only support synthetic source"); - } if (mode.get() != null) { throw new MapperParsingException("Cannot set both [mode] and [enabled] parameters"); } } - if (isDefault()) { - return switch (indexMode) { - case TIME_SERIES -> enableRecoverySource ? TSDB_DEFAULT : TSDB_DEFAULT_NO_RECOVERY_SOURCE; - case LOGSDB -> enableRecoverySource ? LOGSDB_DEFAULT : LOGSDB_DEFAULT_NO_RECOVERY_SOURCE; - default -> enableRecoverySource ? DEFAULT : DEFAULT_NO_RECOVERY_SOURCE; - }; + // NOTE: if the `index.mapper.source.mode` exists it takes precedence to determine the source mode for `_source` + // otherwise the mode is determined according to `index.mode` and `_source.mode`. + final Mode sourceMode = INDEX_MAPPER_SOURCE_MODE_SETTING.exists(settings) + ? INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings) + : mode.get(); + if (isDefault(sourceMode)) { + return resolveSourceMode(indexMode, sourceMode, enableRecoverySource); + } if (supportsNonDefaultParameterValues == false) { List disallowed = new ArrayList<>(); @@ -271,8 +322,9 @@ public SourceFieldMapper build() { ); } } + SourceFieldMapper sourceFieldMapper = new SourceFieldMapper( - mode.get(), + sourceMode, enabled.get(), includes.getValue().toArray(Strings.EMPTY_ARRAY), excludes.getValue().toArray(Strings.EMPTY_ARRAY), @@ -287,21 +339,39 @@ public SourceFieldMapper build() { } + private static SourceFieldMapper resolveSourceMode(final IndexMode indexMode, final Mode sourceMode, boolean enableRecoverySource) { + if (indexMode == IndexMode.STANDARD) { + return enableRecoverySource ? DEFAULT : DEFAULT_NO_RECOVERY_SOURCE; + } + final SourceFieldMapper syntheticWithoutRecoverySource = indexMode == IndexMode.TIME_SERIES + ? TSDB_DEFAULT_NO_RECOVERY_SOURCE + : LOGSDB_DEFAULT_NO_RECOVERY_SOURCE; + final SourceFieldMapper syntheticWithRecoverySource = indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT : LOGSDB_DEFAULT; + final SourceFieldMapper storedWithoutRecoverySource = indexMode == IndexMode.TIME_SERIES + ? TSDB_DEFAULT_NO_RECOVERY_SOURCE_STORED + : LOGSDB_DEFAULT_NO_RECOVERY_SOURCE_STORED; + final SourceFieldMapper storedWithRecoverySource = indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT_STORED : LOGSDB_DEFAULT_STORED; + + return switch (sourceMode) { + case SYNTHETIC -> enableRecoverySource ? syntheticWithRecoverySource : syntheticWithoutRecoverySource; + case STORED -> enableRecoverySource ? storedWithRecoverySource : storedWithoutRecoverySource; + case DISABLED -> throw new IllegalArgumentException( + "_source can not be disabled in index using [" + indexMode + "] index mode" + ); + }; + } + public static final TypeParser PARSER = new ConfigurableTypeParser(c -> { - var indexMode = c.getIndexSettings().getMode(); + final IndexMode indexMode = c.getIndexSettings().getMode(); boolean enableRecoverySource = INDICES_RECOVERY_SOURCE_ENABLED_SETTING.get(c.getSettings()); + final Mode settingSourceMode = INDEX_MAPPER_SOURCE_MODE_SETTING.get(c.getSettings()); + if (indexMode.isSyntheticSourceEnabled()) { - if (indexMode == IndexMode.TIME_SERIES) { - if (c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.V_8_7_0)) { - return enableRecoverySource ? TSDB_DEFAULT : TSDB_DEFAULT_NO_RECOVERY_SOURCE; - } else { - return enableRecoverySource ? TSDB_LEGACY_DEFAULT : TSDB_LEGACY_DEFAULT_NO_RECOVERY_SOURCE; - } - } else if (indexMode == IndexMode.LOGSDB) { - return enableRecoverySource ? LOGSDB_DEFAULT : LOGSDB_DEFAULT_NO_RECOVERY_SOURCE; + if (indexMode == IndexMode.TIME_SERIES && c.getIndexSettings().getIndexVersionCreated().before(IndexVersions.V_8_7_0)) { + return enableRecoverySource ? TSDB_LEGACY_DEFAULT : TSDB_LEGACY_DEFAULT_NO_RECOVERY_SOURCE; } } - return enableRecoverySource ? DEFAULT : DEFAULT_NO_RECOVERY_SOURCE; + return resolveSourceMode(indexMode, settingSourceMode, enableRecoverySource); }, c -> new Builder( c.getIndexSettings().getMode(), diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 4adfe619ca4e1..d7353584706d8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -139,32 +139,27 @@ public static class Builder extends FieldMapper.Builder { if (o instanceof Integer == false) { throw new MapperParsingException("Property [dims] on field [" + n + "] must be an integer but got [" + o + "]"); } - int dims = XContentMapValues.nodeIntegerValue(o); - int maxDims = elementType.getValue() == ElementType.BIT ? MAX_DIMS_COUNT_BIT : MAX_DIMS_COUNT; - int minDims = elementType.getValue() == ElementType.BIT ? Byte.SIZE : 1; - if (dims < minDims || dims > maxDims) { - throw new MapperParsingException( - "The number of dimensions for field [" - + n - + "] should be in the range [" - + minDims - + ", " - + maxDims - + "] but was [" - + dims - + "]" - ); - } - if (elementType.getValue() == ElementType.BIT) { - if (dims % Byte.SIZE != 0) { + + return XContentMapValues.nodeIntegerValue(o); + }, m -> toType(m).fieldType().dims, XContentBuilder::field, Object::toString).setSerializerCheck((id, ic, v) -> v != null) + .setMergeValidator((previous, current, c) -> previous == null || Objects.equals(previous, current)) + .addValidator(dims -> { + if (dims == null) { + return; + } + int maxDims = elementType.getValue() == ElementType.BIT ? MAX_DIMS_COUNT_BIT : MAX_DIMS_COUNT; + int minDims = elementType.getValue() == ElementType.BIT ? Byte.SIZE : 1; + if (dims < minDims || dims > maxDims) { throw new MapperParsingException( - "The number of dimensions for field [" + n + "] should be a multiple of 8 but was [" + dims + "]" + "The number of dimensions should be in the range [" + minDims + ", " + maxDims + "] but was [" + dims + "]" ); } - } - return dims; - }, m -> toType(m).fieldType().dims, XContentBuilder::field, Object::toString).setSerializerCheck((id, ic, v) -> v != null) - .setMergeValidator((previous, current, c) -> previous == null || Objects.equals(previous, current)); + if (elementType.getValue() == ElementType.BIT) { + if (dims % Byte.SIZE != 0) { + throw new MapperParsingException("The number of dimensions for should be a multiple of 8 but was [" + dims + "]"); + } + } + }); private final Parameter similarity; private final Parameter indexOptions; diff --git a/server/src/main/java/org/elasticsearch/inference/EmptySecretSettings.java b/server/src/main/java/org/elasticsearch/inference/EmptySecretSettings.java index 0e5b3a555b800..9c666bd4a35f5 100644 --- a/server/src/main/java/org/elasticsearch/inference/EmptySecretSettings.java +++ b/server/src/main/java/org/elasticsearch/inference/EmptySecretSettings.java @@ -16,6 +16,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Map; /** * This class defines an empty secret settings object. This is useful for services that do not have any secret settings. @@ -48,4 +49,9 @@ public TransportVersion getMinimalSupportedVersion() { @Override public void writeTo(StreamOutput out) throws IOException {} + + @Override + public SecretSettings newSecretSettings(Map newSecrets) { + return INSTANCE; + } } diff --git a/server/src/main/java/org/elasticsearch/inference/EmptyTaskSettings.java b/server/src/main/java/org/elasticsearch/inference/EmptyTaskSettings.java index 0c863932c6afe..cba0282f7fed8 100644 --- a/server/src/main/java/org/elasticsearch/inference/EmptyTaskSettings.java +++ b/server/src/main/java/org/elasticsearch/inference/EmptyTaskSettings.java @@ -16,6 +16,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Map; /** * This class defines an empty task settings object. This is useful for services that do not have any task settings. @@ -53,4 +54,9 @@ public TransportVersion getMinimalSupportedVersion() { @Override public void writeTo(StreamOutput out) throws IOException {} + + @Override + public TaskSettings updatedTaskSettings(Map newSettings) { + return INSTANCE; + } } diff --git a/server/src/main/java/org/elasticsearch/inference/SecretSettings.java b/server/src/main/java/org/elasticsearch/inference/SecretSettings.java index e2c0c8b58c69b..90ca92bb0e2ef 100644 --- a/server/src/main/java/org/elasticsearch/inference/SecretSettings.java +++ b/server/src/main/java/org/elasticsearch/inference/SecretSettings.java @@ -12,6 +12,9 @@ import org.elasticsearch.common.io.stream.VersionedNamedWriteable; import org.elasticsearch.xcontent.ToXContentObject; +import java.util.Map; + public interface SecretSettings extends ToXContentObject, VersionedNamedWriteable { + SecretSettings newSecretSettings(Map newSecrets); } diff --git a/server/src/main/java/org/elasticsearch/inference/TaskSettings.java b/server/src/main/java/org/elasticsearch/inference/TaskSettings.java index 9862abce2332c..7dd20688245ba 100644 --- a/server/src/main/java/org/elasticsearch/inference/TaskSettings.java +++ b/server/src/main/java/org/elasticsearch/inference/TaskSettings.java @@ -12,6 +12,11 @@ import org.elasticsearch.common.io.stream.VersionedNamedWriteable; import org.elasticsearch.xcontent.ToXContentObject; +import java.util.Map; + public interface TaskSettings extends ToXContentObject, VersionedNamedWriteable { + boolean isEmpty(); + + TaskSettings updatedTaskSettings(Map newSettings); } diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestGeoIpFeatures.java b/server/src/main/java/org/elasticsearch/ingest/IngestGeoIpFeatures.java index 1933d285d7870..77b11357d79b1 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestGeoIpFeatures.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestGeoIpFeatures.java @@ -22,7 +22,13 @@ public class IngestGeoIpFeatures implements FeatureSpecification { "get_database_configuration_action.multi_node" ); + public static final NodeFeature PUT_DATABASE_CONFIGURATION_ACTION_IPINFO = new NodeFeature("put_database_configuration_action.ipinfo"); + public Set getFeatures() { - return Set.of(GEOIP_DOWNLOADER_DATABASE_CONFIGURATION, GET_DATABASE_CONFIGURATION_ACTION_MULTI_NODE); + return Set.of( + GEOIP_DOWNLOADER_DATABASE_CONFIGURATION, + GET_DATABASE_CONFIGURATION_ACTION_MULTI_NODE, + PUT_DATABASE_CONFIGURATION_ACTION_IPINFO + ); } } diff --git a/server/src/main/java/org/elasticsearch/script/ScriptTermStats.java b/server/src/main/java/org/elasticsearch/script/ScriptTermStats.java index 9dde32cc75e6a..b27019765e33b 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptTermStats.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptTermStats.java @@ -12,9 +12,8 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermStates; -import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.elasticsearch.common.util.CachedSupplier; import org.elasticsearch.features.NodeFeature; @@ -71,17 +70,15 @@ public int uniqueTermsCount() { public int matchedTermsCount() { final int docId = docIdSupplier.getAsInt(); int matchedTerms = 0; + advancePostings(docId); - try { - for (PostingsEnum postingsEnum : postingsSupplier.get()) { - if (postingsEnum != null && postingsEnum.advance(docId) == docId && postingsEnum.freq() > 0) { - matchedTerms++; - } + for (PostingsEnum postingsEnum : postingsSupplier.get()) { + if (postingsEnum != null && postingsEnum.docID() == docId) { + matchedTerms++; } - return matchedTerms; - } catch (IOException e) { - throw new UncheckedIOException(e); } + + return matchedTerms; } /** @@ -150,8 +147,9 @@ public StatsSummary termFreq() { final int docId = docIdSupplier.getAsInt(); try { + advancePostings(docId); for (PostingsEnum postingsEnum : postingsSupplier.get()) { - if (postingsEnum == null || postingsEnum.advance(docId) != docId) { + if (postingsEnum == null || postingsEnum.docID() != docId) { statsSummary.accept(0); } else { statsSummary.accept(postingsEnum.freq()); @@ -170,12 +168,13 @@ public StatsSummary termFreq() { * @return statistics on termPositions for the terms of the query in the current dac */ public StatsSummary termPositions() { - try { - statsSummary.reset(); - int docId = docIdSupplier.getAsInt(); + statsSummary.reset(); + int docId = docIdSupplier.getAsInt(); + try { + advancePostings(docId); for (PostingsEnum postingsEnum : postingsSupplier.get()) { - if (postingsEnum == null || postingsEnum.advance(docId) != docId) { + if (postingsEnum == null || postingsEnum.docID() != docId) { continue; } for (int i = 0; i < postingsEnum.freq(); i++) { @@ -206,25 +205,9 @@ private TermStates[] loadTermContexts() { private PostingsEnum[] loadPostings() { try { PostingsEnum[] postings = new PostingsEnum[terms.length]; - TermStates[] contexts = termContextsSupplier.get(); for (int i = 0; i < terms.length; i++) { - TermStates termStates = contexts[i]; - if (termStates.docFreq() == 0) { - postings[i] = null; - continue; - } - - TermState state = termStates.get(leafReaderContext); - if (state == null) { - postings[i] = null; - continue; - } - - TermsEnum termsEnum = leafReaderContext.reader().terms(terms[i].field()).iterator(); - termsEnum.seekExact(terms[i].bytes(), state); - - postings[i] = termsEnum.postings(null, PostingsEnum.ALL); + postings[i] = leafReaderContext.reader().postings(terms[i], PostingsEnum.POSITIONS); } return postings; @@ -232,4 +215,16 @@ private PostingsEnum[] loadPostings() { throw new UncheckedIOException(e); } } + + private void advancePostings(int targetDocId) { + try { + for (PostingsEnum posting : postingsSupplier.get()) { + if (posting != null && posting.docID() < targetDocId && posting.docID() != DocIdSetIterator.NO_MORE_DOCS) { + posting.advance(targetDocId); + } + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } } diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index 6fff4318e5b35..be96b4e25d841 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -17,7 +17,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TopDocs; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.ElasticsearchTimeoutException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.ResolvedIndices; @@ -33,6 +32,7 @@ import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.component.AbstractLifecycleComponent; +import org.elasticsearch.common.logging.LoggerMessageFormat; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; @@ -112,6 +112,7 @@ import org.elasticsearch.search.query.QuerySearchRequest; import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.query.ScrollQuerySearchResult; +import org.elasticsearch.search.query.SearchTimeoutException; import org.elasticsearch.search.rank.feature.RankFeatureResult; import org.elasticsearch.search.rank.feature.RankFeatureShardPhase; import org.elasticsearch.search.rank.feature.RankFeatureShardRequest; @@ -598,9 +599,13 @@ private void ensureAfterSeqNoRefreshed( final TimeValue timeout = request.getWaitForCheckpointsTimeout(); final Scheduler.ScheduledCancellable timeoutTask = NO_TIMEOUT.equals(timeout) ? null : threadPool.schedule(() -> { if (isDone.compareAndSet(false, true)) { - listener.onFailure( - new ElasticsearchTimeoutException("Wait for seq_no [{}] refreshed timed out [{}]", waitForCheckpoint, timeout) + var shardTarget = new SearchShardTarget( + shard.routingEntry().currentNodeId(), + shard.shardId(), + request.getClusterAlias() ); + var message = LoggerMessageFormat.format("Wait for seq_no [{}] refreshed timed out [{}]", waitForCheckpoint, timeout); + listener.onFailure(new SearchTimeoutException(shardTarget, message)); } }, timeout, EsExecutors.DIRECT_EXECUTOR_SERVICE); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java index ff1594de523cf..6bc667d4359b1 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java @@ -75,7 +75,10 @@ public void execute(SearchContext context, int[] docIdsToLoad, RankDocShardInfo return; } - Profiler profiler = context.getProfilers() == null ? Profiler.NOOP : Profilers.startProfilingFetchPhase(); + Profiler profiler = context.getProfilers() == null + || (context.request().source() != null && context.request().source().rankBuilder() != null) + ? Profiler.NOOP + : Profilers.startProfilingFetchPhase(); SearchHits hits = null; try { hits = buildSearchHits(context, docIdsToLoad, profiler, rankDocs); diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 423fad92483ed..d17cd4f69dec7 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -60,6 +60,13 @@ private QueryPhase() {} public static void execute(SearchContext searchContext) throws QueryPhaseExecutionException { if (searchContext.queryPhaseRankShardContext() == null) { + if (searchContext.request().source() != null && searchContext.request().source().rankBuilder() != null) { + // if we have a RankBuilder provided, we want to fetch all rankWindowSize results + // and rerank the documents as per the RankBuilder's instructions. + // Pagination will take place later once they're all (re)ranked. + searchContext.size(searchContext.request().source().rankBuilder().rankWindowSize()); + searchContext.from(0); + } executeQuery(searchContext); } else { executeRank(searchContext); diff --git a/server/src/test/java/org/elasticsearch/action/support/CancellableFanOutTests.java b/server/src/test/java/org/elasticsearch/action/support/CancellableFanOutTests.java index 5a8aa5acf09ba..4882d4d8fd5db 100644 --- a/server/src/test/java/org/elasticsearch/action/support/CancellableFanOutTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/CancellableFanOutTests.java @@ -248,6 +248,7 @@ public void testConcurrency() throws InterruptedException { final var itemsProcessed = new AtomicInteger(); final var completionLatch = new CountDownLatch(1); + final var onCompletionCalled = new AtomicBoolean(); new CancellableFanOut() { @Override protected void sendItemRequest(String s, ActionListener listener) { @@ -261,6 +262,7 @@ protected void onItemResponse(String s, String response) { assertCurrentThread(isProcessorThread); assertEquals(s, response); assertThat(itemsProcessed.incrementAndGet(), lessThanOrEqualTo(items.size())); + assertFalse(onCompletionCalled.get()); } @Override @@ -269,10 +271,12 @@ protected void onItemFailure(String s, Exception e) { assertThat(e, instanceOf(ElasticsearchException.class)); assertEquals("sendItemRequest", e.getMessage()); assertThat(itemsProcessed.incrementAndGet(), lessThanOrEqualTo(items.size())); + assertFalse(onCompletionCalled.get()); } @Override protected String onCompletion() { + assertTrue(onCompletionCalled.compareAndSet(false, true)); assertEquals(items.size(), itemsProcessed.get()); assertCurrentThread(anyOf(isTestThread, isProcessorThread)); if (randomBoolean()) { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamOptionsTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamOptionsTests.java index 020955d226a0f..9b0eb93b496a4 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamOptionsTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamOptionsTests.java @@ -24,7 +24,16 @@ protected Writeable.Reader instanceReader() { @Override protected DataStreamOptions createTestInstance() { - return new DataStreamOptions(randomBoolean() ? null : DataStreamFailureStoreTests.randomFailureStore()); + return randomDataStreamOptions(); + } + + public static DataStreamOptions randomDataStreamOptions() { + return switch (randomIntBetween(0, 2)) { + case 0 -> DataStreamOptions.EMPTY; + case 1 -> DataStreamOptions.FAILURE_STORE_DISABLED; + case 2 -> DataStreamOptions.FAILURE_STORE_ENABLED; + default -> throw new IllegalArgumentException("Illegal randomisation branch"); + }; } @Override diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java index 92c1103c950c0..276c20d2d1322 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java @@ -422,6 +422,39 @@ public void testUpdateLifecycle() { } } + public void testUpdateDataStreamOptions() { + String dataStream = randomAlphaOfLength(5); + // we want the data stream options to be non-empty, so we can see the removal in action + DataStreamOptions dataStreamOptions = randomValueOtherThan( + DataStreamOptions.EMPTY, + DataStreamOptionsTests::randomDataStreamOptions + ); + ClusterState before = DataStreamTestHelper.getClusterStateWithDataStreams(List.of(new Tuple<>(dataStream, 2)), List.of()); + MetadataDataStreamsService service = new MetadataDataStreamsService( + mock(ClusterService.class), + mock(IndicesService.class), + DataStreamGlobalRetentionSettings.create(ClusterSettings.createBuiltInClusterSettings()) + ); + + // Ensure no data stream options are stored + DataStream updatedDataStream = before.metadata().dataStreams().get(dataStream); + assertNotNull(updatedDataStream); + assertThat(updatedDataStream.getDataStreamOptions(), equalTo(DataStreamOptions.EMPTY)); + + // Set non-empty data stream options + ClusterState after = service.updateDataStreamOptions(before, List.of(dataStream), dataStreamOptions); + updatedDataStream = after.metadata().dataStreams().get(dataStream); + assertNotNull(updatedDataStream); + assertThat(updatedDataStream.getDataStreamOptions(), equalTo(dataStreamOptions)); + before = after; + + // Remove data stream options + after = service.updateDataStreamOptions(before, List.of(dataStream), null); + updatedDataStream = after.metadata().dataStreams().get(dataStream); + assertNotNull(updatedDataStream); + assertThat(updatedDataStream.getDataStreamOptions(), equalTo(DataStreamOptions.EMPTY)); + } + private MapperService getMapperService(IndexMetadata im) { try { String mapping = im.mapping().source().toString(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java index 7f430cf676809..7e9a196faaa26 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java @@ -1377,6 +1377,7 @@ public void testSubobjectsFalseWithInnerNestedFromDynamicTemplate() { } public void testSubobjectsAutoFlatPaths() throws IOException { + assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()); MapperService mapperService = createDynamicTemplateAutoSubobjects(); ParsedDocument doc = mapperService.documentMapper().parse(source(b -> { b.field("foo.metric.count", 10); @@ -1389,6 +1390,7 @@ public void testSubobjectsAutoFlatPaths() throws IOException { } public void testSubobjectsAutoStructuredPaths() throws IOException { + assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()); MapperService mapperService = createDynamicTemplateAutoSubobjects(); ParsedDocument doc = mapperService.documentMapper().parse(source(b -> { b.startObject("foo"); @@ -1411,6 +1413,7 @@ public void testSubobjectsAutoStructuredPaths() throws IOException { } public void testSubobjectsAutoArrayOfObjects() throws IOException { + assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()); MapperService mapperService = createDynamicTemplateAutoSubobjects(); ParsedDocument doc = mapperService.documentMapper().parse(source(b -> { b.startObject("foo"); @@ -1444,6 +1447,7 @@ public void testSubobjectsAutoArrayOfObjects() throws IOException { } public void testSubobjectAutoDynamicNested() throws IOException { + assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()); DocumentMapper mapper = createDocumentMapper(topMapping(b -> { b.startArray("dynamic_templates"); { @@ -1482,6 +1486,7 @@ public void testSubobjectAutoDynamicNested() throws IOException { } public void testRootSubobjectAutoDynamicNested() throws IOException { + assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()); DocumentMapper mapper = createDocumentMapper(topMapping(b -> { b.startArray("dynamic_templates"); { @@ -1515,6 +1520,7 @@ public void testRootSubobjectAutoDynamicNested() throws IOException { } public void testDynamicSubobjectsAutoDynamicFalse() throws Exception { + assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()); // verify that we read the dynamic value properly from the parent mapper. DocumentParser#dynamicOrDefault splits the field // name where dots are found, but it does that only for the parent prefix e.g. metrics.service and not for the leaf suffix time.max DocumentMapper mapper = createDocumentMapper(topMapping(b -> { @@ -1578,6 +1584,7 @@ public void testDynamicSubobjectsAutoDynamicFalse() throws Exception { } public void testSubobjectsAutoWithInnerNestedFromDynamicTemplate() throws IOException { + assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()); DocumentMapper mapper = createDocumentMapper(topMapping(b -> { b.startArray("dynamic_templates"); { @@ -2045,6 +2052,7 @@ public void testSubobjectsFalseFlattened() throws IOException { } public void testSubobjectsAutoFlattened() throws IOException { + assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()); String mapping = """ { "_doc": { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index 205ff08c397b2..5eac5acdca286 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -901,6 +901,37 @@ public void testNestedArray() throws IOException { ); } + public void testConflictingFieldNameAfterArray() throws IOException { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + b.startObject("path").startObject("properties"); + { + b.startObject("to").startObject("properties"); + { + b.startObject("id").field("type", "integer").field("synthetic_source_keep", "arrays").endObject(); + } + b.endObject().endObject(); + b.startObject("id").field("type", "float").endObject(); + } + b.endObject().endObject(); + })).documentMapper(); + + var syntheticSource = syntheticSource(documentMapper, b -> { + b.startObject("path"); + { + b.startArray("to"); + { + b.startObject().array("id", 1, 20, 3).endObject(); + b.startObject().field("id", 10).endObject(); + } + b.endArray(); + b.field("id", "0.1"); + } + b.endObject(); + }); + assertEquals(""" + {"path":{"id":0.1,"to":{"id":[1,20,3,10]}}}""", syntheticSource); + } + public void testArrayWithinArray() throws IOException { DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java index 64eee39532c31..3b77015fde415 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java @@ -169,27 +169,29 @@ public void testMergeEnabledForIndexTemplates() throws IOException { assertEquals(ObjectMapper.Subobjects.ENABLED, objectMapper.subobjects()); assertTrue(objectMapper.sourceKeepMode().isEmpty()); - // Setting 'enabled' to true is allowed, and updates the mapping. - update = Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject("object") - .field("type", "object") - .field("enabled", true) - .field("subobjects", "auto") - .field(ObjectMapper.STORE_ARRAY_SOURCE_PARAM, true) - .endObject() - .endObject() - .endObject() - ); - mapper = mapperService.merge("type", new CompressedXContent(update), MergeReason.INDEX_TEMPLATE); - - objectMapper = mapper.mappers().objectMappers().get("object"); - assertNotNull(objectMapper); - assertTrue(objectMapper.isEnabled()); - assertEquals(ObjectMapper.Subobjects.AUTO, objectMapper.subobjects()); - assertEquals(Mapper.SourceKeepMode.ARRAYS, objectMapper.sourceKeepMode().orElse(Mapper.SourceKeepMode.NONE)); + if (ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()) { + // Setting 'enabled' to true is allowed, and updates the mapping. + update = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject("object") + .field("type", "object") + .field("enabled", true) + .field("subobjects", "auto") + .field(ObjectMapper.STORE_ARRAY_SOURCE_PARAM, true) + .endObject() + .endObject() + .endObject() + ); + mapper = mapperService.merge("type", new CompressedXContent(update), MergeReason.INDEX_TEMPLATE); + + objectMapper = mapper.mappers().objectMappers().get("object"); + assertNotNull(objectMapper); + assertTrue(objectMapper.isEnabled()); + assertEquals(ObjectMapper.Subobjects.AUTO, objectMapper.subobjects()); + assertEquals(Mapper.SourceKeepMode.ARRAYS, objectMapper.sourceKeepMode().orElse(Mapper.SourceKeepMode.NONE)); + } } public void testFieldReplacementForIndexTemplates() throws IOException { @@ -503,6 +505,7 @@ public void testSubobjectsCannotBeUpdatedOnRoot() throws IOException { } public void testSubobjectsAuto() throws Exception { + assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()); MapperService mapperService = createMapperService(mapping(b -> { b.startObject("metrics.service"); { @@ -532,6 +535,7 @@ public void testSubobjectsAuto() throws Exception { } public void testSubobjectsAutoWithInnerObject() throws IOException { + assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()); MapperService mapperService = createMapperService(mapping(b -> { b.startObject("metrics.service"); { @@ -565,6 +569,7 @@ public void testSubobjectsAutoWithInnerObject() throws IOException { } public void testSubobjectsAutoWithInnerNested() throws IOException { + assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()); MapperService mapperService = createMapperService(mapping(b -> { b.startObject("metrics.service"); { @@ -586,6 +591,7 @@ public void testSubobjectsAutoWithInnerNested() throws IOException { } public void testSubobjectsAutoRoot() throws Exception { + assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()); MapperService mapperService = createMapperService(mappingWithSubobjects(b -> { b.startObject("metrics.service.time"); b.field("type", "long"); @@ -606,6 +612,7 @@ public void testSubobjectsAutoRoot() throws Exception { } public void testSubobjectsAutoRootWithInnerObject() throws IOException { + assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()); MapperService mapperService = createMapperService(mappingWithSubobjects(b -> { b.startObject("metrics.service.time"); { @@ -626,6 +633,7 @@ public void testSubobjectsAutoRootWithInnerObject() throws IOException { } public void testSubobjectsAutoRootWithInnerNested() throws IOException { + assumeTrue("only test when feature flag for subobjects auto is enabled", ObjectMapper.SUB_OBJECTS_AUTO_FEATURE_FLAG.isEnabled()); MapperService mapperService = createMapperService(mappingWithSubobjects(b -> { b.startObject("metrics.service"); b.field("type", "nested"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java index 8aede4940443c..04b9b05ecfe3a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java @@ -175,7 +175,7 @@ protected void registerParameters(ParameterChecker checker) throws IOException { ), fieldMapping( b -> b.field("type", "dense_vector") - .field("dims", dims) + .field("dims", dims * 8) .field("index", true) .field("similarity", "l2_norm") .field("element_type", "bit") @@ -192,7 +192,7 @@ protected void registerParameters(ParameterChecker checker) throws IOException { ), fieldMapping( b -> b.field("type", "dense_vector") - .field("dims", dims) + .field("dims", dims * 8) .field("index", true) .field("similarity", "l2_norm") .field("element_type", "bit") @@ -891,9 +891,7 @@ public void testDims() { }))); assertThat( e.getMessage(), - equalTo( - "Failed to parse mapping: " + "The number of dimensions for field [field] should be in the range [1, 4096] but was [0]" - ) + equalTo("Failed to parse mapping: " + "The number of dimensions should be in the range [1, 4096] but was [0]") ); } // test max limit for non-indexed vectors @@ -904,10 +902,7 @@ public void testDims() { }))); assertThat( e.getMessage(), - equalTo( - "Failed to parse mapping: " - + "The number of dimensions for field [field] should be in the range [1, 4096] but was [5000]" - ) + equalTo("Failed to parse mapping: " + "The number of dimensions should be in the range [1, 4096] but was [5000]") ); } // test max limit for indexed vectors @@ -919,10 +914,7 @@ public void testDims() { }))); assertThat( e.getMessage(), - equalTo( - "Failed to parse mapping: " - + "The number of dimensions for field [field] should be in the range [1, 4096] but was [5000]" - ) + equalTo("Failed to parse mapping: " + "The number of dimensions should be in the range [1, 4096] but was [5000]") ); } } @@ -955,6 +947,14 @@ public void testMergeDims() throws IOException { ); } + public void testLargeDimsBit() throws IOException { + createMapperService(fieldMapping(b -> { + b.field("type", "dense_vector"); + b.field("dims", 1024 * Byte.SIZE); + b.field("element_type", ElementType.BIT.toString()); + })); + } + public void testDefaults() throws Exception { DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "dense_vector").field("dims", 3))); diff --git a/server/src/test/java/org/elasticsearch/script/ScriptTermStatsTests.java b/server/src/test/java/org/elasticsearch/script/ScriptTermStatsTests.java index b1b6a11764120..239c90bdee2fd 100644 --- a/server/src/test/java/org/elasticsearch/script/ScriptTermStatsTests.java +++ b/server/src/test/java/org/elasticsearch/script/ScriptTermStatsTests.java @@ -48,9 +48,9 @@ public void testMatchedTermsCount() throws IOException { // Partial match assertAllDocs( - Set.of(new Term("field", "foo"), new Term("field", "baz")), + Set.of(new Term("field", "foo"), new Term("field", "qux"), new Term("field", "baz")), ScriptTermStats::matchedTermsCount, - Map.of("doc-1", equalTo(1), "doc-2", equalTo(1), "doc-3", equalTo(0)) + Map.of("doc-1", equalTo(2), "doc-2", equalTo(1), "doc-3", equalTo(0)) ); // Always returns 0 when no term is provided. @@ -211,12 +211,12 @@ public void testTermFreq() throws IOException { // With missing terms { assertAllDocs( - Set.of(new Term("field", "foo"), new Term("field", "baz")), + Set.of(new Term("field", "foo"), new Term("field", "qux"), new Term("field", "baz")), ScriptTermStats::termFreq, Map.ofEntries( - Map.entry("doc-1", equalTo(new StatsSummary(2, 1, 0, 1))), - Map.entry("doc-2", equalTo(new StatsSummary(2, 2, 0, 2))), - Map.entry("doc-3", equalTo(new StatsSummary(2, 0, 0, 0))) + Map.entry("doc-1", equalTo(new StatsSummary(3, 2, 0, 1))), + Map.entry("doc-2", equalTo(new StatsSummary(3, 2, 0, 2))), + Map.entry("doc-3", equalTo(new StatsSummary(3, 0, 0, 0))) ) ); } @@ -274,10 +274,10 @@ public void testTermPositions() throws IOException { // With missing terms { assertAllDocs( - Set.of(new Term("field", "foo"), new Term("field", "baz")), + Set.of(new Term("field", "foo"), new Term("field", "qux"), new Term("field", "baz")), ScriptTermStats::termPositions, Map.ofEntries( - Map.entry("doc-1", equalTo(new StatsSummary(1, 1, 1, 1))), + Map.entry("doc-1", equalTo(new StatsSummary(2, 4, 1, 3))), Map.entry("doc-2", equalTo(new StatsSummary(2, 3, 1, 2))), Map.entry("doc-3", equalTo(new StatsSummary())) ) @@ -311,7 +311,7 @@ private void withIndexSearcher(CheckedConsumer consu Document doc = new Document(); doc.add(new TextField("id", "doc-1", Field.Store.YES)); - doc.add(new TextField("field", "foo bar", Field.Store.YES)); + doc.add(new TextField("field", "foo bar qux", Field.Store.YES)); w.addDocument(doc); doc = new Document(); diff --git a/settings.gradle b/settings.gradle index a47751fd499c0..be0844de1164a 100644 --- a/settings.gradle +++ b/settings.gradle @@ -75,8 +75,6 @@ List projects = [ 'distribution:docker:ubi-docker-export', 'distribution:docker:wolfi-docker-aarch64-export', 'distribution:docker:wolfi-docker-export', - 'distribution:docker:wolfi-ess-docker-aarch64-export', - 'distribution:docker:wolfi-ess-docker-export', 'distribution:packages:aarch64-deb', 'distribution:packages:deb', 'distribution:packages:aarch64-rpm', diff --git a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java index f50fe16c500dd..008a056e87901 100644 --- a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java +++ b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java @@ -129,6 +129,15 @@ public void testSortByManyLongsTooMuchMemoryAsync() throws IOException { try { resp = client().performRequest(fetch); } catch (ResponseException e) { + if (e.getResponse().getStatusLine().getStatusCode() == 403) { + /* + * There's a bug when loading from the translog with security + * enabled. If we retry a few times we'll load from the index + * itself and should succeed. + */ + logger.error("polled for results got 403"); + continue; + } if (e.getResponse().getStatusLine().getStatusCode() == 404) { logger.error("polled for results got 404"); continue; @@ -166,7 +175,7 @@ public void testSortByManyLongsTooMuchMemoryAsync() throws IOException { "error", matchesMap().extraOk() .entry("bytes_wanted", greaterThan(1000)) - .entry("reason", matchesRegex("\\[request] Data too large, data for \\[(topn|esql_block_factory)] would .+")) + .entry("reason", matchesRegex("\\[request] Data too large, data for \\[.+] would be .+")) .entry("durability", "TRANSIENT") .entry("type", "circuit_breaking_exception") .entry("bytes_limit", greaterThan(1000)) diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java index 57a6d1e09c52d..c4e1c6c7a0681 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java @@ -80,6 +80,8 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; @@ -104,6 +106,7 @@ public class MockTransportService extends TransportService { private final Map> openConnections = new HashMap<>(); private final List onStopListeners = new CopyOnWriteArrayList<>(); + private final AtomicReference> onConnectionClosedCallback = new AtomicReference<>(); public static class TestPlugin extends Plugin { @Override @@ -788,6 +791,19 @@ public void openConnection(DiscoveryNode node, ConnectionProfile connectionProfi })); } + public void setOnConnectionClosedCallback(Consumer callback) { + onConnectionClosedCallback.set(callback); + } + + @Override + public void onConnectionClosed(Transport.Connection connection) { + final Consumer callback = onConnectionClosedCallback.get(); + if (callback != null) { + callback.accept(connection); + } + super.onConnectionClosed(connection); + } + public void addOnStopListener(Runnable listener) { onStopListeners.add(listener); } diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java index aa72d3248812e..ca2300611b4fd 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java @@ -18,6 +18,7 @@ public enum FeatureFlag { TIME_SERIES_MODE("es.index_mode_feature_flag_registered=true", Version.fromString("8.0.0"), null), FAILURE_STORE_ENABLED("es.failure_store_feature_flag_enabled=true", Version.fromString("8.12.0"), null), + SUB_OBJECTS_AUTO_ENABLED("es.sub_objects_auto_feature_flag_enabled=true", Version.fromString("8.16.0"), null), CHUNKING_SETTINGS_ENABLED("es.inference_chunking_settings_feature_flag_enabled=true", Version.fromString("8.16.0"), null), INFERENCE_DEFAULT_ELSER("es.inference_default_elser_feature_flag_enabled=true", Version.fromString("8.16.0"), null); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java index b4be0b33a464e..ef03fd0ba6f0e 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.index.MapperTestUtils; import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ccr.Ccr; import org.elasticsearch.xpack.ccr.CcrSettings; @@ -332,8 +333,10 @@ public void testDynamicIndexSettingsAreClassified() { replicatedSettings.add(IndexSettings.MAX_SHINGLE_DIFF_SETTING); replicatedSettings.add(IndexSettings.TIME_SERIES_END_TIME); replicatedSettings.add(IndexSettings.PREFER_ILM_SETTING); + replicatedSettings.add(IndexSettings.SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING); replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING); replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING); + replicatedSettings.add(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING); for (Setting setting : IndexScopedSettings.BUILT_IN_INDEX_SETTINGS) { // removed settings have no effect, they are only there for BWC diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/UpdateInferenceModelAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/UpdateInferenceModelAction.java new file mode 100644 index 0000000000000..cc59ae890467b --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/UpdateInferenceModelAction.java @@ -0,0 +1,278 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.inference.action; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.ml.job.messages.Messages; +import org.elasticsearch.xpack.core.ml.utils.MlStrings; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.inference.ModelConfigurations.SERVICE_SETTINGS; +import static org.elasticsearch.inference.ModelConfigurations.TASK_SETTINGS; + +public class UpdateInferenceModelAction extends ActionType { + + public static final UpdateInferenceModelAction INSTANCE = new UpdateInferenceModelAction(); + public static final String NAME = "cluster:admin/xpack/inference/update"; + + public UpdateInferenceModelAction() { + super(NAME); + } + + public record Settings( + @Nullable Map serviceSettings, + @Nullable Map taskSettings, + @Nullable TaskType taskType + ) {} + + public static class Request extends AcknowledgedRequest { + + private final String inferenceEntityId; + private final BytesReference content; + private final XContentType contentType; + private final TaskType taskType; + private Settings settings; + + public Request(String inferenceEntityId, BytesReference content, XContentType contentType, TaskType taskType, TimeValue timeout) { + super(timeout, DEFAULT_ACK_TIMEOUT); + this.inferenceEntityId = inferenceEntityId; + this.content = content; + this.contentType = contentType; + this.taskType = taskType; + } + + public Request(StreamInput in) throws IOException { + super(in); + this.inferenceEntityId = in.readString(); + this.content = in.readBytesReference(); + this.taskType = TaskType.fromStream(in); + this.contentType = in.readEnum(XContentType.class); + } + + public String getInferenceEntityId() { + return inferenceEntityId; + } + + public TaskType getTaskType() { + return taskType; + } + + /** + * The body of the request. + * For in-cluster models, this is expected to contain some of the following: + * "number_of_allocations": `an integer` + * + * For third-party services, this is expected to contain: + * "service_settings": { + * "api_key": `a string` // service settings can only contain an api key + * } + * "task_settings": { a map of settings } + * + */ + public BytesReference getContent() { + return content; + } + + /** + * The body of the request as a map. + * The map is validated such that only allowed fields are present. + * If any fields in the body are not on the allow list, this function will throw an exception. + */ + public Settings getContentAsSettings() { + if (settings == null) { // settings is deterministic on content, so we only need to compute it once + Map unvalidatedMap = XContentHelper.convertToMap(content, false, contentType).v2(); + Map serviceSettings = new HashMap<>(); + Map taskSettings = new HashMap<>(); + TaskType taskType = null; + + if (unvalidatedMap.isEmpty()) { + throw new ElasticsearchStatusException("Request body is empty", RestStatus.BAD_REQUEST); + } + + if (unvalidatedMap.containsKey("task_type")) { + if (unvalidatedMap.get("task_type") instanceof String taskTypeString) { + taskType = TaskType.fromStringOrStatusException(taskTypeString); + } else { + throw new ElasticsearchStatusException( + "Failed to parse [task_type] in update request [{}]", + RestStatus.INTERNAL_SERVER_ERROR, + unvalidatedMap.toString() + ); + } + unvalidatedMap.remove("task_type"); + } + + if (unvalidatedMap.containsKey(SERVICE_SETTINGS)) { + if (unvalidatedMap.get(SERVICE_SETTINGS) instanceof Map tempMap) { + for (Map.Entry entry : (tempMap).entrySet()) { + if (entry.getKey() instanceof String key && entry.getValue() instanceof Object value) { + serviceSettings.put(key, value); + } else { + throw new ElasticsearchStatusException( + "Failed to parse update request [{}]", + RestStatus.INTERNAL_SERVER_ERROR, + unvalidatedMap.toString() + ); + } + } + unvalidatedMap.remove(SERVICE_SETTINGS); + } else { + throw new ElasticsearchStatusException( + "Unable to parse service settings in the request [{}]", + RestStatus.BAD_REQUEST, + unvalidatedMap.toString() + ); + } + } + + if (unvalidatedMap.containsKey(TASK_SETTINGS)) { + if (unvalidatedMap.get(TASK_SETTINGS) instanceof Map tempMap) { + for (Map.Entry entry : (tempMap).entrySet()) { + if (entry.getKey() instanceof String key && entry.getValue() instanceof Object value) { + taskSettings.put(key, value); + } else { + throw new ElasticsearchStatusException( + "Failed to parse update request [{}]", + RestStatus.INTERNAL_SERVER_ERROR, + unvalidatedMap.toString() + ); + } + } + unvalidatedMap.remove(TASK_SETTINGS); + } else { + throw new ElasticsearchStatusException( + "Unable to parse task settings in the request [{}]", + RestStatus.BAD_REQUEST, + unvalidatedMap.toString() + ); + } + } + + if (unvalidatedMap.isEmpty() == false) { + throw new ElasticsearchStatusException( + "Request contained fields which cannot be updated, remove these fields and try again [{}]", + RestStatus.BAD_REQUEST, + unvalidatedMap.toString() + ); + } + + this.settings = new Settings( + serviceSettings.isEmpty() == false ? Collections.unmodifiableMap(serviceSettings) : null, + taskSettings.isEmpty() == false ? Collections.unmodifiableMap(taskSettings) : null, + taskType + ); + } + return this.settings; + } + + public XContentType getContentType() { + return contentType; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(inferenceEntityId); + taskType.writeTo(out); + out.writeBytesReference(content); + XContentHelper.writeTo(out, contentType); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = new ActionRequestValidationException(); + if (MlStrings.isValidId(this.inferenceEntityId) == false) { + validationException.addValidationError(Messages.getMessage(Messages.INVALID_ID, "inference_id", this.inferenceEntityId)); + } + + if (validationException.validationErrors().isEmpty() == false) { + return validationException; + } else { + return null; + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Objects.equals(inferenceEntityId, request.inferenceEntityId) + && Objects.equals(content, request.content) + && contentType == request.contentType + && taskType == request.taskType; + } + + @Override + public int hashCode() { + return Objects.hash(inferenceEntityId, content, contentType, taskType); + } + } + + public static class Response extends ActionResponse implements ToXContentObject { + + private final ModelConfigurations model; + + public Response(ModelConfigurations model) { + this.model = model; + } + + public Response(StreamInput in) throws IOException { + super(in); + model = new ModelConfigurations(in); + } + + public ModelConfigurations getModel() { + return model; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + model.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return model.toFilteredXContent(builder, params); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Response response = (Response) o; + return Objects.equals(model, response.model); + } + + @Override + public int hashCode() { + return Objects.hash(model); + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java index 6ebed55451ae7..9f9def6a0678d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java @@ -281,6 +281,9 @@ public final class Messages { public static final String FIELD_CANNOT_BE_NULL = "Field [{0}] cannot be null"; public static final String MODEL_ID_MATCHES_EXISTING_MODEL_IDS_BUT_MUST_NOT = "Model IDs must be unique. Requested model ID [{}] matches existing model IDs but must not."; + public static final String MODEL_ID_DOES_NOT_MATCH_EXISTING_MODEL_IDS_BUT_MUST_FOR_IN_CLUSTER_SERVICE = + "Requested model ID [{}] does not have a matching trained model and thus cannot be updated."; + public static final String INFERENCE_ENTITY_NON_EXISTANT_NO_UPDATE = "The inference endpoint [{}] does not exist and cannot be updated"; private Messages() {} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java index fb75d95aeed1b..73e3c31297fbf 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java @@ -93,6 +93,10 @@ public static ElasticsearchStatusException badRequestException(String msg, Objec return new ElasticsearchStatusException(msg, RestStatus.BAD_REQUEST, args); } + public static ElasticsearchStatusException entityNotFoundException(String msg, Object... args) { + return new ElasticsearchStatusException(msg, RestStatus.NOT_FOUND, args); + } + public static ElasticsearchStatusException taskOperationFailureToStatusException(TaskOperationFailure failure) { return new ElasticsearchStatusException(failure.getCause().getMessage(), failure.getStatus(), failure.getCause()); } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java index ab05a71b0e1c6..b817ec17c7bda 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java @@ -155,18 +155,19 @@ public static TypeResolution isNotNullAndFoldable(Expression e, String operation return resolution; } - public static TypeResolution isNotFoldable(Expression e, String operationName, ParamOrdinal paramOrd) { - if (e.foldable()) { + public static TypeResolution isNotNull(Expression e, String operationName, ParamOrdinal paramOrd) { + if (e.dataType() == DataType.NULL) { return new TypeResolution( format( null, - "{}argument of [{}] must be a table column, found constant [{}]", + "{}argument of [{}] cannot be null, received [{}]", paramOrd == null || paramOrd == DEFAULT ? "" : paramOrd.name().toLowerCase(Locale.ROOT) + " ", operationName, Expressions.name(e) ) ); } + return TypeResolution.TYPE_RESOLVED; } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index c0092caeb9d5d..b23703c6d8b66 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -425,6 +425,10 @@ public static boolean isRepresentable(DataType t) { && t.isCounter() == false; } + public static boolean isCounter(DataType t) { + return t == COUNTER_DOUBLE || t == COUNTER_INTEGER || t == COUNTER_LONG; + } + public static boolean isSpatialPoint(DataType t) { return t == GEO_POINT || t == CARTESIAN_POINT; } @@ -437,6 +441,10 @@ public static boolean isSpatial(DataType t) { return t == GEO_POINT || t == CARTESIAN_POINT || t == GEO_SHAPE || t == CARTESIAN_SHAPE; } + public static boolean isSortable(DataType t) { + return false == (t == SOURCE || isCounter(t) || isSpatial(t)); + } + public String nameUpper() { return name; } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/BytesRefBlockHash.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/BytesRefBlockHash.java index 7fcb412de5e2b..3c5bf2c18c915 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/BytesRefBlockHash.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/BytesRefBlockHash.java @@ -23,15 +23,18 @@ import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.OrdinalBytesRefBlock; +import org.elasticsearch.compute.data.OrdinalBytesRefVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.mvdedupe.MultivalueDedupe; import org.elasticsearch.compute.operator.mvdedupe.MultivalueDedupeBytesRef; +import org.elasticsearch.compute.operator.mvdedupe.MultivalueDedupeInt; import org.elasticsearch.core.ReleasableIterator; import java.io.IOException; /** * Maps a {@link BytesRefBlock} column to group ids. + * This class is generated. Do not edit it. */ final class BytesRefBlockHash extends BlockHash { private final int channel; @@ -54,6 +57,7 @@ final class BytesRefBlockHash extends BlockHash { @Override public void add(Page page, GroupingAggregatorFunction.AddInput addInput) { + // TODO track raw counts and which implementation we pick for the profiler - #114008 var block = page.getBlock(channel); if (block.areAllValuesNull()) { seenNull = true; @@ -76,6 +80,10 @@ public void add(Page page, GroupingAggregatorFunction.AddInput addInput) { } IntVector add(BytesRefVector vector) { + var ordinals = vector.asOrdinals(); + if (ordinals != null) { + return addOrdinalsVector(ordinals); + } BytesRef scratch = new BytesRef(); int positions = vector.getPositionCount(); try (var builder = blockFactory.newIntVectorFixedBuilder(positions)) { @@ -113,15 +121,29 @@ public ReleasableIterator lookup(Page page, ByteSizeValue targetBlockS return ReleasableIterator.single(lookup(vector)); } - private IntBlock addOrdinalsBlock(OrdinalBytesRefBlock inputBlock) { - var inputOrds = inputBlock.getOrdinalsBlock(); + private IntVector addOrdinalsVector(OrdinalBytesRefVector inputBlock) { + IntVector inputOrds = inputBlock.getOrdinalsVector(); try ( - var builder = blockFactory.newIntBlockBuilder(inputOrds.getPositionCount()); + var builder = blockFactory.newIntVectorBuilder(inputOrds.getPositionCount()); var hashOrds = add(inputBlock.getDictionaryVector()) ) { - for (int i = 0; i < inputOrds.getPositionCount(); i++) { - int valueCount = inputOrds.getValueCount(i); - int firstIndex = inputOrds.getFirstValueIndex(i); + for (int p = 0; p < inputOrds.getPositionCount(); p++) { + int ord = hashOrds.getInt(inputOrds.getInt(p)); + builder.appendInt(ord); + } + return builder.build(); + } + } + + private IntBlock addOrdinalsBlock(OrdinalBytesRefBlock inputBlock) { + try ( + IntBlock inputOrds = new MultivalueDedupeInt(inputBlock.getOrdinalsBlock()).dedupeToBlockAdaptive(blockFactory); + IntBlock.Builder builder = blockFactory.newIntBlockBuilder(inputOrds.getPositionCount()); + IntVector hashOrds = add(inputBlock.getDictionaryVector()) + ) { + for (int p = 0; p < inputOrds.getPositionCount(); p++) { + int valueCount = inputOrds.getValueCount(p); + int firstIndex = inputOrds.getFirstValueIndex(p); switch (valueCount) { case 0 -> { builder.appendInt(0); @@ -132,9 +154,11 @@ private IntBlock addOrdinalsBlock(OrdinalBytesRefBlock inputBlock) { builder.appendInt(ord); } default -> { + int start = firstIndex; + int end = firstIndex + valueCount; builder.beginPositionEntry(); - for (int v = 0; v < valueCount; v++) { - int ord = hashOrds.getInt(inputOrds.getInt(firstIndex + i)); + for (int i = start; i < end; i++) { + int ord = hashOrds.getInt(inputOrds.getInt(i)); builder.appendInt(ord); } builder.endPositionEntry(); diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/DoubleBlockHash.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/DoubleBlockHash.java index bd9d752302ae3..c9c672112a630 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/DoubleBlockHash.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/DoubleBlockHash.java @@ -28,6 +28,7 @@ /** * Maps a {@link DoubleBlock} column to group ids. + * This class is generated. Do not edit it. */ final class DoubleBlockHash extends BlockHash { private final int channel; @@ -50,6 +51,7 @@ final class DoubleBlockHash extends BlockHash { @Override public void add(Page page, GroupingAggregatorFunction.AddInput addInput) { + // TODO track raw counts and which implementation we pick for the profiler - #114008 var block = page.getBlock(channel); if (block.areAllValuesNull()) { seenNull = true; diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/IntBlockHash.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/IntBlockHash.java index 5b1b48bd270ab..13b60c6f1fec5 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/IntBlockHash.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/IntBlockHash.java @@ -26,6 +26,7 @@ /** * Maps a {@link IntBlock} column to group ids. + * This class is generated. Do not edit it. */ final class IntBlockHash extends BlockHash { private final int channel; @@ -48,6 +49,7 @@ final class IntBlockHash extends BlockHash { @Override public void add(Page page, GroupingAggregatorFunction.AddInput addInput) { + // TODO track raw counts and which implementation we pick for the profiler - #114008 var block = page.getBlock(channel); if (block.areAllValuesNull()) { seenNull = true; diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/LongBlockHash.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/LongBlockHash.java index 074ccb2f7cd7d..5252bd742ec51 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/LongBlockHash.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/blockhash/LongBlockHash.java @@ -28,6 +28,7 @@ /** * Maps a {@link LongBlock} column to group ids. + * This class is generated. Do not edit it. */ final class LongBlockHash extends BlockHash { private final int channel; @@ -50,6 +51,7 @@ final class LongBlockHash extends BlockHash { @Override public void add(Page page, GroupingAggregatorFunction.AddInput addInput) { + // TODO track raw counts and which implementation we pick for the profiler - #114008 var block = page.getBlock(channel); if (block.areAllValuesNull()) { seenNull = true; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BlockHash.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BlockHash.java index fe1a07e8e16a6..919cb92f79260 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BlockHash.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BlockHash.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.BitArray; import org.elasticsearch.common.util.BytesRefHash; +import org.elasticsearch.common.util.Int3Hash; import org.elasticsearch.common.util.LongHash; import org.elasticsearch.common.util.LongLongHash; import org.elasticsearch.compute.aggregation.GroupingAggregatorFunction; @@ -28,14 +29,37 @@ import java.util.List; /** - * A specialized hash table implementation maps values of a {@link Block} to ids (in longs). - * This class delegates to {@link LongHash} or {@link BytesRefHash}. - * - * @see LongHash - * @see BytesRefHash + * Specialized hash table implementations that map rows to a set + * of bucket IDs to which they belong to implement {@code GROUP BY} expressions. + *

    + * A row is always in at least one bucket so the results are never {@code null}. + * {@code null} valued key columns will map to some integer bucket id. + * If none of key columns are multivalued then the output is always an + * {@link IntVector}. If any of the key are multivalued then a row is + * in a bucket for each value. If more than one key is multivalued then + * the row is in the combinatorial explosion of all value combinations. + * Luckily for the number of values rows can only be in each bucket once. + * Unluckily, it's the responsibility of {@link BlockHash} to remove those + * duplicates. + *

    + *

    + * These classes typically delegate to some combination of {@link BytesRefHash}, + * {@link LongHash}, {@link LongLongHash}, {@link Int3Hash}. They don't + * technically have to be hash tables, so long as they + * implement the deduplication semantics above and vend integer ids. + *

    + *

    + * The integer ids are assigned to offsets into arrays of aggregation states + * so its permissible to have gaps in the ints. But large gaps are a bad + * idea because they'll waste space in the aggregations that use these + * positions. For example, {@link BooleanBlockHash} assigns {@code 0} to + * {@code null}, {@code 1} to {@code false}, and {@code 1} to {@code true} + * and that's fine and simple and good because it'll never + * leave a big gap, even if we never see {@code null}. + *

    */ public abstract sealed class BlockHash implements Releasable, SeenGroupIds // - permits BooleanBlockHash, BytesRefBlockHash, DoubleBlockHash, IntBlockHash, LongBlockHash, BytesRef3BlockHash, // + permits BooleanBlockHash, BytesRefBlockHash, DoubleBlockHash, IntBlockHash, LongBlockHash, BytesRef2BlockHash, BytesRef3BlockHash, // NullBlockHash, PackedValuesBlockHash, BytesRefLongBlockHash, LongLongBlockHash, TimeSeriesBlockHash { protected final BlockFactory blockFactory; @@ -98,8 +122,19 @@ public static BlockHash build(List groups, BlockFactory blockFactory, if (groups.size() == 1) { return newForElementType(groups.get(0).channel(), groups.get(0).elementType(), blockFactory); } - if (groups.size() == 3 && groups.stream().allMatch(g -> g.elementType == ElementType.BYTES_REF)) { - return new BytesRef3BlockHash(blockFactory, groups.get(0).channel, groups.get(1).channel, groups.get(2).channel, emitBatchSize); + if (groups.stream().allMatch(g -> g.elementType == ElementType.BYTES_REF)) { + switch (groups.size()) { + case 2: + return new BytesRef2BlockHash(blockFactory, groups.get(0).channel, groups.get(1).channel, emitBatchSize); + case 3: + return new BytesRef3BlockHash( + blockFactory, + groups.get(0).channel, + groups.get(1).channel, + groups.get(2).channel, + emitBatchSize + ); + } } if (allowBrokenOptimizations && groups.size() == 2) { var g1 = groups.get(0); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BooleanBlockHash.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BooleanBlockHash.java index 4c2817588904a..ecbf292b077ea 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BooleanBlockHash.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BooleanBlockHash.java @@ -25,8 +25,9 @@ import static org.elasticsearch.compute.operator.mvdedupe.MultivalueDedupeBoolean.TRUE_ORD; /** - * Maps a {@link BooleanBlock} column to group ids. Assigns group - * {@code 0} to {@code false} and group {@code 1} to {@code true}. + * Maps a {@link BooleanBlock} column to group ids. Assigns + * {@code 0} to {@code null}, {@code 1} to {@code false}, and + * {@code 2} to {@code true}. */ final class BooleanBlockHash extends BlockHash { private final int channel; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BytesRef2BlockHash.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BytesRef2BlockHash.java new file mode 100644 index 0000000000000..ff25aa1381004 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BytesRef2BlockHash.java @@ -0,0 +1,196 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation.blockhash; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.BitArray; +import org.elasticsearch.common.util.LongHash; +import org.elasticsearch.compute.aggregation.GroupingAggregatorFunction; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.core.ReleasableIterator; +import org.elasticsearch.core.Releasables; + +import java.util.Locale; + +/** + * Maps two {@link BytesRefBlock}s to group ids. + */ +final class BytesRef2BlockHash extends BlockHash { + private final int emitBatchSize; + private final int channel1; + private final int channel2; + private final BytesRefBlockHash hash1; + private final BytesRefBlockHash hash2; + private final LongHash finalHash; + + BytesRef2BlockHash(BlockFactory blockFactory, int channel1, int channel2, int emitBatchSize) { + super(blockFactory); + this.emitBatchSize = emitBatchSize; + this.channel1 = channel1; + this.channel2 = channel2; + boolean success = false; + try { + this.hash1 = new BytesRefBlockHash(channel1, blockFactory); + this.hash2 = new BytesRefBlockHash(channel2, blockFactory); + this.finalHash = new LongHash(1, blockFactory.bigArrays()); + success = true; + } finally { + if (success == false) { + close(); + } + } + } + + @Override + public void close() { + Releasables.close(hash1, hash2, finalHash); + } + + @Override + public void add(Page page, GroupingAggregatorFunction.AddInput addInput) { + BytesRefBlock b1 = page.getBlock(channel1); + BytesRefBlock b2 = page.getBlock(channel2); + BytesRefVector v1 = b1.asVector(); + BytesRefVector v2 = b2.asVector(); + if (v1 != null && v2 != null) { + addVectors(v1, v2, addInput); + } else { + try (IntBlock k1 = hash1.add(b1); IntBlock k2 = hash2.add(b2)) { + try (AddWork work = new AddWork(k1, k2, addInput)) { + work.add(); + } + } + } + } + + private void addVectors(BytesRefVector v1, BytesRefVector v2, GroupingAggregatorFunction.AddInput addInput) { + final int positionCount = v1.getPositionCount(); + try (IntVector.FixedBuilder ordsBuilder = blockFactory.newIntVectorFixedBuilder(positionCount)) { + try (IntVector k1 = hash1.add(v1); IntVector k2 = hash2.add(v2)) { + for (int p = 0; p < positionCount; p++) { + long ord = ord(k1.getInt(p), k2.getInt(p)); + ordsBuilder.appendInt(p, Math.toIntExact(ord)); + } + } + try (IntVector ords = ordsBuilder.build()) { + addInput.add(0, ords); + } + } + } + + private class AddWork extends AddPage { + final IntBlock b1; + final IntBlock b2; + + AddWork(IntBlock b1, IntBlock b2, GroupingAggregatorFunction.AddInput addInput) { + super(blockFactory, emitBatchSize, addInput); + this.b1 = b1; + this.b2 = b2; + } + + void add() { + int positionCount = b1.getPositionCount(); + for (int i = 0; i < positionCount; i++) { + int v1 = b1.getValueCount(i); + int v2 = b2.getValueCount(i); + int first1 = b1.getFirstValueIndex(i); + int first2 = b2.getFirstValueIndex(i); + if (v1 == 1 && v2 == 1) { + long ord = ord(b1.getInt(first1), b2.getInt(first2)); + appendOrdSv(i, Math.toIntExact(ord)); + continue; + } + for (int i1 = 0; i1 < v1; i1++) { + int k1 = b1.getInt(first1 + i1); + for (int i2 = 0; i2 < v2; i2++) { + int k2 = b2.getInt(first2 + i2); + long ord = ord(k1, k2); + appendOrdInMv(i, Math.toIntExact(ord)); + } + } + finishMv(); + } + flushRemaining(); + } + } + + private long ord(int k1, int k2) { + return hashOrdToGroup(finalHash.add((long) k2 << 32 | k1)); + } + + @Override + public ReleasableIterator lookup(Page page, ByteSizeValue targetBlockSize) { + throw new UnsupportedOperationException("TODO"); + } + + @Override + public Block[] getKeys() { + // TODO Build Ordinals blocks #114010 + final int positions = (int) finalHash.size(); + final BytesRef scratch = new BytesRef(); + final BytesRefBlock[] outputBlocks = new BytesRefBlock[2]; + try { + try (BytesRefBlock.Builder b1 = blockFactory.newBytesRefBlockBuilder(positions)) { + for (int i = 0; i < positions; i++) { + int k1 = (int) (finalHash.get(i) & 0xffffL); + if (k1 == 0) { + b1.appendNull(); + } else { + b1.appendBytesRef(hash1.hash.get(k1 - 1, scratch)); + } + } + outputBlocks[0] = b1.build(); + } + try (BytesRefBlock.Builder b2 = blockFactory.newBytesRefBlockBuilder(positions)) { + for (int i = 0; i < positions; i++) { + int k2 = (int) (finalHash.get(i) >>> 32); + if (k2 == 0) { + b2.appendNull(); + } else { + b2.appendBytesRef(hash2.hash.get(k2 - 1, scratch)); + } + } + outputBlocks[1] = b2.build(); + } + return outputBlocks; + } finally { + if (outputBlocks[outputBlocks.length - 1] == null) { + Releasables.close(outputBlocks); + } + } + } + + @Override + public BitArray seenGroupIds(BigArrays bigArrays) { + return new Range(0, Math.toIntExact(finalHash.size())).seenGroupIds(bigArrays); + } + + @Override + public IntVector nonEmpty() { + return IntVector.range(0, Math.toIntExact(finalHash.size()), blockFactory); + } + + @Override + public String toString() { + return String.format( + Locale.ROOT, + "BytesRef2BlockHash{keys=[channel1=%d, channel2=%d], entries=%d}", + channel1, + channel2, + finalHash.size() + ); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BytesRef3BlockHash.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BytesRef3BlockHash.java index 54bf068b4de33..987a808ed7950 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BytesRef3BlockHash.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BytesRef3BlockHash.java @@ -85,7 +85,6 @@ public void add(Page page, GroupingAggregatorFunction.AddInput addInput) { private void addVectors(BytesRefVector v1, BytesRefVector v2, BytesRefVector v3, GroupingAggregatorFunction.AddInput addInput) { final int positionCount = v1.getPositionCount(); try (IntVector.FixedBuilder ordsBuilder = blockFactory.newIntVectorFixedBuilder(positionCount)) { - // TODO: enable ordinal vectors in BytesRefBlockHash try (IntVector k1 = hash1.add(v1); IntVector k2 = hash2.add(v2); IntVector k3 = hash3.add(v3)) { for (int p = 0; p < positionCount; p++) { long ord = hashOrdToGroup(finalHash.add(k1.getInt(p), k2.getInt(p), k3.getInt(p))); @@ -148,6 +147,7 @@ public ReleasableIterator lookup(Page page, ByteSizeValue targetBlockS @Override public Block[] getKeys() { + // TODO Build Ordinals blocks #114010 final int positions = (int) finalHash.size(); final BytesRef scratch = new BytesRef(); final BytesRefBlock[] outputBlocks = new BytesRefBlock[3]; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/X-BlockHash.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/X-BlockHash.java.st index b4f700980558e..7c21cff56d7bb 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/X-BlockHash.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/X-BlockHash.java.st @@ -28,6 +28,7 @@ import org.elasticsearch.compute.data.BytesRefVector; import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.OrdinalBytesRefBlock; +import org.elasticsearch.compute.data.OrdinalBytesRefVector; $elseif(double)$ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; @@ -51,6 +52,9 @@ $endif$ import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.mvdedupe.MultivalueDedupe; import org.elasticsearch.compute.operator.mvdedupe.MultivalueDedupe$Type$; +$if(BytesRef)$ +import org.elasticsearch.compute.operator.mvdedupe.MultivalueDedupeInt; +$endif$ import org.elasticsearch.core.ReleasableIterator; $if(BytesRef)$ @@ -62,6 +66,7 @@ import java.util.BitSet; $endif$ /** * Maps a {@link $Type$Block} column to group ids. + * This class is generated. Do not edit it. */ final class $Type$BlockHash extends BlockHash { private final int channel; @@ -84,6 +89,7 @@ final class $Type$BlockHash extends BlockHash { @Override public void add(Page page, GroupingAggregatorFunction.AddInput addInput) { + // TODO track raw counts and which implementation we pick for the profiler - #114008 var block = page.getBlock(channel); if (block.areAllValuesNull()) { seenNull = true; @@ -107,6 +113,10 @@ final class $Type$BlockHash extends BlockHash { IntVector add($Type$Vector vector) { $if(BytesRef)$ + var ordinals = vector.asOrdinals(); + if (ordinals != null) { + return addOrdinalsVector(ordinals); + } BytesRef scratch = new BytesRef(); $endif$ int positions = vector.getPositionCount(); @@ -154,15 +164,29 @@ $endif$ } $if(BytesRef)$ - private IntBlock addOrdinalsBlock(OrdinalBytesRefBlock inputBlock) { - var inputOrds = inputBlock.getOrdinalsBlock(); + private IntVector addOrdinalsVector(OrdinalBytesRefVector inputBlock) { + IntVector inputOrds = inputBlock.getOrdinalsVector(); try ( - var builder = blockFactory.newIntBlockBuilder(inputOrds.getPositionCount()); + var builder = blockFactory.newIntVectorBuilder(inputOrds.getPositionCount()); var hashOrds = add(inputBlock.getDictionaryVector()) ) { - for (int i = 0; i < inputOrds.getPositionCount(); i++) { - int valueCount = inputOrds.getValueCount(i); - int firstIndex = inputOrds.getFirstValueIndex(i); + for (int p = 0; p < inputOrds.getPositionCount(); p++) { + int ord = hashOrds.getInt(inputOrds.getInt(p)); + builder.appendInt(ord); + } + return builder.build(); + } + } + + private IntBlock addOrdinalsBlock(OrdinalBytesRefBlock inputBlock) { + try ( + IntBlock inputOrds = new MultivalueDedupeInt(inputBlock.getOrdinalsBlock()).dedupeToBlockAdaptive(blockFactory); + IntBlock.Builder builder = blockFactory.newIntBlockBuilder(inputOrds.getPositionCount()); + IntVector hashOrds = add(inputBlock.getDictionaryVector()) + ) { + for (int p = 0; p < inputOrds.getPositionCount(); p++) { + int valueCount = inputOrds.getValueCount(p); + int firstIndex = inputOrds.getFirstValueIndex(p); switch (valueCount) { case 0 -> { builder.appendInt(0); @@ -173,9 +197,11 @@ $if(BytesRef)$ builder.appendInt(ord); } default -> { + int start = firstIndex; + int end = firstIndex + valueCount; builder.beginPositionEntry(); - for (int v = 0; v < valueCount; v++) { - int ord = hashOrds.getInt(inputOrds.getInt(firstIndex + i)); + for (int i = start; i < end; i++) { + int ord = hashOrds.getInt(inputOrds.getInt(i)); builder.appendInt(ord); } builder.endPositionEntry(); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/AggregatorFunctionTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/AggregatorFunctionTestCase.java index a4eb252dbf35c..e2c9c255b67bd 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/AggregatorFunctionTestCase.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/AggregatorFunctionTestCase.java @@ -22,6 +22,7 @@ import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.data.TestBlockFactory; +import org.elasticsearch.compute.operator.AddGarbageRowsSourceOperator; import org.elasticsearch.compute.operator.AggregationOperator; import org.elasticsearch.compute.operator.CannedSourceOperator; import org.elasticsearch.compute.operator.Driver; @@ -203,6 +204,22 @@ public void testNoneFiltered() { assertSimpleOutput(origInput, results); } + public void testSomeFiltered() { + Operator.OperatorFactory factory = simpleWithMode( + AggregatorMode.SINGLE, + agg -> new FilteredAggregatorFunctionSupplier(agg, AddGarbageRowsSourceOperator.filterFactory()) + ); + DriverContext driverContext = driverContext(); + // Build the test data + List input = CannedSourceOperator.collectPages(simpleInput(driverContext.blockFactory(), 10)); + List origInput = BlockTestUtils.deepCopyOf(input, TestBlockFactory.getNonBreakingInstance()); + // Sprinkle garbage into it + input = CannedSourceOperator.collectPages(new AddGarbageRowsSourceOperator(new CannedSourceOperator(input.iterator()))); + List results = drive(factory.get(driverContext), input.iterator(), driverContext); + assertThat(results, hasSize(1)); + assertSimpleOutput(origInput, results); + } + // Returns an intermediate state that is equivalent to what the local execution planner will emit // if it determines that certain shards have no relevant data. List nullIntermediateState(BlockFactory blockFactory) { diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/FilteredAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/FilteredAggregatorFunctionTests.java index 7e1575fb81726..da2c3502144db 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/FilteredAggregatorFunctionTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/FilteredAggregatorFunctionTests.java @@ -103,4 +103,9 @@ public void testNoneFiltered() { public void testAllFiltered() { assumeFalse("can't double filter. tests already filter.", true); } + + @Override + public void testSomeFiltered() { + assumeFalse("can't double filter. tests already filter.", true); + } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java index 316058e57e089..9414e076a26e6 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java @@ -26,6 +26,7 @@ import org.elasticsearch.compute.data.LongVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.data.TestBlockFactory; +import org.elasticsearch.compute.operator.AddGarbageRowsSourceOperator; import org.elasticsearch.compute.operator.CannedSourceOperator; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.ForkingOperatorTestCase; @@ -53,6 +54,7 @@ import static org.elasticsearch.compute.data.BlockTestUtils.append; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.in; /** * Shared tests for testing grouped aggregations. @@ -160,11 +162,17 @@ protected long randomGroupId(int pageSize) { @Override protected final void assertSimpleOutput(List input, List results) { + assertSimpleOutput(input, results, true); + } + + private void assertSimpleOutput(List input, List results, boolean assertGroupCount) { SeenGroups seenGroups = seenGroups(input); assertThat(results, hasSize(1)); assertThat(results.get(0).getBlockCount(), equalTo(2)); - assertThat(results.get(0).getPositionCount(), equalTo(seenGroups.size())); + if (assertGroupCount) { + assertThat(results.get(0).getPositionCount(), equalTo(seenGroups.size())); + } Block groups = results.get(0).getBlock(0); Block result = results.get(0).getBlock(1); @@ -394,6 +402,23 @@ public final void testNoneFiltered() { assertSimpleOutput(origInput, results); } + public void testSomeFiltered() { + Operator.OperatorFactory factory = simpleWithMode( + AggregatorMode.SINGLE, + agg -> new FilteredAggregatorFunctionSupplier(agg, AddGarbageRowsSourceOperator.filterFactory()) + ); + DriverContext driverContext = driverContext(); + // Build the test data + List input = CannedSourceOperator.collectPages(simpleInput(driverContext.blockFactory(), 10)); + List origInput = BlockTestUtils.deepCopyOf(input, TestBlockFactory.getNonBreakingInstance()); + // Sprinkle garbage into it + input = CannedSourceOperator.collectPages(new AddGarbageRowsSourceOperator(new CannedSourceOperator(input.iterator()))); + List results = drive(factory.get(driverContext), input.iterator(), driverContext); + assertThat(results, hasSize(1)); + + assertSimpleOutput(origInput, results, false); + } + /** * Asserts that the output from an empty input is a {@link Block} containing * only {@code null}. Override for {@code count} style aggregations that diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashRandomizedTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashRandomizedTests.java index 27ec0b979e8ae..76d4caf810eb8 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashRandomizedTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashRandomizedTests.java @@ -21,10 +21,13 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.BlockTestUtils; +import org.elasticsearch.compute.data.BytesRefVector; import org.elasticsearch.compute.data.ElementType; import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.MockBlockFactory; +import org.elasticsearch.compute.data.OrdinalBytesRefBlock; import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.data.TestBlockFactory; import org.elasticsearch.compute.operator.mvdedupe.MultivalueDedupeTests; import org.elasticsearch.core.ReleasableIterator; import org.elasticsearch.core.Releasables; @@ -38,11 +41,13 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.NavigableSet; import java.util.Set; import java.util.TreeSet; +import java.util.stream.Stream; import static org.elasticsearch.test.ListMatcher.matchesList; import static org.elasticsearch.test.MapMatcher.assertMap; @@ -58,26 +63,40 @@ public class BlockHashRandomizedTests extends ESTestCase { @ParametersFactory public static List params() { - List params = new ArrayList<>(); + List> allowedTypesChoices = List.of( + /* + * Run with only `LONG` elements because we have some + * optimizations that hit if you only have those. + */ + List.of(new Basic(ElementType.LONG)), + /* + * Run with only `BYTES_REF` elements because we have some + * optimizations that hit if you only have those. + */ + List.of(new Basic(ElementType.BYTES_REF)), + /* + * Run with only `BYTES_REF` elements in an OrdinalBytesRefBlock + * because we have a few optimizations that use it. + */ + List.of(new Ordinals(10)), + /* + * Run with only `LONG` and `BYTES_REF` elements because + * we have some optimizations that hit if you only have + * those. + */ + List.of(new Basic(ElementType.LONG), new Basic(ElementType.BYTES_REF)), + /* + * Any random source. + */ + Stream.concat(Stream.of(new Ordinals(10)), MultivalueDedupeTests.supportedTypes().stream().map(Basic::new)).toList() + ); + List params = new ArrayList<>(); for (boolean forcePackedHash : new boolean[] { false, true }) { for (int groups : new int[] { 1, 2, 3, 4, 5, 10 }) { for (int maxValuesPerPosition : new int[] { 1, 3 }) { for (int dups : new int[] { 0, 2 }) { - for (List allowedTypes : List.of( - /* - * Run with only `LONG` elements because we have some - * optimizations that hit if you only have those. - */ - List.of(ElementType.LONG), - /* - * Run with only `LONG` and `BYTES_REF` elements because - * we have some optimizations that hit if you only have - * those. - */ - List.of(ElementType.LONG, ElementType.BYTES_REF), - MultivalueDedupeTests.supportedTypes() - )) { + for (List allowedTypes : allowedTypesChoices) { params.add(new Object[] { forcePackedHash, groups, maxValuesPerPosition, dups, allowedTypes }); } } @@ -87,18 +106,33 @@ public static List params() { return params; } + /** + * The type of {@link Block} being tested. + */ + interface Type { + /** + * The type of the {@link ElementType elements} in the {@link Block}. + */ + ElementType elementType(); + + /** + * Build a random {@link Block}. + */ + BasicBlockTests.RandomBlock randomBlock(int positionCount, int maxValuesPerPosition, int dups); + } + private final boolean forcePackedHash; private final int groups; private final int maxValuesPerPosition; private final int dups; - private final List allowedTypes; + private final List allowedTypes; public BlockHashRandomizedTests( @Name("forcePackedHash") boolean forcePackedHash, @Name("groups") int groups, @Name("maxValuesPerPosition") int maxValuesPerPosition, @Name("dups") int dups, - @Name("allowedTypes") List allowedTypes + @Name("allowedTypes") List allowedTypes ) { this.forcePackedHash = forcePackedHash; this.groups = groups; @@ -127,21 +161,22 @@ public void testWithCranky() { } private void test(MockBlockFactory blockFactory) { - List types = randomList(groups, groups, () -> randomFrom(allowedTypes)); + List types = randomList(groups, groups, () -> randomFrom(allowedTypes)); + List elementTypes = types.stream().map(Type::elementType).toList(); BasicBlockTests.RandomBlock[] randomBlocks = new BasicBlockTests.RandomBlock[types.size()]; Block[] blocks = new Block[types.size()]; - int pageCount = between(1, 10); + int pageCount = between(1, groups < 10 ? 10 : 5); int positionCount = 100; int emitBatchSize = 100; - try (BlockHash blockHash = newBlockHash(blockFactory, emitBatchSize, types)) { + try (BlockHash blockHash = newBlockHash(blockFactory, emitBatchSize, elementTypes)) { /* * Only the long/long, long/bytes_ref, and bytes_ref/long implementations don't collect nulls. */ Oracle oracle = new Oracle( forcePackedHash - || false == (types.equals(List.of(ElementType.LONG, ElementType.LONG)) - || types.equals(List.of(ElementType.LONG, ElementType.BYTES_REF)) - || types.equals(List.of(ElementType.BYTES_REF, ElementType.LONG))) + || false == (elementTypes.equals(List.of(ElementType.LONG, ElementType.LONG)) + || elementTypes.equals(List.of(ElementType.LONG, ElementType.BYTES_REF)) + || elementTypes.equals(List.of(ElementType.BYTES_REF, ElementType.LONG))) ); /* * Expected ordinals for checking lookup. Skipped if we have more than 5 groups because @@ -151,15 +186,7 @@ private void test(MockBlockFactory blockFactory) { for (int p = 0; p < pageCount; p++) { for (int g = 0; g < blocks.length; g++) { - randomBlocks[g] = BasicBlockTests.randomBlock( - types.get(g), - positionCount, - types.get(g) == ElementType.NULL ? true : randomBoolean(), - 1, - maxValuesPerPosition, - 0, - dups - ); + randomBlocks[g] = types.get(g).randomBlock(positionCount, maxValuesPerPosition, dups); blocks[g] = randomBlocks[g].block(); } oracle.add(randomBlocks); @@ -209,6 +236,7 @@ private void test(MockBlockFactory blockFactory) { if (blockHash instanceof LongLongBlockHash == false && blockHash instanceof BytesRefLongBlockHash == false + && blockHash instanceof BytesRef2BlockHash == false && blockHash instanceof BytesRef3BlockHash == false) { assertLookup(blockFactory, expectedOrds, types, blockHash, oracle); } @@ -235,14 +263,14 @@ private BlockHash newBlockHash(BlockFactory blockFactory, int emitBatchSize, Lis private void assertLookup( BlockFactory blockFactory, Map, Set> expectedOrds, - List types, + List types, BlockHash blockHash, Oracle oracle ) { Block.Builder[] builders = new Block.Builder[types.size()]; try { for (int b = 0; b < builders.length; b++) { - builders[b] = types.get(b).newBlockBuilder(LOOKUP_POSITIONS, blockFactory); + builders[b] = types.get(b).elementType().newBlockBuilder(LOOKUP_POSITIONS, blockFactory); } for (int p = 0; p < LOOKUP_POSITIONS; p++) { /* @@ -408,8 +436,8 @@ static CircuitBreakerService mockBreakerService(CircuitBreaker breaker) { return breakerService; } - private static List randomKey(List types) { - return types.stream().map(BlockHashRandomizedTests::randomKeyElement).toList(); + private static List randomKey(List types) { + return types.stream().map(t -> randomKeyElement(t.elementType())).toList(); } public static Object randomKeyElement(ElementType type) { @@ -423,4 +451,75 @@ public static Object randomKeyElement(ElementType type) { default -> throw new IllegalArgumentException("unsupported element type [" + type + "]"); }; } + + private record Basic(ElementType elementType) implements Type { + @Override + public BasicBlockTests.RandomBlock randomBlock(int positionCount, int maxValuesPerPosition, int dups) { + return BasicBlockTests.randomBlock( + elementType, + positionCount, + elementType == ElementType.NULL | randomBoolean(), + 1, + maxValuesPerPosition, + 0, + dups + ); + } + } + + private record Ordinals(int dictionarySize) implements Type { + @Override + public ElementType elementType() { + return ElementType.BYTES_REF; + } + + @Override + public BasicBlockTests.RandomBlock randomBlock(int positionCount, int maxValuesPerPosition, int dups) { + List> dictionary = new ArrayList<>(); + List> values = new ArrayList<>(positionCount); + try ( + IntBlock.Builder ordinals = TestBlockFactory.getNonBreakingInstance() + .newIntBlockBuilder(positionCount * maxValuesPerPosition); + BytesRefVector.Builder bytes = TestBlockFactory.getNonBreakingInstance().newBytesRefVectorBuilder(maxValuesPerPosition); + ) { + for (String value : dictionary(maxValuesPerPosition)) { + bytes.appendBytesRef(new BytesRef(value)); + dictionary.add(Map.entry(value, dictionary.size())); + } + for (int p = 0; p < positionCount; p++) { + int valueCount = between(1, maxValuesPerPosition); + int dupCount = between(0, dups); + + List ordsAtPosition = new ArrayList<>(); + List valuesAtPosition = new ArrayList<>(); + values.add(valuesAtPosition); + if (valueCount != 1 || dupCount != 0) { + ordinals.beginPositionEntry(); + } + for (int v = 0; v < valueCount; v++) { + Map.Entry value = randomFrom(dictionary); + valuesAtPosition.add(new BytesRef(value.getKey())); + ordinals.appendInt(value.getValue()); + ordsAtPosition.add(value.getValue()); + } + for (int v = 0; v < dupCount; v++) { + ordinals.appendInt(randomFrom(ordsAtPosition)); + } + if (valueCount != 1 || dupCount != 0) { + ordinals.endPositionEntry(); + } + } + return new BasicBlockTests.RandomBlock(values, new OrdinalBytesRefBlock(ordinals.build(), bytes.build())); + } + } + + private Set dictionary(int maxValuesPerPosition) { + int count = Math.max(dictionarySize, maxValuesPerPosition); + Set values = new HashSet<>(); + while (values.size() < count) { + values.add(randomAlphaOfLength(5)); + } + return values; + } + } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashTests.java index 800683c696c0f..aeea18e52da0f 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashTests.java @@ -20,12 +20,15 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BooleanBlock; import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; import org.elasticsearch.compute.data.DoubleBlock; import org.elasticsearch.compute.data.ElementType; import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.MockBlockFactory; +import org.elasticsearch.compute.data.OrdinalBytesRefBlock; +import org.elasticsearch.compute.data.OrdinalBytesRefVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.data.TestBlockFactory; import org.elasticsearch.core.Releasable; @@ -460,6 +463,133 @@ public void testBytesRefHashWithMultiValuedFields() { } } + public void testBasicOrdinals() { + try ( + IntVector.Builder ords = blockFactory.newIntVectorFixedBuilder(8); + BytesRefVector.Builder bytes = blockFactory.newBytesRefVectorBuilder(8) + ) { + ords.appendInt(1); + ords.appendInt(0); + ords.appendInt(3); + ords.appendInt(1); + ords.appendInt(3); + ords.appendInt(0); + ords.appendInt(2); + ords.appendInt(3); + bytes.appendBytesRef(new BytesRef("item-1")); + bytes.appendBytesRef(new BytesRef("item-2")); + bytes.appendBytesRef(new BytesRef("item-3")); + bytes.appendBytesRef(new BytesRef("item-4")); + + hash(ordsAndKeys -> { + if (forcePackedHash) { + assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:BYTES_REF], entries=4, size=")); + assertThat(ordsAndKeys.description, endsWith("b}")); + assertOrds(ordsAndKeys.ords, 0, 1, 2, 0, 2, 1, 3, 2); + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 4))); + assertKeys(ordsAndKeys.keys, "item-2", "item-1", "item-4", "item-3"); + } else { + assertThat(ordsAndKeys.description, startsWith("BytesRefBlockHash{channel=0, entries=4, size=")); + assertThat(ordsAndKeys.description, endsWith("b, seenNull=false}")); + assertOrds(ordsAndKeys.ords, 2, 1, 4, 2, 4, 1, 3, 4); + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(1, 5))); + assertKeys(ordsAndKeys.keys, "item-1", "item-2", "item-3", "item-4"); + } + }, new OrdinalBytesRefVector(ords.build(), bytes.build()).asBlock()); + } + } + + public void testOrdinalsWithNulls() { + try ( + IntBlock.Builder ords = blockFactory.newIntBlockBuilder(4); + BytesRefVector.Builder bytes = blockFactory.newBytesRefVectorBuilder(2) + ) { + ords.appendInt(0); + ords.appendNull(); + ords.appendInt(1); + ords.appendNull(); + bytes.appendBytesRef(new BytesRef("cat")); + bytes.appendBytesRef(new BytesRef("dog")); + + hash(ordsAndKeys -> { + if (forcePackedHash) { + assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:BYTES_REF], entries=3, size=")); + assertThat(ordsAndKeys.description, endsWith("b}")); + assertOrds(ordsAndKeys.ords, 0, 1, 2, 1); + assertKeys(ordsAndKeys.keys, "cat", null, "dog"); + } else { + assertThat(ordsAndKeys.description, startsWith("BytesRefBlockHash{channel=0, entries=2, size=")); + assertThat(ordsAndKeys.description, endsWith("b, seenNull=true}")); + assertOrds(ordsAndKeys.ords, 1, 0, 2, 0); + assertKeys(ordsAndKeys.keys, null, "cat", "dog"); + } + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 3))); + }, new OrdinalBytesRefBlock(ords.build(), bytes.build())); + } + } + + public void testOrdinalsWithMultiValuedFields() { + try ( + IntBlock.Builder ords = blockFactory.newIntBlockBuilder(4); + BytesRefVector.Builder bytes = blockFactory.newBytesRefVectorBuilder(2) + ) { + ords.appendInt(0); + ords.beginPositionEntry(); + ords.appendInt(0); + ords.appendInt(1); + ords.endPositionEntry(); + ords.beginPositionEntry(); + ords.appendInt(1); + ords.appendInt(2); + ords.endPositionEntry(); + ords.beginPositionEntry(); + ords.appendInt(2); + ords.appendInt(1); + ords.endPositionEntry(); + ords.appendNull(); + ords.beginPositionEntry(); + ords.appendInt(2); + ords.appendInt(2); + ords.appendInt(1); + ords.endPositionEntry(); + + bytes.appendBytesRef(new BytesRef("foo")); + bytes.appendBytesRef(new BytesRef("bar")); + bytes.appendBytesRef(new BytesRef("bort")); + + hash(ordsAndKeys -> { + if (forcePackedHash) { + assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:BYTES_REF], entries=4, size=")); + assertThat(ordsAndKeys.description, endsWith("b}")); + assertOrds( + ordsAndKeys.ords, + new int[] { 0 }, + new int[] { 0, 1 }, + new int[] { 1, 2 }, + new int[] { 2, 1 }, + new int[] { 3 }, + new int[] { 2, 1 } + ); + assertKeys(ordsAndKeys.keys, "foo", "bar", "bort", null); + } else { + assertThat(ordsAndKeys.description, startsWith("BytesRefBlockHash{channel=0, entries=3, size=")); + assertThat(ordsAndKeys.description, endsWith("b, seenNull=true}")); + assertOrds( + ordsAndKeys.ords, + new int[] { 1 }, + new int[] { 1, 2 }, + new int[] { 2, 3 }, + new int[] { 3, 2 }, + new int[] { 0 }, + new int[] { 3, 2 } + ); + assertKeys(ordsAndKeys.keys, null, "foo", "bar", "bort"); + } + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 4))); + }, new OrdinalBytesRefBlock(ords.build(), bytes.build())); + } + } + public void testBooleanHashFalseFirst() { boolean[] values = new boolean[] { false, true, true, true, true }; hash(ordsAndKeys -> { @@ -1315,6 +1445,7 @@ public void close() { }); if (blockHash instanceof LongLongBlockHash == false && blockHash instanceof BytesRefLongBlockHash == false + && blockHash instanceof BytesRef2BlockHash == false && blockHash instanceof BytesRef3BlockHash == false) { Block[] keys = blockHash.getKeys(); try (ReleasableIterator lookup = blockHash.lookup(new Page(keys), ByteSizeValue.ofKb(between(1, 100)))) { diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AddGarbageRowsSourceOperator.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AddGarbageRowsSourceOperator.java new file mode 100644 index 0000000000000..079be87161421 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AddGarbageRowsSourceOperator.java @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.operator; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.test.ESTestCase; + +/** + * A {@link SourceOperator} that inserts random garbage into data from another + * {@link SourceOperator}. It also inserts an extra channel at the end of the page + * containing a {@code boolean} column. If it is {@code true} then the data came + * from the original operator. If it's {@code false} then the data is random + * garbage inserted by this operator. + */ +public class AddGarbageRowsSourceOperator extends SourceOperator { + public static EvalOperator.ExpressionEvaluator.Factory filterFactory() { + /* + * Grabs the filter from the last block. That's where we put it. + */ + return ctx -> new EvalOperator.ExpressionEvaluator() { + @Override + public Block eval(Page page) { + Block block = page.getBlock(page.getBlockCount() - 1); + block.incRef(); + return block; + } + + @Override + public void close() {} + }; + } + + private final SourceOperator next; + + public AddGarbageRowsSourceOperator(SourceOperator next) { + this.next = next; + } + + @Override + public void finish() { + next.finish(); + } + + @Override + public boolean isFinished() { + return next.isFinished(); + } + + @Override + public Page getOutput() { + Page page = next.getOutput(); + if (page == null) { + return null; + } + Block.Builder[] newBlocks = new Block.Builder[page.getBlockCount() + 1]; + try { + for (int b = 0; b < page.getBlockCount(); b++) { + Block block = page.getBlock(b); + newBlocks[b] = block.elementType().newBlockBuilder(page.getPositionCount(), block.blockFactory()); + } + newBlocks[page.getBlockCount()] = page.getBlock(0).blockFactory().newBooleanBlockBuilder(page.getPositionCount()); + + for (int p = 0; p < page.getPositionCount(); p++) { + if (ESTestCase.randomBoolean()) { + insertGarbageRows(newBlocks, page); + } + copyPosition(newBlocks, page, p); + if (ESTestCase.randomBoolean()) { + insertGarbageRows(newBlocks, page); + } + } + + return new Page(Block.Builder.buildAll(newBlocks)); + } finally { + Releasables.close(Releasables.wrap(newBlocks), page::releaseBlocks); + } + } + + private void copyPosition(Block.Builder[] newBlocks, Page page, int p) { + for (int b = 0; b < page.getBlockCount(); b++) { + Block block = page.getBlock(b); + newBlocks[b].copyFrom(block, p, p + 1); + } + signalKeep(newBlocks, true); + } + + private void insertGarbageRows(Block.Builder[] newBlocks, Page page) { + int count = ESTestCase.between(1, 5); + for (int c = 0; c < count; c++) { + insertGarbageRow(newBlocks, page); + } + } + + private void insertGarbageRow(Block.Builder[] newBlocks, Page page) { + for (int b = 0; b < page.getBlockCount(); b++) { + Block block = page.getBlock(b); + switch (block.elementType()) { + case BOOLEAN -> ((BooleanBlock.Builder) newBlocks[b]).appendBoolean(ESTestCase.randomBoolean()); + case BYTES_REF -> ((BytesRefBlock.Builder) newBlocks[b]).appendBytesRef(new BytesRef(ESTestCase.randomAlphaOfLength(5))); + case COMPOSITE, DOC, UNKNOWN -> throw new UnsupportedOperationException(); + case INT -> ((IntBlock.Builder) newBlocks[b]).appendInt(ESTestCase.randomInt()); + case LONG -> ((LongBlock.Builder) newBlocks[b]).appendLong(ESTestCase.randomLong()); + case NULL -> newBlocks[b].appendNull(); + case DOUBLE -> ((DoubleBlock.Builder) newBlocks[b]).appendDouble(ESTestCase.randomDouble()); + case FLOAT -> ((FloatBlock.Builder) newBlocks[b]).appendFloat(ESTestCase.randomFloat()); + } + } + signalKeep(newBlocks, false); + } + + private void signalKeep(Block.Builder[] newBlocks, boolean shouldKeep) { + ((BooleanBlock.Builder) newBlocks[newBlocks.length - 1]).appendBoolean(shouldKeep); + } + + @Override + public void close() { + next.close(); + } +} diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java index 454f3962c07ea..1f72827057c5b 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java @@ -127,8 +127,18 @@ void indexDocs(RestClient client, String index, List docs) throws IOExcepti refresh(client, index); } - private Map run(String query) throws IOException { - Map resp = runEsql(new RestEsqlTestCase.RequestObjectBuilder().query(query).build()); + private Map run(String query, boolean includeCCSMetadata) throws IOException { + Map resp = runEsql( + new RestEsqlTestCase.RequestObjectBuilder().query(query).includeCCSMetadata(includeCCSMetadata).build() + ); + logger.info("--> query {} response {}", query, resp); + return resp; + } + + private Map runWithColumnarAndIncludeCCSMetadata(String query) throws IOException { + Map resp = runEsql( + new RestEsqlTestCase.RequestObjectBuilder().query(query).includeCCSMetadata(true).columnar(true).build() + ); logger.info("--> query {} response {}", query, resp); return resp; } @@ -147,62 +157,77 @@ private Map runEsql(RestEsqlTestCase.RequestObjectBuilder reques public void testCount() throws Exception { { - Map result = run("FROM test-local-index,*:test-remote-index | STATS c = COUNT(*)"); + boolean includeCCSMetadata = randomBoolean(); + Map result = run("FROM test-local-index,*:test-remote-index | STATS c = COUNT(*)", includeCCSMetadata); var columns = List.of(Map.of("name", "c", "type", "long")); var values = List.of(List.of(localDocs.size() + remoteDocs.size())); MapMatcher mapMatcher = matchesMap(); - assertMap( - result, - mapMatcher.entry("columns", columns) - .entry("values", values) - .entry("took", greaterThanOrEqualTo(0)) - .entry("_clusters", any(Map.class)) - ); - assertClusterDetailsMap(result, false); + if (includeCCSMetadata) { + mapMatcher = mapMatcher.entry("_clusters", any(Map.class)); + } + assertMap(result, mapMatcher.entry("columns", columns).entry("values", values).entry("took", greaterThanOrEqualTo(0))); + if (includeCCSMetadata) { + assertClusterDetailsMap(result, false); + } } { - Map result = run("FROM *:test-remote-index | STATS c = COUNT(*)"); + boolean includeCCSMetadata = randomBoolean(); + Map result = run("FROM *:test-remote-index | STATS c = COUNT(*)", includeCCSMetadata); var columns = List.of(Map.of("name", "c", "type", "long")); var values = List.of(List.of(remoteDocs.size())); MapMatcher mapMatcher = matchesMap(); - assertMap( - result, - mapMatcher.entry("columns", columns) - .entry("values", values) - .entry("took", greaterThanOrEqualTo(0)) - .entry("_clusters", any(Map.class)) - ); - assertClusterDetailsMap(result, true); + if (includeCCSMetadata) { + mapMatcher = mapMatcher.entry("_clusters", any(Map.class)); + } + assertMap(result, mapMatcher.entry("columns", columns).entry("values", values).entry("took", greaterThanOrEqualTo(0))); + if (includeCCSMetadata) { + assertClusterDetailsMap(result, true); + } } } public void testUngroupedAggs() throws Exception { { - Map result = run("FROM test-local-index,*:test-remote-index | STATS total = SUM(data)"); + boolean includeCCSMetadata = randomBoolean(); + Map result = run("FROM test-local-index,*:test-remote-index | STATS total = SUM(data)", includeCCSMetadata); var columns = List.of(Map.of("name", "total", "type", "long")); long sum = Stream.concat(localDocs.stream(), remoteDocs.stream()).mapToLong(d -> d.data).sum(); var values = List.of(List.of(Math.toIntExact(sum))); // check all sections of map except _cluster/details MapMatcher mapMatcher = matchesMap(); - assertMap( - result, - mapMatcher.entry("columns", columns) - .entry("values", values) - .entry("took", greaterThanOrEqualTo(0)) - .entry("_clusters", any(Map.class)) - ); - assertClusterDetailsMap(result, false); + if (includeCCSMetadata) { + mapMatcher = mapMatcher.entry("_clusters", any(Map.class)); + } + assertMap(result, mapMatcher.entry("columns", columns).entry("values", values).entry("took", greaterThanOrEqualTo(0))); + if (includeCCSMetadata) { + assertClusterDetailsMap(result, false); + } } { - Map result = run("FROM *:test-remote-index | STATS total = SUM(data)"); + boolean includeCCSMetadata = randomBoolean(); + Map result = run("FROM *:test-remote-index | STATS total = SUM(data)", includeCCSMetadata); + var columns = List.of(Map.of("name", "total", "type", "long")); + long sum = remoteDocs.stream().mapToLong(d -> d.data).sum(); + var values = List.of(List.of(Math.toIntExact(sum))); + + MapMatcher mapMatcher = matchesMap(); + if (includeCCSMetadata) { + mapMatcher = mapMatcher.entry("_clusters", any(Map.class)); + } + assertMap(result, mapMatcher.entry("columns", columns).entry("values", values).entry("took", greaterThanOrEqualTo(0))); + if (includeCCSMetadata) { + assertClusterDetailsMap(result, true); + } + } + { + Map result = runWithColumnarAndIncludeCCSMetadata("FROM *:test-remote-index | STATS total = SUM(data)"); var columns = List.of(Map.of("name", "total", "type", "long")); long sum = remoteDocs.stream().mapToLong(d -> d.data).sum(); var values = List.of(List.of(Math.toIntExact(sum))); - // check all sections of map except _cluster/details MapMatcher mapMatcher = matchesMap(); assertMap( result, @@ -269,7 +294,11 @@ private void assertClusterDetailsMap(Map result, boolean remoteO public void testGroupedAggs() throws Exception { { - Map result = run("FROM test-local-index,*:test-remote-index | STATS total = SUM(data) BY color | SORT color"); + boolean includeCCSMetadata = randomBoolean(); + Map result = run( + "FROM test-local-index,*:test-remote-index | STATS total = SUM(data) BY color | SORT color", + includeCCSMetadata + ); var columns = List.of(Map.of("name", "total", "type", "long"), Map.of("name", "color", "type", "keyword")); var values = Stream.concat(localDocs.stream(), remoteDocs.stream()) .collect(Collectors.toMap(d -> d.color, Doc::data, Long::sum)) @@ -280,17 +309,20 @@ public void testGroupedAggs() throws Exception { .toList(); MapMatcher mapMatcher = matchesMap(); - assertMap( - result, - mapMatcher.entry("columns", columns) - .entry("values", values) - .entry("took", greaterThanOrEqualTo(0)) - .entry("_clusters", any(Map.class)) - ); - assertClusterDetailsMap(result, false); + if (includeCCSMetadata) { + mapMatcher = mapMatcher.entry("_clusters", any(Map.class)); + } + assertMap(result, mapMatcher.entry("columns", columns).entry("values", values).entry("took", greaterThanOrEqualTo(0))); + if (includeCCSMetadata) { + assertClusterDetailsMap(result, false); + } } { - Map result = run("FROM *:test-remote-index | STATS total = SUM(data) by color | SORT color"); + boolean includeCCSMetadata = randomBoolean(); + Map result = run( + "FROM *:test-remote-index | STATS total = SUM(data) by color | SORT color", + includeCCSMetadata + ); var columns = List.of(Map.of("name", "total", "type", "long"), Map.of("name", "color", "type", "keyword")); var values = remoteDocs.stream() .collect(Collectors.toMap(d -> d.color, Doc::data, Long::sum)) @@ -300,16 +332,15 @@ public void testGroupedAggs() throws Exception { .map(e -> List.of(Math.toIntExact(e.getValue()), e.getKey())) .toList(); - // check all sections of map except _cluster/details + // check all sections of map except _clusters/details MapMatcher mapMatcher = matchesMap(); - assertMap( - result, - mapMatcher.entry("columns", columns) - .entry("values", values) - .entry("took", greaterThanOrEqualTo(0)) - .entry("_clusters", any(Map.class)) - ); - assertClusterDetailsMap(result, true); + if (includeCCSMetadata) { + mapMatcher = mapMatcher.entry("_clusters", any(Map.class)); + } + assertMap(result, mapMatcher.entry("columns", columns).entry("values", values).entry("took", greaterThanOrEqualTo(0))); + if (includeCCSMetadata) { + assertClusterDetailsMap(result, true); + } } } diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java index 974180526d750..7de4ee4ccae28 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java @@ -45,6 +45,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.startsWith; @@ -75,6 +76,7 @@ public void testBasicEsql() throws IOException { indexTimestampData(1); RequestObjectBuilder builder = requestObjectBuilder().query(fromIndex() + " | stats avg(value)"); + requestObjectBuilder().includeCCSMetadata(randomBoolean()); if (Build.current().isSnapshot()) { builder.pragmas(Settings.builder().put("data_partitioning", "shard").build()); } @@ -331,14 +333,13 @@ public void testProfile() throws IOException { } public void testProfileOrdinalsGroupingOperator() throws IOException { + assumeTrue("requires pragmas", Build.current().isSnapshot()); indexTimestampData(1); RequestObjectBuilder builder = requestObjectBuilder().query(fromIndex() + " | STATS AVG(value) BY test.keyword"); builder.profile(true); - if (Build.current().isSnapshot()) { - // Lock to shard level partitioning, so we get consistent profile output - builder.pragmas(Settings.builder().put("data_partitioning", "shard").build()); - } + // Lock to shard level partitioning, so we get consistent profile output + builder.pragmas(Settings.builder().put("data_partitioning", "shard").build()); Map result = runEsql(builder); List> signatures = new ArrayList<>(); @@ -356,7 +357,7 @@ public void testProfileOrdinalsGroupingOperator() throws IOException { signatures.add(sig); } - assertThat(signatures.get(0).get(2), equalTo("OrdinalsGroupingOperator[aggregators=[\"sum of longs\", \"count\"]]")); + assertThat(signatures, hasItem(hasItem("OrdinalsGroupingOperator[aggregators=[\"sum of longs\", \"count\"]]"))); } public void testInlineStatsProfile() throws IOException { @@ -423,11 +424,9 @@ public void testInlineStatsProfile() throws IOException { .item("ProjectOperator") .item("OutputOperator"), // Second pass read and join via eval - matchesList().item("LuceneSourceOperator") + matchesList().item("LuceneTopNSourceOperator") .item("EvalOperator") .item("ValuesSourceReaderOperator") - .item("TopNOperator") - .item("ValuesSourceReaderOperator") .item("ProjectOperator") .item("ExchangeSinkOperator"), // Second pass node level reduce @@ -591,6 +590,16 @@ private String checkOperatorProfile(Map o) { case "TopNOperator" -> matchesMap().entry("occupied_rows", 0) .entry("ram_used", instanceOf(String.class)) .entry("ram_bytes_used", greaterThan(0)); + case "LuceneTopNSourceOperator" -> matchesMap().entry("pages_emitted", greaterThan(0)) + .entry("current", greaterThan(0)) + .entry("processed_slices", greaterThan(0)) + .entry("processed_shards", List.of("rest-esql-test:0")) + .entry("total_slices", greaterThan(0)) + .entry("slice_max", 0) + .entry("slice_min", 0) + .entry("processing_nanos", greaterThan(0)) + .entry("processed_queries", List.of("*:*")) + .entry("slice_index", 0); default -> throw new AssertionError("unexpected status: " + o); }; MapMatcher expectedOp = matchesMap().entry("operator", startsWith(name)); diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java index 8163e73078c71..4fa6ac3009654 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java @@ -128,6 +128,7 @@ public static class RequestObjectBuilder { private Boolean keepOnCompletion = null; private Boolean profile = null; + private Boolean includeCCSMetadata = null; private CheckedConsumer filter; @@ -197,6 +198,11 @@ public RequestObjectBuilder profile(boolean profile) { return this; } + public RequestObjectBuilder includeCCSMetadata(boolean includeCCSMetadata) { + this.includeCCSMetadata = includeCCSMetadata; + return this; + } + public RequestObjectBuilder filter(CheckedConsumer filter) { this.filter = filter; return this; @@ -220,6 +226,9 @@ public RequestObjectBuilder build() throws IOException { if (profile != null) { builder.field("profile", profile); } + if (includeCCSMetadata != null) { + builder.field("include_ccs_metadata", includeCCSMetadata); + } if (filter != null) { builder.startObject("filter"); filter.accept(builder); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec index 1fdb6150a0e81..36035c48f182c 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec @@ -369,7 +369,7 @@ date1:date | dd_ms:integer evalDateDiffMonthAsWhole0Months -ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-01-01T00:00:00"::DATETIME +ROW from=TO_DATETIME("2023-12-31T23:59:59.999Z"), to=TO_DATETIME("2024-01-01T00:00:00") | EVAL msecs=DATE_DIFF("milliseconds", from, to), months=DATE_DIFF("month", from, to) ; @@ -380,7 +380,7 @@ ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-01-01T00:00:00"::DATETIM evalDateDiffMonthAsWhole1Month -ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-02-01T00:00:00"::DATETIME +ROW from=TO_DATETIME("2023-12-31T23:59:59.999Z"), to=TO_DATETIME("2024-02-01T00:00:00") | EVAL secs=DATE_DIFF("seconds", from, to), months=DATE_DIFF("month", from, to) ; @@ -392,7 +392,7 @@ ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-02-01T00:00:00"::DATETIM evalDateDiffYearAsWhole0Years required_capability: date_diff_year_calendarial -ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-01-01T00:00:00"::DATETIME +ROW from=TO_DATETIME("2023-12-31T23:59:59.999Z"), to=TO_DATETIME("2024-01-01T00:00:00") | EVAL msecs=DATE_DIFF("milliseconds", from, to), years=DATE_DIFF("year", from, to) ; @@ -403,7 +403,7 @@ ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-01-01T00:00:00"::DATETIM evalDateDiffYearAsWhole1Year required_capability: date_diff_year_calendarial -ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2025-01-01T00:00:00"::DATETIME +ROW from=TO_DATETIME("2023-12-31T23:59:59.999Z"), to=TO_DATETIME("2025-01-01T00:00:00") | EVAL secs=DATE_DIFF("seconds", from, to), years=DATE_DIFF("year", from, to) ; @@ -414,7 +414,7 @@ ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2025-01-01T00:00:00"::DATETIM evalDateDiffYearAsWhole1Year required_capability: date_diff_year_calendarial -ROW from="2024-01-01T00:00:00Z"::DATETIME, to="2025-01-01T00:00:00"::DATETIME +ROW from=TO_DATETIME("2024-01-01T00:00:00Z"), to=TO_DATETIME("2025-01-01T00:00:00") | EVAL secs=DATE_DIFF("seconds", from, to), years=DATE_DIFF("year", from, to) ; @@ -426,9 +426,9 @@ evalDateDiffYearForDocs required_capability: date_diff_year_calendarial // tag::evalDateDiffYearForDocs[] -ROW end_23="2023-12-31T23:59:59.999Z"::DATETIME, - start_24="2024-01-01T00:00:00.000Z"::DATETIME, - end_24="2024-12-31T23:59:59.999"::DATETIME +ROW end_23=TO_DATETIME("2023-12-31T23:59:59.999Z"), + start_24=TO_DATETIME("2024-01-01T00:00:00.000Z"), + end_24=TO_DATETIME("2024-12-31T23:59:59.999") | EVAL end23_to_start24=DATE_DIFF("year", end_23, start_24) | EVAL end23_to_end24=DATE_DIFF("year", end_23, end_24) | EVAL start_to_end_24=DATE_DIFF("year", start_24, end_24) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec new file mode 100644 index 0000000000000..b0578aa1a4ed0 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec @@ -0,0 +1,199 @@ +############################################### +# Tests for Match function +# + +matchWithField +required_capability: match_function + +// tag::match-with-field[] +from books +| where match(author, "Faulkner") +| keep book_no, author +| sort book_no +| limit 5; +// end::match-with-field[] + +// tag::match-with-field-result[] +book_no:keyword | author:text +2378 | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] +2713 | William Faulkner +2847 | Colleen Faulkner +2883 | William Faulkner +3293 | Danny Faulkner +; +// end::match-with-field-result[] + +matchWithMultipleFunctions +required_capability: match_function + +from books +| where match(title, "Return") AND match(author, "Tolkien") +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +2714 | Return of the King Being the Third Part of The Lord of the Rings +7350 | Return of the Shadow +; + +matchWithQueryExpressions +required_capability: match_function + +from books +| where match(title, CONCAT("Return ", " King")) +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +2714 | Return of the King Being the Third Part of The Lord of the Rings +7350 | Return of the Shadow +; + +matchAfterKeep +required_capability: match_function + +from books +| keep book_no, author +| where match(author, "Faulkner") +| sort book_no +| limit 5; + +book_no:keyword | author:text +2378 | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] +2713 | William Faulkner +2847 | Colleen Faulkner +2883 | William Faulkner +3293 | Danny Faulkner +; + +matchAfterDrop +required_capability: match_function + +from books +| drop ratings, description, year, publisher, title, author.keyword +| where match(author, "Faulkner") +| keep book_no, author +| sort book_no +| limit 5; + +book_no:keyword | author:text +2378 | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] +2713 | William Faulkner +2847 | Colleen Faulkner +2883 | William Faulkner +3293 | Danny Faulkner +; + +matchAfterEval +required_capability: match_function + +from books +| eval stars = to_long(ratings / 2.0) +| where match(author, "Faulkner") +| sort book_no +| keep book_no, author, stars +| limit 5; + +book_no:keyword | author:text | stars:long +2378 | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] | 3 +2713 | William Faulkner | 2 +2847 | Colleen Faulkner | 3 +2883 | William Faulkner | 2 +3293 | Danny Faulkner | 2 +; + +matchWithConjunction +required_capability: match_function + +from books +| where match(title, "Rings") and ratings > 4.6 +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +4023 |A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings +7140 |The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1) +; + +matchWithFunctionPushedToLucene +required_capability: match_function + +from hosts +| where match(host, "beta") and cidr_match(ip1, "127.0.0.2/32", "127.0.0.3/32") +| keep card, host, ip0, ip1; +ignoreOrder:true + +card:keyword |host:keyword |ip0:ip |ip1:ip +eth1 |beta |127.0.0.1 |127.0.0.2 +; + +matchWithNonPushableConjunction +required_capability: match_function + +from books +| where match(title, "Rings") and length(title) > 75 +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +4023 | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings +; + +matchWithMultipleWhereClauses +required_capability: match_function + +from books +| where match(title, "rings") +| where match(title, "lord") +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +2675 | The Lord of the Rings - Boxed Set +2714 | Return of the King Being the Third Part of The Lord of the Rings +4023 | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings +7140 | The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1) +; + +matchMultivaluedField +required_capability: match_function + +from employees +| where match(job_positions, "Tech Lead") and match(job_positions, "Reporting Analyst") +| keep emp_no, first_name, last_name; +ignoreOrder:true + +emp_no:integer | first_name:keyword | last_name:keyword +10004 | Chirstian | Koblick +10010 | Duangkaew | Piveteau +10011 | Mary | Sluis +10088 | Jungsoon | Syrzycki +10093 | Sailaja | Desikan +10097 | Remzi | Waschkowski +; + +testMultiValuedFieldWithConjunction +required_capability: match_function + +from employees +| where match(job_positions, "Data Scientist") and match(job_positions, "Support Engineer") +| keep emp_no, first_name, last_name; +ignoreOrder:true + +emp_no:integer | first_name:keyword | last_name:keyword +10043 | Yishay | Tzvieli +; + +testMatchAndQueryStringFunctions +required_capability: match_function +required_capability: qstr_function + +from employees +| where match(job_positions, "Data Scientist") and qstr("job_positions: (Support Engineer) and gender: F") +| keep emp_no, first_name, last_name; +ignoreOrder:true + +emp_no:integer | first_name:keyword | last_name:keyword +10041 | Uri | Lenart +10043 | Yishay | Tzvieli +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/math.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/math.csv-spec index b00bb5143726c..da069836504d4 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/math.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/math.csv-spec @@ -1379,6 +1379,63 @@ d:double | c:double -0.0 | -0.0 ; +hypot +required_capability: fn_hypot +// tag::hypot[] +ROW a = 3.0, b = 4.0 +| EVAL c = HYPOT(a, b) +// end::hypot[] +; + +// tag::hypot-result[] +a:double | b:double | c:double +3.0 | 4.0 | 5.0 +// end::hypot-result[] +; + +hypotWithFrom +required_capability: fn_hypot +FROM ul_logs +| WHERE id > 95 +| EVAL bytes_hypot = HYPOT(bytes_in, bytes_out) +| SORT id ASC +| LIMIT 5 +| KEEP id, bytes_in, bytes_out, bytes_hypot +; + +id:integer | bytes_in:unsigned_long | bytes_out:unsigned_long | bytes_hypot:double +96 | 9932469097722733505 | 14925592145374204307 | 1.792839209932874E19 +97 | 11620953158540412267 | 3809712277266935082 | 1.2229491401875583E19 +98 | 3448205404634246112 | 5409549730889481641 | 6.415087591258227E18 +99 | 1957665857956635540 | 352442273299370793 | 1.9891382977102218E18 +100 | 16462768484251021236 | 15616395223975497926 | 2.2691287886707827E19 +; + +hypotBothNull +required_capability: fn_hypot +FROM ul_logs +| WHERE bytes_in IS NULL and bytes_out IS NULL +| LIMIT 1 +| EVAL bytes_hypot = HYPOT(bytes_in, bytes_out) +| KEEP bytes_in, bytes_out, bytes_hypot +; + +bytes_in:unsigned_long | bytes_out:unsigned_long | bytes_hypot:double +null | null | null +; + +hypotOneNull +required_capability: fn_hypot +FROM ul_logs +| WHERE id == 41 +| EVAL confused_hypot = HYPOT(id, bytes_in) +| KEEP id, bytes_in, confused_hypot +; + +id:integer | bytes_in:unsigned_long | confused_hypot:double +41 | null | null +; + least // tag::least[] ROW a = 10, b = 20 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec index 2f6313925032e..6dc03d0debcfa 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec @@ -49,20 +49,6 @@ book_no:keyword | title:text 7350 | Return of the Shadow ; -qstrWithDisjunction -required_capability: qstr_function - -from books -| where qstr("title:Return") or year > 2020 -| keep book_no, title; -ignoreOrder:true - -book_no:keyword | title:text -2714 | Return of the King Being the Third Part of The Lord of the Rings -6818 | Hadji Murad -7350 | Return of the Shadow -; - qstrWithConjunction required_capability: qstr_function @@ -88,17 +74,16 @@ card:keyword |host:keyword |ip0:ip |ip1:ip eth1 |beta |127.0.0.1 |127.0.0.2 ; -qstrWithFunctionNotPushedToLucene +qstrWithNonPushableConjunction required_capability: qstr_function from books -| where qstr("title: rings") and length(description) > 600 +| where qstr("title: Rings") and length(title) > 75 | keep book_no, title; ignoreOrder:true book_no:keyword | title:text -2675 | The Lord of the Rings - Boxed Set -2714 | Return of the King Being the Third Part of The Lord of the Rings +4023 |A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings ; qstrWithMultipleWhereClauses @@ -114,3 +99,55 @@ book_no:keyword | title:text 4023 | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings 7140 | The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1) ; + + +matchMultivaluedTextField +required_capability: match_function + +from employees +| where qstr("job_positions: (Tech Lead) AND job_positions:(Reporting Analyst)") +| keep emp_no, first_name, last_name; +ignoreOrder:true + +emp_no:integer | first_name:keyword | last_name:keyword +10004 | Chirstian | Koblick +10010 | Duangkaew | Piveteau +10011 | Mary | Sluis +10088 | Jungsoon | Syrzycki +10093 | Sailaja | Desikan +10097 | Remzi | Waschkowski +; + +matchMultivaluedNumericField +required_capability: match_function + +from employees +| where qstr("salary_change: [14 TO *]") +| keep emp_no, first_name, last_name, salary_change; +ignoreOrder:true + +emp_no:integer | first_name:keyword | last_name:keyword | salary_change:double +10003 | Parto | Bamford | [12.82, 14.68] +10015 | Guoxiang | Nooteboom | [12.4, 14.25] +10023 | Bojan | Montemayor | [0.8, 14.63] +10040 | Weiyi | Meriste | [-8.94, 1.92, 6.97, 14.74] +10061 | Tse | Herber | [-2.58, -0.95, 14.39] +10065 | Satosi | Awdeh | [-9.81, -1.47, 14.44] +10099 | Valter | Sullins | [-8.78, -3.98, 10.71, 14.26] +; + +testMultiValuedFieldWithConjunction +required_capability: match_function + +from employees +| where (qstr("job_positions: (Data Scientist) OR job_positions:(Support Engineer)")) and gender == "F" +| keep emp_no, first_name, last_name; +ignoreOrder:true + +emp_no:integer | first_name:keyword | last_name:keyword +10023 | Bojan | Montemayor +10041 | Uri | Lenart +10044 | Mingsen | Casley +10053 | Sanjiv | Zschoche +10069 | Margareta | Bierman +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec index 35416c7945128..4c40808a4ff96 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec @@ -1116,9 +1116,147 @@ count:long | country:k 1 | Poland ; +airportsWithinEvalDistanceBandCopenhagenTrainStationCount +required_capability: st_distance + +FROM airports +| EVAL distance = ST_DISTANCE(location, TO_GEOPOINT("POINT(12.565 55.673)")) +| WHERE distance < 600000 AND distance > 400000 +| STATS count=COUNT() BY country +| SORT count DESC, country ASC +; + +count:long | country:k +3 | Sweden +2 | Norway +1 | Germany +1 | Lithuania +1 | Poland +; + +airportsWithinEvalDistanceBandCopenhagenTrainStationKeepDistance +required_capability: st_distance + +FROM airports +| EVAL distance = ST_DISTANCE(location, TO_GEOPOINT("POINT(12.565 55.673)")), importance = 10 - scalerank +| WHERE distance < 500000 AND distance > 400000 +| STATS count=COUNT() BY distance, importance +| SORT distance ASC, importance DESC, count DESC +; + +count:long | distance:double | importance:integer +1 | 402611.1308019835 | 4 +1 | 433987.3301951482 | 3 +; + +airportsWithinEvalDistanceBandCopenhagenTrainStationCountNonPushableEval +required_capability: st_distance + +FROM airports +| EVAL distance = ST_DISTANCE(location, TO_GEOPOINT("POINT(12.565 55.673)")), position = location::keyword +| WHERE distance < 600000 AND distance > 400000 AND SUBSTRING(position, 1, 5) == "POINT" +| STATS count=COUNT() BY country +| SORT count DESC, country ASC +; + +count:long | country:k +3 | Sweden +2 | Norway +1 | Germany +1 | Lithuania +1 | Poland +; + +airportsWithinEvalDistanceBandCopenhagenTrainStationCountNonPushableWhereConjunctive +required_capability: st_distance + +FROM airports +| EVAL distance = ST_DISTANCE(location, TO_GEOPOINT("POINT(12.565 55.673)")) +| WHERE distance < 500000 AND 0.5*distance < 300000 +| STATS count=COUNT() BY country +| SORT count DESC, country ASC +; + +count:long | country:k +3 | Germany +3 | Sweden +1 | Denmark +1 | Poland +; + +airportsWithinEvalDistanceBandCopenhagenTrainStationCountNonPushableWhereDisjunctive +required_capability: st_distance + +FROM airports +| EVAL distance = ST_DISTANCE(location, TO_GEOPOINT("POINT(12.565 55.673)")) +| WHERE distance < 500000 OR 0.5*distance < 300000 +| STATS count=COUNT() BY country +| SORT count DESC, country ASC +; + +count:long | country:k +5 | Sweden +4 | Germany +2 | Norway +1 | Denmark +1 | Lithuania +1 | Poland +; + +airportsSortCityName +FROM airports +| SORT abbrev +| LIMIT 5 +| KEEP abbrev, name, location, country, city +; + +abbrev:keyword | name:text | location:geo_point | country:keyword | city:keyword +ABJ | Abidjan Port Bouet | POINT(-3.93221929167636 5.2543984451492) | Côte d'Ivoire | Abidjan +ABQ | Albuquerque Int'l | POINT(-106.6166851616 35.0491578018276) | United States | Albuquerque +ABV | Abuja Int'l | POINT(7.27025993974356 9.00437659781094) | Nigeria | Abuja +ACA | General Juan N Alvarez Int'l | POINT(-99.7545085619681 16.76196735278) | Mexico | Acapulco de Juárez +ACC | Kotoka Int'l | POINT(-0.171402855660817 5.60698152381193) | Ghana | Accra +; + airportsSortDistanceFromCopenhagenTrainStation required_capability: st_distance +FROM airports +| EVAL distance = ST_DISTANCE(location, TO_GEOPOINT("POINT(12.565 55.673)")) +| SORT distance ASC +| LIMIT 5 +| KEEP abbrev, name, location, country, city +; + +abbrev:k | name:text | location:geo_point | country:k | city:k +CPH | Copenhagen | POINT(12.6493508684508 55.6285017221528) | Denmark | Copenhagen +GOT | Gothenburg | POINT(12.2938269092573 57.6857493534879) | Sweden | Gothenburg +HAM | Hamburg | POINT(10.005647830925 53.6320011640866) | Germany | Norderstedt +TXL | Berlin-Tegel Int'l | POINT(13.2903090925074 52.5544287044101) | Germany | Hohen Neuendorf +BRE | Bremen | POINT(8.7858617703132 53.052287104156) | Germany | Bremen +; + +airportsSortDistanceFromCopenhagenTrainStationInline +required_capability: st_distance +required_capability: spatial_distance_pushdown_enhancements + +FROM airports +| SORT ST_DISTANCE(location, TO_GEOPOINT("POINT(12.565 55.673)")) ASC +| LIMIT 5 +| KEEP abbrev, name, location, country, city +; + +abbrev:k | name:text | location:geo_point | country:k | city:k +CPH | Copenhagen | POINT(12.6493508684508 55.6285017221528) | Denmark | Copenhagen +GOT | Gothenburg | POINT(12.2938269092573 57.6857493534879) | Sweden | Gothenburg +HAM | Hamburg | POINT(10.005647830925 53.6320011640866) | Germany | Norderstedt +TXL | Berlin-Tegel Int'l | POINT(13.2903090925074 52.5544287044101) | Germany | Hohen Neuendorf +BRE | Bremen | POINT(8.7858617703132 53.052287104156) | Germany | Bremen +; + +airportsSortDistanceFromCopenhagenTrainStationDetails +required_capability: st_distance + FROM airports | EVAL distance = ST_DISTANCE(location, TO_GEOPOINT("POINT(12.565 55.673)")) | SORT distance ASC @@ -1136,6 +1274,26 @@ TXL | Berlin-Tegel Int'l | POINT(13.2903090925074 52.5544287044101) | BRE | Bremen | POINT(8.7858617703132 53.052287104156) | Germany | Bremen | POINT(8.8 53.0833) | 380.5 | 377.22 ; +airportsSortDistanceFromCopenhagenTrainStationDetailsAndNonPushableEval +required_capability: st_distance + +FROM airports +| EVAL distance = ST_DISTANCE(location, TO_GEOPOINT("POINT(12.565 55.673)")), position = location::keyword +| WHERE distance < 600000 AND SUBSTRING(position, 1, 5) == "POINT" +| SORT distance ASC +| LIMIT 5 +| EVAL distance = ROUND(distance/1000,2) +| KEEP abbrev, name, position, distance +; + +abbrev:k | name:text | position:keyword | distance:d +CPH | Copenhagen | POINT (12.6493508684508 55.6285017221528) | 7.24 +GOT | Gothenburg | POINT (12.2938269092573 57.6857493534879) | 224.42 +HAM | Hamburg | POINT (10.005647830925 53.6320011640866) | 280.34 +TXL | Berlin-Tegel Int'l | POINT (13.2903090925074 52.5544287044101) | 349.97 +BRE | Bremen | POINT (8.7858617703132 53.052287104156) | 380.5 +; + airportsSortDistanceFromAirportToCity required_capability: st_distance diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/topN.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/topN.csv-spec index e09bc933340d1..3d4d890546050 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/topN.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/topN.csv-spec @@ -134,3 +134,37 @@ Otmar |Herbst |[-8.19, -1.9, -0.32] |[-0.32, -1.90, null |Swan |-8.46 |-8.46 |-8 |10034 Sanjiv |Zschoche |[-7.67, -3.25] |[-3.25, -7.67] |[-3, -8] |10053 ; + +sortingOnSwappedFields +FROM employees +| EVAL name = last_name, last_name = first_name, first_name = name +| WHERE first_name > "B" AND last_name IS NOT NULL +| SORT name +| LIMIT 10 +| KEEP name, last_name, first_name +; + +name:keyword | last_name:keyword | first_name:keyword +Baek | Premal | Baek +Bamford | Parto | Bamford +Bernatsky | Mokhtar | Bernatsky +Bernini | Brendon | Bernini +Berztiss | Yongqiao | Berztiss +Bierman | Margareta | Bierman +Billingsley | Breannda | Billingsley +Bouloucos | Cristinel | Bouloucos +Brattka | Charlene | Brattka +Bridgland | Patricio | Bridgland +; + +sortingOnSwappedFieldsNoKeep +// Note that this test requires all fields to be returned in order to test a specific code path in physical planning +FROM employees +| EVAL name = first_name, first_name = last_name, last_name = name +| WHERE first_name == "Bernini" AND last_name == "Brendon" +| SORT name +; + +avg_worked_seconds:long | birth_date:date | emp_no:i | gender:k | height:d | height.float:d | height.half_float:d | height.scaled_float:d | hire_date:date | is_rehired:bool | job_positions:k | languages:i | languages.byte:i | languages.long:l | languages.short:short | salary:i | salary_change:d | salary_change.int:i | salary_change.keyword:k | salary_change.long:l | still_hired:bool | name:k | first_name:k | last_name:k +349086555 | 1961-09-01T00:00:00Z | 10056 | F | 1.57 | 1.5700000524520874 | 1.5703125 | 1.57 | 1990-02-01T00:00:00Z | [false, false, true] | [Senior Team Lead] | 2 | 2 | 2 | 2 | 33370 | [-5.17, 10.99] | [-5, 10] | [-5.17, 10.99] | [-5, 10] | true | Brendon | Bernini | Brendon +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/tsdb-mapping.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/tsdb-mapping.json index dd4073d5dc7cf..39b1b10edd916 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/tsdb-mapping.json +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/tsdb-mapping.json @@ -27,6 +27,10 @@ "message_in": { "type": "float", "time_series_metric": "counter" + }, + "message_out": { + "type": "integer", + "time_series_metric": "counter" } } } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java index 452d56680e8da..7d8bb738098d3 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java @@ -14,6 +14,7 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.core.Tuple; import org.elasticsearch.ingest.common.IngestCommonPlugin; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.license.LicenseService; @@ -220,7 +221,7 @@ static String enrichVendors(Enrich.Mode mode) { public void testWithHostsPolicy() { for (var mode : Enrich.Mode.values()) { String query = "FROM events | eval ip= TO_STR(host) | " + enrichHosts(mode) + " | stats c = COUNT(*) by os | SORT os"; - try (EsqlQueryResponse resp = runQuery(query)) { + try (EsqlQueryResponse resp = runQuery(query, null)) { List> rows = getValuesList(resp); assertThat( rows, @@ -237,9 +238,14 @@ public void testWithHostsPolicy() { assertFalse(resp.getExecutionInfo().isCrossClusterSearch()); } } + + Tuple includeCCSMetadata = randomIncludeCCSMetadata(); + Boolean requestIncludeMeta = includeCCSMetadata.v1(); + boolean responseExpectMeta = includeCCSMetadata.v2(); + for (var mode : Enrich.Mode.values()) { String query = "FROM *:events | eval ip= TO_STR(host) | " + enrichHosts(mode) + " | stats c = COUNT(*) by os | SORT os"; - try (EsqlQueryResponse resp = runQuery(query)) { + try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) { List> rows = getValuesList(resp); assertThat( rows, @@ -255,6 +261,7 @@ public void testWithHostsPolicy() { ) ); EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); assertThat(executionInfo.clusterAliases(), equalTo(Set.of("c1", "c2"))); assertCCSExecutionInfoDetails(executionInfo); } @@ -262,7 +269,7 @@ public void testWithHostsPolicy() { for (var mode : Enrich.Mode.values()) { String query = "FROM *:events,events | eval ip= TO_STR(host) | " + enrichHosts(mode) + " | stats c = COUNT(*) by os | SORT os"; - try (EsqlQueryResponse resp = runQuery(query)) { + try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) { List> rows = getValuesList(resp); assertThat( rows, @@ -278,6 +285,7 @@ public void testWithHostsPolicy() { ) ); EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); assertThat(executionInfo.clusterAliases(), equalTo(Set.of("", "c1", "c2"))); assertCCSExecutionInfoDetails(executionInfo); } @@ -285,6 +293,10 @@ public void testWithHostsPolicy() { } public void testEnrichHostsAggThenEnrichVendorCoordinator() { + Tuple includeCCSMetadata = randomIncludeCCSMetadata(); + Boolean requestIncludeMeta = includeCCSMetadata.v1(); + boolean responseExpectMeta = includeCCSMetadata.v2(); + for (var hostMode : Enrich.Mode.values()) { String query = String.format(Locale.ROOT, """ FROM *:events,events @@ -295,7 +307,7 @@ public void testEnrichHostsAggThenEnrichVendorCoordinator() { | stats c = SUM(c) by vendor | sort vendor """, enrichHosts(hostMode), enrichVendors(Enrich.Mode.COORDINATOR)); - try (EsqlQueryResponse resp = runQuery(query)) { + try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) { assertThat( getValuesList(resp), equalTo( @@ -309,6 +321,7 @@ public void testEnrichHostsAggThenEnrichVendorCoordinator() { ) ); EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); assertThat(executionInfo.clusterAliases(), equalTo(Set.of("", "c1", "c2"))); assertCCSExecutionInfoDetails(executionInfo); } @@ -316,6 +329,10 @@ public void testEnrichHostsAggThenEnrichVendorCoordinator() { } public void testEnrichTwiceThenAggs() { + Tuple includeCCSMetadata = randomIncludeCCSMetadata(); + Boolean requestIncludeMeta = includeCCSMetadata.v1(); + boolean responseExpectMeta = includeCCSMetadata.v2(); + for (var hostMode : Enrich.Mode.values()) { String query = String.format(Locale.ROOT, """ FROM *:events,events @@ -325,7 +342,7 @@ public void testEnrichTwiceThenAggs() { | stats c = COUNT(*) by vendor | sort vendor """, enrichHosts(hostMode), enrichVendors(Enrich.Mode.COORDINATOR)); - try (EsqlQueryResponse resp = runQuery(query)) { + try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) { assertThat( getValuesList(resp), equalTo( @@ -339,6 +356,7 @@ public void testEnrichTwiceThenAggs() { ) ); EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); assertThat(executionInfo.clusterAliases(), equalTo(Set.of("", "c1", "c2"))); assertCCSExecutionInfoDetails(executionInfo); } @@ -346,6 +364,10 @@ public void testEnrichTwiceThenAggs() { } public void testEnrichCoordinatorThenAny() { + Tuple includeCCSMetadata = randomIncludeCCSMetadata(); + Boolean requestIncludeMeta = includeCCSMetadata.v1(); + boolean responseExpectMeta = includeCCSMetadata.v2(); + String query = String.format(Locale.ROOT, """ FROM *:events,events | eval ip= TO_STR(host) @@ -354,7 +376,7 @@ public void testEnrichCoordinatorThenAny() { | stats c = COUNT(*) by vendor | sort vendor """, enrichHosts(Enrich.Mode.COORDINATOR), enrichVendors(Enrich.Mode.ANY)); - try (EsqlQueryResponse resp = runQuery(query)) { + try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) { assertThat( getValuesList(resp), equalTo( @@ -368,12 +390,17 @@ public void testEnrichCoordinatorThenAny() { ) ); EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); assertThat(executionInfo.clusterAliases(), equalTo(Set.of("", "c1", "c2"))); assertCCSExecutionInfoDetails(executionInfo); } } public void testEnrichCoordinatorWithVendor() { + Tuple includeCCSMetadata = randomIncludeCCSMetadata(); + Boolean requestIncludeMeta = includeCCSMetadata.v1(); + boolean responseExpectMeta = includeCCSMetadata.v2(); + for (Enrich.Mode hostMode : Enrich.Mode.values()) { String query = String.format(Locale.ROOT, """ FROM *:events,events @@ -383,7 +410,7 @@ public void testEnrichCoordinatorWithVendor() { | stats c = COUNT(*) by vendor | sort vendor """, enrichHosts(hostMode), enrichVendors(Enrich.Mode.COORDINATOR)); - try (EsqlQueryResponse resp = runQuery(query)) { + try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) { assertThat( getValuesList(resp), equalTo( @@ -397,6 +424,7 @@ public void testEnrichCoordinatorWithVendor() { ) ); EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); assertThat(executionInfo.clusterAliases(), equalTo(Set.of("", "c1", "c2"))); assertCCSExecutionInfoDetails(executionInfo); } @@ -405,6 +433,10 @@ public void testEnrichCoordinatorWithVendor() { } public void testEnrichRemoteWithVendor() { + Tuple includeCCSMetadata = randomIncludeCCSMetadata(); + Boolean requestIncludeMeta = includeCCSMetadata.v1(); + boolean responseExpectMeta = includeCCSMetadata.v2(); + for (Enrich.Mode hostMode : List.of(Enrich.Mode.ANY, Enrich.Mode.REMOTE)) { var query = String.format(Locale.ROOT, """ FROM *:events,events @@ -414,7 +446,7 @@ public void testEnrichRemoteWithVendor() { | stats c = COUNT(*) by vendor | sort vendor """, enrichHosts(hostMode), enrichVendors(Enrich.Mode.REMOTE)); - try (EsqlQueryResponse resp = runQuery(query)) { + try (EsqlQueryResponse resp = runQuery(query, requestIncludeMeta)) { assertThat( getValuesList(resp), equalTo( @@ -430,6 +462,7 @@ public void testEnrichRemoteWithVendor() { ) ); EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); assertThat(executionInfo.clusterAliases(), equalTo(Set.of("", "c1", "c2"))); assertCCSExecutionInfoDetails(executionInfo); } @@ -444,7 +477,7 @@ public void testTopNThenEnrichRemote() { | LIMIT 5 | %s """, enrichHosts(Enrich.Mode.REMOTE)); - var error = expectThrows(VerificationException.class, () -> runQuery(query).close()); + var error = expectThrows(VerificationException.class, () -> runQuery(query, randomBoolean()).close()); assertThat(error.getMessage(), containsString("ENRICH with remote policy can't be executed after LIMIT")); } @@ -455,7 +488,7 @@ public void testLimitThenEnrichRemote() { | eval ip= TO_STR(host) | %s """, enrichHosts(Enrich.Mode.REMOTE)); - var error = expectThrows(VerificationException.class, () -> runQuery(query).close()); + var error = expectThrows(VerificationException.class, () -> runQuery(query, randomBoolean()).close()); assertThat(error.getMessage(), containsString("ENRICH with remote policy can't be executed after LIMIT")); } @@ -468,7 +501,7 @@ public void testAggThenEnrichRemote() { | %s | sort vendor """, enrichHosts(Enrich.Mode.ANY), enrichVendors(Enrich.Mode.REMOTE)); - var error = expectThrows(VerificationException.class, () -> runQuery(query).close()); + var error = expectThrows(VerificationException.class, () -> runQuery(query, randomBoolean()).close()); assertThat(error.getMessage(), containsString("ENRICH with remote policy can't be executed after STATS")); } @@ -480,20 +513,23 @@ public void testEnrichCoordinatorThenEnrichRemote() { | %s | sort vendor """, enrichHosts(Enrich.Mode.COORDINATOR), enrichVendors(Enrich.Mode.REMOTE)); - var error = expectThrows(VerificationException.class, () -> runQuery(query).close()); + var error = expectThrows(VerificationException.class, () -> runQuery(query, randomBoolean()).close()); assertThat( error.getMessage(), containsString("ENRICH with remote policy can't be executed after another ENRICH with coordinator policy") ); } - protected EsqlQueryResponse runQuery(String query) { + protected EsqlQueryResponse runQuery(String query, Boolean ccsMetadataInResponse) { EsqlQueryRequest request = EsqlQueryRequest.syncEsqlQueryRequest(); request.query(query); request.pragmas(AbstractEsqlIntegTestCase.randomPragmas()); if (randomBoolean()) { request.profile(true); } + if (ccsMetadataInResponse != null) { + request.includeCCSMetadata(ccsMetadataInResponse); + } return client(LOCAL_CLUSTER).execute(EsqlQueryAction.INSTANCE, request).actionGet(30, TimeUnit.SECONDS); } @@ -516,6 +552,15 @@ private static void assertCCSExecutionInfoDetails(EsqlExecutionInfo executionInf } } + public static Tuple randomIncludeCCSMetadata() { + return switch (randomIntBetween(1, 3)) { + case 1 -> new Tuple<>(Boolean.TRUE, Boolean.TRUE); + case 2 -> new Tuple<>(Boolean.FALSE, Boolean.FALSE); + case 3 -> new Tuple<>(null, Boolean.FALSE); + default -> throw new AssertionError("should not get here"); + }; + } + public static class LocalStateEnrich extends LocalStateCompositeXPackPlugin { public LocalStateEnrich(final Settings settings, final Path configPath) throws Exception { diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java index 4f4f3d112247e..adfa2fc7273cd 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java @@ -10,7 +10,6 @@ import org.elasticsearch.Build; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Priority; @@ -21,13 +20,16 @@ import org.elasticsearch.compute.operator.DriverProfile; import org.elasticsearch.compute.operator.exchange.ExchangeService; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.Tuple; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.AbstractMultiClustersTestCase; import org.elasticsearch.test.InternalTestCluster; +import org.elasticsearch.test.XContentTestUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; import org.elasticsearch.xpack.esql.plugin.QueryPragmas; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -79,12 +81,15 @@ public List> getSettings() { } } - public void testSimple() { + public void testSuccessfulPathways() { Map testClusterInfo = setupTwoClusters(); int localNumShards = (Integer) testClusterInfo.get("local.num_shards"); int remoteNumShards = (Integer) testClusterInfo.get("remote.num_shards"); - try (EsqlQueryResponse resp = runQuery("from logs-*,*:logs-* | stats sum (v)")) { + Tuple includeCCSMetadata = randomIncludeCCSMetadata(); + Boolean requestIncludeMeta = includeCCSMetadata.v1(); + boolean responseExpectMeta = includeCCSMetadata.v2(); + try (EsqlQueryResponse resp = runQuery("from logs-*,*:logs-* | stats sum (v)", requestIncludeMeta)) { List> values = getValuesList(resp); assertThat(values, hasSize(1)); assertThat(values.get(0), equalTo(List.of(330L))); @@ -93,6 +98,7 @@ public void testSimple() { assertNotNull(executionInfo); assertThat(executionInfo.isCrossClusterSearch(), is(true)); assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER))); @@ -113,9 +119,12 @@ public void testSimple() { assertThat(localCluster.getSuccessfulShards(), equalTo(localNumShards)); assertThat(localCluster.getSkippedShards(), equalTo(0)); assertThat(localCluster.getFailedShards(), equalTo(0)); + + // ensure that the _clusters metadata is present only if requested + assertClusterMetadataInResponse(resp, responseExpectMeta); } - try (EsqlQueryResponse resp = runQuery("from logs-*,*:logs-* | stats count(*) by tag | sort tag | keep tag")) { + try (EsqlQueryResponse resp = runQuery("from logs-*,*:logs-* | stats count(*) by tag | sort tag | keep tag", requestIncludeMeta)) { List> values = getValuesList(resp); assertThat(values, hasSize(2)); assertThat(values.get(0), equalTo(List.of("local"))); @@ -125,6 +134,7 @@ public void testSimple() { assertNotNull(executionInfo); assertThat(executionInfo.isCrossClusterSearch(), is(true)); assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER))); @@ -145,6 +155,9 @@ public void testSimple() { assertThat(localCluster.getSuccessfulShards(), equalTo(localNumShards)); assertThat(localCluster.getSkippedShards(), equalTo(0)); assertThat(localCluster.getFailedShards(), equalTo(0)); + + // ensure that the _clusters metadata is present only if requested + assertClusterMetadataInResponse(resp, responseExpectMeta); } } @@ -153,9 +166,13 @@ public void testSearchesWhereMissingIndicesAreSpecified() { int localNumShards = (Integer) testClusterInfo.get("local.num_shards"); int remoteNumShards = (Integer) testClusterInfo.get("remote.num_shards"); + Tuple includeCCSMetadata = randomIncludeCCSMetadata(); + Boolean requestIncludeMeta = includeCCSMetadata.v1(); + boolean responseExpectMeta = includeCCSMetadata.v2(); + // since a valid local index was specified, the invalid index on cluster-a does not throw an exception, // but instead is simply ignored - ensure this is captured in the EsqlExecutionInfo - try (EsqlQueryResponse resp = runQuery("from logs-*,cluster-a:no_such_index | stats sum (v)")) { + try (EsqlQueryResponse resp = runQuery("from logs-*,cluster-a:no_such_index | stats sum (v)", requestIncludeMeta)) { EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); List> values = getValuesList(resp); assertThat(values, hasSize(1)); @@ -164,6 +181,7 @@ public void testSearchesWhereMissingIndicesAreSpecified() { assertNotNull(executionInfo); assertThat(executionInfo.isCrossClusterSearch(), is(true)); assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER))); @@ -188,7 +206,12 @@ public void testSearchesWhereMissingIndicesAreSpecified() { // since the remote cluster has a valid index expression, the missing local index is ignored // make this is captured in the EsqlExecutionInfo - try (EsqlQueryResponse resp = runQuery("from no_such_index,*:logs-* | stats count(*) by tag | sort tag | keep tag")) { + try ( + EsqlQueryResponse resp = runQuery( + "from no_such_index,*:logs-* | stats count(*) by tag | sort tag | keep tag", + requestIncludeMeta + ) + ) { List> values = getValuesList(resp); assertThat(values, hasSize(1)); assertThat(values.get(0), equalTo(List.of("remote"))); @@ -197,6 +220,7 @@ public void testSearchesWhereMissingIndicesAreSpecified() { assertNotNull(executionInfo); assertThat(executionInfo.isCrossClusterSearch(), is(true)); assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER))); @@ -223,7 +247,8 @@ public void testSearchesWhereMissingIndicesAreSpecified() { // in the index expression of the EsqlExecutionInfo and with an indication that zero shards were searched try ( EsqlQueryResponse resp = runQuery( - "FROM no_such_index*,*:no_such_index1,*:no_such_index2,logs-1 | STATS COUNT(*) by tag | SORT tag | KEEP tag" + "FROM no_such_index*,*:no_such_index1,*:no_such_index2,logs-1 | STATS COUNT(*) by tag | SORT tag | KEEP tag", + requestIncludeMeta ) ) { List> values = getValuesList(resp); @@ -234,6 +259,7 @@ public void testSearchesWhereMissingIndicesAreSpecified() { assertNotNull(executionInfo); assertThat(executionInfo.isCrossClusterSearch(), is(true)); assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER))); @@ -257,7 +283,7 @@ public void testSearchesWhereMissingIndicesAreSpecified() { } // wildcard on remote cluster that matches nothing - should be present in EsqlExecutionInfo marked as SKIPPED, no shards searched - try (EsqlQueryResponse resp = runQuery("from cluster-a:no_such_index*,logs-* | stats sum (v)")) { + try (EsqlQueryResponse resp = runQuery("from cluster-a:no_such_index*,logs-* | stats sum (v)", requestIncludeMeta)) { EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); List> values = getValuesList(resp); assertThat(values, hasSize(1)); @@ -266,6 +292,7 @@ public void testSearchesWhereMissingIndicesAreSpecified() { assertNotNull(executionInfo); assertThat(executionInfo.isCrossClusterSearch(), is(true)); assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER))); @@ -293,8 +320,12 @@ public void testSearchesWhereNonExistentClusterIsSpecifiedWithWildcards() { Map testClusterInfo = setupTwoClusters(); int localNumShards = (Integer) testClusterInfo.get("local.num_shards"); + Tuple includeCCSMetadata = randomIncludeCCSMetadata(); + Boolean requestIncludeMeta = includeCCSMetadata.v1(); + boolean responseExpectMeta = includeCCSMetadata.v2(); + // a query which matches no remote cluster is not a cross cluster search - try (EsqlQueryResponse resp = runQuery("from logs-*,x*:no_such_index* | stats sum (v)")) { + try (EsqlQueryResponse resp = runQuery("from logs-*,x*:no_such_index* | stats sum (v)", requestIncludeMeta)) { EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); List> values = getValuesList(resp); assertThat(values, hasSize(1)); @@ -303,12 +334,18 @@ public void testSearchesWhereNonExistentClusterIsSpecifiedWithWildcards() { assertNotNull(executionInfo); assertThat(executionInfo.clusterAliases(), equalTo(Set.of(LOCAL_CLUSTER))); assertThat(executionInfo.isCrossClusterSearch(), is(false)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); // since this not a CCS, only the overall took time in the EsqlExecutionInfo matters assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L)); } // cluster-foo* matches nothing and so should not be present in the EsqlExecutionInfo - try (EsqlQueryResponse resp = runQuery("from logs-*,no_such_index*,cluster-a:no_such_index*,cluster-foo*:* | stats sum (v)")) { + try ( + EsqlQueryResponse resp = runQuery( + "from logs-*,no_such_index*,cluster-a:no_such_index*,cluster-foo*:* | stats sum (v)", + requestIncludeMeta + ) + ) { EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); List> values = getValuesList(resp); assertThat(values, hasSize(1)); @@ -317,6 +354,7 @@ public void testSearchesWhereNonExistentClusterIsSpecifiedWithWildcards() { assertNotNull(executionInfo); assertThat(executionInfo.isCrossClusterSearch(), is(true)); assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER))); @@ -349,8 +387,12 @@ public void testSearchesWhereNonExistentClusterIsSpecifiedWithWildcards() { public void testCCSExecutionOnSearchesWithLimit0() { setupTwoClusters(); + Tuple includeCCSMetadata = randomIncludeCCSMetadata(); + Boolean requestIncludeMeta = includeCCSMetadata.v1(); + boolean responseExpectMeta = includeCCSMetadata.v2(); + // Ensure non-cross cluster queries have overall took time - try (EsqlQueryResponse resp = runQuery("FROM logs* | LIMIT 0")) { + try (EsqlQueryResponse resp = runQuery("FROM logs* | LIMIT 0", requestIncludeMeta)) { EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); assertNotNull(executionInfo); assertThat(executionInfo.isCrossClusterSearch(), is(false)); @@ -358,12 +400,13 @@ public void testCCSExecutionOnSearchesWithLimit0() { } // ensure cross-cluster searches have overall took time and correct per-cluster details in EsqlExecutionInfo - try (EsqlQueryResponse resp = runQuery("FROM logs*,cluster-a:* | LIMIT 0")) { + try (EsqlQueryResponse resp = runQuery("FROM logs*,cluster-a:* | LIMIT 0", requestIncludeMeta)) { EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); assertNotNull(executionInfo); assertThat(executionInfo.isCrossClusterSearch(), is(true)); long overallTookMillis = executionInfo.overallTook().millis(); assertThat(overallTookMillis, greaterThanOrEqualTo(0L)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER))); EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER); @@ -387,12 +430,13 @@ public void testCCSExecutionOnSearchesWithLimit0() { assertNull(localCluster.getFailedShards()); } - try (EsqlQueryResponse resp = runQuery("FROM logs*,cluster-a:nomatch* | LIMIT 0")) { + try (EsqlQueryResponse resp = runQuery("FROM logs*,cluster-a:nomatch* | LIMIT 0", requestIncludeMeta)) { EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); assertNotNull(executionInfo); assertThat(executionInfo.isCrossClusterSearch(), is(true)); long overallTookMillis = executionInfo.overallTook().millis(); assertThat(overallTookMillis, greaterThanOrEqualTo(0L)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER))); EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER); @@ -415,12 +459,13 @@ public void testCCSExecutionOnSearchesWithLimit0() { assertNull(localCluster.getFailedShards()); } - try (EsqlQueryResponse resp = runQuery("FROM nomatch*,cluster-a:* | LIMIT 0")) { + try (EsqlQueryResponse resp = runQuery("FROM nomatch*,cluster-a:* | LIMIT 0", requestIncludeMeta)) { EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); assertNotNull(executionInfo); assertThat(executionInfo.isCrossClusterSearch(), is(true)); long overallTookMillis = executionInfo.overallTook().millis(); assertThat(overallTookMillis, greaterThanOrEqualTo(0L)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER))); EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER); @@ -447,7 +492,16 @@ public void testMetadataIndex() { int localNumShards = (Integer) testClusterInfo.get("local.num_shards"); int remoteNumShards = (Integer) testClusterInfo.get("remote.num_shards"); - try (EsqlQueryResponse resp = runQuery("FROM logs*,*:logs* METADATA _index | stats sum(v) by _index | sort _index")) { + Tuple includeCCSMetadata = randomIncludeCCSMetadata(); + Boolean requestIncludeMeta = includeCCSMetadata.v1(); + boolean responseExpectMeta = includeCCSMetadata.v2(); + + try ( + EsqlQueryResponse resp = runQuery( + "FROM logs*,*:logs* METADATA _index | stats sum(v) by _index | sort _index", + requestIncludeMeta + ) + ) { List> values = getValuesList(resp); assertThat(values.get(0), equalTo(List.of(285L, "cluster-a:logs-2"))); assertThat(values.get(1), equalTo(List.of(45L, "logs-1"))); @@ -455,6 +509,7 @@ public void testMetadataIndex() { EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); assertNotNull(executionInfo); assertThat(executionInfo.isCrossClusterSearch(), is(true)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L)); EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER); @@ -477,18 +532,6 @@ public void testMetadataIndex() { } } - void waitForNoInitializingShards(Client client, TimeValue timeout, String... indices) { - ClusterHealthResponse resp = client.admin() - .cluster() - .prepareHealth(TEST_REQUEST_TIMEOUT, indices) - .setWaitForEvents(Priority.LANGUID) - .setWaitForNoRelocatingShards(true) - .setWaitForNoInitializingShards(true) - .setTimeout(timeout) - .get(); - assertFalse(Strings.toString(resp, true, true), resp.isTimedOut()); - } - public void testProfile() { Map testClusterInfo = setupTwoClusters(); int localNumShards = (Integer) testClusterInfo.get("local.num_shards"); @@ -529,6 +572,7 @@ public void testProfile() { EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER); assertNull(remoteCluster); assertThat(executionInfo.isCrossClusterSearch(), is(false)); + assertThat(executionInfo.includeCCSMetadata(), is(false)); // since this not a CCS, only the overall took time in the EsqlExecutionInfo matters assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L)); } @@ -550,6 +594,7 @@ public void testProfile() { EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); assertNotNull(executionInfo); assertThat(executionInfo.isCrossClusterSearch(), is(true)); + assertThat(executionInfo.includeCCSMetadata(), is(false)); assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L)); EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER); @@ -582,6 +627,7 @@ public void testProfile() { EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); assertNotNull(executionInfo); assertThat(executionInfo.isCrossClusterSearch(), is(true)); + assertThat(executionInfo.includeCCSMetadata(), is(false)); assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L)); EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER); @@ -608,14 +654,11 @@ public void testProfile() { public void testWarnings() throws Exception { Map testClusterInfo = setupTwoClusters(); - String localIndex = (String) testClusterInfo.get("local.index"); - String remoteIndex = (String) testClusterInfo.get("remote.index"); int localNumShards = (Integer) testClusterInfo.get("local.num_shards"); int remoteNumShards = (Integer) testClusterInfo.get("remote.num_shards"); EsqlQueryRequest request = EsqlQueryRequest.syncEsqlQueryRequest(); request.query("FROM logs*,*:logs* | EVAL ip = to_ip(id) | STATS total = sum(v) by ip | LIMIT 10"); - PlainActionFuture future = new PlainActionFuture<>(); InternalTestCluster cluster = cluster(LOCAL_CLUSTER); String node = randomFrom(cluster.getNodeNames()); CountDownLatch latch = new CountDownLatch(1); @@ -634,6 +677,7 @@ public void testWarnings() throws Exception { EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); assertNotNull(executionInfo); assertThat(executionInfo.isCrossClusterSearch(), is(true)); + assertThat(executionInfo.includeCCSMetadata(), is(false)); assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L)); EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER); @@ -662,11 +706,34 @@ public void testWarnings() throws Exception { assertTrue(latch.await(30, TimeUnit.SECONDS)); } - protected EsqlQueryResponse runQuery(String query) { + private static void assertClusterMetadataInResponse(EsqlQueryResponse resp, boolean responseExpectMeta) { + try { + final Map esqlResponseAsMap = XContentTestUtils.convertToMap(resp); + final Object clusters = esqlResponseAsMap.get("_clusters"); + if (responseExpectMeta) { + assertNotNull(clusters); + // test a few entries to ensure it looks correct (other tests do a full analysis of the metadata in the response) + @SuppressWarnings("unchecked") + Map inner = (Map) clusters; + assertTrue(inner.containsKey("total")); + assertTrue(inner.containsKey("details")); + } else { + assertNull(clusters); + } + } catch (IOException e) { + fail("Could not convert ESQL response to Map: " + e); + } + } + + protected EsqlQueryResponse runQuery(String query, Boolean ccsMetadataInResponse) { EsqlQueryRequest request = EsqlQueryRequest.syncEsqlQueryRequest(); request.query(query); request.pragmas(AbstractEsqlIntegTestCase.randomPragmas()); - request.profile(true); + request.profile(randomInt(5) == 2); + request.columnar(randomBoolean()); + if (ccsMetadataInResponse != null) { + request.includeCCSMetadata(ccsMetadataInResponse); + } return runQuery(request); } @@ -674,6 +741,32 @@ protected EsqlQueryResponse runQuery(EsqlQueryRequest request) { return client(LOCAL_CLUSTER).execute(EsqlQueryAction.INSTANCE, request).actionGet(30, TimeUnit.SECONDS); } + /** + * v1: value to send to runQuery (can be null; null means use default value) + * v2: whether to expect CCS Metadata in the response (cannot be null) + * @return + */ + public static Tuple randomIncludeCCSMetadata() { + return switch (randomIntBetween(1, 3)) { + case 1 -> new Tuple<>(Boolean.TRUE, Boolean.TRUE); + case 2 -> new Tuple<>(Boolean.FALSE, Boolean.FALSE); + case 3 -> new Tuple<>(null, Boolean.FALSE); + default -> throw new AssertionError("should not get here"); + }; + } + + void waitForNoInitializingShards(Client client, TimeValue timeout, String... indices) { + ClusterHealthResponse resp = client.admin() + .cluster() + .prepareHealth(TEST_REQUEST_TIMEOUT, indices) + .setWaitForEvents(Priority.LANGUID) + .setWaitForNoRelocatingShards(true) + .setWaitForNoInitializingShards(true) + .setTimeout(timeout) + .get(); + assertFalse(Strings.toString(resp, true, true), resp.isTimedOut()); + } + Map setupTwoClusters() { String localIndex = "logs-1"; int numShardsLocal = randomIntBetween(1, 5); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringFunctionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringIT.java similarity index 98% rename from x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringFunctionIT.java rename to x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringIT.java index e6f11ca1f44d2..53b833c7e8a15 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringFunctionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringIT.java @@ -29,7 +29,7 @@ import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.Matchers.equalTo; -public class QueryStringFunctionIT extends AbstractEsqlIntegTestCase { +public class QueryStringIT extends AbstractEsqlIntegTestCase { @Before public void setupIndex() { diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 index 0d8d3abf77ecc..ce3947875e6c7 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 @@ -86,7 +86,6 @@ WHERE : 'where' -> pushMode(EXPRESSION_MODE); // MYCOMMAND : 'mycommand' -> ... DEV_INLINESTATS : {this.isDevVersion()}? 'inlinestats' -> pushMode(EXPRESSION_MODE); DEV_LOOKUP : {this.isDevVersion()}? 'lookup' -> pushMode(LOOKUP_MODE); -DEV_MATCH : {this.isDevVersion()}? 'match' -> pushMode(EXPRESSION_MODE); DEV_METRICS : {this.isDevVersion()}? 'metrics' -> pushMode(METRICS_MODE); // @@ -209,8 +208,7 @@ ASTERISK : '*'; SLASH : '/'; PERCENT : '%'; -// move it in the main section if the feature gets promoted -DEV_MATCH_OP : {this.isDevVersion()}? DEV_MATCH -> type(DEV_MATCH); +DEV_MATCH : {this.isDevVersion()}? 'match'; NAMED_OR_POSITIONAL_PARAM : PARAM (LETTER | UNDERSCORE) UNQUOTED_ID_BODY* diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens index 4fd37ab9900f2..2fe262a6983f7 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens @@ -16,51 +16,51 @@ STATS=15 WHERE=16 DEV_INLINESTATS=17 DEV_LOOKUP=18 -DEV_MATCH=19 -DEV_METRICS=20 -UNKNOWN_CMD=21 -LINE_COMMENT=22 -MULTILINE_COMMENT=23 -WS=24 -PIPE=25 -QUOTED_STRING=26 -INTEGER_LITERAL=27 -DECIMAL_LITERAL=28 -BY=29 -AND=30 -ASC=31 -ASSIGN=32 -CAST_OP=33 -COMMA=34 -DESC=35 -DOT=36 -FALSE=37 -FIRST=38 -IN=39 -IS=40 -LAST=41 -LIKE=42 -LP=43 -NOT=44 -NULL=45 -NULLS=46 -OR=47 -PARAM=48 -RLIKE=49 -RP=50 -TRUE=51 -EQ=52 -CIEQ=53 -NEQ=54 -LT=55 -LTE=56 -GT=57 -GTE=58 -PLUS=59 -MINUS=60 -ASTERISK=61 -SLASH=62 -PERCENT=63 +DEV_METRICS=19 +UNKNOWN_CMD=20 +LINE_COMMENT=21 +MULTILINE_COMMENT=22 +WS=23 +PIPE=24 +QUOTED_STRING=25 +INTEGER_LITERAL=26 +DECIMAL_LITERAL=27 +BY=28 +AND=29 +ASC=30 +ASSIGN=31 +CAST_OP=32 +COMMA=33 +DESC=34 +DOT=35 +FALSE=36 +FIRST=37 +IN=38 +IS=39 +LAST=40 +LIKE=41 +LP=42 +NOT=43 +NULL=44 +NULLS=45 +OR=46 +PARAM=47 +RLIKE=48 +RP=49 +TRUE=50 +EQ=51 +CIEQ=52 +NEQ=53 +LT=54 +LTE=55 +GT=56 +GTE=57 +PLUS=58 +MINUS=59 +ASTERISK=60 +SLASH=61 +PERCENT=62 +DEV_MATCH=63 NAMED_OR_POSITIONAL_PARAM=64 OPENING_BRACKET=65 CLOSING_BRACKET=66 @@ -134,42 +134,42 @@ CLOSING_METRICS_WS=120 'sort'=14 'stats'=15 'where'=16 -'|'=25 -'by'=29 -'and'=30 -'asc'=31 -'='=32 -'::'=33 -','=34 -'desc'=35 -'.'=36 -'false'=37 -'first'=38 -'in'=39 -'is'=40 -'last'=41 -'like'=42 -'('=43 -'not'=44 -'null'=45 -'nulls'=46 -'or'=47 -'?'=48 -'rlike'=49 -')'=50 -'true'=51 -'=='=52 -'=~'=53 -'!='=54 -'<'=55 -'<='=56 -'>'=57 -'>='=58 -'+'=59 -'-'=60 -'*'=61 -'/'=62 -'%'=63 +'|'=24 +'by'=28 +'and'=29 +'asc'=30 +'='=31 +'::'=32 +','=33 +'desc'=34 +'.'=35 +'false'=36 +'first'=37 +'in'=38 +'is'=39 +'last'=40 +'like'=41 +'('=42 +'not'=43 +'null'=44 +'nulls'=45 +'or'=46 +'?'=47 +'rlike'=48 +')'=49 +'true'=50 +'=='=51 +'=~'=52 +'!='=53 +'<'=54 +'<='=55 +'>'=56 +'>='=57 +'+'=58 +'-'=59 +'*'=60 +'/'=61 +'%'=62 ']'=66 'metadata'=75 'as'=84 diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 index b720ba98babf0..c053824861a96 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 @@ -101,7 +101,13 @@ primaryExpression ; functionExpression - : identifierOrParameter LP (ASTERISK | (booleanExpression (COMMA booleanExpression)*))? RP + : functionName LP (ASTERISK | (booleanExpression (COMMA booleanExpression)*))? RP + ; + +functionName + // Additional function identifiers that are already a reserved word in the language + : {this.isDevVersion()}? DEV_MATCH + | identifierOrParameter ; dataType diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens index 4fd37ab9900f2..2fe262a6983f7 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens @@ -16,51 +16,51 @@ STATS=15 WHERE=16 DEV_INLINESTATS=17 DEV_LOOKUP=18 -DEV_MATCH=19 -DEV_METRICS=20 -UNKNOWN_CMD=21 -LINE_COMMENT=22 -MULTILINE_COMMENT=23 -WS=24 -PIPE=25 -QUOTED_STRING=26 -INTEGER_LITERAL=27 -DECIMAL_LITERAL=28 -BY=29 -AND=30 -ASC=31 -ASSIGN=32 -CAST_OP=33 -COMMA=34 -DESC=35 -DOT=36 -FALSE=37 -FIRST=38 -IN=39 -IS=40 -LAST=41 -LIKE=42 -LP=43 -NOT=44 -NULL=45 -NULLS=46 -OR=47 -PARAM=48 -RLIKE=49 -RP=50 -TRUE=51 -EQ=52 -CIEQ=53 -NEQ=54 -LT=55 -LTE=56 -GT=57 -GTE=58 -PLUS=59 -MINUS=60 -ASTERISK=61 -SLASH=62 -PERCENT=63 +DEV_METRICS=19 +UNKNOWN_CMD=20 +LINE_COMMENT=21 +MULTILINE_COMMENT=22 +WS=23 +PIPE=24 +QUOTED_STRING=25 +INTEGER_LITERAL=26 +DECIMAL_LITERAL=27 +BY=28 +AND=29 +ASC=30 +ASSIGN=31 +CAST_OP=32 +COMMA=33 +DESC=34 +DOT=35 +FALSE=36 +FIRST=37 +IN=38 +IS=39 +LAST=40 +LIKE=41 +LP=42 +NOT=43 +NULL=44 +NULLS=45 +OR=46 +PARAM=47 +RLIKE=48 +RP=49 +TRUE=50 +EQ=51 +CIEQ=52 +NEQ=53 +LT=54 +LTE=55 +GT=56 +GTE=57 +PLUS=58 +MINUS=59 +ASTERISK=60 +SLASH=61 +PERCENT=62 +DEV_MATCH=63 NAMED_OR_POSITIONAL_PARAM=64 OPENING_BRACKET=65 CLOSING_BRACKET=66 @@ -134,42 +134,42 @@ CLOSING_METRICS_WS=120 'sort'=14 'stats'=15 'where'=16 -'|'=25 -'by'=29 -'and'=30 -'asc'=31 -'='=32 -'::'=33 -','=34 -'desc'=35 -'.'=36 -'false'=37 -'first'=38 -'in'=39 -'is'=40 -'last'=41 -'like'=42 -'('=43 -'not'=44 -'null'=45 -'nulls'=46 -'or'=47 -'?'=48 -'rlike'=49 -')'=50 -'true'=51 -'=='=52 -'=~'=53 -'!='=54 -'<'=55 -'<='=56 -'>'=57 -'>='=58 -'+'=59 -'-'=60 -'*'=61 -'/'=62 -'%'=63 +'|'=24 +'by'=28 +'and'=29 +'asc'=30 +'='=31 +'::'=32 +','=33 +'desc'=34 +'.'=35 +'false'=36 +'first'=37 +'in'=38 +'is'=39 +'last'=40 +'like'=41 +'('=42 +'not'=43 +'null'=44 +'nulls'=45 +'or'=46 +'?'=47 +'rlike'=48 +')'=49 +'true'=50 +'=='=51 +'=~'=52 +'!='=53 +'<'=54 +'<='=55 +'>'=56 +'>='=57 +'+'=58 +'-'=59 +'*'=60 +'/'=61 +'%'=62 ']'=66 'metadata'=75 'as'=84 diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/HypotEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/HypotEvaluator.java new file mode 100644 index 0000000000000..22094f7e623e6 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/HypotEvaluator.java @@ -0,0 +1,146 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.math; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.DoubleVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link Hypot}. + * This class is generated. Do not edit it. + */ +public final class HypotEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator n1; + + private final EvalOperator.ExpressionEvaluator n2; + + private final DriverContext driverContext; + + private Warnings warnings; + + public HypotEvaluator(Source source, EvalOperator.ExpressionEvaluator n1, + EvalOperator.ExpressionEvaluator n2, DriverContext driverContext) { + this.source = source; + this.n1 = n1; + this.n2 = n2; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (DoubleBlock n1Block = (DoubleBlock) n1.eval(page)) { + try (DoubleBlock n2Block = (DoubleBlock) n2.eval(page)) { + DoubleVector n1Vector = n1Block.asVector(); + if (n1Vector == null) { + return eval(page.getPositionCount(), n1Block, n2Block); + } + DoubleVector n2Vector = n2Block.asVector(); + if (n2Vector == null) { + return eval(page.getPositionCount(), n1Block, n2Block); + } + return eval(page.getPositionCount(), n1Vector, n2Vector).asBlock(); + } + } + } + + public DoubleBlock eval(int positionCount, DoubleBlock n1Block, DoubleBlock n2Block) { + try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + if (n1Block.isNull(p)) { + result.appendNull(); + continue position; + } + if (n1Block.getValueCount(p) != 1) { + if (n1Block.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (n2Block.isNull(p)) { + result.appendNull(); + continue position; + } + if (n2Block.getValueCount(p) != 1) { + if (n2Block.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + result.appendDouble(Hypot.process(n1Block.getDouble(n1Block.getFirstValueIndex(p)), n2Block.getDouble(n2Block.getFirstValueIndex(p)))); + } + return result.build(); + } + } + + public DoubleVector eval(int positionCount, DoubleVector n1Vector, DoubleVector n2Vector) { + try(DoubleVector.FixedBuilder result = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + result.appendDouble(p, Hypot.process(n1Vector.getDouble(p), n2Vector.getDouble(p))); + } + return result.build(); + } + } + + @Override + public String toString() { + return "HypotEvaluator[" + "n1=" + n1 + ", n2=" + n2 + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(n1, n2); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory n1; + + private final EvalOperator.ExpressionEvaluator.Factory n2; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory n1, + EvalOperator.ExpressionEvaluator.Factory n2) { + this.source = source; + this.n1 = n1; + this.n2 = n2; + } + + @Override + public HypotEvaluator get(DriverContext context) { + return new HypotEvaluator(source, n1.get(context), n2.get(context), context); + } + + @Override + public String toString() { + return "HypotEvaluator[" + "n1=" + n1 + ", n2=" + n2 + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index f7454e41a6a8b..1d6d81077b9be 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -37,6 +37,11 @@ public enum Cap { */ FN_CBRT, + /** + * Support for function {@code HYPOT}. + */ + FN_HYPOT, + /** * Support for {@code MV_APPEND} function. #107001 */ @@ -165,6 +170,11 @@ public enum Cap { */ SPATIAL_PREDICATES_SUPPORT_MULTIVALUES, + /** + * Support a number of fixes and enhancements to spatial distance pushdown. Done in #112938. + */ + SPATIAL_DISTANCE_PUSHDOWN_ENHANCEMENTS, + /** * Fix to GROK and DISSECT that allows extracting attributes with the same name as the input * https://github.com/elastic/elasticsearch/issues/110184 @@ -336,6 +346,11 @@ public enum Cap { */ QSTR_FUNCTION(true), + /** + * MATCH function + */ + MATCH_FUNCTION(true), + /** * Don't optimize CASE IS NOT NULL function by not requiring the fields to be not null as well. * https://github.com/elastic/elasticsearch/issues/112704 @@ -350,7 +365,12 @@ public enum Cap { /** * Support named parameters for field names. */ - NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES; + NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES, + + /** + * Fix sorting not allowed on _source and counters. + */ + SORTING_ON_SOURCE_AND_COUNTERS_FORBIDDEN; private final boolean snapshotOnly; private final FeatureFlag featureFlag; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java index f7966ff5ae9ec..dabccd4ffeb17 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.action; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -63,24 +64,29 @@ public class EsqlExecutionInfo implements ChunkedToXContentObject, Writeable { private final transient Predicate skipUnavailablePredicate; private TimeValue overallTook; - public EsqlExecutionInfo() { - this(Predicates.always()); // default all clusters to skip_unavailable=true + // whether the user has asked for CCS metadata to be in the JSON response (the overall took will always be present) + private final boolean includeCCSMetadata; + + public EsqlExecutionInfo(boolean includeCCSMetadata) { + this(Predicates.always(), includeCCSMetadata); // default all clusters to skip_unavailable=true } /** * @param skipUnavailablePredicate provide lookup for whether a given cluster has skip_unavailable set to true or false */ - public EsqlExecutionInfo(Predicate skipUnavailablePredicate) { + public EsqlExecutionInfo(Predicate skipUnavailablePredicate, boolean includeCCSMetadata) { this.clusterInfo = ConcurrentCollections.newConcurrentMap(); this.skipUnavailablePredicate = skipUnavailablePredicate; + this.includeCCSMetadata = includeCCSMetadata; } /** * For testing use with fromXContent parsing only * @param clusterInfo */ - EsqlExecutionInfo(ConcurrentMap clusterInfo) { + EsqlExecutionInfo(ConcurrentMap clusterInfo, boolean includeCCSMetadata) { this.clusterInfo = clusterInfo; + this.includeCCSMetadata = includeCCSMetadata; this.skipUnavailablePredicate = Predicates.always(); } @@ -94,6 +100,11 @@ public EsqlExecutionInfo(StreamInput in) throws IOException { clusterList.forEach(c -> m.put(c.getClusterAlias(), c)); this.clusterInfo = m; } + if (in.getTransportVersion().onOrAfter(TransportVersions.OPT_IN_ESQL_CCS_EXECUTION_INFO)) { + this.includeCCSMetadata = in.readBoolean(); + } else { + this.includeCCSMetadata = false; + } this.skipUnavailablePredicate = Predicates.always(); } @@ -105,6 +116,13 @@ public void writeTo(StreamOutput out) throws IOException { } else { out.writeCollection(Collections.emptyList()); } + if (out.getTransportVersion().onOrAfter(TransportVersions.OPT_IN_ESQL_CCS_EXECUTION_INFO)) { + out.writeBoolean(includeCCSMetadata); + } + } + + public boolean includeCCSMetadata() { + return includeCCSMetadata; } public void overallTook(TimeValue took) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java index 4ab310863c61d..239f9e2696f88 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java @@ -42,6 +42,7 @@ public class EsqlQueryRequest extends org.elasticsearch.xpack.core.esql.action.E private String query; private boolean columnar; private boolean profile; + private boolean includeCCSMetadata; private Locale locale; private QueryBuilder filter; private QueryPragmas pragmas = new QueryPragmas(Settings.EMPTY); @@ -128,6 +129,14 @@ public void profile(boolean profile) { this.profile = profile; } + public void includeCCSMetadata(boolean include) { + this.includeCCSMetadata = include; + } + + public boolean includeCCSMetadata() { + return includeCCSMetadata; + } + /** * Is profiling enabled? */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java index 3232f3a9118d4..4e59d5419fe6f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java @@ -206,7 +206,7 @@ public Iterator toXContentChunked(ToXContent.Params params b.append(ResponseXContentUtils.allColumns(columns, "columns")); } b.array("values", ResponseXContentUtils.columnValues(this.columns, this.pages, columnar, nullColumns)); - if (executionInfo != null && executionInfo.isCrossClusterSearch()) { + if (executionInfo != null && executionInfo.isCrossClusterSearch() && executionInfo.includeCCSMetadata()) { b.field("_clusters", executionInfo); } if (profile != null) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java index b930fa5823404..7224aa049093d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java @@ -80,6 +80,7 @@ String fields() { private static final ParseField LOCALE_FIELD = new ParseField("locale"); private static final ParseField PROFILE_FIELD = new ParseField("profile"); private static final ParseField ACCEPT_PRAGMA_RISKS = new ParseField("accept_pragma_risks"); + private static final ParseField INCLUDE_CCS_METADATA_FIELD = new ParseField("include_ccs_metadata"); static final ParseField TABLES_FIELD = new ParseField("tables"); static final ParseField WAIT_FOR_COMPLETION_TIMEOUT = new ParseField("wait_for_completion_timeout"); @@ -117,6 +118,7 @@ private static void objectParserCommon(ObjectParser parser) parser.declareBoolean(EsqlQueryRequest::columnar, COLUMNAR_FIELD); parser.declareObject(EsqlQueryRequest::filter, (p, c) -> AbstractQueryBuilder.parseTopLevelQuery(p), FILTER_FIELD); parser.declareBoolean(EsqlQueryRequest::acceptedPragmaRisks, ACCEPT_PRAGMA_RISKS); + parser.declareBoolean(EsqlQueryRequest::includeCCSMetadata, INCLUDE_CCS_METADATA_FIELD); parser.declareObject( EsqlQueryRequest::pragmas, (p, c) -> new QueryPragmas(Settings.builder().loadFromMap(p.map()).build()), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index a29e16139dde7..dd2b72b4d35d9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -19,8 +19,12 @@ import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.expression.function.Function; import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryOperator; import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MatchQueryPredicate; +import org.elasticsearch.xpack.esql.core.expression.predicate.logical.BinaryLogic; +import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not; +import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.Holder; @@ -28,6 +32,8 @@ import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.esql.expression.function.aggregate.Rate; import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; +import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; +import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryString; import org.elasticsearch.xpack.esql.expression.function.grouping.GroupingFunction; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Neg; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; @@ -55,6 +61,7 @@ import java.util.List; import java.util.Locale; import java.util.Set; +import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.stream.Stream; @@ -183,7 +190,7 @@ else if (p instanceof Lookup lookup) { checkOperationsOnUnsignedLong(p, failures); checkBinaryComparison(p, failures); - checkForSortOnSpatialTypes(p, failures); + checkForSortableDataTypes(p, failures); checkFilterMatchConditions(p, failures); checkFullTextQueryFunctions(p, failures); @@ -548,14 +555,13 @@ private static Failure validateUnsignedLongNegation(Neg neg) { } /** - * Makes sure that spatial types do not appear in sorting contexts. + * Some datatypes are not sortable */ - private static void checkForSortOnSpatialTypes(LogicalPlan p, Set localFailures) { + private static void checkForSortableDataTypes(LogicalPlan p, Set localFailures) { if (p instanceof OrderBy ob) { - ob.forEachExpression(Attribute.class, attr -> { - DataType dataType = attr.dataType(); - if (DataType.isSpatial(dataType)) { - localFailures.add(fail(attr, "cannot sort on " + dataType.typeName())); + ob.order().forEach(order -> { + if (DataType.isSortable(order.dataType()) == false) { + localFailures.add(fail(order, "cannot sort on " + order.dataType().typeName())); } }); } @@ -645,27 +651,105 @@ private static void checkFilterMatchConditions(LogicalPlan plan, Set fa private static void checkFullTextQueryFunctions(LogicalPlan plan, Set failures) { if (plan instanceof Filter f) { Expression condition = f.condition(); - if (condition instanceof FullTextFunction ftf) { - // Similar to cases present in org.elasticsearch.xpack.esql.optimizer.rules.PushDownAndCombineFilters - - // we can't check if it can be pushed down as we don't have yet information about the fields present in the - // StringQueryPredicate - plan.forEachDown(LogicalPlan.class, lp -> { - if ((lp instanceof Filter || lp instanceof OrderBy || lp instanceof EsRelation) == false) { - failures.add( - fail( - plan, - "[{}] function cannot be used after {}", - ftf.functionName(), - lp.sourceText().split(" ")[0].toUpperCase(Locale.ROOT) - ) - ); - } - }); - } + checkCommandsBeforeQueryStringFunction(plan, condition, failures); + checkCommandsBeforeMatchFunction(plan, condition, failures); + checkFullTextFunctionsConditions(condition, failures); + checkFullTextFunctionsParents(condition, failures); } else { plan.forEachExpression(FullTextFunction.class, ftf -> { failures.add(fail(ftf, "[{}] function is only supported in WHERE commands", ftf.functionName())); }); } } + + private static void checkCommandsBeforeQueryStringFunction(LogicalPlan plan, Expression condition, Set failures) { + condition.forEachDown(QueryString.class, qsf -> { + plan.forEachDown(LogicalPlan.class, lp -> { + if ((lp instanceof Filter || lp instanceof OrderBy || lp instanceof EsRelation) == false) { + failures.add( + fail( + plan, + "[{}] function cannot be used after {}", + qsf.functionName(), + lp.sourceText().split(" ")[0].toUpperCase(Locale.ROOT) + ) + ); + } + }); + }); + } + + private static void checkCommandsBeforeMatchFunction(LogicalPlan plan, Expression condition, Set failures) { + condition.forEachDown(Match.class, qsf -> { + plan.forEachDown(LogicalPlan.class, lp -> { + if (lp instanceof Limit) { + failures.add( + fail( + plan, + "[{}] function cannot be used after {}", + qsf.functionName(), + lp.sourceText().split(" ")[0].toUpperCase(Locale.ROOT) + ) + ); + } + }); + }); + } + + private static void checkFullTextFunctionsConditions(Expression condition, Set failures) { + condition.forEachUp(Or.class, or -> { + checkFullTextFunctionInDisjunction(failures, or, or.left()); + checkFullTextFunctionInDisjunction(failures, or, or.right()); + }); + } + + private static void checkFullTextFunctionInDisjunction(Set failures, Or or, Expression left) { + left.forEachDown(FullTextFunction.class, ftf -> { + failures.add( + fail( + or, + "Invalid condition [{}]. Function {} can't be used as part of an or condition", + or.sourceText(), + ftf.functionName() + ) + ); + }); + } + + private static void checkFullTextFunctionsParents(Expression condition, Set failures) { + forEachFullTextFunctionParent(condition, (ftf, parent) -> { + if ((parent instanceof FullTextFunction == false) + && (parent instanceof BinaryLogic == false) + && (parent instanceof Not == false)) { + failures.add( + fail( + condition, + "Invalid condition [{}]. Function {} can't be used with {}", + condition.sourceText(), + ftf.functionName(), + ((Function) parent).functionName() + ) + ); + } + }); + } + + /** + * Executes the action on every parent of a FullTextFunction in the condition if it is found + * + * @param action the action to execute for each parent of a FullTextFunction + */ + private static FullTextFunction forEachFullTextFunctionParent(Expression condition, BiConsumer action) { + if (condition instanceof FullTextFunction ftf) { + return ftf; + } + for (Expression child : condition.children()) { + FullTextFunction foundMatchingChild = forEachFullTextFunctionParent(child, action); + if (foundMatchingChild != null) { + action.accept(foundMatchingChild, condition); + return foundMatchingChild; + } + } + return null; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 8e238f9ed760c..e8921c68b8913 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -32,7 +32,8 @@ import org.elasticsearch.xpack.esql.expression.function.aggregate.Top; import org.elasticsearch.xpack.esql.expression.function.aggregate.Values; import org.elasticsearch.xpack.esql.expression.function.aggregate.WeightedAvg; -import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryStringFunction; +import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; +import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryString; import org.elasticsearch.xpack.esql.expression.function.grouping.Bucket; import org.elasticsearch.xpack.esql.expression.function.grouping.Categorize; import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case; @@ -78,6 +79,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.math.E; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Exp; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Floor; +import org.elasticsearch.xpack.esql.expression.function.scalar.math.Hypot; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Log; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Log10; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Pi; @@ -285,6 +287,7 @@ private FunctionDefinition[][] functions() { def(Exp.class, Exp::new, "exp"), def(Floor.class, Floor::new, "floor"), def(Greatest.class, Greatest::new, "greatest"), + def(Hypot.class, Hypot::new, "hypot"), def(Log.class, Log::new, "log"), def(Log10.class, Log10::new, "log10"), def(Least.class, Least::new, "least"), @@ -393,7 +396,8 @@ private static FunctionDefinition[][] snapshotFunctions() { def(Categorize.class, Categorize::new, "categorize"), def(Rate.class, Rate::withUnresolvedTimestamp, "rate"), // Full text functions - def(QueryStringFunction.class, QueryStringFunction::new, "qstr") } }; + def(QueryString.class, QueryString::new, "qstr"), + def(Match.class, Match::new, "match") } }; } public EsqlFunctionRegistry snapshotRegistry() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java index f5597b7d64e81..135264c448f10 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java @@ -115,7 +115,7 @@ public DataType dataType() { protected TypeResolution resolveType() { TypeResolution resolution = isType( field(), - dt -> dt == DataType.COUNTER_LONG || dt == DataType.COUNTER_INTEGER || dt == DataType.COUNTER_DOUBLE, + dt -> DataType.isCounter(dt), sourceText(), FIRST, "counter_long", diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java index 54730eec4f317..a39c0d7bc6b50 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java @@ -7,22 +7,20 @@ package org.elasticsearch.xpack.esql.expression.function.fulltext; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Nullability; +import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.expression.function.Function; -import org.elasticsearch.xpack.esql.core.querydsl.query.Query; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.core.util.PlanStreamInput; -import java.io.IOException; import java.util.ArrayList; import java.util.List; -import static java.util.Collections.singletonList; +import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNotNullAndFoldable; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; @@ -36,45 +34,84 @@ public abstract class FullTextFunction extends Function { public static List getNamedWriteables() { List entries = new ArrayList<>(); if (EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()) { - entries.add(QueryStringFunction.ENTRY); + entries.add(QueryString.ENTRY); + } + if (EsqlCapabilities.Cap.MATCH_FUNCTION.isEnabled()) { + entries.add(Match.ENTRY); } return entries; } private final Expression query; - protected FullTextFunction(Source source, Expression query) { - super(source, singletonList(query)); + protected FullTextFunction(Source source, Expression query, List children) { + super(source, children); this.query = query; } - protected FullTextFunction(StreamInput in) throws IOException { - this(Source.readFrom((StreamInput & PlanStreamInput) in), in.readNamedWriteable(Expression.class)); - } - @Override public DataType dataType() { return DataType.BOOLEAN; } @Override - protected TypeResolution resolveType() { + protected final TypeResolution resolveType() { if (childrenResolved() == false) { return new TypeResolution("Unresolved children"); } - return isString(query(), sourceText(), DEFAULT).and(isNotNullAndFoldable(query(), functionName(), DEFAULT)); + return resolveNonQueryParamTypes().and(resolveQueryParamType()); + } + + /** + * Resolves the type for the query parameter, as part of the type resolution for the function + * + * @return type resolution for query parameter + */ + private TypeResolution resolveQueryParamType() { + return isString(query(), sourceText(), queryParamOrdinal()).and(isNotNullAndFoldable(query(), sourceText(), queryParamOrdinal())); + } + + /** + * Subclasses can override this method for custom type resolution for additional function parameters + * + * @return type resolution for non-query parameter types + */ + protected TypeResolution resolveNonQueryParamTypes() { + return TypeResolution.TYPE_RESOLVED; } public Expression query() { return query; } - @Override - public void writeTo(StreamOutput out) throws IOException { - source().writeTo(out); - out.writeNamedWriteable(query); + /** + * Returns the resulting query as a String + * + * @return query expression as a string + */ + public final String queryAsText() { + Object queryAsObject = query().fold(); + if (queryAsObject instanceof BytesRef bytesRef) { + return bytesRef.utf8ToString(); + } + + throw new IllegalArgumentException( + format(null, "{} argument in {} function needs to be resolved to a string", queryParamOrdinal(), functionName()) + ); } - public abstract Query asQuery(); + /** + * Returns the param ordinal for the query parameter so it can be used in error messages + * + * @return Query ordinal for the + */ + protected TypeResolutions.ParamOrdinal queryParamOrdinal() { + return DEFAULT; + } + + @Override + public Nullability nullable() { + return Nullability.FALSE; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java new file mode 100644 index 0000000000000..b4e0f3c743216 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.fulltext; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.esql.capabilities.Validatable; +import org.elasticsearch.xpack.esql.common.Failure; +import org.elasticsearch.xpack.esql.common.Failures; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.querydsl.query.QueryStringQuery; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNotNull; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; + +/** + * Full text function that performs a {@link QueryStringQuery} . + */ +public class Match extends FullTextFunction implements Validatable { + + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Match", Match::new); + + private final Expression field; + + @FunctionInfo( + returnType = "boolean", + preview = true, + description = "Performs a match query on the specified field. Returns true if the provided query matches the row.", + examples = { @Example(file = "match-function", tag = "match-with-field") } + ) + public Match( + Source source, + @Param(name = "field", type = { "keyword", "text" }, description = "Field that the query will target.") Expression field, + @Param( + name = "query", + type = { "keyword", "text" }, + description = "Text you wish to find in the provided field." + ) Expression matchQuery + ) { + super(source, matchQuery, List.of(field, matchQuery)); + this.field = field; + } + + private Match(StreamInput in) throws IOException { + this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), in.readNamedWriteable(Expression.class)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + out.writeNamedWriteable(field); + out.writeNamedWriteable(query()); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + protected TypeResolution resolveNonQueryParamTypes() { + return isNotNull(field, sourceText(), FIRST).and(isString(field, sourceText(), FIRST)).and(super.resolveNonQueryParamTypes()); + } + + @Override + public void validate(Failures failures) { + if (field instanceof FieldAttribute == false) { + failures.add( + Failure.fail( + field, + "[{}] cannot operate on [{}], which is not a field from an index mapping", + functionName(), + field.sourceText() + ) + ); + } + } + + @Override + public Expression replaceChildren(List newChildren) { + // Query is the first child, field is the second child + return new Match(source(), newChildren.get(0), newChildren.get(1)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Match::new, field, query()); + } + + protected TypeResolutions.ParamOrdinal queryParamOrdinal() { + return SECOND; + } + + public Expression field() { + return field; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryString.java similarity index 66% rename from x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunction.java rename to x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryString.java index fa331acd08655..0d7d15a13dd80 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryString.java @@ -7,32 +7,27 @@ package org.elasticsearch.xpack.esql.expression.function.fulltext; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.querydsl.query.Query; import org.elasticsearch.xpack.esql.core.querydsl.query.QueryStringQuery; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import java.io.IOException; import java.util.List; -import java.util.Map; /** * Full text function that performs a {@link QueryStringQuery} . */ -public class QueryStringFunction extends FullTextFunction { +public class QueryString extends FullTextFunction { - public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( - Expression.class, - "QStr", - QueryStringFunction::new - ); + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "QStr", QueryString::new); @FunctionInfo( returnType = "boolean", @@ -40,7 +35,7 @@ public class QueryStringFunction extends FullTextFunction { description = "Performs a query string query. Returns true if the provided query string matches the row.", examples = { @Example(file = "qstr-function", tag = "qstr-with-field") } ) - public QueryStringFunction( + public QueryString( Source source, @Param( name = "query", @@ -48,40 +43,37 @@ public QueryStringFunction( description = "Query string in Lucene query string format." ) Expression queryString ) { - super(source, queryString); + super(source, queryString, List.of(queryString)); } - private QueryStringFunction(StreamInput in) throws IOException { - super(in); + private QueryString(StreamInput in) throws IOException { + this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class)); } @Override - public String functionName() { - return "QSTR"; + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + out.writeNamedWriteable(query()); } @Override - public Query asQuery() { - Object queryAsObject = query().fold(); - if (queryAsObject instanceof BytesRef queryAsBytesRef) { - return new QueryStringQuery(source(), queryAsBytesRef.utf8ToString(), Map.of(), null); - } else { - throw new IllegalArgumentException("Query in QSTR needs to be resolved to a string"); - } + public String getWriteableName() { + return ENTRY.name; } @Override - public Expression replaceChildren(List newChildren) { - return new QueryStringFunction(source(), newChildren.get(0)); + public String functionName() { + return "QSTR"; } @Override - protected NodeInfo info() { - return NodeInfo.create(this, QueryStringFunction::new, query()); + public Expression replaceChildren(List newChildren) { + return new QueryString(source(), newChildren.get(0)); } @Override - public String getWriteableName() { - return ENTRY.name; + protected NodeInfo info() { + return NodeInfo.create(this, QueryString::new, query()); } + } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java index afe9bf6e45eda..e4e1fbb6e5aac 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java @@ -30,6 +30,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.ip.IpPrefix; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Atan2; import org.elasticsearch.xpack.esql.expression.function.scalar.math.E; +import org.elasticsearch.xpack.esql.expression.function.scalar.math.Hypot; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Log; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Pi; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Pow; @@ -82,6 +83,7 @@ public static List getNamedWriteables() { entries.add(E.ENTRY); entries.add(EndsWith.ENTRY); entries.add(Greatest.ENTRY); + entries.add(Hypot.ENTRY); entries.add(In.ENTRY); entries.add(InsensitiveEquals.ENTRY); entries.add(DateExtract.ENTRY); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Hypot.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Hypot.java new file mode 100644 index 0000000000000..1a644c929f3c3 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Hypot.java @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.math; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNumeric; + +/** + * Returns the hypotenuse of the numbers given as parameters. + */ +public class Hypot extends EsqlScalarFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Hypot", Hypot::new); + + private final Expression n1; + private final Expression n2; + + @FunctionInfo(returnType = "double", description = """ + Returns the hypotenuse of two numbers. The input can be any numeric values, the return value is always a double. + Hypotenuses of infinities are null.""", examples = @Example(file = "math", tag = "hypot")) + public Hypot( + Source source, + @Param( + name = "number1", + type = { "double", "integer", "long", "unsigned_long" }, + description = "Numeric expression. If `null`, the function returns `null`." + ) Expression n1, + @Param( + name = "number2", + type = { "double", "integer", "long", "unsigned_long" }, + description = "Numeric expression. If `null`, the function returns `null`." + ) Expression n2 + ) { + super(source, List.of(n1, n2)); + this.n1 = n1; + this.n2 = n2; + } + + private Hypot(StreamInput in) throws IOException { + this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), in.readNamedWriteable(Expression.class)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + out.writeNamedWriteable(n1); + out.writeNamedWriteable(n2); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new Hypot(source(), newChildren.get(0), newChildren.get(1)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Hypot::new, n1, n2); + } + + @Evaluator + static double process(double n1, double n2) { + return Math.hypot(n1, n2); + } + + @Override + public DataType dataType() { + return DataType.DOUBLE; + } + + @Override + protected TypeResolution resolveType() { + if (childrenResolved() == false) { + return new TypeResolution("Unresolved children"); + } + + TypeResolution resolution = isNumeric(n1, sourceText(), TypeResolutions.ParamOrdinal.FIRST); + if (resolution.unresolved()) { + return resolution; + } + return isNumeric(n2, sourceText(), TypeResolutions.ParamOrdinal.SECOND); + } + + @Override + public boolean foldable() { + return Expressions.foldable(children()); + } + + @Override + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + var n1Eval = Cast.cast(source(), n1.dataType(), DataType.DOUBLE, toEvaluator.apply(n1)); + var n2Eval = Cast.cast(source(), n2.dataType(), DataType.DOUBLE, toEvaluator.apply(n2)); + return new HypotEvaluator.Factory(source(), n1Eval, n2Eval); + } + + public Expression n1() { + return n1; + } + + public Expression n2() { + return n2; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java index 72dd052fc7637..5e8d39217fcca 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.util.List; +import java.util.Objects; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -80,6 +81,29 @@ public void writeTo(StreamOutput out) throws IOException { // The CRS type is re-resolved from the combination of left and right fields, and also not necessary to serialize } + /** + * Mark the function as expecting the specified fields to arrive as doc-values. + */ + public abstract BinarySpatialFunction withDocValues(boolean foundLeft, boolean foundRight); + + @Override + public int hashCode() { + // NB: the hashcode is currently used for key generation so + // to avoid clashes between aggs with the same arguments, add the class name as variation + return Objects.hash(getClass(), children(), leftDocValues, rightDocValues); + } + + @Override + public boolean equals(Object obj) { + if (super.equals(obj)) { + BinarySpatialFunction other = (BinarySpatialFunction) obj; + return Objects.equals(other.children(), children()) + && Objects.equals(other.leftDocValues, leftDocValues) + && Objects.equals(other.rightDocValues, rightDocValues); + } + return false; + } + @Override protected TypeResolution resolveType() { return spatialTypeResolver.resolveType(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java index 7f578565f81f2..9189c6a7b8f70 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java @@ -26,7 +26,6 @@ import org.elasticsearch.lucene.spatial.CoordinateEncoder; import org.elasticsearch.lucene.spatial.GeometryDocValueReader; import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -38,7 +37,6 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; -import java.util.Set; import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT; import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE; @@ -198,10 +196,10 @@ public ShapeRelation queryRelation() { } @Override - public SpatialContains withDocValues(Set attributes) { + public SpatialContains withDocValues(boolean foundLeft, boolean foundRight) { // Only update the docValues flags if the field is found in the attributes - boolean leftDV = leftDocValues || foundField(left(), attributes); - boolean rightDV = rightDocValues || foundField(right(), attributes); + boolean leftDV = leftDocValues || foundLeft; + boolean rightDV = rightDocValues || foundRight; return new SpatialContains(source(), left(), right(), leftDV, rightDV); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java index 47d19ebae884b..ee78f50c4d6bd 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java @@ -23,7 +23,6 @@ import org.elasticsearch.lucene.spatial.CoordinateEncoder; import org.elasticsearch.lucene.spatial.GeometryDocValueReader; import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -35,7 +34,6 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; -import java.util.Set; import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT; import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE; @@ -113,10 +111,10 @@ public ShapeRelation queryRelation() { } @Override - public SpatialDisjoint withDocValues(Set attributes) { + public SpatialDisjoint withDocValues(boolean foundLeft, boolean foundRight) { // Only update the docValues flags if the field is found in the attributes - boolean leftDV = leftDocValues || foundField(left(), attributes); - boolean rightDV = rightDocValues || foundField(right(), attributes); + boolean leftDV = leftDocValues || foundLeft; + boolean rightDV = rightDocValues || foundRight; return new SpatialDisjoint(source(), left(), right(), leftDV, rightDV); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java index 8e287baeaa9b8..8d54e5ee443c2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java @@ -23,7 +23,6 @@ import org.elasticsearch.lucene.spatial.CoordinateEncoder; import org.elasticsearch.lucene.spatial.GeometryDocValueReader; import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -35,7 +34,6 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; -import java.util.Set; import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT; import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE; @@ -111,10 +109,10 @@ public ShapeRelation queryRelation() { } @Override - public SpatialIntersects withDocValues(Set attributes) { + public SpatialIntersects withDocValues(boolean foundLeft, boolean foundRight) { // Only update the docValues flags if the field is found in the attributes - boolean leftDV = leftDocValues || foundField(left(), attributes); - boolean rightDV = rightDocValues || foundField(right(), attributes); + boolean leftDV = leftDocValues || foundLeft; + boolean rightDV = rightDocValues || foundRight; return new SpatialIntersects(source(), left(), right(), leftDV, rightDV); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java index ee2b4450a64ff..8ca89334b059b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java @@ -22,7 +22,6 @@ import org.elasticsearch.lucene.spatial.CoordinateEncoder; import org.elasticsearch.lucene.spatial.GeometryDocValueReader; import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes; @@ -30,9 +29,6 @@ import java.io.IOException; import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.function.Predicate; import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asGeometryDocValueReader; import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asLuceneComponent2D; @@ -57,47 +53,6 @@ public DataType dataType() { return DataType.BOOLEAN; } - /** - * Mark the function as expecting the specified fields to arrive as doc-values. - */ - public abstract SpatialRelatesFunction withDocValues(Set attributes); - - /** - * Push-down to Lucene is only possible if one field is an indexed spatial field, and the other is a constant spatial or string column. - */ - public boolean canPushToSource(Predicate isAggregatable) { - // The use of foldable here instead of SpatialEvaluatorFieldKey.isConstant is intentional to match the behavior of the - // Lucene pushdown code in EsqlTranslationHandler::SpatialRelatesTranslator - // We could enhance both places to support ReferenceAttributes that refer to constants, but that is a larger change - return isPushableFieldAttribute(left(), isAggregatable) && right().foldable() - || isPushableFieldAttribute(right(), isAggregatable) && left().foldable(); - } - - private static boolean isPushableFieldAttribute(Expression exp, Predicate isAggregatable) { - return exp instanceof FieldAttribute fa - && fa.getExactInfo().hasExact() - && isAggregatable.test(fa) - && DataType.isSpatial(fa.dataType()); - } - - @Override - public int hashCode() { - // NB: the hashcode is currently used for key generation so - // to avoid clashes between aggs with the same arguments, add the class name as variation - return Objects.hash(getClass(), children(), leftDocValues, rightDocValues); - } - - @Override - public boolean equals(Object obj) { - if (super.equals(obj)) { - SpatialRelatesFunction other = (SpatialRelatesFunction) obj; - return Objects.equals(other.children(), children()) - && Objects.equals(other.leftDocValues, leftDocValues) - && Objects.equals(other.rightDocValues, rightDocValues); - } - return false; - } - /** * Produce a map of rules defining combinations of incoming types to the evaluator factory that should be used. */ @@ -115,19 +70,6 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua return SpatialEvaluatorFactory.makeSpatialEvaluator(this, evaluatorRules(), toEvaluator); } - /** - * When performing local physical plan optimization, it is necessary to know if this function has a field attribute. - * This is because the planner might push down a spatial aggregation to lucene, which results in the field being provided - * as doc-values instead of source values, and this function needs to know if it should use doc-values or not. - */ - public boolean hasFieldAttribute(Set foundAttributes) { - return foundField(left(), foundAttributes) || foundField(right(), foundAttributes); - } - - protected boolean foundField(Expression expression, Set foundAttributes) { - return expression instanceof FieldAttribute field && foundAttributes.contains(field); - } - protected static class SpatialRelations extends BinarySpatialComparator { protected final ShapeField.QueryRelation queryRelation; protected final ShapeIndexer shapeIndexer; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithin.java index 84ea5b86f1d40..2005709cd37e9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithin.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithin.java @@ -23,7 +23,6 @@ import org.elasticsearch.lucene.spatial.CoordinateEncoder; import org.elasticsearch.lucene.spatial.GeometryDocValueReader; import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -36,7 +35,6 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; -import java.util.Set; import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT; import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE; @@ -113,10 +111,10 @@ public ShapeRelation queryRelation() { } @Override - public SpatialWithin withDocValues(Set attributes) { + public SpatialWithin withDocValues(boolean foundLeft, boolean foundRight) { // Only update the docValues flags if the field is found in the attributes - boolean leftDV = leftDocValues || foundField(left(), attributes); - boolean rightDV = rightDocValues || foundField(right(), attributes); + boolean leftDV = leftDocValues || foundLeft; + boolean rightDV = rightDocValues || foundRight; return new SpatialWithin(source(), left(), right(), leftDV, rightDV); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java index 17bcc68004bff..ae9d3383bad39 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java @@ -148,6 +148,14 @@ private StDistance(StreamInput in) throws IOException { super(in, false, false, true); } + @Override + public StDistance withDocValues(boolean foundLeft, boolean foundRight) { + // Only update the docValues flags if the field is found in the attributes + boolean leftDV = leftDocValues || foundLeft; + boolean rightDV = rightDocValues || foundRight; + return new StDistance(source(), left(), right(), leftDV, rightDV); + } + @Override public String getWriteableName() { return ENTRY.name; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFilters.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFilters.java index e39b590228d57..ed09d0bc16754 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFilters.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFilters.java @@ -7,10 +7,13 @@ package org.elasticsearch.xpack.esql.optimizer.rules.logical; +import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.AttributeMap; import org.elasticsearch.xpack.esql.core.expression.AttributeSet; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.expression.predicate.Predicates; import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; @@ -25,6 +28,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.function.Function; import java.util.function.Predicate; public final class PushDownAndCombineFilters extends OptimizerRules.OptimizerRule { @@ -43,20 +47,37 @@ protected LogicalPlan rule(Filter filter) { filter, agg, e -> e instanceof Attribute && agg.output().contains(e) && agg.groupings().contains(e) == false - || e instanceof AggregateFunction + || e instanceof AggregateFunction, + NO_OP ); } else if (child instanceof Eval eval) { - // Don't push if Filter (still) contains references of Eval's fields. - var attributes = new AttributeSet(Expressions.asAttributes(eval.fields())); - plan = maybePushDownPastUnary(filter, eval, attributes::contains); + // Don't push if Filter (still) contains references to Eval's fields. + // Account for simple aliases in the Eval, though - these shouldn't stop us. + AttributeMap.Builder aliasesBuilder = AttributeMap.builder(); + for (Alias alias : eval.fields()) { + aliasesBuilder.put(alias.toAttribute(), alias.child()); + } + AttributeMap evalAliases = aliasesBuilder.build(); + + Function resolveRenames = expr -> expr.transformDown(ReferenceAttribute.class, r -> { + Expression resolved = evalAliases.resolve(r, null); + // Avoid resolving to an intermediate attribute that only lives inside the Eval - only replace if the attribute existed + // before the Eval. + if (resolved instanceof Attribute && eval.inputSet().contains(resolved)) { + return resolved; + } + return r; + }); + + plan = maybePushDownPastUnary(filter, eval, evalAliases::containsKey, resolveRenames); } else if (child instanceof RegexExtract re) { // Push down filters that do not rely on attributes created by RegexExtract var attributes = new AttributeSet(Expressions.asAttributes(re.extractedFields())); - plan = maybePushDownPastUnary(filter, re, attributes::contains); + plan = maybePushDownPastUnary(filter, re, attributes::contains, NO_OP); } else if (child instanceof Enrich enrich) { // Push down filters that do not rely on attributes created by Enrich var attributes = new AttributeSet(Expressions.asAttributes(enrich.enrichFields())); - plan = maybePushDownPastUnary(filter, enrich, attributes::contains); + plan = maybePushDownPastUnary(filter, enrich, attributes::contains, NO_OP); } else if (child instanceof Project) { return PushDownUtils.pushDownPastProject(filter); } else if (child instanceof OrderBy orderBy) { @@ -67,21 +88,35 @@ protected LogicalPlan rule(Filter filter) { return plan; } - private static LogicalPlan maybePushDownPastUnary(Filter filter, UnaryPlan unary, Predicate cannotPush) { + private static Function NO_OP = expression -> expression; + + private static LogicalPlan maybePushDownPastUnary( + Filter filter, + UnaryPlan unary, + Predicate cannotPush, + Function resolveRenames + ) { LogicalPlan plan; List pushable = new ArrayList<>(); List nonPushable = new ArrayList<>(); for (Expression exp : Predicates.splitAnd(filter.condition())) { - (exp.anyMatch(cannotPush) ? nonPushable : pushable).add(exp); + Expression resolvedExp = resolveRenames.apply(exp); + if (resolvedExp.anyMatch(cannotPush)) { + // Add the original expression to the non-pushables. + nonPushable.add(exp); + } else { + // When we can push down, we use the resolved expression. + pushable.add(resolvedExp); + } } // Push the filter down even if it might not be pushable all the way to ES eventually: eval'ing it closer to the source, // potentially still in the Exec Engine, distributes the computation. - if (pushable.size() > 0) { - if (nonPushable.size() > 0) { - Filter pushed = new Filter(filter.source(), unary.child(), Predicates.combineAnd(pushable)); + if (pushable.isEmpty() == false) { + Filter pushed = filter.with(unary.child(), Predicates.combineAnd(pushable)); + if (nonPushable.isEmpty() == false) { plan = filter.with(unary.replaceChild(pushed), Predicates.combineAnd(nonPushable)); } else { - plan = unary.replaceChild(filter.with(unary.child(), filter.condition())); + plan = unary.replaceChild(pushed); } } else { plan = filter; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java index e27418c2cf6a9..be6e124502ba5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java @@ -12,8 +12,14 @@ import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.Point; import org.elasticsearch.geometry.utils.WellKnownBinary; +import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.AttributeMap; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.NameId; +import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; +import org.elasticsearch.xpack.esql.core.expression.predicate.Predicates; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -25,17 +31,42 @@ import org.elasticsearch.xpack.esql.optimizer.LocalPhysicalOptimizerContext; import org.elasticsearch.xpack.esql.optimizer.PhysicalOptimizerRules; import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec; +import org.elasticsearch.xpack.esql.plan.physical.EvalExec; import org.elasticsearch.xpack.esql.plan.physical.FilterExec; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.xpack.esql.core.expression.predicate.Predicates.splitAnd; +import static org.elasticsearch.xpack.esql.optimizer.rules.physical.local.PushFiltersToSource.canPushSpatialFunctionToSource; +import static org.elasticsearch.xpack.esql.optimizer.rules.physical.local.PushFiltersToSource.canPushToSource; +import static org.elasticsearch.xpack.esql.optimizer.rules.physical.local.PushFiltersToSource.getAliasReplacedBy; /** * When a spatial distance predicate can be pushed down to lucene, this is done by capturing the distance within the same function. * In principle this is like re-writing the predicate: * 
    WHERE ST_DISTANCE(field, TO_GEOPOINT("POINT(0 0)")) <= 10000
    * as: - * 
    WHERE ST_INTERSECTS(field, TO_GEOSHAPE("CIRCLE(0,0,10000)"))
    + * 
    WHERE ST_INTERSECTS(field, TO_GEOSHAPE("CIRCLE(0,0,10000)"))
    . + *

    + * In addition, since the distance could be calculated in a preceding EVAL command, we also need to consider the case: + * 

    + *     FROM index
+ *     | EVAL distance = ST_DISTANCE(field, TO_GEOPOINT("POINT(0 0)"))
+ *     | WHERE distance <= 10000
+ *
    + * And re-write that as: + * 
    + *     FROM index
+ *     | WHERE ST_INTERSECTS(field, TO_GEOSHAPE("CIRCLE(0,0,10000)"))
+ *     | EVAL distance = ST_DISTANCE(field, TO_GEOPOINT("POINT(0 0)"))
+ *
    + * Note that the WHERE clause is both rewritten to an intersection and pushed down closer to the EsQueryExec, + * which allows the predicate to be pushed down to Lucene in a later rule, PushFiltersToSource. */ public class EnableSpatialDistancePushdown extends PhysicalOptimizerRules.ParameterizedOptimizerRule< FilterExec, @@ -44,23 +75,121 @@ public class EnableSpatialDistancePushdown extends PhysicalOptimizerRules.Parame @Override protected PhysicalPlan rule(FilterExec filterExec, LocalPhysicalOptimizerContext ctx) { PhysicalPlan plan = filterExec; - if (filterExec.child() instanceof EsQueryExec) { + if (filterExec.child() instanceof EsQueryExec esQueryExec) { + plan = rewrite(filterExec, esQueryExec); + } else if (filterExec.child() instanceof EvalExec evalExec && evalExec.child() instanceof EsQueryExec esQueryExec) { + plan = rewriteBySplittingFilter(filterExec, evalExec, esQueryExec); + } + + return plan; + } + + private FilterExec rewrite(FilterExec filterExec, EsQueryExec esQueryExec) { + // Find and rewrite any binary comparisons that involve a distance function and a literal + var rewritten = filterExec.condition().transformDown(EsqlBinaryComparison.class, comparison -> { + ComparisonType comparisonType = ComparisonType.from(comparison.getFunctionType()); + if (comparison.left() instanceof StDistance dist && comparison.right().foldable()) { + return rewriteComparison(comparison, dist, comparison.right(), comparisonType); + } else if (comparison.right() instanceof StDistance dist && comparison.left().foldable()) { + return rewriteComparison(comparison, dist, comparison.left(), ComparisonType.invert(comparisonType)); + } + return comparison; + }); + if (rewritten.equals(filterExec.condition()) == false) { + return new FilterExec(filterExec.source(), esQueryExec, rewritten); + } + return filterExec; + } + + /** + * This version of the rewrite will try to split the filter into two parts, one that can be pushed down to the source and + * one that needs to be kept after the EVAL. + * For example: + * 
    +     *     FROM index
+     *     | EVAL distance = ST_DISTANCE(field, TO_GEOPOINT("POINT(0 0)")), other = scale * 2
+     *     | WHERE distance <= 10000 AND distance > 5000 AND other > 10
+     *
    + * Should be rewritten as: + * 
    +     *     FROM index
+     *     | WHERE ST_INTERSECTS(field, TO_GEOSHAPE("CIRCLE(0,0,10000)"))
+     *         AND ST_DISJOINT(field, TO_GEOSHAPE("CIRCLE(0,0,5000)"))
+     *     | EVAL distance = ST_DISTANCE(field, TO_GEOPOINT("POINT(0 0)")), other = scale * 2
+     *     | WHERE other > 10
+     *
    + */ + private PhysicalPlan rewriteBySplittingFilter(FilterExec filterExec, EvalExec evalExec, EsQueryExec esQueryExec) { + // Find all pushable distance functions in the EVAL + Map distances = getPushableDistances(evalExec.fields()); + + // Don't do anything if there are no distances to push down + if (distances.isEmpty()) { + return filterExec; + } + + // Process the EVAL to get all aliases that might be needed in the filter rewrite + AttributeMap aliasReplacedBy = getAliasReplacedBy(evalExec); + + // First we split the filter into multiple AND'd expressions, and then we evaluate each individually for distance rewrites + List pushable = new ArrayList<>(); + List nonPushable = new ArrayList<>(); + for (Expression exp : splitAnd(filterExec.condition())) { + Expression resExp = exp.transformUp(ReferenceAttribute.class, r -> aliasReplacedBy.resolve(r, r)); // Find and rewrite any binary comparisons that involve a distance function and a literal - var rewritten = filterExec.condition().transformDown(EsqlBinaryComparison.class, comparison -> { - ComparisonType comparisonType = ComparisonType.from(comparison.getFunctionType()); - if (comparison.left() instanceof StDistance dist && comparison.right().foldable()) { - return rewriteComparison(comparison, dist, comparison.right(), comparisonType); - } else if (comparison.right() instanceof StDistance dist && comparison.left().foldable()) { - return rewriteComparison(comparison, dist, comparison.left(), ComparisonType.invert(comparisonType)); - } - return comparison; - }); - if (rewritten.equals(filterExec.condition()) == false) { - plan = new FilterExec(filterExec.source(), filterExec.child(), rewritten); + var rewritten = rewriteDistanceFilters(resExp, distances); + // If all pushable StDistance functions were found and re-written, we need to re-write the FILTER/EVAL combination + if (rewritten.equals(resExp) == false && canPushToSource(rewritten, x -> false)) { + pushable.add(rewritten); + } else { + nonPushable.add(exp); } } - return plan; + // If nothing pushable was rewritten, we can return the original filter + if (pushable.isEmpty()) { + return filterExec; + } + + // Move the rewritten pushable filters below the EVAL + var distanceFilter = new FilterExec(filterExec.source(), esQueryExec, Predicates.combineAnd(pushable)); + var newEval = new EvalExec(evalExec.source(), distanceFilter, evalExec.fields()); + if (nonPushable.isEmpty()) { + // No other filters found, we can just return the original eval with the new filter as child + return newEval; + } else { + // Some other filters found, we need to return two filters with the eval in between + return new FilterExec(filterExec.source(), newEval, Predicates.combineAnd(nonPushable)); + } + } + + private Map getPushableDistances(List aliases) { + Map distances = new LinkedHashMap<>(); + aliases.forEach(alias -> { + if (alias.child() instanceof StDistance distance && canPushSpatialFunctionToSource(distance)) { + distances.put(alias.id(), distance); + } else if (alias.child() instanceof ReferenceAttribute ref && distances.containsKey(ref.id())) { + StDistance distance = distances.get(ref.id()); + distances.put(alias.id(), distance); + } + }); + return distances; + } + + private Expression rewriteDistanceFilters(Expression expr, Map distances) { + return expr.transformDown(EsqlBinaryComparison.class, comparison -> { + ComparisonType comparisonType = ComparisonType.from(comparison.getFunctionType()); + if (comparison.left() instanceof ReferenceAttribute r && distances.containsKey(r.id()) && comparison.right().foldable()) { + StDistance dist = distances.get(r.id()); + return rewriteComparison(comparison, dist, comparison.right(), comparisonType); + } else if (comparison.right() instanceof ReferenceAttribute r + && distances.containsKey(r.id()) + && comparison.left().foldable()) { + StDistance dist = distances.get(r.id()); + return rewriteComparison(comparison, dist, comparison.left(), ComparisonType.invert(comparisonType)); + } + return comparison; + }); } private Expression rewriteComparison( @@ -117,7 +246,7 @@ private Literal makeCircleLiteral(Point point, double distance, Expression liter /** * This enum captures the key differences between various inequalities as perceived from the spatial distance function. * In particular, we need to know which direction the inequality points, with lt=true meaning the left is expected to be smaller - * than the right. And eq=true meaning we expect euality as well. We currently don't support Equals and NotEquals, so the third + * than the right. And eq=true meaning we expect equality as well. We currently don't support Equals and NotEquals, so the third * field disables those. */ enum ComparisonType { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java index 0a71bce2575fa..2209dffe5af06 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java @@ -8,10 +8,14 @@ package org.elasticsearch.xpack.esql.optimizer.rules.physical.local; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.AttributeMap; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; +import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.expression.function.scalar.UnaryScalarFunction; import org.elasticsearch.xpack.esql.core.expression.predicate.Predicates; import org.elasticsearch.xpack.esql.core.expression.predicate.Range; @@ -28,8 +32,10 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.CollectionUtils; import org.elasticsearch.xpack.esql.core.util.Queries; -import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; +import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; +import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryString; import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.BinarySpatialFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison; @@ -43,6 +49,7 @@ import org.elasticsearch.xpack.esql.optimizer.LocalPhysicalOptimizerContext; import org.elasticsearch.xpack.esql.optimizer.PhysicalOptimizerRules; import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec; +import org.elasticsearch.xpack.esql.plan.physical.EvalExec; import org.elasticsearch.xpack.esql.plan.physical.FilterExec; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.esql.planner.PlannerUtils; @@ -53,6 +60,7 @@ import static java.util.Arrays.asList; import static org.elasticsearch.xpack.esql.core.expression.predicate.Predicates.splitAnd; +import static org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushDownUtils.isAggregatable; public class PushFiltersToSource extends PhysicalOptimizerRules.ParameterizedOptimizerRule { @@ -60,40 +68,88 @@ public class PushFiltersToSource extends PhysicalOptimizerRules.ParameterizedOpt protected PhysicalPlan rule(FilterExec filterExec, LocalPhysicalOptimizerContext ctx) { PhysicalPlan plan = filterExec; if (filterExec.child() instanceof EsQueryExec queryExec) { - List pushable = new ArrayList<>(); - List nonPushable = new ArrayList<>(); - for (Expression exp : splitAnd(filterExec.condition())) { - (canPushToSource(exp, x -> LucenePushDownUtils.hasIdenticalDelegate(x, ctx.searchStats())) ? pushable : nonPushable).add( - exp - ); - } - // Combine GT, GTE, LT and LTE in pushable to Range if possible - List newPushable = combineEligiblePushableToRange(pushable); - if (newPushable.size() > 0) { // update the executable with pushable conditions - Query queryDSL = PlannerUtils.TRANSLATOR_HANDLER.asQuery(Predicates.combineAnd(newPushable)); - QueryBuilder planQuery = queryDSL.asBuilder(); - var query = Queries.combine(Queries.Clause.FILTER, asList(queryExec.query(), planQuery)); - queryExec = new EsQueryExec( - queryExec.source(), - queryExec.index(), - queryExec.indexMode(), - queryExec.output(), - query, - queryExec.limit(), - queryExec.sorts(), - queryExec.estimatedRowSize() - ); - if (nonPushable.size() > 0) { // update filter with remaining non-pushable conditions - plan = new FilterExec(filterExec.source(), queryExec, Predicates.combineAnd(nonPushable)); - } else { // prune Filter entirely - plan = queryExec; - } - } // else: nothing changes + plan = planFilterExec(filterExec, queryExec, ctx); + } else if (filterExec.child() instanceof EvalExec evalExec && evalExec.child() instanceof EsQueryExec queryExec) { + plan = planFilterExec(filterExec, evalExec, queryExec, ctx); } - return plan; } + private static PhysicalPlan planFilterExec(FilterExec filterExec, EsQueryExec queryExec, LocalPhysicalOptimizerContext ctx) { + List pushable = new ArrayList<>(); + List nonPushable = new ArrayList<>(); + for (Expression exp : splitAnd(filterExec.condition())) { + (canPushToSource(exp, x -> LucenePushDownUtils.hasIdenticalDelegate(x, ctx.searchStats())) ? pushable : nonPushable).add(exp); + } + return rewrite(filterExec, queryExec, pushable, nonPushable, List.of()); + } + + private static PhysicalPlan planFilterExec( + FilterExec filterExec, + EvalExec evalExec, + EsQueryExec queryExec, + LocalPhysicalOptimizerContext ctx + ) { + AttributeMap aliasReplacedBy = getAliasReplacedBy(evalExec); + List pushable = new ArrayList<>(); + List nonPushable = new ArrayList<>(); + for (Expression exp : splitAnd(filterExec.condition())) { + Expression resExp = exp.transformUp(ReferenceAttribute.class, r -> aliasReplacedBy.resolve(r, r)); + (canPushToSource(resExp, x -> LucenePushDownUtils.hasIdenticalDelegate(x, ctx.searchStats())) ? pushable : nonPushable).add( + exp + ); + } + // Replace field references with their actual field attributes + pushable.replaceAll(e -> e.transformDown(ReferenceAttribute.class, r -> aliasReplacedBy.resolve(r, r))); + return rewrite(filterExec, queryExec, pushable, nonPushable, evalExec.fields()); + } + + static AttributeMap getAliasReplacedBy(EvalExec evalExec) { + AttributeMap.Builder aliasReplacedByBuilder = AttributeMap.builder(); + evalExec.fields().forEach(alias -> { + if (alias.child() instanceof Attribute attr) { + aliasReplacedByBuilder.put(alias.toAttribute(), attr); + } + }); + return aliasReplacedByBuilder.build(); + } + + private static PhysicalPlan rewrite( + FilterExec filterExec, + EsQueryExec queryExec, + List pushable, + List nonPushable, + List evalFields + ) { + // Combine GT, GTE, LT and LTE in pushable to Range if possible + List newPushable = combineEligiblePushableToRange(pushable); + if (newPushable.size() > 0) { // update the executable with pushable conditions + Query queryDSL = PlannerUtils.TRANSLATOR_HANDLER.asQuery(Predicates.combineAnd(newPushable)); + QueryBuilder planQuery = queryDSL.asBuilder(); + var query = Queries.combine(Queries.Clause.FILTER, asList(queryExec.query(), planQuery)); + queryExec = new EsQueryExec( + queryExec.source(), + queryExec.index(), + queryExec.indexMode(), + queryExec.output(), + query, + queryExec.limit(), + queryExec.sorts(), + queryExec.estimatedRowSize() + ); + // If the eval contains other aliases, not just field attributes, we need to keep them in the plan + PhysicalPlan plan = evalFields.isEmpty() ? queryExec : new EvalExec(filterExec.source(), queryExec, evalFields); + if (nonPushable.size() > 0) { + // update filter with remaining non-pushable conditions + return new FilterExec(filterExec.source(), plan, Predicates.combineAnd(nonPushable)); + } else { + // prune Filter entirely + return plan; + } + } // else: nothing changes + return filterExec; + } + private static List combineEligiblePushableToRange(List pushable) { List bcs = new ArrayList<>(); List ranges = new ArrayList<>(); @@ -189,18 +245,34 @@ public static boolean canPushToSource(Expression exp, Predicate } } else if (exp instanceof CIDRMatch cidrMatch) { return isAttributePushable(cidrMatch.ipField(), cidrMatch, hasIdenticalDelegate) && Expressions.foldable(cidrMatch.matches()); - } else if (exp instanceof SpatialRelatesFunction bc) { - return bc.canPushToSource(LucenePushDownUtils::isAggregatable); + } else if (exp instanceof SpatialRelatesFunction spatial) { + return canPushSpatialFunctionToSource(spatial); } else if (exp instanceof MatchQueryPredicate mqp) { return mqp.field() instanceof FieldAttribute && DataType.isString(mqp.field().dataType()); } else if (exp instanceof StringQueryPredicate) { return true; - } else if (exp instanceof FullTextFunction) { + } else if (exp instanceof QueryString) { return true; + } else if (exp instanceof Match mf) { + return mf.field() instanceof FieldAttribute && DataType.isString(mf.field().dataType()); } return false; } + /** + * Push-down to Lucene is only possible if one field is an indexed spatial field, and the other is a constant spatial or string column. + */ + public static boolean canPushSpatialFunctionToSource(BinarySpatialFunction s) { + // The use of foldable here instead of SpatialEvaluatorFieldKey.isConstant is intentional to match the behavior of the + // Lucene pushdown code in EsqlTranslationHandler::SpatialRelatesTranslator + // We could enhance both places to support ReferenceAttributes that refer to constants, but that is a larger change + return isPushableSpatialAttribute(s.left()) && s.right().foldable() || isPushableSpatialAttribute(s.right()) && s.left().foldable(); + } + + private static boolean isPushableSpatialAttribute(Expression exp) { + return exp instanceof FieldAttribute fa && fa.getExactInfo().hasExact() && isAggregatable(fa) && DataType.isSpatial(fa.dataType()); + } + private static boolean isAttributePushable( Expression expression, Expression operation, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java index 87bc344c397c1..6db35fa0a06d6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java @@ -7,55 +7,217 @@ package org.elasticsearch.xpack.esql.optimizer.rules.physical.local; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.AttributeMap; +import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.NameId; +import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.expression.Order; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.BinarySpatialFunction; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StDistance; import org.elasticsearch.xpack.esql.optimizer.LocalPhysicalOptimizerContext; import org.elasticsearch.xpack.esql.optimizer.PhysicalOptimizerRules; import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec; +import org.elasticsearch.xpack.esql.plan.physical.EvalExec; import org.elasticsearch.xpack.esql.plan.physical.ExchangeExec; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.esql.plan.physical.TopNExec; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; import java.util.function.Predicate; +/** + * We handle two main scenarios here: + *
      + *
    1. + * Queries like `FROM index | SORT field` will be pushed to the source if the field is an indexed field. + *
      2. + *
    2. + * Queries like `FROM index | EVAL ref = ... | SORT ref` will be pushed to the source if the reference function is pushable, + * which can happen under two conditions: + *
        + *
      • + * The reference refers linearly to an indexed field. + * For example: `FROM index | EVAL ref = field | SORT ref` + *
        • + *
      • + * The reference refers to a distance function that refers to an indexed field and a constant expression. + * For example `FROM index | EVAL distance = ST_DISTANCE(field, POINT(0, 0)) | SORT distance`. + * As with the previous condition, both the attribute and the constant can be further aliased. + *
        • + *
      + *
      3. + *
    3. + *
      4. + *
    + */ public class PushTopNToSource extends PhysicalOptimizerRules.ParameterizedOptimizerRule { @Override protected PhysicalPlan rule(TopNExec topNExec, LocalPhysicalOptimizerContext ctx) { - PhysicalPlan plan = topNExec; - PhysicalPlan child = topNExec.child(); - if (canPushSorts(child) - && canPushDownOrders(topNExec.order(), x -> LucenePushDownUtils.hasIdenticalDelegate(x, ctx.searchStats()))) { + Pushable pushable = evaluatePushable(topNExec, x -> LucenePushDownUtils.hasIdenticalDelegate(x, ctx.searchStats())); + return pushable.rewrite(topNExec); + } + + /** + * Multiple scenarios for pushing down TopN to Lucene source. Each involve checking a combination of conditions and then + * performing an associated rewrite specific to that scenario. This interface should be extended by each scenario, and + * include the appropriate rewrite logic. + */ + interface Pushable { + PhysicalPlan rewrite(TopNExec topNExec); + } + + private static final Pushable NO_OP = new NoOpPushable(); + + record NoOpPushable() implements Pushable { + public PhysicalPlan rewrite(TopNExec topNExec) { + return topNExec; + } + } + + /** + * TODO: Consider deleting this case entirely. We do not know if this is ever hit. + */ + record PushableExchangeExec(ExchangeExec exchangeExec, EsQueryExec queryExec) implements Pushable { + public PhysicalPlan rewrite(TopNExec topNExec) { + var sorts = buildFieldSorts(topNExec.order()); + var limit = topNExec.limit(); + return exchangeExec.replaceChild(queryExec.withSorts(sorts).withLimit(limit)); + } + } + + record PushableQueryExec(EsQueryExec queryExec) implements Pushable { + public PhysicalPlan rewrite(TopNExec topNExec) { var sorts = buildFieldSorts(topNExec.order()); var limit = topNExec.limit(); + return queryExec.withSorts(sorts).withLimit(limit); + } + } + + record PushableGeoDistance(FieldAttribute fieldAttribute, Order order, Point point) { + private EsQueryExec.Sort sort() { + return new EsQueryExec.GeoDistanceSort(fieldAttribute.exactAttribute(), order.direction(), point.getLat(), point.getLon()); + } - if (child instanceof ExchangeExec exchangeExec && exchangeExec.child() instanceof EsQueryExec queryExec) { - plan = exchangeExec.replaceChild(queryExec.withSorts(sorts).withLimit(limit)); - } else { - plan = ((EsQueryExec) child).withSorts(sorts).withLimit(limit); + private static PushableGeoDistance from(StDistance distance, Order order) { + if (distance.left() instanceof Attribute attr && distance.right().foldable()) { + return from(attr, distance.right(), order); + } else if (distance.right() instanceof Attribute attr && distance.left().foldable()) { + return from(attr, distance.left(), order); } + return null; + } + + private static PushableGeoDistance from(Attribute attr, Expression foldable, Order order) { + if (attr instanceof FieldAttribute fieldAttribute) { + Geometry geometry = SpatialRelatesUtils.makeGeometryFromLiteral(foldable); + if (geometry instanceof Point point) { + return new PushableGeoDistance(fieldAttribute, order, point); + } + } + return null; } - return plan; } - private static boolean canPushSorts(PhysicalPlan plan) { - if (plan instanceof EsQueryExec queryExec) { - return queryExec.canPushSorts(); + record PushableCompoundExec(EvalExec evalExec, EsQueryExec queryExec, List pushableSorts) implements Pushable { + public PhysicalPlan rewrite(TopNExec topNExec) { + // We need to keep the EVAL in place because the coordinator will have its own TopNExec so we need to keep the distance + return evalExec.replaceChild(queryExec.withSorts(pushableSorts).withLimit(topNExec.limit())); } - if (plan instanceof ExchangeExec exchangeExec && exchangeExec.child() instanceof EsQueryExec queryExec) { - return queryExec.canPushSorts(); + } + + private static Pushable evaluatePushable(TopNExec topNExec, Predicate hasIdenticalDelegate) { + PhysicalPlan child = topNExec.child(); + if (child instanceof EsQueryExec queryExec + && queryExec.canPushSorts() + && canPushDownOrders(topNExec.order(), hasIdenticalDelegate)) { + // With the simplest case of `FROM index | SORT ...` we only allow pushing down if the sort is on a field + return new PushableQueryExec(queryExec); + } + if (child instanceof ExchangeExec exchangeExec + && exchangeExec.child() instanceof EsQueryExec queryExec + && queryExec.canPushSorts() + && canPushDownOrders(topNExec.order(), hasIdenticalDelegate)) { + // When we have an exchange between the FROM and the SORT, we also only allow pushing down if the sort is on a field + return new PushableExchangeExec(exchangeExec, queryExec); + } + if (child instanceof EvalExec evalExec && evalExec.child() instanceof EsQueryExec queryExec && queryExec.canPushSorts()) { + // When we have an EVAL between the FROM and the SORT, we consider pushing down if the sort is on a field and/or + // a distance function defined in the EVAL. We also move the EVAL to after the SORT. + List orders = topNExec.order(); + List fields = evalExec.fields(); + LinkedHashMap distances = new LinkedHashMap<>(); + AttributeMap.Builder aliasReplacedByBuilder = AttributeMap.builder(); + fields.forEach(alias -> { + // TODO: can we support CARTESIAN also? + if (alias.child() instanceof StDistance distance && distance.crsType() == BinarySpatialFunction.SpatialCrsType.GEO) { + distances.put(alias.id(), distance); + } else if (alias.child() instanceof Attribute attr) { + aliasReplacedByBuilder.put(alias.toAttribute(), attr.toAttribute()); + } + }); + AttributeMap aliasReplacedBy = aliasReplacedByBuilder.build(); + + List pushableSorts = new ArrayList<>(); + for (Order order : orders) { + if (LucenePushDownUtils.isPushableFieldAttribute(order.child(), hasIdenticalDelegate)) { + pushableSorts.add( + new EsQueryExec.FieldSort( + ((FieldAttribute) order.child()).exactAttribute(), + order.direction(), + order.nullsPosition() + ) + ); + } else if (order.child() instanceof ReferenceAttribute referenceAttribute) { + Attribute resolvedAttribute = aliasReplacedBy.resolve(referenceAttribute, referenceAttribute); + if (distances.containsKey(resolvedAttribute.id())) { + StDistance distance = distances.get(resolvedAttribute.id()); + StDistance d = (StDistance) distance.transformDown(ReferenceAttribute.class, r -> aliasReplacedBy.resolve(r, r)); + PushableGeoDistance pushableGeoDistance = PushableGeoDistance.from(d, order); + if (pushableGeoDistance != null) { + pushableSorts.add(pushableGeoDistance.sort()); + } else { + // As soon as we see a non-pushable sort, we know we need a final SORT command + break; + } + } else if (aliasReplacedBy.resolve(referenceAttribute, referenceAttribute) instanceof FieldAttribute fieldAttribute + && LucenePushDownUtils.isPushableFieldAttribute(fieldAttribute, hasIdenticalDelegate)) { + // If the SORT refers to a reference to a pushable field, we can push it down + pushableSorts.add( + new EsQueryExec.FieldSort(fieldAttribute.exactAttribute(), order.direction(), order.nullsPosition()) + ); + } else { + // If the SORT refers to a non-pushable reference function, the EVAL must remain before the SORT, + // and we can no longer push down anything + break; + } + } else { + // As soon as we see a non-pushable sort, we know we need a final SORT command + break; + } + } + // TODO: We can push down partial sorts where `pushableSorts.size() < orders.size()`, but that should involve benchmarks + if (pushableSorts.size() > 0 && pushableSorts.size() == orders.size()) { + return new PushableCompoundExec(evalExec, queryExec, pushableSorts); + } } - return false; + return NO_OP; } - private boolean canPushDownOrders(List orders, Predicate hasIdenticalDelegate) { + private static boolean canPushDownOrders(List orders, Predicate hasIdenticalDelegate) { // allow only exact FieldAttributes (no expressions) for sorting return orders.stream().allMatch(o -> LucenePushDownUtils.isPushableFieldAttribute(o.child(), hasIdenticalDelegate)); } - private List buildFieldSorts(List orders) { - List sorts = new ArrayList<>(orders.size()); + private static List buildFieldSorts(List orders) { + List sorts = new ArrayList<>(orders.size()); for (Order o : orders) { sorts.add(new EsQueryExec.FieldSort(((FieldAttribute) o.child()).exactAttribute(), o.direction(), o.nullsPosition())); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialDocValuesExtraction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialDocValuesExtraction.java index ea6541326458e..d03cd9ef7cb0b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialDocValuesExtraction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SpatialDocValuesExtraction.java @@ -9,9 +9,11 @@ import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.expression.function.aggregate.SpatialAggregateFunction; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.BinarySpatialFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction; import org.elasticsearch.xpack.esql.optimizer.PhysicalOptimizerRules; import org.elasticsearch.xpack.esql.plan.physical.AggregateExec; @@ -26,6 +28,41 @@ import java.util.List; import java.util.Set; +/** + * This rule is responsible for marking spatial fields to be extracted from doc-values instead of source values. + * This is a very specific optimization that is only used in the context of spatial aggregations. + * Normally spatial fields are extracted from source values because this maintains original precision, but is very slow. + * Simply loading from doc-values loses precision for points, and loses the geometry topological information for shapes. + * For this reason we only consider loading from doc values under very specific conditions: + *
      + *
    • The spatial data is consumed by a spatial aggregation (eg. ST_CENTROIDS_AGG, negating the need for precision.
      • + *
    • This aggregation is planned to run on the data node, so the doc-values Blocks are never transmit to the coordinator node.
      • + *
    • The data node index in question has doc-values stored for the field in question.
      • + *
    + * While we do not support transmitting spatial doc-values to the coordinator node, it is still important on the data node to ensure + * that all spatial functions that will receive these doc-values are aware of this fact. For this reason, if the above conditions are met, + * we need to make four edits to the local physical plan to consistently support spatial doc-values: + *
      + *
    • The spatial aggregation function itself is marked using withDocValues() to enable its + * toEvaluator() method to produce the correct doc-values aware Evaluator functions.
      • + *
    • Any spatial functions called within EVAL commands before the doc-values are consumed by the aggregation + * also need to be marked using withDocValues() so their evaluators are correct.
      • + *
    • Any spatial functions used within filters, WHERE commands, are similarly marked for the same reason.
      • + *
    • The FieldExtractExec that will extract the field is marked with withDocValuesAttributes(...) + * so that it calls the FieldType.blockReader() method with the correct FieldExtractPreference
      • + *
    + * The question has been raised why the spatial functions need to know if they are using doc-values or not. At first glance one might + * perceive ES|QL functions as being logical planning only constructs, reflecting only the intent of the user. This, however, is not true. + * The ES|QL functions all contain the runtime implementation of the functions behaviour, in the form of one or more static methods, + * as well as a toEvaluator() instance method that is used to generates Block traversal code to call these runtime + * implementations, based on some internal state of the instance of the function. In most cases this internal state contains information + * determined during the logical planning phase, such as the field name and type, and whether it is a literal and can be folded. + * In the case of spatial functions, the internal state also contains information about whether the function is using doc-values or not. + * This knowledge is determined in the class being described here, and is only determined during local physical planning on each data + * node. This is because the decision to use doc-values is based on the local data node's index configuration, and the local physical plan + * is the only place where this information is available. This also means that the knowledge of the usage of doc-values does not need + * to be serialized between nodes, and is only used locally. + */ public class SpatialDocValuesExtraction extends PhysicalOptimizerRules.OptimizerRule { @Override protected PhysicalPlan rule(AggregateExec aggregate) { @@ -65,14 +102,7 @@ && allowedForDocValues(fieldAttribute, agg, foundAttributes)) { if (exec instanceof EvalExec evalExec) { List fields = evalExec.fields(); List changed = fields.stream() - .map( - f -> (Alias) f.transformDown( - SpatialRelatesFunction.class, - spatialRelatesFunction -> (spatialRelatesFunction.hasFieldAttribute(foundAttributes)) - ? spatialRelatesFunction.withDocValues(foundAttributes) - : spatialRelatesFunction - ) - ) + .map(f -> (Alias) f.transformDown(BinarySpatialFunction.class, s -> withDocValues(s, foundAttributes))) .toList(); if (changed.equals(fields) == false) { exec = new EvalExec(exec.source(), exec.child(), changed); @@ -81,13 +111,7 @@ && allowedForDocValues(fieldAttribute, agg, foundAttributes)) { if (exec instanceof FilterExec filterExec) { // Note that ST_CENTROID does not support shapes, but SpatialRelatesFunction does, so when we extend the centroid // to support shapes, we need to consider loading shape doc-values for both centroid and relates (ST_INTERSECTS) - var condition = filterExec.condition() - .transformDown( - SpatialRelatesFunction.class, - spatialRelatesFunction -> (spatialRelatesFunction.hasFieldAttribute(foundAttributes)) - ? spatialRelatesFunction.withDocValues(foundAttributes) - : spatialRelatesFunction - ); + var condition = filterExec.condition().transformDown(BinarySpatialFunction.class, s -> withDocValues(s, foundAttributes)); if (filterExec.condition().equals(condition) == false) { exec = new FilterExec(filterExec.source(), filterExec.child(), condition); } @@ -110,6 +134,21 @@ && allowedForDocValues(fieldAttribute, agg, foundAttributes)) { return plan; } + private BinarySpatialFunction withDocValues(BinarySpatialFunction spatial, Set foundAttributes) { + // Only update the docValues flags if the field is found in the attributes + boolean foundLeft = foundField(spatial.left(), foundAttributes); + boolean foundRight = foundField(spatial.right(), foundAttributes); + return foundLeft || foundRight ? spatial.withDocValues(foundLeft, foundRight) : spatial; + } + + private boolean hasFieldAttribute(BinarySpatialFunction spatial, Set foundAttributes) { + return foundField(spatial.left(), foundAttributes) || foundField(spatial.right(), foundAttributes); + } + + private boolean foundField(Expression expression, Set foundAttributes) { + return expression instanceof FieldAttribute field && foundAttributes.contains(field); + } + /** * This function disallows the use of more than one field for doc-values extraction in the same spatial relation function. * This is because comparing two doc-values fields is not supported in the current implementation. @@ -123,7 +162,7 @@ private boolean allowedForDocValues(FieldAttribute fieldAttribute, AggregateExec var spatialRelatesAttributes = new HashSet(); agg.forEachExpressionDown(SpatialRelatesFunction.class, relatesFunction -> { candidateDocValuesAttributes.forEach(candidate -> { - if (relatesFunction.hasFieldAttribute(Set.of(candidate))) { + if (hasFieldAttribute(relatesFunction, Set.of(candidate))) { spatialRelatesAttributes.add(candidate); } }); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp index b5ca44826c051..e9e6f45bdc30f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp @@ -23,7 +23,6 @@ null null null null -null '|' null null @@ -65,6 +64,7 @@ null '%' null null +null ']' null null @@ -141,7 +141,6 @@ STATS WHERE DEV_INLINESTATS DEV_LOOKUP -DEV_MATCH DEV_METRICS UNKNOWN_CMD LINE_COMMENT @@ -186,6 +185,7 @@ MINUS ASTERISK SLASH PERCENT +DEV_MATCH NAMED_OR_POSITIONAL_PARAM OPENING_BRACKET CLOSING_BRACKET @@ -263,7 +263,6 @@ STATS WHERE DEV_INLINESTATS DEV_LOOKUP -DEV_MATCH DEV_METRICS UNKNOWN_CMD LINE_COMMENT @@ -318,7 +317,7 @@ MINUS ASTERISK SLASH PERCENT -DEV_MATCH_OP +DEV_MATCH NAMED_OR_POSITIONAL_PARAM OPENING_BRACKET CLOSING_BRACKET @@ -466,4 +465,4 @@ METRICS_MODE CLOSING_METRICS_MODE atn: -[4, 0, 120, 1475, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 2, 68, 7, 68, 2, 69, 7, 69, 2, 70, 7, 70, 2, 71, 7, 71, 2, 72, 7, 72, 2, 73, 7, 73, 2, 74, 7, 74, 2, 75, 7, 75, 2, 76, 7, 76, 2, 77, 7, 77, 2, 78, 7, 78, 2, 79, 7, 79, 2, 80, 7, 80, 2, 81, 7, 81, 2, 82, 7, 82, 2, 83, 7, 83, 2, 84, 7, 84, 2, 85, 7, 85, 2, 86, 7, 86, 2, 87, 7, 87, 2, 88, 7, 88, 2, 89, 7, 89, 2, 90, 7, 90, 2, 91, 7, 91, 2, 92, 7, 92, 2, 93, 7, 93, 2, 94, 7, 94, 2, 95, 7, 95, 2, 96, 7, 96, 2, 97, 7, 97, 2, 98, 7, 98, 2, 99, 7, 99, 2, 100, 7, 100, 2, 101, 7, 101, 2, 102, 7, 102, 2, 103, 7, 103, 2, 104, 7, 104, 2, 105, 7, 105, 2, 106, 7, 106, 2, 107, 7, 107, 2, 108, 7, 108, 2, 109, 7, 109, 2, 110, 7, 110, 2, 111, 7, 111, 2, 112, 7, 112, 2, 113, 7, 113, 2, 114, 7, 114, 2, 115, 7, 115, 2, 116, 7, 116, 2, 117, 7, 117, 2, 118, 7, 118, 2, 119, 7, 119, 2, 120, 7, 120, 2, 121, 7, 121, 2, 122, 7, 122, 2, 123, 7, 123, 2, 124, 7, 124, 2, 125, 7, 125, 2, 126, 7, 126, 2, 127, 7, 127, 2, 128, 7, 128, 2, 129, 7, 129, 2, 130, 7, 130, 2, 131, 7, 131, 2, 132, 7, 132, 2, 133, 7, 133, 2, 134, 7, 134, 2, 135, 7, 135, 2, 136, 7, 136, 2, 137, 7, 137, 2, 138, 7, 138, 2, 139, 7, 139, 2, 140, 7, 140, 2, 141, 7, 141, 2, 142, 7, 142, 2, 143, 7, 143, 2, 144, 7, 144, 2, 145, 7, 145, 2, 146, 7, 146, 2, 147, 7, 147, 2, 148, 7, 148, 2, 149, 7, 149, 2, 150, 7, 150, 2, 151, 7, 151, 2, 152, 7, 152, 2, 153, 7, 153, 2, 154, 7, 154, 2, 155, 7, 155, 2, 156, 7, 156, 2, 157, 7, 157, 2, 158, 7, 158, 2, 159, 7, 159, 2, 160, 7, 160, 2, 161, 7, 161, 2, 162, 7, 162, 2, 163, 7, 163, 2, 164, 7, 164, 2, 165, 7, 165, 2, 166, 7, 166, 2, 167, 7, 167, 2, 168, 7, 168, 2, 169, 7, 169, 2, 170, 7, 170, 2, 171, 7, 171, 2, 172, 7, 172, 2, 173, 7, 173, 2, 174, 7, 174, 2, 175, 7, 175, 2, 176, 7, 176, 2, 177, 7, 177, 2, 178, 7, 178, 2, 179, 7, 179, 2, 180, 7, 180, 2, 181, 7, 181, 2, 182, 7, 182, 2, 183, 7, 183, 2, 184, 7, 184, 2, 185, 7, 185, 2, 186, 7, 186, 2, 187, 7, 187, 2, 188, 7, 188, 2, 189, 7, 189, 2, 190, 7, 190, 2, 191, 7, 191, 2, 192, 7, 192, 2, 193, 7, 193, 2, 194, 7, 194, 2, 195, 7, 195, 2, 196, 7, 196, 2, 197, 7, 197, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 20, 4, 20, 587, 8, 20, 11, 20, 12, 20, 588, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 1, 21, 5, 21, 597, 8, 21, 10, 21, 12, 21, 600, 9, 21, 1, 21, 3, 21, 603, 8, 21, 1, 21, 3, 21, 606, 8, 21, 1, 21, 1, 21, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 5, 22, 615, 8, 22, 10, 22, 12, 22, 618, 9, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 23, 4, 23, 626, 8, 23, 11, 23, 12, 23, 627, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 1, 24, 1, 25, 1, 25, 1, 26, 1, 26, 1, 27, 1, 27, 1, 27, 1, 28, 1, 28, 1, 29, 1, 29, 3, 29, 647, 8, 29, 1, 29, 4, 29, 650, 8, 29, 11, 29, 12, 29, 651, 1, 30, 1, 30, 1, 31, 1, 31, 1, 32, 1, 32, 1, 32, 3, 32, 661, 8, 32, 1, 33, 1, 33, 1, 34, 1, 34, 1, 34, 3, 34, 668, 8, 34, 1, 35, 1, 35, 1, 35, 5, 35, 673, 8, 35, 10, 35, 12, 35, 676, 9, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 5, 35, 684, 8, 35, 10, 35, 12, 35, 687, 9, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 3, 35, 694, 8, 35, 1, 35, 3, 35, 697, 8, 35, 3, 35, 699, 8, 35, 1, 36, 4, 36, 702, 8, 36, 11, 36, 12, 36, 703, 1, 37, 4, 37, 707, 8, 37, 11, 37, 12, 37, 708, 1, 37, 1, 37, 5, 37, 713, 8, 37, 10, 37, 12, 37, 716, 9, 37, 1, 37, 1, 37, 4, 37, 720, 8, 37, 11, 37, 12, 37, 721, 1, 37, 4, 37, 725, 8, 37, 11, 37, 12, 37, 726, 1, 37, 1, 37, 5, 37, 731, 8, 37, 10, 37, 12, 37, 734, 9, 37, 3, 37, 736, 8, 37, 1, 37, 1, 37, 1, 37, 1, 37, 4, 37, 742, 8, 37, 11, 37, 12, 37, 743, 1, 37, 1, 37, 3, 37, 748, 8, 37, 1, 38, 1, 38, 1, 38, 1, 39, 1, 39, 1, 39, 1, 39, 1, 40, 1, 40, 1, 40, 1, 40, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 44, 1, 44, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 1, 47, 1, 47, 1, 47, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 49, 1, 50, 1, 50, 1, 50, 1, 50, 1, 50, 1, 51, 1, 51, 1, 51, 1, 51, 1, 51, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 1, 58, 1, 58, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 63, 1, 64, 1, 64, 1, 65, 1, 65, 1, 65, 1, 66, 1, 66, 1, 67, 1, 67, 1, 67, 1, 68, 1, 68, 1, 69, 1, 69, 1, 70, 1, 70, 1, 71, 1, 71, 1, 72, 1, 72, 1, 73, 1, 73, 1, 73, 1, 73, 1, 73, 1, 74, 1, 74, 1, 74, 3, 74, 875, 8, 74, 1, 74, 5, 74, 878, 8, 74, 10, 74, 12, 74, 881, 9, 74, 1, 74, 1, 74, 4, 74, 885, 8, 74, 11, 74, 12, 74, 886, 3, 74, 889, 8, 74, 1, 75, 1, 75, 1, 75, 1, 75, 1, 75, 1, 76, 1, 76, 1, 76, 1, 76, 1, 76, 1, 77, 1, 77, 5, 77, 903, 8, 77, 10, 77, 12, 77, 906, 9, 77, 1, 77, 1, 77, 3, 77, 910, 8, 77, 1, 77, 4, 77, 913, 8, 77, 11, 77, 12, 77, 914, 3, 77, 917, 8, 77, 1, 78, 1, 78, 4, 78, 921, 8, 78, 11, 78, 12, 78, 922, 1, 78, 1, 78, 1, 79, 1, 79, 1, 80, 1, 80, 1, 80, 1, 80, 1, 81, 1, 81, 1, 81, 1, 81, 1, 82, 1, 82, 1, 82, 1, 82, 1, 83, 1, 83, 1, 83, 1, 83, 1, 83, 1, 84, 1, 84, 1, 84, 1, 84, 1, 84, 1, 85, 1, 85, 1, 85, 1, 85, 1, 86, 1, 86, 1, 86, 1, 86, 1, 87, 1, 87, 1, 87, 1, 87, 1, 88, 1, 88, 1, 88, 1, 88, 1, 88, 1, 89, 1, 89, 1, 89, 1, 89, 1, 90, 1, 90, 1, 90, 1, 90, 1, 91, 1, 91, 1, 91, 1, 91, 1, 92, 1, 92, 1, 92, 1, 92, 1, 93, 1, 93, 1, 93, 1, 93, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 95, 1, 95, 1, 95, 3, 95, 1000, 8, 95, 1, 96, 4, 96, 1003, 8, 96, 11, 96, 12, 96, 1004, 1, 97, 1, 97, 1, 97, 1, 97, 1, 98, 1, 98, 1, 98, 1, 98, 1, 99, 1, 99, 1, 99, 1, 99, 1, 100, 1, 100, 1, 100, 1, 100, 1, 101, 1, 101, 1, 101, 1, 101, 1, 102, 1, 102, 1, 102, 1, 102, 1, 102, 1, 103, 1, 103, 1, 103, 1, 103, 1, 104, 1, 104, 1, 104, 1, 104, 1, 105, 1, 105, 1, 105, 1, 105, 1, 106, 1, 106, 1, 106, 1, 106, 1, 107, 1, 107, 1, 107, 1, 107, 3, 107, 1052, 8, 107, 1, 108, 1, 108, 3, 108, 1056, 8, 108, 1, 108, 5, 108, 1059, 8, 108, 10, 108, 12, 108, 1062, 9, 108, 1, 108, 1, 108, 3, 108, 1066, 8, 108, 1, 108, 4, 108, 1069, 8, 108, 11, 108, 12, 108, 1070, 3, 108, 1073, 8, 108, 1, 109, 1, 109, 4, 109, 1077, 8, 109, 11, 109, 12, 109, 1078, 1, 110, 1, 110, 1, 110, 1, 110, 1, 111, 1, 111, 1, 111, 1, 111, 1, 112, 1, 112, 1, 112, 1, 112, 1, 113, 1, 113, 1, 113, 1, 113, 1, 113, 1, 114, 1, 114, 1, 114, 1, 114, 1, 115, 1, 115, 1, 115, 1, 115, 1, 116, 1, 116, 1, 116, 1, 116, 1, 117, 1, 117, 1, 117, 1, 117, 1, 118, 1, 118, 1, 118, 1, 118, 1, 119, 1, 119, 1, 119, 1, 120, 1, 120, 1, 120, 1, 120, 1, 121, 1, 121, 1, 121, 1, 121, 1, 122, 1, 122, 1, 122, 1, 122, 1, 123, 1, 123, 1, 123, 1, 123, 1, 124, 1, 124, 1, 124, 1, 124, 1, 124, 1, 125, 1, 125, 1, 125, 1, 125, 1, 125, 1, 126, 1, 126, 1, 126, 1, 126, 1, 126, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 128, 1, 128, 1, 129, 4, 129, 1162, 8, 129, 11, 129, 12, 129, 1163, 1, 129, 1, 129, 3, 129, 1168, 8, 129, 1, 129, 4, 129, 1171, 8, 129, 11, 129, 12, 129, 1172, 1, 130, 1, 130, 1, 130, 1, 130, 1, 131, 1, 131, 1, 131, 1, 131, 1, 132, 1, 132, 1, 132, 1, 132, 1, 133, 1, 133, 1, 133, 1, 133, 1, 134, 1, 134, 1, 134, 1, 134, 1, 134, 1, 134, 1, 135, 1, 135, 1, 135, 1, 135, 1, 136, 1, 136, 1, 136, 1, 136, 1, 137, 1, 137, 1, 137, 1, 137, 1, 138, 1, 138, 1, 138, 1, 138, 1, 139, 1, 139, 1, 139, 1, 139, 1, 140, 1, 140, 1, 140, 1, 140, 1, 141, 1, 141, 1, 141, 1, 141, 1, 142, 1, 142, 1, 142, 1, 142, 1, 143, 1, 143, 1, 143, 1, 143, 1, 144, 1, 144, 1, 144, 1, 144, 1, 145, 1, 145, 1, 145, 1, 145, 1, 146, 1, 146, 1, 146, 1, 146, 1, 146, 1, 147, 1, 147, 1, 147, 1, 147, 1, 148, 1, 148, 1, 148, 1, 148, 1, 149, 1, 149, 1, 149, 1, 149, 1, 150, 1, 150, 1, 150, 1, 150, 1, 151, 1, 151, 1, 151, 1, 151, 1, 152, 1, 152, 1, 152, 1, 152, 1, 153, 1, 153, 1, 153, 1, 153, 1, 154, 1, 154, 1, 154, 1, 154, 1, 155, 1, 155, 1, 155, 1, 155, 1, 155, 1, 156, 1, 156, 1, 156, 1, 156, 1, 156, 1, 157, 1, 157, 1, 157, 1, 157, 1, 158, 1, 158, 1, 158, 1, 158, 1, 159, 1, 159, 1, 159, 1, 159, 1, 160, 1, 160, 1, 160, 1, 160, 1, 160, 1, 161, 1, 161, 1, 162, 1, 162, 1, 162, 1, 162, 1, 162, 4, 162, 1312, 8, 162, 11, 162, 12, 162, 1313, 1, 163, 1, 163, 1, 163, 1, 163, 1, 164, 1, 164, 1, 164, 1, 164, 1, 165, 1, 165, 1, 165, 1, 165, 1, 166, 1, 166, 1, 166, 1, 166, 1, 166, 1, 167, 1, 167, 1, 167, 1, 167, 1, 168, 1, 168, 1, 168, 1, 168, 1, 169, 1, 169, 1, 169, 1, 169, 1, 170, 1, 170, 1, 170, 1, 170, 1, 170, 1, 171, 1, 171, 1, 171, 1, 171, 1, 172, 1, 172, 1, 172, 1, 172, 1, 173, 1, 173, 1, 173, 1, 173, 1, 174, 1, 174, 1, 174, 1, 174, 1, 175, 1, 175, 1, 175, 1, 175, 1, 176, 1, 176, 1, 176, 1, 176, 1, 176, 1, 176, 1, 177, 1, 177, 1, 177, 1, 177, 1, 178, 1, 178, 1, 178, 1, 178, 1, 179, 1, 179, 1, 179, 1, 179, 1, 180, 1, 180, 1, 180, 1, 180, 1, 181, 1, 181, 1, 181, 1, 181, 1, 182, 1, 182, 1, 182, 1, 182, 1, 183, 1, 183, 1, 183, 1, 183, 1, 183, 1, 184, 1, 184, 1, 184, 1, 184, 1, 184, 1, 184, 1, 185, 1, 185, 1, 185, 1, 185, 1, 185, 1, 185, 1, 186, 1, 186, 1, 186, 1, 186, 1, 187, 1, 187, 1, 187, 1, 187, 1, 188, 1, 188, 1, 188, 1, 188, 1, 189, 1, 189, 1, 189, 1, 189, 1, 189, 1, 189, 1, 190, 1, 190, 1, 190, 1, 190, 1, 190, 1, 190, 1, 191, 1, 191, 1, 191, 1, 191, 1, 192, 1, 192, 1, 192, 1, 192, 1, 193, 1, 193, 1, 193, 1, 193, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 195, 1, 195, 1, 195, 1, 195, 1, 195, 1, 195, 1, 196, 1, 196, 1, 196, 1, 196, 1, 196, 1, 196, 1, 197, 1, 197, 1, 197, 1, 197, 1, 197, 2, 616, 685, 0, 198, 15, 1, 17, 2, 19, 3, 21, 4, 23, 5, 25, 6, 27, 7, 29, 8, 31, 9, 33, 10, 35, 11, 37, 12, 39, 13, 41, 14, 43, 15, 45, 16, 47, 17, 49, 18, 51, 19, 53, 20, 55, 21, 57, 22, 59, 23, 61, 24, 63, 25, 65, 0, 67, 0, 69, 0, 71, 0, 73, 0, 75, 0, 77, 0, 79, 0, 81, 0, 83, 0, 85, 26, 87, 27, 89, 28, 91, 29, 93, 30, 95, 31, 97, 32, 99, 33, 101, 34, 103, 35, 105, 36, 107, 37, 109, 38, 111, 39, 113, 40, 115, 41, 117, 42, 119, 43, 121, 44, 123, 45, 125, 46, 127, 47, 129, 48, 131, 49, 133, 50, 135, 51, 137, 52, 139, 53, 141, 54, 143, 55, 145, 56, 147, 57, 149, 58, 151, 59, 153, 60, 155, 61, 157, 62, 159, 63, 161, 0, 163, 64, 165, 65, 167, 66, 169, 67, 171, 0, 173, 68, 175, 69, 177, 70, 179, 71, 181, 0, 183, 0, 185, 72, 187, 73, 189, 74, 191, 0, 193, 0, 195, 0, 197, 0, 199, 0, 201, 0, 203, 75, 205, 0, 207, 76, 209, 0, 211, 0, 213, 77, 215, 78, 217, 79, 219, 0, 221, 0, 223, 0, 225, 0, 227, 0, 229, 0, 231, 0, 233, 80, 235, 81, 237, 82, 239, 83, 241, 0, 243, 0, 245, 0, 247, 0, 249, 0, 251, 0, 253, 84, 255, 0, 257, 85, 259, 86, 261, 87, 263, 0, 265, 0, 267, 88, 269, 89, 271, 0, 273, 90, 275, 0, 277, 91, 279, 92, 281, 93, 283, 0, 285, 0, 287, 0, 289, 0, 291, 0, 293, 0, 295, 0, 297, 0, 299, 0, 301, 94, 303, 95, 305, 96, 307, 0, 309, 0, 311, 0, 313, 0, 315, 0, 317, 0, 319, 97, 321, 98, 323, 99, 325, 0, 327, 100, 329, 101, 331, 102, 333, 103, 335, 0, 337, 104, 339, 105, 341, 106, 343, 107, 345, 108, 347, 0, 349, 0, 351, 0, 353, 0, 355, 0, 357, 0, 359, 0, 361, 109, 363, 110, 365, 111, 367, 0, 369, 0, 371, 0, 373, 0, 375, 112, 377, 113, 379, 114, 381, 0, 383, 0, 385, 0, 387, 115, 389, 116, 391, 117, 393, 0, 395, 0, 397, 118, 399, 119, 401, 120, 403, 0, 405, 0, 407, 0, 409, 0, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 35, 2, 0, 68, 68, 100, 100, 2, 0, 73, 73, 105, 105, 2, 0, 83, 83, 115, 115, 2, 0, 69, 69, 101, 101, 2, 0, 67, 67, 99, 99, 2, 0, 84, 84, 116, 116, 2, 0, 82, 82, 114, 114, 2, 0, 79, 79, 111, 111, 2, 0, 80, 80, 112, 112, 2, 0, 78, 78, 110, 110, 2, 0, 72, 72, 104, 104, 2, 0, 86, 86, 118, 118, 2, 0, 65, 65, 97, 97, 2, 0, 76, 76, 108, 108, 2, 0, 88, 88, 120, 120, 2, 0, 70, 70, 102, 102, 2, 0, 77, 77, 109, 109, 2, 0, 71, 71, 103, 103, 2, 0, 75, 75, 107, 107, 2, 0, 87, 87, 119, 119, 2, 0, 85, 85, 117, 117, 6, 0, 9, 10, 13, 13, 32, 32, 47, 47, 91, 91, 93, 93, 2, 0, 10, 10, 13, 13, 3, 0, 9, 10, 13, 13, 32, 32, 1, 0, 48, 57, 2, 0, 65, 90, 97, 122, 8, 0, 34, 34, 78, 78, 82, 82, 84, 84, 92, 92, 110, 110, 114, 114, 116, 116, 4, 0, 10, 10, 13, 13, 34, 34, 92, 92, 2, 0, 43, 43, 45, 45, 1, 0, 96, 96, 2, 0, 66, 66, 98, 98, 2, 0, 89, 89, 121, 121, 11, 0, 9, 10, 13, 13, 32, 32, 34, 34, 44, 44, 47, 47, 58, 58, 61, 61, 91, 91, 93, 93, 124, 124, 2, 0, 42, 42, 47, 47, 11, 0, 9, 10, 13, 13, 32, 32, 34, 35, 44, 44, 47, 47, 58, 58, 60, 60, 62, 63, 92, 92, 124, 124, 1503, 0, 15, 1, 0, 0, 0, 0, 17, 1, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 23, 1, 0, 0, 0, 0, 25, 1, 0, 0, 0, 0, 27, 1, 0, 0, 0, 0, 29, 1, 0, 0, 0, 0, 31, 1, 0, 0, 0, 0, 33, 1, 0, 0, 0, 0, 35, 1, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 39, 1, 0, 0, 0, 0, 41, 1, 0, 0, 0, 0, 43, 1, 0, 0, 0, 0, 45, 1, 0, 0, 0, 0, 47, 1, 0, 0, 0, 0, 49, 1, 0, 0, 0, 0, 51, 1, 0, 0, 0, 0, 53, 1, 0, 0, 0, 0, 55, 1, 0, 0, 0, 0, 57, 1, 0, 0, 0, 0, 59, 1, 0, 0, 0, 0, 61, 1, 0, 0, 0, 1, 63, 1, 0, 0, 0, 1, 85, 1, 0, 0, 0, 1, 87, 1, 0, 0, 0, 1, 89, 1, 0, 0, 0, 1, 91, 1, 0, 0, 0, 1, 93, 1, 0, 0, 0, 1, 95, 1, 0, 0, 0, 1, 97, 1, 0, 0, 0, 1, 99, 1, 0, 0, 0, 1, 101, 1, 0, 0, 0, 1, 103, 1, 0, 0, 0, 1, 105, 1, 0, 0, 0, 1, 107, 1, 0, 0, 0, 1, 109, 1, 0, 0, 0, 1, 111, 1, 0, 0, 0, 1, 113, 1, 0, 0, 0, 1, 115, 1, 0, 0, 0, 1, 117, 1, 0, 0, 0, 1, 119, 1, 0, 0, 0, 1, 121, 1, 0, 0, 0, 1, 123, 1, 0, 0, 0, 1, 125, 1, 0, 0, 0, 1, 127, 1, 0, 0, 0, 1, 129, 1, 0, 0, 0, 1, 131, 1, 0, 0, 0, 1, 133, 1, 0, 0, 0, 1, 135, 1, 0, 0, 0, 1, 137, 1, 0, 0, 0, 1, 139, 1, 0, 0, 0, 1, 141, 1, 0, 0, 0, 1, 143, 1, 0, 0, 0, 1, 145, 1, 0, 0, 0, 1, 147, 1, 0, 0, 0, 1, 149, 1, 0, 0, 0, 1, 151, 1, 0, 0, 0, 1, 153, 1, 0, 0, 0, 1, 155, 1, 0, 0, 0, 1, 157, 1, 0, 0, 0, 1, 159, 1, 0, 0, 0, 1, 161, 1, 0, 0, 0, 1, 163, 1, 0, 0, 0, 1, 165, 1, 0, 0, 0, 1, 167, 1, 0, 0, 0, 1, 169, 1, 0, 0, 0, 1, 173, 1, 0, 0, 0, 1, 175, 1, 0, 0, 0, 1, 177, 1, 0, 0, 0, 1, 179, 1, 0, 0, 0, 2, 181, 1, 0, 0, 0, 2, 183, 1, 0, 0, 0, 2, 185, 1, 0, 0, 0, 2, 187, 1, 0, 0, 0, 2, 189, 1, 0, 0, 0, 3, 191, 1, 0, 0, 0, 3, 193, 1, 0, 0, 0, 3, 195, 1, 0, 0, 0, 3, 197, 1, 0, 0, 0, 3, 199, 1, 0, 0, 0, 3, 201, 1, 0, 0, 0, 3, 203, 1, 0, 0, 0, 3, 207, 1, 0, 0, 0, 3, 209, 1, 0, 0, 0, 3, 211, 1, 0, 0, 0, 3, 213, 1, 0, 0, 0, 3, 215, 1, 0, 0, 0, 3, 217, 1, 0, 0, 0, 4, 219, 1, 0, 0, 0, 4, 221, 1, 0, 0, 0, 4, 223, 1, 0, 0, 0, 4, 225, 1, 0, 0, 0, 4, 227, 1, 0, 0, 0, 4, 233, 1, 0, 0, 0, 4, 235, 1, 0, 0, 0, 4, 237, 1, 0, 0, 0, 4, 239, 1, 0, 0, 0, 5, 241, 1, 0, 0, 0, 5, 243, 1, 0, 0, 0, 5, 245, 1, 0, 0, 0, 5, 247, 1, 0, 0, 0, 5, 249, 1, 0, 0, 0, 5, 251, 1, 0, 0, 0, 5, 253, 1, 0, 0, 0, 5, 255, 1, 0, 0, 0, 5, 257, 1, 0, 0, 0, 5, 259, 1, 0, 0, 0, 5, 261, 1, 0, 0, 0, 6, 263, 1, 0, 0, 0, 6, 265, 1, 0, 0, 0, 6, 267, 1, 0, 0, 0, 6, 269, 1, 0, 0, 0, 6, 273, 1, 0, 0, 0, 6, 275, 1, 0, 0, 0, 6, 277, 1, 0, 0, 0, 6, 279, 1, 0, 0, 0, 6, 281, 1, 0, 0, 0, 7, 283, 1, 0, 0, 0, 7, 285, 1, 0, 0, 0, 7, 287, 1, 0, 0, 0, 7, 289, 1, 0, 0, 0, 7, 291, 1, 0, 0, 0, 7, 293, 1, 0, 0, 0, 7, 295, 1, 0, 0, 0, 7, 297, 1, 0, 0, 0, 7, 299, 1, 0, 0, 0, 7, 301, 1, 0, 0, 0, 7, 303, 1, 0, 0, 0, 7, 305, 1, 0, 0, 0, 8, 307, 1, 0, 0, 0, 8, 309, 1, 0, 0, 0, 8, 311, 1, 0, 0, 0, 8, 313, 1, 0, 0, 0, 8, 315, 1, 0, 0, 0, 8, 317, 1, 0, 0, 0, 8, 319, 1, 0, 0, 0, 8, 321, 1, 0, 0, 0, 8, 323, 1, 0, 0, 0, 9, 325, 1, 0, 0, 0, 9, 327, 1, 0, 0, 0, 9, 329, 1, 0, 0, 0, 9, 331, 1, 0, 0, 0, 9, 333, 1, 0, 0, 0, 10, 335, 1, 0, 0, 0, 10, 337, 1, 0, 0, 0, 10, 339, 1, 0, 0, 0, 10, 341, 1, 0, 0, 0, 10, 343, 1, 0, 0, 0, 10, 345, 1, 0, 0, 0, 11, 347, 1, 0, 0, 0, 11, 349, 1, 0, 0, 0, 11, 351, 1, 0, 0, 0, 11, 353, 1, 0, 0, 0, 11, 355, 1, 0, 0, 0, 11, 357, 1, 0, 0, 0, 11, 359, 1, 0, 0, 0, 11, 361, 1, 0, 0, 0, 11, 363, 1, 0, 0, 0, 11, 365, 1, 0, 0, 0, 12, 367, 1, 0, 0, 0, 12, 369, 1, 0, 0, 0, 12, 371, 1, 0, 0, 0, 12, 373, 1, 0, 0, 0, 12, 375, 1, 0, 0, 0, 12, 377, 1, 0, 0, 0, 12, 379, 1, 0, 0, 0, 13, 381, 1, 0, 0, 0, 13, 383, 1, 0, 0, 0, 13, 385, 1, 0, 0, 0, 13, 387, 1, 0, 0, 0, 13, 389, 1, 0, 0, 0, 13, 391, 1, 0, 0, 0, 14, 393, 1, 0, 0, 0, 14, 395, 1, 0, 0, 0, 14, 397, 1, 0, 0, 0, 14, 399, 1, 0, 0, 0, 14, 401, 1, 0, 0, 0, 14, 403, 1, 0, 0, 0, 14, 405, 1, 0, 0, 0, 14, 407, 1, 0, 0, 0, 14, 409, 1, 0, 0, 0, 15, 411, 1, 0, 0, 0, 17, 421, 1, 0, 0, 0, 19, 428, 1, 0, 0, 0, 21, 437, 1, 0, 0, 0, 23, 444, 1, 0, 0, 0, 25, 454, 1, 0, 0, 0, 27, 461, 1, 0, 0, 0, 29, 468, 1, 0, 0, 0, 31, 475, 1, 0, 0, 0, 33, 483, 1, 0, 0, 0, 35, 495, 1, 0, 0, 0, 37, 504, 1, 0, 0, 0, 39, 510, 1, 0, 0, 0, 41, 517, 1, 0, 0, 0, 43, 524, 1, 0, 0, 0, 45, 532, 1, 0, 0, 0, 47, 540, 1, 0, 0, 0, 49, 555, 1, 0, 0, 0, 51, 565, 1, 0, 0, 0, 53, 574, 1, 0, 0, 0, 55, 586, 1, 0, 0, 0, 57, 592, 1, 0, 0, 0, 59, 609, 1, 0, 0, 0, 61, 625, 1, 0, 0, 0, 63, 631, 1, 0, 0, 0, 65, 635, 1, 0, 0, 0, 67, 637, 1, 0, 0, 0, 69, 639, 1, 0, 0, 0, 71, 642, 1, 0, 0, 0, 73, 644, 1, 0, 0, 0, 75, 653, 1, 0, 0, 0, 77, 655, 1, 0, 0, 0, 79, 660, 1, 0, 0, 0, 81, 662, 1, 0, 0, 0, 83, 667, 1, 0, 0, 0, 85, 698, 1, 0, 0, 0, 87, 701, 1, 0, 0, 0, 89, 747, 1, 0, 0, 0, 91, 749, 1, 0, 0, 0, 93, 752, 1, 0, 0, 0, 95, 756, 1, 0, 0, 0, 97, 760, 1, 0, 0, 0, 99, 762, 1, 0, 0, 0, 101, 765, 1, 0, 0, 0, 103, 767, 1, 0, 0, 0, 105, 772, 1, 0, 0, 0, 107, 774, 1, 0, 0, 0, 109, 780, 1, 0, 0, 0, 111, 786, 1, 0, 0, 0, 113, 789, 1, 0, 0, 0, 115, 792, 1, 0, 0, 0, 117, 797, 1, 0, 0, 0, 119, 802, 1, 0, 0, 0, 121, 804, 1, 0, 0, 0, 123, 808, 1, 0, 0, 0, 125, 813, 1, 0, 0, 0, 127, 819, 1, 0, 0, 0, 129, 822, 1, 0, 0, 0, 131, 824, 1, 0, 0, 0, 133, 830, 1, 0, 0, 0, 135, 832, 1, 0, 0, 0, 137, 837, 1, 0, 0, 0, 139, 840, 1, 0, 0, 0, 141, 843, 1, 0, 0, 0, 143, 846, 1, 0, 0, 0, 145, 848, 1, 0, 0, 0, 147, 851, 1, 0, 0, 0, 149, 853, 1, 0, 0, 0, 151, 856, 1, 0, 0, 0, 153, 858, 1, 0, 0, 0, 155, 860, 1, 0, 0, 0, 157, 862, 1, 0, 0, 0, 159, 864, 1, 0, 0, 0, 161, 866, 1, 0, 0, 0, 163, 888, 1, 0, 0, 0, 165, 890, 1, 0, 0, 0, 167, 895, 1, 0, 0, 0, 169, 916, 1, 0, 0, 0, 171, 918, 1, 0, 0, 0, 173, 926, 1, 0, 0, 0, 175, 928, 1, 0, 0, 0, 177, 932, 1, 0, 0, 0, 179, 936, 1, 0, 0, 0, 181, 940, 1, 0, 0, 0, 183, 945, 1, 0, 0, 0, 185, 950, 1, 0, 0, 0, 187, 954, 1, 0, 0, 0, 189, 958, 1, 0, 0, 0, 191, 962, 1, 0, 0, 0, 193, 967, 1, 0, 0, 0, 195, 971, 1, 0, 0, 0, 197, 975, 1, 0, 0, 0, 199, 979, 1, 0, 0, 0, 201, 983, 1, 0, 0, 0, 203, 987, 1, 0, 0, 0, 205, 999, 1, 0, 0, 0, 207, 1002, 1, 0, 0, 0, 209, 1006, 1, 0, 0, 0, 211, 1010, 1, 0, 0, 0, 213, 1014, 1, 0, 0, 0, 215, 1018, 1, 0, 0, 0, 217, 1022, 1, 0, 0, 0, 219, 1026, 1, 0, 0, 0, 221, 1031, 1, 0, 0, 0, 223, 1035, 1, 0, 0, 0, 225, 1039, 1, 0, 0, 0, 227, 1043, 1, 0, 0, 0, 229, 1051, 1, 0, 0, 0, 231, 1072, 1, 0, 0, 0, 233, 1076, 1, 0, 0, 0, 235, 1080, 1, 0, 0, 0, 237, 1084, 1, 0, 0, 0, 239, 1088, 1, 0, 0, 0, 241, 1092, 1, 0, 0, 0, 243, 1097, 1, 0, 0, 0, 245, 1101, 1, 0, 0, 0, 247, 1105, 1, 0, 0, 0, 249, 1109, 1, 0, 0, 0, 251, 1113, 1, 0, 0, 0, 253, 1117, 1, 0, 0, 0, 255, 1120, 1, 0, 0, 0, 257, 1124, 1, 0, 0, 0, 259, 1128, 1, 0, 0, 0, 261, 1132, 1, 0, 0, 0, 263, 1136, 1, 0, 0, 0, 265, 1141, 1, 0, 0, 0, 267, 1146, 1, 0, 0, 0, 269, 1151, 1, 0, 0, 0, 271, 1158, 1, 0, 0, 0, 273, 1167, 1, 0, 0, 0, 275, 1174, 1, 0, 0, 0, 277, 1178, 1, 0, 0, 0, 279, 1182, 1, 0, 0, 0, 281, 1186, 1, 0, 0, 0, 283, 1190, 1, 0, 0, 0, 285, 1196, 1, 0, 0, 0, 287, 1200, 1, 0, 0, 0, 289, 1204, 1, 0, 0, 0, 291, 1208, 1, 0, 0, 0, 293, 1212, 1, 0, 0, 0, 295, 1216, 1, 0, 0, 0, 297, 1220, 1, 0, 0, 0, 299, 1224, 1, 0, 0, 0, 301, 1228, 1, 0, 0, 0, 303, 1232, 1, 0, 0, 0, 305, 1236, 1, 0, 0, 0, 307, 1240, 1, 0, 0, 0, 309, 1245, 1, 0, 0, 0, 311, 1249, 1, 0, 0, 0, 313, 1253, 1, 0, 0, 0, 315, 1257, 1, 0, 0, 0, 317, 1261, 1, 0, 0, 0, 319, 1265, 1, 0, 0, 0, 321, 1269, 1, 0, 0, 0, 323, 1273, 1, 0, 0, 0, 325, 1277, 1, 0, 0, 0, 327, 1282, 1, 0, 0, 0, 329, 1287, 1, 0, 0, 0, 331, 1291, 1, 0, 0, 0, 333, 1295, 1, 0, 0, 0, 335, 1299, 1, 0, 0, 0, 337, 1304, 1, 0, 0, 0, 339, 1311, 1, 0, 0, 0, 341, 1315, 1, 0, 0, 0, 343, 1319, 1, 0, 0, 0, 345, 1323, 1, 0, 0, 0, 347, 1327, 1, 0, 0, 0, 349, 1332, 1, 0, 0, 0, 351, 1336, 1, 0, 0, 0, 353, 1340, 1, 0, 0, 0, 355, 1344, 1, 0, 0, 0, 357, 1349, 1, 0, 0, 0, 359, 1353, 1, 0, 0, 0, 361, 1357, 1, 0, 0, 0, 363, 1361, 1, 0, 0, 0, 365, 1365, 1, 0, 0, 0, 367, 1369, 1, 0, 0, 0, 369, 1375, 1, 0, 0, 0, 371, 1379, 1, 0, 0, 0, 373, 1383, 1, 0, 0, 0, 375, 1387, 1, 0, 0, 0, 377, 1391, 1, 0, 0, 0, 379, 1395, 1, 0, 0, 0, 381, 1399, 1, 0, 0, 0, 383, 1404, 1, 0, 0, 0, 385, 1410, 1, 0, 0, 0, 387, 1416, 1, 0, 0, 0, 389, 1420, 1, 0, 0, 0, 391, 1424, 1, 0, 0, 0, 393, 1428, 1, 0, 0, 0, 395, 1434, 1, 0, 0, 0, 397, 1440, 1, 0, 0, 0, 399, 1444, 1, 0, 0, 0, 401, 1448, 1, 0, 0, 0, 403, 1452, 1, 0, 0, 0, 405, 1458, 1, 0, 0, 0, 407, 1464, 1, 0, 0, 0, 409, 1470, 1, 0, 0, 0, 411, 412, 7, 0, 0, 0, 412, 413, 7, 1, 0, 0, 413, 414, 7, 2, 0, 0, 414, 415, 7, 2, 0, 0, 415, 416, 7, 3, 0, 0, 416, 417, 7, 4, 0, 0, 417, 418, 7, 5, 0, 0, 418, 419, 1, 0, 0, 0, 419, 420, 6, 0, 0, 0, 420, 16, 1, 0, 0, 0, 421, 422, 7, 0, 0, 0, 422, 423, 7, 6, 0, 0, 423, 424, 7, 7, 0, 0, 424, 425, 7, 8, 0, 0, 425, 426, 1, 0, 0, 0, 426, 427, 6, 1, 1, 0, 427, 18, 1, 0, 0, 0, 428, 429, 7, 3, 0, 0, 429, 430, 7, 9, 0, 0, 430, 431, 7, 6, 0, 0, 431, 432, 7, 1, 0, 0, 432, 433, 7, 4, 0, 0, 433, 434, 7, 10, 0, 0, 434, 435, 1, 0, 0, 0, 435, 436, 6, 2, 2, 0, 436, 20, 1, 0, 0, 0, 437, 438, 7, 3, 0, 0, 438, 439, 7, 11, 0, 0, 439, 440, 7, 12, 0, 0, 440, 441, 7, 13, 0, 0, 441, 442, 1, 0, 0, 0, 442, 443, 6, 3, 0, 0, 443, 22, 1, 0, 0, 0, 444, 445, 7, 3, 0, 0, 445, 446, 7, 14, 0, 0, 446, 447, 7, 8, 0, 0, 447, 448, 7, 13, 0, 0, 448, 449, 7, 12, 0, 0, 449, 450, 7, 1, 0, 0, 450, 451, 7, 9, 0, 0, 451, 452, 1, 0, 0, 0, 452, 453, 6, 4, 3, 0, 453, 24, 1, 0, 0, 0, 454, 455, 7, 15, 0, 0, 455, 456, 7, 6, 0, 0, 456, 457, 7, 7, 0, 0, 457, 458, 7, 16, 0, 0, 458, 459, 1, 0, 0, 0, 459, 460, 6, 5, 4, 0, 460, 26, 1, 0, 0, 0, 461, 462, 7, 17, 0, 0, 462, 463, 7, 6, 0, 0, 463, 464, 7, 7, 0, 0, 464, 465, 7, 18, 0, 0, 465, 466, 1, 0, 0, 0, 466, 467, 6, 6, 0, 0, 467, 28, 1, 0, 0, 0, 468, 469, 7, 18, 0, 0, 469, 470, 7, 3, 0, 0, 470, 471, 7, 3, 0, 0, 471, 472, 7, 8, 0, 0, 472, 473, 1, 0, 0, 0, 473, 474, 6, 7, 1, 0, 474, 30, 1, 0, 0, 0, 475, 476, 7, 13, 0, 0, 476, 477, 7, 1, 0, 0, 477, 478, 7, 16, 0, 0, 478, 479, 7, 1, 0, 0, 479, 480, 7, 5, 0, 0, 480, 481, 1, 0, 0, 0, 481, 482, 6, 8, 0, 0, 482, 32, 1, 0, 0, 0, 483, 484, 7, 16, 0, 0, 484, 485, 7, 11, 0, 0, 485, 486, 5, 95, 0, 0, 486, 487, 7, 3, 0, 0, 487, 488, 7, 14, 0, 0, 488, 489, 7, 8, 0, 0, 489, 490, 7, 12, 0, 0, 490, 491, 7, 9, 0, 0, 491, 492, 7, 0, 0, 0, 492, 493, 1, 0, 0, 0, 493, 494, 6, 9, 5, 0, 494, 34, 1, 0, 0, 0, 495, 496, 7, 6, 0, 0, 496, 497, 7, 3, 0, 0, 497, 498, 7, 9, 0, 0, 498, 499, 7, 12, 0, 0, 499, 500, 7, 16, 0, 0, 500, 501, 7, 3, 0, 0, 501, 502, 1, 0, 0, 0, 502, 503, 6, 10, 6, 0, 503, 36, 1, 0, 0, 0, 504, 505, 7, 6, 0, 0, 505, 506, 7, 7, 0, 0, 506, 507, 7, 19, 0, 0, 507, 508, 1, 0, 0, 0, 508, 509, 6, 11, 0, 0, 509, 38, 1, 0, 0, 0, 510, 511, 7, 2, 0, 0, 511, 512, 7, 10, 0, 0, 512, 513, 7, 7, 0, 0, 513, 514, 7, 19, 0, 0, 514, 515, 1, 0, 0, 0, 515, 516, 6, 12, 7, 0, 516, 40, 1, 0, 0, 0, 517, 518, 7, 2, 0, 0, 518, 519, 7, 7, 0, 0, 519, 520, 7, 6, 0, 0, 520, 521, 7, 5, 0, 0, 521, 522, 1, 0, 0, 0, 522, 523, 6, 13, 0, 0, 523, 42, 1, 0, 0, 0, 524, 525, 7, 2, 0, 0, 525, 526, 7, 5, 0, 0, 526, 527, 7, 12, 0, 0, 527, 528, 7, 5, 0, 0, 528, 529, 7, 2, 0, 0, 529, 530, 1, 0, 0, 0, 530, 531, 6, 14, 0, 0, 531, 44, 1, 0, 0, 0, 532, 533, 7, 19, 0, 0, 533, 534, 7, 10, 0, 0, 534, 535, 7, 3, 0, 0, 535, 536, 7, 6, 0, 0, 536, 537, 7, 3, 0, 0, 537, 538, 1, 0, 0, 0, 538, 539, 6, 15, 0, 0, 539, 46, 1, 0, 0, 0, 540, 541, 4, 16, 0, 0, 541, 542, 7, 1, 0, 0, 542, 543, 7, 9, 0, 0, 543, 544, 7, 13, 0, 0, 544, 545, 7, 1, 0, 0, 545, 546, 7, 9, 0, 0, 546, 547, 7, 3, 0, 0, 547, 548, 7, 2, 0, 0, 548, 549, 7, 5, 0, 0, 549, 550, 7, 12, 0, 0, 550, 551, 7, 5, 0, 0, 551, 552, 7, 2, 0, 0, 552, 553, 1, 0, 0, 0, 553, 554, 6, 16, 0, 0, 554, 48, 1, 0, 0, 0, 555, 556, 4, 17, 1, 0, 556, 557, 7, 13, 0, 0, 557, 558, 7, 7, 0, 0, 558, 559, 7, 7, 0, 0, 559, 560, 7, 18, 0, 0, 560, 561, 7, 20, 0, 0, 561, 562, 7, 8, 0, 0, 562, 563, 1, 0, 0, 0, 563, 564, 6, 17, 8, 0, 564, 50, 1, 0, 0, 0, 565, 566, 4, 18, 2, 0, 566, 567, 7, 16, 0, 0, 567, 568, 7, 12, 0, 0, 568, 569, 7, 5, 0, 0, 569, 570, 7, 4, 0, 0, 570, 571, 7, 10, 0, 0, 571, 572, 1, 0, 0, 0, 572, 573, 6, 18, 0, 0, 573, 52, 1, 0, 0, 0, 574, 575, 4, 19, 3, 0, 575, 576, 7, 16, 0, 0, 576, 577, 7, 3, 0, 0, 577, 578, 7, 5, 0, 0, 578, 579, 7, 6, 0, 0, 579, 580, 7, 1, 0, 0, 580, 581, 7, 4, 0, 0, 581, 582, 7, 2, 0, 0, 582, 583, 1, 0, 0, 0, 583, 584, 6, 19, 9, 0, 584, 54, 1, 0, 0, 0, 585, 587, 8, 21, 0, 0, 586, 585, 1, 0, 0, 0, 587, 588, 1, 0, 0, 0, 588, 586, 1, 0, 0, 0, 588, 589, 1, 0, 0, 0, 589, 590, 1, 0, 0, 0, 590, 591, 6, 20, 0, 0, 591, 56, 1, 0, 0, 0, 592, 593, 5, 47, 0, 0, 593, 594, 5, 47, 0, 0, 594, 598, 1, 0, 0, 0, 595, 597, 8, 22, 0, 0, 596, 595, 1, 0, 0, 0, 597, 600, 1, 0, 0, 0, 598, 596, 1, 0, 0, 0, 598, 599, 1, 0, 0, 0, 599, 602, 1, 0, 0, 0, 600, 598, 1, 0, 0, 0, 601, 603, 5, 13, 0, 0, 602, 601, 1, 0, 0, 0, 602, 603, 1, 0, 0, 0, 603, 605, 1, 0, 0, 0, 604, 606, 5, 10, 0, 0, 605, 604, 1, 0, 0, 0, 605, 606, 1, 0, 0, 0, 606, 607, 1, 0, 0, 0, 607, 608, 6, 21, 10, 0, 608, 58, 1, 0, 0, 0, 609, 610, 5, 47, 0, 0, 610, 611, 5, 42, 0, 0, 611, 616, 1, 0, 0, 0, 612, 615, 3, 59, 22, 0, 613, 615, 9, 0, 0, 0, 614, 612, 1, 0, 0, 0, 614, 613, 1, 0, 0, 0, 615, 618, 1, 0, 0, 0, 616, 617, 1, 0, 0, 0, 616, 614, 1, 0, 0, 0, 617, 619, 1, 0, 0, 0, 618, 616, 1, 0, 0, 0, 619, 620, 5, 42, 0, 0, 620, 621, 5, 47, 0, 0, 621, 622, 1, 0, 0, 0, 622, 623, 6, 22, 10, 0, 623, 60, 1, 0, 0, 0, 624, 626, 7, 23, 0, 0, 625, 624, 1, 0, 0, 0, 626, 627, 1, 0, 0, 0, 627, 625, 1, 0, 0, 0, 627, 628, 1, 0, 0, 0, 628, 629, 1, 0, 0, 0, 629, 630, 6, 23, 10, 0, 630, 62, 1, 0, 0, 0, 631, 632, 5, 124, 0, 0, 632, 633, 1, 0, 0, 0, 633, 634, 6, 24, 11, 0, 634, 64, 1, 0, 0, 0, 635, 636, 7, 24, 0, 0, 636, 66, 1, 0, 0, 0, 637, 638, 7, 25, 0, 0, 638, 68, 1, 0, 0, 0, 639, 640, 5, 92, 0, 0, 640, 641, 7, 26, 0, 0, 641, 70, 1, 0, 0, 0, 642, 643, 8, 27, 0, 0, 643, 72, 1, 0, 0, 0, 644, 646, 7, 3, 0, 0, 645, 647, 7, 28, 0, 0, 646, 645, 1, 0, 0, 0, 646, 647, 1, 0, 0, 0, 647, 649, 1, 0, 0, 0, 648, 650, 3, 65, 25, 0, 649, 648, 1, 0, 0, 0, 650, 651, 1, 0, 0, 0, 651, 649, 1, 0, 0, 0, 651, 652, 1, 0, 0, 0, 652, 74, 1, 0, 0, 0, 653, 654, 5, 64, 0, 0, 654, 76, 1, 0, 0, 0, 655, 656, 5, 96, 0, 0, 656, 78, 1, 0, 0, 0, 657, 661, 8, 29, 0, 0, 658, 659, 5, 96, 0, 0, 659, 661, 5, 96, 0, 0, 660, 657, 1, 0, 0, 0, 660, 658, 1, 0, 0, 0, 661, 80, 1, 0, 0, 0, 662, 663, 5, 95, 0, 0, 663, 82, 1, 0, 0, 0, 664, 668, 3, 67, 26, 0, 665, 668, 3, 65, 25, 0, 666, 668, 3, 81, 33, 0, 667, 664, 1, 0, 0, 0, 667, 665, 1, 0, 0, 0, 667, 666, 1, 0, 0, 0, 668, 84, 1, 0, 0, 0, 669, 674, 5, 34, 0, 0, 670, 673, 3, 69, 27, 0, 671, 673, 3, 71, 28, 0, 672, 670, 1, 0, 0, 0, 672, 671, 1, 0, 0, 0, 673, 676, 1, 0, 0, 0, 674, 672, 1, 0, 0, 0, 674, 675, 1, 0, 0, 0, 675, 677, 1, 0, 0, 0, 676, 674, 1, 0, 0, 0, 677, 699, 5, 34, 0, 0, 678, 679, 5, 34, 0, 0, 679, 680, 5, 34, 0, 0, 680, 681, 5, 34, 0, 0, 681, 685, 1, 0, 0, 0, 682, 684, 8, 22, 0, 0, 683, 682, 1, 0, 0, 0, 684, 687, 1, 0, 0, 0, 685, 686, 1, 0, 0, 0, 685, 683, 1, 0, 0, 0, 686, 688, 1, 0, 0, 0, 687, 685, 1, 0, 0, 0, 688, 689, 5, 34, 0, 0, 689, 690, 5, 34, 0, 0, 690, 691, 5, 34, 0, 0, 691, 693, 1, 0, 0, 0, 692, 694, 5, 34, 0, 0, 693, 692, 1, 0, 0, 0, 693, 694, 1, 0, 0, 0, 694, 696, 1, 0, 0, 0, 695, 697, 5, 34, 0, 0, 696, 695, 1, 0, 0, 0, 696, 697, 1, 0, 0, 0, 697, 699, 1, 0, 0, 0, 698, 669, 1, 0, 0, 0, 698, 678, 1, 0, 0, 0, 699, 86, 1, 0, 0, 0, 700, 702, 3, 65, 25, 0, 701, 700, 1, 0, 0, 0, 702, 703, 1, 0, 0, 0, 703, 701, 1, 0, 0, 0, 703, 704, 1, 0, 0, 0, 704, 88, 1, 0, 0, 0, 705, 707, 3, 65, 25, 0, 706, 705, 1, 0, 0, 0, 707, 708, 1, 0, 0, 0, 708, 706, 1, 0, 0, 0, 708, 709, 1, 0, 0, 0, 709, 710, 1, 0, 0, 0, 710, 714, 3, 105, 45, 0, 711, 713, 3, 65, 25, 0, 712, 711, 1, 0, 0, 0, 713, 716, 1, 0, 0, 0, 714, 712, 1, 0, 0, 0, 714, 715, 1, 0, 0, 0, 715, 748, 1, 0, 0, 0, 716, 714, 1, 0, 0, 0, 717, 719, 3, 105, 45, 0, 718, 720, 3, 65, 25, 0, 719, 718, 1, 0, 0, 0, 720, 721, 1, 0, 0, 0, 721, 719, 1, 0, 0, 0, 721, 722, 1, 0, 0, 0, 722, 748, 1, 0, 0, 0, 723, 725, 3, 65, 25, 0, 724, 723, 1, 0, 0, 0, 725, 726, 1, 0, 0, 0, 726, 724, 1, 0, 0, 0, 726, 727, 1, 0, 0, 0, 727, 735, 1, 0, 0, 0, 728, 732, 3, 105, 45, 0, 729, 731, 3, 65, 25, 0, 730, 729, 1, 0, 0, 0, 731, 734, 1, 0, 0, 0, 732, 730, 1, 0, 0, 0, 732, 733, 1, 0, 0, 0, 733, 736, 1, 0, 0, 0, 734, 732, 1, 0, 0, 0, 735, 728, 1, 0, 0, 0, 735, 736, 1, 0, 0, 0, 736, 737, 1, 0, 0, 0, 737, 738, 3, 73, 29, 0, 738, 748, 1, 0, 0, 0, 739, 741, 3, 105, 45, 0, 740, 742, 3, 65, 25, 0, 741, 740, 1, 0, 0, 0, 742, 743, 1, 0, 0, 0, 743, 741, 1, 0, 0, 0, 743, 744, 1, 0, 0, 0, 744, 745, 1, 0, 0, 0, 745, 746, 3, 73, 29, 0, 746, 748, 1, 0, 0, 0, 747, 706, 1, 0, 0, 0, 747, 717, 1, 0, 0, 0, 747, 724, 1, 0, 0, 0, 747, 739, 1, 0, 0, 0, 748, 90, 1, 0, 0, 0, 749, 750, 7, 30, 0, 0, 750, 751, 7, 31, 0, 0, 751, 92, 1, 0, 0, 0, 752, 753, 7, 12, 0, 0, 753, 754, 7, 9, 0, 0, 754, 755, 7, 0, 0, 0, 755, 94, 1, 0, 0, 0, 756, 757, 7, 12, 0, 0, 757, 758, 7, 2, 0, 0, 758, 759, 7, 4, 0, 0, 759, 96, 1, 0, 0, 0, 760, 761, 5, 61, 0, 0, 761, 98, 1, 0, 0, 0, 762, 763, 5, 58, 0, 0, 763, 764, 5, 58, 0, 0, 764, 100, 1, 0, 0, 0, 765, 766, 5, 44, 0, 0, 766, 102, 1, 0, 0, 0, 767, 768, 7, 0, 0, 0, 768, 769, 7, 3, 0, 0, 769, 770, 7, 2, 0, 0, 770, 771, 7, 4, 0, 0, 771, 104, 1, 0, 0, 0, 772, 773, 5, 46, 0, 0, 773, 106, 1, 0, 0, 0, 774, 775, 7, 15, 0, 0, 775, 776, 7, 12, 0, 0, 776, 777, 7, 13, 0, 0, 777, 778, 7, 2, 0, 0, 778, 779, 7, 3, 0, 0, 779, 108, 1, 0, 0, 0, 780, 781, 7, 15, 0, 0, 781, 782, 7, 1, 0, 0, 782, 783, 7, 6, 0, 0, 783, 784, 7, 2, 0, 0, 784, 785, 7, 5, 0, 0, 785, 110, 1, 0, 0, 0, 786, 787, 7, 1, 0, 0, 787, 788, 7, 9, 0, 0, 788, 112, 1, 0, 0, 0, 789, 790, 7, 1, 0, 0, 790, 791, 7, 2, 0, 0, 791, 114, 1, 0, 0, 0, 792, 793, 7, 13, 0, 0, 793, 794, 7, 12, 0, 0, 794, 795, 7, 2, 0, 0, 795, 796, 7, 5, 0, 0, 796, 116, 1, 0, 0, 0, 797, 798, 7, 13, 0, 0, 798, 799, 7, 1, 0, 0, 799, 800, 7, 18, 0, 0, 800, 801, 7, 3, 0, 0, 801, 118, 1, 0, 0, 0, 802, 803, 5, 40, 0, 0, 803, 120, 1, 0, 0, 0, 804, 805, 7, 9, 0, 0, 805, 806, 7, 7, 0, 0, 806, 807, 7, 5, 0, 0, 807, 122, 1, 0, 0, 0, 808, 809, 7, 9, 0, 0, 809, 810, 7, 20, 0, 0, 810, 811, 7, 13, 0, 0, 811, 812, 7, 13, 0, 0, 812, 124, 1, 0, 0, 0, 813, 814, 7, 9, 0, 0, 814, 815, 7, 20, 0, 0, 815, 816, 7, 13, 0, 0, 816, 817, 7, 13, 0, 0, 817, 818, 7, 2, 0, 0, 818, 126, 1, 0, 0, 0, 819, 820, 7, 7, 0, 0, 820, 821, 7, 6, 0, 0, 821, 128, 1, 0, 0, 0, 822, 823, 5, 63, 0, 0, 823, 130, 1, 0, 0, 0, 824, 825, 7, 6, 0, 0, 825, 826, 7, 13, 0, 0, 826, 827, 7, 1, 0, 0, 827, 828, 7, 18, 0, 0, 828, 829, 7, 3, 0, 0, 829, 132, 1, 0, 0, 0, 830, 831, 5, 41, 0, 0, 831, 134, 1, 0, 0, 0, 832, 833, 7, 5, 0, 0, 833, 834, 7, 6, 0, 0, 834, 835, 7, 20, 0, 0, 835, 836, 7, 3, 0, 0, 836, 136, 1, 0, 0, 0, 837, 838, 5, 61, 0, 0, 838, 839, 5, 61, 0, 0, 839, 138, 1, 0, 0, 0, 840, 841, 5, 61, 0, 0, 841, 842, 5, 126, 0, 0, 842, 140, 1, 0, 0, 0, 843, 844, 5, 33, 0, 0, 844, 845, 5, 61, 0, 0, 845, 142, 1, 0, 0, 0, 846, 847, 5, 60, 0, 0, 847, 144, 1, 0, 0, 0, 848, 849, 5, 60, 0, 0, 849, 850, 5, 61, 0, 0, 850, 146, 1, 0, 0, 0, 851, 852, 5, 62, 0, 0, 852, 148, 1, 0, 0, 0, 853, 854, 5, 62, 0, 0, 854, 855, 5, 61, 0, 0, 855, 150, 1, 0, 0, 0, 856, 857, 5, 43, 0, 0, 857, 152, 1, 0, 0, 0, 858, 859, 5, 45, 0, 0, 859, 154, 1, 0, 0, 0, 860, 861, 5, 42, 0, 0, 861, 156, 1, 0, 0, 0, 862, 863, 5, 47, 0, 0, 863, 158, 1, 0, 0, 0, 864, 865, 5, 37, 0, 0, 865, 160, 1, 0, 0, 0, 866, 867, 4, 73, 4, 0, 867, 868, 3, 51, 18, 0, 868, 869, 1, 0, 0, 0, 869, 870, 6, 73, 12, 0, 870, 162, 1, 0, 0, 0, 871, 874, 3, 129, 57, 0, 872, 875, 3, 67, 26, 0, 873, 875, 3, 81, 33, 0, 874, 872, 1, 0, 0, 0, 874, 873, 1, 0, 0, 0, 875, 879, 1, 0, 0, 0, 876, 878, 3, 83, 34, 0, 877, 876, 1, 0, 0, 0, 878, 881, 1, 0, 0, 0, 879, 877, 1, 0, 0, 0, 879, 880, 1, 0, 0, 0, 880, 889, 1, 0, 0, 0, 881, 879, 1, 0, 0, 0, 882, 884, 3, 129, 57, 0, 883, 885, 3, 65, 25, 0, 884, 883, 1, 0, 0, 0, 885, 886, 1, 0, 0, 0, 886, 884, 1, 0, 0, 0, 886, 887, 1, 0, 0, 0, 887, 889, 1, 0, 0, 0, 888, 871, 1, 0, 0, 0, 888, 882, 1, 0, 0, 0, 889, 164, 1, 0, 0, 0, 890, 891, 5, 91, 0, 0, 891, 892, 1, 0, 0, 0, 892, 893, 6, 75, 0, 0, 893, 894, 6, 75, 0, 0, 894, 166, 1, 0, 0, 0, 895, 896, 5, 93, 0, 0, 896, 897, 1, 0, 0, 0, 897, 898, 6, 76, 11, 0, 898, 899, 6, 76, 11, 0, 899, 168, 1, 0, 0, 0, 900, 904, 3, 67, 26, 0, 901, 903, 3, 83, 34, 0, 902, 901, 1, 0, 0, 0, 903, 906, 1, 0, 0, 0, 904, 902, 1, 0, 0, 0, 904, 905, 1, 0, 0, 0, 905, 917, 1, 0, 0, 0, 906, 904, 1, 0, 0, 0, 907, 910, 3, 81, 33, 0, 908, 910, 3, 75, 30, 0, 909, 907, 1, 0, 0, 0, 909, 908, 1, 0, 0, 0, 910, 912, 1, 0, 0, 0, 911, 913, 3, 83, 34, 0, 912, 911, 1, 0, 0, 0, 913, 914, 1, 0, 0, 0, 914, 912, 1, 0, 0, 0, 914, 915, 1, 0, 0, 0, 915, 917, 1, 0, 0, 0, 916, 900, 1, 0, 0, 0, 916, 909, 1, 0, 0, 0, 917, 170, 1, 0, 0, 0, 918, 920, 3, 77, 31, 0, 919, 921, 3, 79, 32, 0, 920, 919, 1, 0, 0, 0, 921, 922, 1, 0, 0, 0, 922, 920, 1, 0, 0, 0, 922, 923, 1, 0, 0, 0, 923, 924, 1, 0, 0, 0, 924, 925, 3, 77, 31, 0, 925, 172, 1, 0, 0, 0, 926, 927, 3, 171, 78, 0, 927, 174, 1, 0, 0, 0, 928, 929, 3, 57, 21, 0, 929, 930, 1, 0, 0, 0, 930, 931, 6, 80, 10, 0, 931, 176, 1, 0, 0, 0, 932, 933, 3, 59, 22, 0, 933, 934, 1, 0, 0, 0, 934, 935, 6, 81, 10, 0, 935, 178, 1, 0, 0, 0, 936, 937, 3, 61, 23, 0, 937, 938, 1, 0, 0, 0, 938, 939, 6, 82, 10, 0, 939, 180, 1, 0, 0, 0, 940, 941, 3, 165, 75, 0, 941, 942, 1, 0, 0, 0, 942, 943, 6, 83, 13, 0, 943, 944, 6, 83, 14, 0, 944, 182, 1, 0, 0, 0, 945, 946, 3, 63, 24, 0, 946, 947, 1, 0, 0, 0, 947, 948, 6, 84, 15, 0, 948, 949, 6, 84, 11, 0, 949, 184, 1, 0, 0, 0, 950, 951, 3, 61, 23, 0, 951, 952, 1, 0, 0, 0, 952, 953, 6, 85, 10, 0, 953, 186, 1, 0, 0, 0, 954, 955, 3, 57, 21, 0, 955, 956, 1, 0, 0, 0, 956, 957, 6, 86, 10, 0, 957, 188, 1, 0, 0, 0, 958, 959, 3, 59, 22, 0, 959, 960, 1, 0, 0, 0, 960, 961, 6, 87, 10, 0, 961, 190, 1, 0, 0, 0, 962, 963, 3, 63, 24, 0, 963, 964, 1, 0, 0, 0, 964, 965, 6, 88, 15, 0, 965, 966, 6, 88, 11, 0, 966, 192, 1, 0, 0, 0, 967, 968, 3, 165, 75, 0, 968, 969, 1, 0, 0, 0, 969, 970, 6, 89, 13, 0, 970, 194, 1, 0, 0, 0, 971, 972, 3, 167, 76, 0, 972, 973, 1, 0, 0, 0, 973, 974, 6, 90, 16, 0, 974, 196, 1, 0, 0, 0, 975, 976, 3, 337, 161, 0, 976, 977, 1, 0, 0, 0, 977, 978, 6, 91, 17, 0, 978, 198, 1, 0, 0, 0, 979, 980, 3, 101, 43, 0, 980, 981, 1, 0, 0, 0, 981, 982, 6, 92, 18, 0, 982, 200, 1, 0, 0, 0, 983, 984, 3, 97, 41, 0, 984, 985, 1, 0, 0, 0, 985, 986, 6, 93, 19, 0, 986, 202, 1, 0, 0, 0, 987, 988, 7, 16, 0, 0, 988, 989, 7, 3, 0, 0, 989, 990, 7, 5, 0, 0, 990, 991, 7, 12, 0, 0, 991, 992, 7, 0, 0, 0, 992, 993, 7, 12, 0, 0, 993, 994, 7, 5, 0, 0, 994, 995, 7, 12, 0, 0, 995, 204, 1, 0, 0, 0, 996, 1000, 8, 32, 0, 0, 997, 998, 5, 47, 0, 0, 998, 1000, 8, 33, 0, 0, 999, 996, 1, 0, 0, 0, 999, 997, 1, 0, 0, 0, 1000, 206, 1, 0, 0, 0, 1001, 1003, 3, 205, 95, 0, 1002, 1001, 1, 0, 0, 0, 1003, 1004, 1, 0, 0, 0, 1004, 1002, 1, 0, 0, 0, 1004, 1005, 1, 0, 0, 0, 1005, 208, 1, 0, 0, 0, 1006, 1007, 3, 207, 96, 0, 1007, 1008, 1, 0, 0, 0, 1008, 1009, 6, 97, 20, 0, 1009, 210, 1, 0, 0, 0, 1010, 1011, 3, 85, 35, 0, 1011, 1012, 1, 0, 0, 0, 1012, 1013, 6, 98, 21, 0, 1013, 212, 1, 0, 0, 0, 1014, 1015, 3, 57, 21, 0, 1015, 1016, 1, 0, 0, 0, 1016, 1017, 6, 99, 10, 0, 1017, 214, 1, 0, 0, 0, 1018, 1019, 3, 59, 22, 0, 1019, 1020, 1, 0, 0, 0, 1020, 1021, 6, 100, 10, 0, 1021, 216, 1, 0, 0, 0, 1022, 1023, 3, 61, 23, 0, 1023, 1024, 1, 0, 0, 0, 1024, 1025, 6, 101, 10, 0, 1025, 218, 1, 0, 0, 0, 1026, 1027, 3, 63, 24, 0, 1027, 1028, 1, 0, 0, 0, 1028, 1029, 6, 102, 15, 0, 1029, 1030, 6, 102, 11, 0, 1030, 220, 1, 0, 0, 0, 1031, 1032, 3, 105, 45, 0, 1032, 1033, 1, 0, 0, 0, 1033, 1034, 6, 103, 22, 0, 1034, 222, 1, 0, 0, 0, 1035, 1036, 3, 101, 43, 0, 1036, 1037, 1, 0, 0, 0, 1037, 1038, 6, 104, 18, 0, 1038, 224, 1, 0, 0, 0, 1039, 1040, 3, 129, 57, 0, 1040, 1041, 1, 0, 0, 0, 1041, 1042, 6, 105, 23, 0, 1042, 226, 1, 0, 0, 0, 1043, 1044, 3, 163, 74, 0, 1044, 1045, 1, 0, 0, 0, 1045, 1046, 6, 106, 24, 0, 1046, 228, 1, 0, 0, 0, 1047, 1052, 3, 67, 26, 0, 1048, 1052, 3, 65, 25, 0, 1049, 1052, 3, 81, 33, 0, 1050, 1052, 3, 155, 70, 0, 1051, 1047, 1, 0, 0, 0, 1051, 1048, 1, 0, 0, 0, 1051, 1049, 1, 0, 0, 0, 1051, 1050, 1, 0, 0, 0, 1052, 230, 1, 0, 0, 0, 1053, 1056, 3, 67, 26, 0, 1054, 1056, 3, 155, 70, 0, 1055, 1053, 1, 0, 0, 0, 1055, 1054, 1, 0, 0, 0, 1056, 1060, 1, 0, 0, 0, 1057, 1059, 3, 229, 107, 0, 1058, 1057, 1, 0, 0, 0, 1059, 1062, 1, 0, 0, 0, 1060, 1058, 1, 0, 0, 0, 1060, 1061, 1, 0, 0, 0, 1061, 1073, 1, 0, 0, 0, 1062, 1060, 1, 0, 0, 0, 1063, 1066, 3, 81, 33, 0, 1064, 1066, 3, 75, 30, 0, 1065, 1063, 1, 0, 0, 0, 1065, 1064, 1, 0, 0, 0, 1066, 1068, 1, 0, 0, 0, 1067, 1069, 3, 229, 107, 0, 1068, 1067, 1, 0, 0, 0, 1069, 1070, 1, 0, 0, 0, 1070, 1068, 1, 0, 0, 0, 1070, 1071, 1, 0, 0, 0, 1071, 1073, 1, 0, 0, 0, 1072, 1055, 1, 0, 0, 0, 1072, 1065, 1, 0, 0, 0, 1073, 232, 1, 0, 0, 0, 1074, 1077, 3, 231, 108, 0, 1075, 1077, 3, 171, 78, 0, 1076, 1074, 1, 0, 0, 0, 1076, 1075, 1, 0, 0, 0, 1077, 1078, 1, 0, 0, 0, 1078, 1076, 1, 0, 0, 0, 1078, 1079, 1, 0, 0, 0, 1079, 234, 1, 0, 0, 0, 1080, 1081, 3, 57, 21, 0, 1081, 1082, 1, 0, 0, 0, 1082, 1083, 6, 110, 10, 0, 1083, 236, 1, 0, 0, 0, 1084, 1085, 3, 59, 22, 0, 1085, 1086, 1, 0, 0, 0, 1086, 1087, 6, 111, 10, 0, 1087, 238, 1, 0, 0, 0, 1088, 1089, 3, 61, 23, 0, 1089, 1090, 1, 0, 0, 0, 1090, 1091, 6, 112, 10, 0, 1091, 240, 1, 0, 0, 0, 1092, 1093, 3, 63, 24, 0, 1093, 1094, 1, 0, 0, 0, 1094, 1095, 6, 113, 15, 0, 1095, 1096, 6, 113, 11, 0, 1096, 242, 1, 0, 0, 0, 1097, 1098, 3, 97, 41, 0, 1098, 1099, 1, 0, 0, 0, 1099, 1100, 6, 114, 19, 0, 1100, 244, 1, 0, 0, 0, 1101, 1102, 3, 101, 43, 0, 1102, 1103, 1, 0, 0, 0, 1103, 1104, 6, 115, 18, 0, 1104, 246, 1, 0, 0, 0, 1105, 1106, 3, 105, 45, 0, 1106, 1107, 1, 0, 0, 0, 1107, 1108, 6, 116, 22, 0, 1108, 248, 1, 0, 0, 0, 1109, 1110, 3, 129, 57, 0, 1110, 1111, 1, 0, 0, 0, 1111, 1112, 6, 117, 23, 0, 1112, 250, 1, 0, 0, 0, 1113, 1114, 3, 163, 74, 0, 1114, 1115, 1, 0, 0, 0, 1115, 1116, 6, 118, 24, 0, 1116, 252, 1, 0, 0, 0, 1117, 1118, 7, 12, 0, 0, 1118, 1119, 7, 2, 0, 0, 1119, 254, 1, 0, 0, 0, 1120, 1121, 3, 233, 109, 0, 1121, 1122, 1, 0, 0, 0, 1122, 1123, 6, 120, 25, 0, 1123, 256, 1, 0, 0, 0, 1124, 1125, 3, 57, 21, 0, 1125, 1126, 1, 0, 0, 0, 1126, 1127, 6, 121, 10, 0, 1127, 258, 1, 0, 0, 0, 1128, 1129, 3, 59, 22, 0, 1129, 1130, 1, 0, 0, 0, 1130, 1131, 6, 122, 10, 0, 1131, 260, 1, 0, 0, 0, 1132, 1133, 3, 61, 23, 0, 1133, 1134, 1, 0, 0, 0, 1134, 1135, 6, 123, 10, 0, 1135, 262, 1, 0, 0, 0, 1136, 1137, 3, 63, 24, 0, 1137, 1138, 1, 0, 0, 0, 1138, 1139, 6, 124, 15, 0, 1139, 1140, 6, 124, 11, 0, 1140, 264, 1, 0, 0, 0, 1141, 1142, 3, 165, 75, 0, 1142, 1143, 1, 0, 0, 0, 1143, 1144, 6, 125, 13, 0, 1144, 1145, 6, 125, 26, 0, 1145, 266, 1, 0, 0, 0, 1146, 1147, 7, 7, 0, 0, 1147, 1148, 7, 9, 0, 0, 1148, 1149, 1, 0, 0, 0, 1149, 1150, 6, 126, 27, 0, 1150, 268, 1, 0, 0, 0, 1151, 1152, 7, 19, 0, 0, 1152, 1153, 7, 1, 0, 0, 1153, 1154, 7, 5, 0, 0, 1154, 1155, 7, 10, 0, 0, 1155, 1156, 1, 0, 0, 0, 1156, 1157, 6, 127, 27, 0, 1157, 270, 1, 0, 0, 0, 1158, 1159, 8, 34, 0, 0, 1159, 272, 1, 0, 0, 0, 1160, 1162, 3, 271, 128, 0, 1161, 1160, 1, 0, 0, 0, 1162, 1163, 1, 0, 0, 0, 1163, 1161, 1, 0, 0, 0, 1163, 1164, 1, 0, 0, 0, 1164, 1165, 1, 0, 0, 0, 1165, 1166, 3, 337, 161, 0, 1166, 1168, 1, 0, 0, 0, 1167, 1161, 1, 0, 0, 0, 1167, 1168, 1, 0, 0, 0, 1168, 1170, 1, 0, 0, 0, 1169, 1171, 3, 271, 128, 0, 1170, 1169, 1, 0, 0, 0, 1171, 1172, 1, 0, 0, 0, 1172, 1170, 1, 0, 0, 0, 1172, 1173, 1, 0, 0, 0, 1173, 274, 1, 0, 0, 0, 1174, 1175, 3, 273, 129, 0, 1175, 1176, 1, 0, 0, 0, 1176, 1177, 6, 130, 28, 0, 1177, 276, 1, 0, 0, 0, 1178, 1179, 3, 57, 21, 0, 1179, 1180, 1, 0, 0, 0, 1180, 1181, 6, 131, 10, 0, 1181, 278, 1, 0, 0, 0, 1182, 1183, 3, 59, 22, 0, 1183, 1184, 1, 0, 0, 0, 1184, 1185, 6, 132, 10, 0, 1185, 280, 1, 0, 0, 0, 1186, 1187, 3, 61, 23, 0, 1187, 1188, 1, 0, 0, 0, 1188, 1189, 6, 133, 10, 0, 1189, 282, 1, 0, 0, 0, 1190, 1191, 3, 63, 24, 0, 1191, 1192, 1, 0, 0, 0, 1192, 1193, 6, 134, 15, 0, 1193, 1194, 6, 134, 11, 0, 1194, 1195, 6, 134, 11, 0, 1195, 284, 1, 0, 0, 0, 1196, 1197, 3, 97, 41, 0, 1197, 1198, 1, 0, 0, 0, 1198, 1199, 6, 135, 19, 0, 1199, 286, 1, 0, 0, 0, 1200, 1201, 3, 101, 43, 0, 1201, 1202, 1, 0, 0, 0, 1202, 1203, 6, 136, 18, 0, 1203, 288, 1, 0, 0, 0, 1204, 1205, 3, 105, 45, 0, 1205, 1206, 1, 0, 0, 0, 1206, 1207, 6, 137, 22, 0, 1207, 290, 1, 0, 0, 0, 1208, 1209, 3, 269, 127, 0, 1209, 1210, 1, 0, 0, 0, 1210, 1211, 6, 138, 29, 0, 1211, 292, 1, 0, 0, 0, 1212, 1213, 3, 233, 109, 0, 1213, 1214, 1, 0, 0, 0, 1214, 1215, 6, 139, 25, 0, 1215, 294, 1, 0, 0, 0, 1216, 1217, 3, 173, 79, 0, 1217, 1218, 1, 0, 0, 0, 1218, 1219, 6, 140, 30, 0, 1219, 296, 1, 0, 0, 0, 1220, 1221, 3, 129, 57, 0, 1221, 1222, 1, 0, 0, 0, 1222, 1223, 6, 141, 23, 0, 1223, 298, 1, 0, 0, 0, 1224, 1225, 3, 163, 74, 0, 1225, 1226, 1, 0, 0, 0, 1226, 1227, 6, 142, 24, 0, 1227, 300, 1, 0, 0, 0, 1228, 1229, 3, 57, 21, 0, 1229, 1230, 1, 0, 0, 0, 1230, 1231, 6, 143, 10, 0, 1231, 302, 1, 0, 0, 0, 1232, 1233, 3, 59, 22, 0, 1233, 1234, 1, 0, 0, 0, 1234, 1235, 6, 144, 10, 0, 1235, 304, 1, 0, 0, 0, 1236, 1237, 3, 61, 23, 0, 1237, 1238, 1, 0, 0, 0, 1238, 1239, 6, 145, 10, 0, 1239, 306, 1, 0, 0, 0, 1240, 1241, 3, 63, 24, 0, 1241, 1242, 1, 0, 0, 0, 1242, 1243, 6, 146, 15, 0, 1243, 1244, 6, 146, 11, 0, 1244, 308, 1, 0, 0, 0, 1245, 1246, 3, 105, 45, 0, 1246, 1247, 1, 0, 0, 0, 1247, 1248, 6, 147, 22, 0, 1248, 310, 1, 0, 0, 0, 1249, 1250, 3, 129, 57, 0, 1250, 1251, 1, 0, 0, 0, 1251, 1252, 6, 148, 23, 0, 1252, 312, 1, 0, 0, 0, 1253, 1254, 3, 163, 74, 0, 1254, 1255, 1, 0, 0, 0, 1255, 1256, 6, 149, 24, 0, 1256, 314, 1, 0, 0, 0, 1257, 1258, 3, 173, 79, 0, 1258, 1259, 1, 0, 0, 0, 1259, 1260, 6, 150, 30, 0, 1260, 316, 1, 0, 0, 0, 1261, 1262, 3, 169, 77, 0, 1262, 1263, 1, 0, 0, 0, 1263, 1264, 6, 151, 31, 0, 1264, 318, 1, 0, 0, 0, 1265, 1266, 3, 57, 21, 0, 1266, 1267, 1, 0, 0, 0, 1267, 1268, 6, 152, 10, 0, 1268, 320, 1, 0, 0, 0, 1269, 1270, 3, 59, 22, 0, 1270, 1271, 1, 0, 0, 0, 1271, 1272, 6, 153, 10, 0, 1272, 322, 1, 0, 0, 0, 1273, 1274, 3, 61, 23, 0, 1274, 1275, 1, 0, 0, 0, 1275, 1276, 6, 154, 10, 0, 1276, 324, 1, 0, 0, 0, 1277, 1278, 3, 63, 24, 0, 1278, 1279, 1, 0, 0, 0, 1279, 1280, 6, 155, 15, 0, 1280, 1281, 6, 155, 11, 0, 1281, 326, 1, 0, 0, 0, 1282, 1283, 7, 1, 0, 0, 1283, 1284, 7, 9, 0, 0, 1284, 1285, 7, 15, 0, 0, 1285, 1286, 7, 7, 0, 0, 1286, 328, 1, 0, 0, 0, 1287, 1288, 3, 57, 21, 0, 1288, 1289, 1, 0, 0, 0, 1289, 1290, 6, 157, 10, 0, 1290, 330, 1, 0, 0, 0, 1291, 1292, 3, 59, 22, 0, 1292, 1293, 1, 0, 0, 0, 1293, 1294, 6, 158, 10, 0, 1294, 332, 1, 0, 0, 0, 1295, 1296, 3, 61, 23, 0, 1296, 1297, 1, 0, 0, 0, 1297, 1298, 6, 159, 10, 0, 1298, 334, 1, 0, 0, 0, 1299, 1300, 3, 167, 76, 0, 1300, 1301, 1, 0, 0, 0, 1301, 1302, 6, 160, 16, 0, 1302, 1303, 6, 160, 11, 0, 1303, 336, 1, 0, 0, 0, 1304, 1305, 5, 58, 0, 0, 1305, 338, 1, 0, 0, 0, 1306, 1312, 3, 75, 30, 0, 1307, 1312, 3, 65, 25, 0, 1308, 1312, 3, 105, 45, 0, 1309, 1312, 3, 67, 26, 0, 1310, 1312, 3, 81, 33, 0, 1311, 1306, 1, 0, 0, 0, 1311, 1307, 1, 0, 0, 0, 1311, 1308, 1, 0, 0, 0, 1311, 1309, 1, 0, 0, 0, 1311, 1310, 1, 0, 0, 0, 1312, 1313, 1, 0, 0, 0, 1313, 1311, 1, 0, 0, 0, 1313, 1314, 1, 0, 0, 0, 1314, 340, 1, 0, 0, 0, 1315, 1316, 3, 57, 21, 0, 1316, 1317, 1, 0, 0, 0, 1317, 1318, 6, 163, 10, 0, 1318, 342, 1, 0, 0, 0, 1319, 1320, 3, 59, 22, 0, 1320, 1321, 1, 0, 0, 0, 1321, 1322, 6, 164, 10, 0, 1322, 344, 1, 0, 0, 0, 1323, 1324, 3, 61, 23, 0, 1324, 1325, 1, 0, 0, 0, 1325, 1326, 6, 165, 10, 0, 1326, 346, 1, 0, 0, 0, 1327, 1328, 3, 63, 24, 0, 1328, 1329, 1, 0, 0, 0, 1329, 1330, 6, 166, 15, 0, 1330, 1331, 6, 166, 11, 0, 1331, 348, 1, 0, 0, 0, 1332, 1333, 3, 337, 161, 0, 1333, 1334, 1, 0, 0, 0, 1334, 1335, 6, 167, 17, 0, 1335, 350, 1, 0, 0, 0, 1336, 1337, 3, 101, 43, 0, 1337, 1338, 1, 0, 0, 0, 1338, 1339, 6, 168, 18, 0, 1339, 352, 1, 0, 0, 0, 1340, 1341, 3, 105, 45, 0, 1341, 1342, 1, 0, 0, 0, 1342, 1343, 6, 169, 22, 0, 1343, 354, 1, 0, 0, 0, 1344, 1345, 3, 267, 126, 0, 1345, 1346, 1, 0, 0, 0, 1346, 1347, 6, 170, 32, 0, 1347, 1348, 6, 170, 33, 0, 1348, 356, 1, 0, 0, 0, 1349, 1350, 3, 207, 96, 0, 1350, 1351, 1, 0, 0, 0, 1351, 1352, 6, 171, 20, 0, 1352, 358, 1, 0, 0, 0, 1353, 1354, 3, 85, 35, 0, 1354, 1355, 1, 0, 0, 0, 1355, 1356, 6, 172, 21, 0, 1356, 360, 1, 0, 0, 0, 1357, 1358, 3, 57, 21, 0, 1358, 1359, 1, 0, 0, 0, 1359, 1360, 6, 173, 10, 0, 1360, 362, 1, 0, 0, 0, 1361, 1362, 3, 59, 22, 0, 1362, 1363, 1, 0, 0, 0, 1363, 1364, 6, 174, 10, 0, 1364, 364, 1, 0, 0, 0, 1365, 1366, 3, 61, 23, 0, 1366, 1367, 1, 0, 0, 0, 1367, 1368, 6, 175, 10, 0, 1368, 366, 1, 0, 0, 0, 1369, 1370, 3, 63, 24, 0, 1370, 1371, 1, 0, 0, 0, 1371, 1372, 6, 176, 15, 0, 1372, 1373, 6, 176, 11, 0, 1373, 1374, 6, 176, 11, 0, 1374, 368, 1, 0, 0, 0, 1375, 1376, 3, 101, 43, 0, 1376, 1377, 1, 0, 0, 0, 1377, 1378, 6, 177, 18, 0, 1378, 370, 1, 0, 0, 0, 1379, 1380, 3, 105, 45, 0, 1380, 1381, 1, 0, 0, 0, 1381, 1382, 6, 178, 22, 0, 1382, 372, 1, 0, 0, 0, 1383, 1384, 3, 233, 109, 0, 1384, 1385, 1, 0, 0, 0, 1385, 1386, 6, 179, 25, 0, 1386, 374, 1, 0, 0, 0, 1387, 1388, 3, 57, 21, 0, 1388, 1389, 1, 0, 0, 0, 1389, 1390, 6, 180, 10, 0, 1390, 376, 1, 0, 0, 0, 1391, 1392, 3, 59, 22, 0, 1392, 1393, 1, 0, 0, 0, 1393, 1394, 6, 181, 10, 0, 1394, 378, 1, 0, 0, 0, 1395, 1396, 3, 61, 23, 0, 1396, 1397, 1, 0, 0, 0, 1397, 1398, 6, 182, 10, 0, 1398, 380, 1, 0, 0, 0, 1399, 1400, 3, 63, 24, 0, 1400, 1401, 1, 0, 0, 0, 1401, 1402, 6, 183, 15, 0, 1402, 1403, 6, 183, 11, 0, 1403, 382, 1, 0, 0, 0, 1404, 1405, 3, 207, 96, 0, 1405, 1406, 1, 0, 0, 0, 1406, 1407, 6, 184, 20, 0, 1407, 1408, 6, 184, 11, 0, 1408, 1409, 6, 184, 34, 0, 1409, 384, 1, 0, 0, 0, 1410, 1411, 3, 85, 35, 0, 1411, 1412, 1, 0, 0, 0, 1412, 1413, 6, 185, 21, 0, 1413, 1414, 6, 185, 11, 0, 1414, 1415, 6, 185, 34, 0, 1415, 386, 1, 0, 0, 0, 1416, 1417, 3, 57, 21, 0, 1417, 1418, 1, 0, 0, 0, 1418, 1419, 6, 186, 10, 0, 1419, 388, 1, 0, 0, 0, 1420, 1421, 3, 59, 22, 0, 1421, 1422, 1, 0, 0, 0, 1422, 1423, 6, 187, 10, 0, 1423, 390, 1, 0, 0, 0, 1424, 1425, 3, 61, 23, 0, 1425, 1426, 1, 0, 0, 0, 1426, 1427, 6, 188, 10, 0, 1427, 392, 1, 0, 0, 0, 1428, 1429, 3, 337, 161, 0, 1429, 1430, 1, 0, 0, 0, 1430, 1431, 6, 189, 17, 0, 1431, 1432, 6, 189, 11, 0, 1432, 1433, 6, 189, 9, 0, 1433, 394, 1, 0, 0, 0, 1434, 1435, 3, 101, 43, 0, 1435, 1436, 1, 0, 0, 0, 1436, 1437, 6, 190, 18, 0, 1437, 1438, 6, 190, 11, 0, 1438, 1439, 6, 190, 9, 0, 1439, 396, 1, 0, 0, 0, 1440, 1441, 3, 57, 21, 0, 1441, 1442, 1, 0, 0, 0, 1442, 1443, 6, 191, 10, 0, 1443, 398, 1, 0, 0, 0, 1444, 1445, 3, 59, 22, 0, 1445, 1446, 1, 0, 0, 0, 1446, 1447, 6, 192, 10, 0, 1447, 400, 1, 0, 0, 0, 1448, 1449, 3, 61, 23, 0, 1449, 1450, 1, 0, 0, 0, 1450, 1451, 6, 193, 10, 0, 1451, 402, 1, 0, 0, 0, 1452, 1453, 3, 173, 79, 0, 1453, 1454, 1, 0, 0, 0, 1454, 1455, 6, 194, 11, 0, 1455, 1456, 6, 194, 0, 0, 1456, 1457, 6, 194, 30, 0, 1457, 404, 1, 0, 0, 0, 1458, 1459, 3, 169, 77, 0, 1459, 1460, 1, 0, 0, 0, 1460, 1461, 6, 195, 11, 0, 1461, 1462, 6, 195, 0, 0, 1462, 1463, 6, 195, 31, 0, 1463, 406, 1, 0, 0, 0, 1464, 1465, 3, 91, 38, 0, 1465, 1466, 1, 0, 0, 0, 1466, 1467, 6, 196, 11, 0, 1467, 1468, 6, 196, 0, 0, 1468, 1469, 6, 196, 35, 0, 1469, 408, 1, 0, 0, 0, 1470, 1471, 3, 63, 24, 0, 1471, 1472, 1, 0, 0, 0, 1472, 1473, 6, 197, 15, 0, 1473, 1474, 6, 197, 11, 0, 1474, 410, 1, 0, 0, 0, 65, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 588, 598, 602, 605, 614, 616, 627, 646, 651, 660, 667, 672, 674, 685, 693, 696, 698, 703, 708, 714, 721, 726, 732, 735, 743, 747, 874, 879, 886, 888, 904, 909, 914, 916, 922, 999, 1004, 1051, 1055, 1060, 1065, 1070, 1072, 1076, 1078, 1163, 1167, 1172, 1311, 1313, 36, 5, 1, 0, 5, 4, 0, 5, 6, 0, 5, 2, 0, 5, 3, 0, 5, 8, 0, 5, 5, 0, 5, 9, 0, 5, 11, 0, 5, 13, 0, 0, 1, 0, 4, 0, 0, 7, 19, 0, 7, 65, 0, 5, 0, 0, 7, 25, 0, 7, 66, 0, 7, 104, 0, 7, 34, 0, 7, 32, 0, 7, 76, 0, 7, 26, 0, 7, 36, 0, 7, 48, 0, 7, 64, 0, 7, 80, 0, 5, 10, 0, 5, 7, 0, 7, 90, 0, 7, 89, 0, 7, 68, 0, 7, 67, 0, 7, 88, 0, 5, 12, 0, 5, 14, 0, 7, 29, 0] \ No newline at end of file +[4, 0, 120, 1466, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 2, 68, 7, 68, 2, 69, 7, 69, 2, 70, 7, 70, 2, 71, 7, 71, 2, 72, 7, 72, 2, 73, 7, 73, 2, 74, 7, 74, 2, 75, 7, 75, 2, 76, 7, 76, 2, 77, 7, 77, 2, 78, 7, 78, 2, 79, 7, 79, 2, 80, 7, 80, 2, 81, 7, 81, 2, 82, 7, 82, 2, 83, 7, 83, 2, 84, 7, 84, 2, 85, 7, 85, 2, 86, 7, 86, 2, 87, 7, 87, 2, 88, 7, 88, 2, 89, 7, 89, 2, 90, 7, 90, 2, 91, 7, 91, 2, 92, 7, 92, 2, 93, 7, 93, 2, 94, 7, 94, 2, 95, 7, 95, 2, 96, 7, 96, 2, 97, 7, 97, 2, 98, 7, 98, 2, 99, 7, 99, 2, 100, 7, 100, 2, 101, 7, 101, 2, 102, 7, 102, 2, 103, 7, 103, 2, 104, 7, 104, 2, 105, 7, 105, 2, 106, 7, 106, 2, 107, 7, 107, 2, 108, 7, 108, 2, 109, 7, 109, 2, 110, 7, 110, 2, 111, 7, 111, 2, 112, 7, 112, 2, 113, 7, 113, 2, 114, 7, 114, 2, 115, 7, 115, 2, 116, 7, 116, 2, 117, 7, 117, 2, 118, 7, 118, 2, 119, 7, 119, 2, 120, 7, 120, 2, 121, 7, 121, 2, 122, 7, 122, 2, 123, 7, 123, 2, 124, 7, 124, 2, 125, 7, 125, 2, 126, 7, 126, 2, 127, 7, 127, 2, 128, 7, 128, 2, 129, 7, 129, 2, 130, 7, 130, 2, 131, 7, 131, 2, 132, 7, 132, 2, 133, 7, 133, 2, 134, 7, 134, 2, 135, 7, 135, 2, 136, 7, 136, 2, 137, 7, 137, 2, 138, 7, 138, 2, 139, 7, 139, 2, 140, 7, 140, 2, 141, 7, 141, 2, 142, 7, 142, 2, 143, 7, 143, 2, 144, 7, 144, 2, 145, 7, 145, 2, 146, 7, 146, 2, 147, 7, 147, 2, 148, 7, 148, 2, 149, 7, 149, 2, 150, 7, 150, 2, 151, 7, 151, 2, 152, 7, 152, 2, 153, 7, 153, 2, 154, 7, 154, 2, 155, 7, 155, 2, 156, 7, 156, 2, 157, 7, 157, 2, 158, 7, 158, 2, 159, 7, 159, 2, 160, 7, 160, 2, 161, 7, 161, 2, 162, 7, 162, 2, 163, 7, 163, 2, 164, 7, 164, 2, 165, 7, 165, 2, 166, 7, 166, 2, 167, 7, 167, 2, 168, 7, 168, 2, 169, 7, 169, 2, 170, 7, 170, 2, 171, 7, 171, 2, 172, 7, 172, 2, 173, 7, 173, 2, 174, 7, 174, 2, 175, 7, 175, 2, 176, 7, 176, 2, 177, 7, 177, 2, 178, 7, 178, 2, 179, 7, 179, 2, 180, 7, 180, 2, 181, 7, 181, 2, 182, 7, 182, 2, 183, 7, 183, 2, 184, 7, 184, 2, 185, 7, 185, 2, 186, 7, 186, 2, 187, 7, 187, 2, 188, 7, 188, 2, 189, 7, 189, 2, 190, 7, 190, 2, 191, 7, 191, 2, 192, 7, 192, 2, 193, 7, 193, 2, 194, 7, 194, 2, 195, 7, 195, 2, 196, 7, 196, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 19, 4, 19, 576, 8, 19, 11, 19, 12, 19, 577, 1, 19, 1, 19, 1, 20, 1, 20, 1, 20, 1, 20, 5, 20, 586, 8, 20, 10, 20, 12, 20, 589, 9, 20, 1, 20, 3, 20, 592, 8, 20, 1, 20, 3, 20, 595, 8, 20, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 5, 21, 604, 8, 21, 10, 21, 12, 21, 607, 9, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 22, 4, 22, 615, 8, 22, 11, 22, 12, 22, 616, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 1, 25, 1, 25, 1, 26, 1, 26, 1, 26, 1, 27, 1, 27, 1, 28, 1, 28, 3, 28, 636, 8, 28, 1, 28, 4, 28, 639, 8, 28, 11, 28, 12, 28, 640, 1, 29, 1, 29, 1, 30, 1, 30, 1, 31, 1, 31, 1, 31, 3, 31, 650, 8, 31, 1, 32, 1, 32, 1, 33, 1, 33, 1, 33, 3, 33, 657, 8, 33, 1, 34, 1, 34, 1, 34, 5, 34, 662, 8, 34, 10, 34, 12, 34, 665, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 673, 8, 34, 10, 34, 12, 34, 676, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 3, 34, 683, 8, 34, 1, 34, 3, 34, 686, 8, 34, 3, 34, 688, 8, 34, 1, 35, 4, 35, 691, 8, 35, 11, 35, 12, 35, 692, 1, 36, 4, 36, 696, 8, 36, 11, 36, 12, 36, 697, 1, 36, 1, 36, 5, 36, 702, 8, 36, 10, 36, 12, 36, 705, 9, 36, 1, 36, 1, 36, 4, 36, 709, 8, 36, 11, 36, 12, 36, 710, 1, 36, 4, 36, 714, 8, 36, 11, 36, 12, 36, 715, 1, 36, 1, 36, 5, 36, 720, 8, 36, 10, 36, 12, 36, 723, 9, 36, 3, 36, 725, 8, 36, 1, 36, 1, 36, 1, 36, 1, 36, 4, 36, 731, 8, 36, 11, 36, 12, 36, 732, 1, 36, 1, 36, 3, 36, 737, 8, 36, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 1, 38, 1, 39, 1, 39, 1, 39, 1, 39, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 49, 1, 49, 1, 49, 1, 50, 1, 50, 1, 50, 1, 50, 1, 50, 1, 51, 1, 51, 1, 52, 1, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 64, 1, 64, 1, 64, 1, 65, 1, 65, 1, 66, 1, 66, 1, 66, 1, 67, 1, 67, 1, 68, 1, 68, 1, 69, 1, 69, 1, 70, 1, 70, 1, 71, 1, 71, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 73, 1, 73, 1, 73, 3, 73, 866, 8, 73, 1, 73, 5, 73, 869, 8, 73, 10, 73, 12, 73, 872, 9, 73, 1, 73, 1, 73, 4, 73, 876, 8, 73, 11, 73, 12, 73, 877, 3, 73, 880, 8, 73, 1, 74, 1, 74, 1, 74, 1, 74, 1, 74, 1, 75, 1, 75, 1, 75, 1, 75, 1, 75, 1, 76, 1, 76, 5, 76, 894, 8, 76, 10, 76, 12, 76, 897, 9, 76, 1, 76, 1, 76, 3, 76, 901, 8, 76, 1, 76, 4, 76, 904, 8, 76, 11, 76, 12, 76, 905, 3, 76, 908, 8, 76, 1, 77, 1, 77, 4, 77, 912, 8, 77, 11, 77, 12, 77, 913, 1, 77, 1, 77, 1, 78, 1, 78, 1, 79, 1, 79, 1, 79, 1, 79, 1, 80, 1, 80, 1, 80, 1, 80, 1, 81, 1, 81, 1, 81, 1, 81, 1, 82, 1, 82, 1, 82, 1, 82, 1, 82, 1, 83, 1, 83, 1, 83, 1, 83, 1, 83, 1, 84, 1, 84, 1, 84, 1, 84, 1, 85, 1, 85, 1, 85, 1, 85, 1, 86, 1, 86, 1, 86, 1, 86, 1, 87, 1, 87, 1, 87, 1, 87, 1, 87, 1, 88, 1, 88, 1, 88, 1, 88, 1, 89, 1, 89, 1, 89, 1, 89, 1, 90, 1, 90, 1, 90, 1, 90, 1, 91, 1, 91, 1, 91, 1, 91, 1, 92, 1, 92, 1, 92, 1, 92, 1, 93, 1, 93, 1, 93, 1, 93, 1, 93, 1, 93, 1, 93, 1, 93, 1, 93, 1, 94, 1, 94, 1, 94, 3, 94, 991, 8, 94, 1, 95, 4, 95, 994, 8, 95, 11, 95, 12, 95, 995, 1, 96, 1, 96, 1, 96, 1, 96, 1, 97, 1, 97, 1, 97, 1, 97, 1, 98, 1, 98, 1, 98, 1, 98, 1, 99, 1, 99, 1, 99, 1, 99, 1, 100, 1, 100, 1, 100, 1, 100, 1, 101, 1, 101, 1, 101, 1, 101, 1, 101, 1, 102, 1, 102, 1, 102, 1, 102, 1, 103, 1, 103, 1, 103, 1, 103, 1, 104, 1, 104, 1, 104, 1, 104, 1, 105, 1, 105, 1, 105, 1, 105, 1, 106, 1, 106, 1, 106, 1, 106, 3, 106, 1043, 8, 106, 1, 107, 1, 107, 3, 107, 1047, 8, 107, 1, 107, 5, 107, 1050, 8, 107, 10, 107, 12, 107, 1053, 9, 107, 1, 107, 1, 107, 3, 107, 1057, 8, 107, 1, 107, 4, 107, 1060, 8, 107, 11, 107, 12, 107, 1061, 3, 107, 1064, 8, 107, 1, 108, 1, 108, 4, 108, 1068, 8, 108, 11, 108, 12, 108, 1069, 1, 109, 1, 109, 1, 109, 1, 109, 1, 110, 1, 110, 1, 110, 1, 110, 1, 111, 1, 111, 1, 111, 1, 111, 1, 112, 1, 112, 1, 112, 1, 112, 1, 112, 1, 113, 1, 113, 1, 113, 1, 113, 1, 114, 1, 114, 1, 114, 1, 114, 1, 115, 1, 115, 1, 115, 1, 115, 1, 116, 1, 116, 1, 116, 1, 116, 1, 117, 1, 117, 1, 117, 1, 117, 1, 118, 1, 118, 1, 118, 1, 119, 1, 119, 1, 119, 1, 119, 1, 120, 1, 120, 1, 120, 1, 120, 1, 121, 1, 121, 1, 121, 1, 121, 1, 122, 1, 122, 1, 122, 1, 122, 1, 123, 1, 123, 1, 123, 1, 123, 1, 123, 1, 124, 1, 124, 1, 124, 1, 124, 1, 124, 1, 125, 1, 125, 1, 125, 1, 125, 1, 125, 1, 126, 1, 126, 1, 126, 1, 126, 1, 126, 1, 126, 1, 126, 1, 127, 1, 127, 1, 128, 4, 128, 1153, 8, 128, 11, 128, 12, 128, 1154, 1, 128, 1, 128, 3, 128, 1159, 8, 128, 1, 128, 4, 128, 1162, 8, 128, 11, 128, 12, 128, 1163, 1, 129, 1, 129, 1, 129, 1, 129, 1, 130, 1, 130, 1, 130, 1, 130, 1, 131, 1, 131, 1, 131, 1, 131, 1, 132, 1, 132, 1, 132, 1, 132, 1, 133, 1, 133, 1, 133, 1, 133, 1, 133, 1, 133, 1, 134, 1, 134, 1, 134, 1, 134, 1, 135, 1, 135, 1, 135, 1, 135, 1, 136, 1, 136, 1, 136, 1, 136, 1, 137, 1, 137, 1, 137, 1, 137, 1, 138, 1, 138, 1, 138, 1, 138, 1, 139, 1, 139, 1, 139, 1, 139, 1, 140, 1, 140, 1, 140, 1, 140, 1, 141, 1, 141, 1, 141, 1, 141, 1, 142, 1, 142, 1, 142, 1, 142, 1, 143, 1, 143, 1, 143, 1, 143, 1, 144, 1, 144, 1, 144, 1, 144, 1, 145, 1, 145, 1, 145, 1, 145, 1, 145, 1, 146, 1, 146, 1, 146, 1, 146, 1, 147, 1, 147, 1, 147, 1, 147, 1, 148, 1, 148, 1, 148, 1, 148, 1, 149, 1, 149, 1, 149, 1, 149, 1, 150, 1, 150, 1, 150, 1, 150, 1, 151, 1, 151, 1, 151, 1, 151, 1, 152, 1, 152, 1, 152, 1, 152, 1, 153, 1, 153, 1, 153, 1, 153, 1, 154, 1, 154, 1, 154, 1, 154, 1, 154, 1, 155, 1, 155, 1, 155, 1, 155, 1, 155, 1, 156, 1, 156, 1, 156, 1, 156, 1, 157, 1, 157, 1, 157, 1, 157, 1, 158, 1, 158, 1, 158, 1, 158, 1, 159, 1, 159, 1, 159, 1, 159, 1, 159, 1, 160, 1, 160, 1, 161, 1, 161, 1, 161, 1, 161, 1, 161, 4, 161, 1303, 8, 161, 11, 161, 12, 161, 1304, 1, 162, 1, 162, 1, 162, 1, 162, 1, 163, 1, 163, 1, 163, 1, 163, 1, 164, 1, 164, 1, 164, 1, 164, 1, 165, 1, 165, 1, 165, 1, 165, 1, 165, 1, 166, 1, 166, 1, 166, 1, 166, 1, 167, 1, 167, 1, 167, 1, 167, 1, 168, 1, 168, 1, 168, 1, 168, 1, 169, 1, 169, 1, 169, 1, 169, 1, 169, 1, 170, 1, 170, 1, 170, 1, 170, 1, 171, 1, 171, 1, 171, 1, 171, 1, 172, 1, 172, 1, 172, 1, 172, 1, 173, 1, 173, 1, 173, 1, 173, 1, 174, 1, 174, 1, 174, 1, 174, 1, 175, 1, 175, 1, 175, 1, 175, 1, 175, 1, 175, 1, 176, 1, 176, 1, 176, 1, 176, 1, 177, 1, 177, 1, 177, 1, 177, 1, 178, 1, 178, 1, 178, 1, 178, 1, 179, 1, 179, 1, 179, 1, 179, 1, 180, 1, 180, 1, 180, 1, 180, 1, 181, 1, 181, 1, 181, 1, 181, 1, 182, 1, 182, 1, 182, 1, 182, 1, 182, 1, 183, 1, 183, 1, 183, 1, 183, 1, 183, 1, 183, 1, 184, 1, 184, 1, 184, 1, 184, 1, 184, 1, 184, 1, 185, 1, 185, 1, 185, 1, 185, 1, 186, 1, 186, 1, 186, 1, 186, 1, 187, 1, 187, 1, 187, 1, 187, 1, 188, 1, 188, 1, 188, 1, 188, 1, 188, 1, 188, 1, 189, 1, 189, 1, 189, 1, 189, 1, 189, 1, 189, 1, 190, 1, 190, 1, 190, 1, 190, 1, 191, 1, 191, 1, 191, 1, 191, 1, 192, 1, 192, 1, 192, 1, 192, 1, 193, 1, 193, 1, 193, 1, 193, 1, 193, 1, 193, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 195, 1, 195, 1, 195, 1, 195, 1, 195, 1, 195, 1, 196, 1, 196, 1, 196, 1, 196, 1, 196, 2, 605, 674, 0, 197, 15, 1, 17, 2, 19, 3, 21, 4, 23, 5, 25, 6, 27, 7, 29, 8, 31, 9, 33, 10, 35, 11, 37, 12, 39, 13, 41, 14, 43, 15, 45, 16, 47, 17, 49, 18, 51, 19, 53, 20, 55, 21, 57, 22, 59, 23, 61, 24, 63, 0, 65, 0, 67, 0, 69, 0, 71, 0, 73, 0, 75, 0, 77, 0, 79, 0, 81, 0, 83, 25, 85, 26, 87, 27, 89, 28, 91, 29, 93, 30, 95, 31, 97, 32, 99, 33, 101, 34, 103, 35, 105, 36, 107, 37, 109, 38, 111, 39, 113, 40, 115, 41, 117, 42, 119, 43, 121, 44, 123, 45, 125, 46, 127, 47, 129, 48, 131, 49, 133, 50, 135, 51, 137, 52, 139, 53, 141, 54, 143, 55, 145, 56, 147, 57, 149, 58, 151, 59, 153, 60, 155, 61, 157, 62, 159, 63, 161, 64, 163, 65, 165, 66, 167, 67, 169, 0, 171, 68, 173, 69, 175, 70, 177, 71, 179, 0, 181, 0, 183, 72, 185, 73, 187, 74, 189, 0, 191, 0, 193, 0, 195, 0, 197, 0, 199, 0, 201, 75, 203, 0, 205, 76, 207, 0, 209, 0, 211, 77, 213, 78, 215, 79, 217, 0, 219, 0, 221, 0, 223, 0, 225, 0, 227, 0, 229, 0, 231, 80, 233, 81, 235, 82, 237, 83, 239, 0, 241, 0, 243, 0, 245, 0, 247, 0, 249, 0, 251, 84, 253, 0, 255, 85, 257, 86, 259, 87, 261, 0, 263, 0, 265, 88, 267, 89, 269, 0, 271, 90, 273, 0, 275, 91, 277, 92, 279, 93, 281, 0, 283, 0, 285, 0, 287, 0, 289, 0, 291, 0, 293, 0, 295, 0, 297, 0, 299, 94, 301, 95, 303, 96, 305, 0, 307, 0, 309, 0, 311, 0, 313, 0, 315, 0, 317, 97, 319, 98, 321, 99, 323, 0, 325, 100, 327, 101, 329, 102, 331, 103, 333, 0, 335, 104, 337, 105, 339, 106, 341, 107, 343, 108, 345, 0, 347, 0, 349, 0, 351, 0, 353, 0, 355, 0, 357, 0, 359, 109, 361, 110, 363, 111, 365, 0, 367, 0, 369, 0, 371, 0, 373, 112, 375, 113, 377, 114, 379, 0, 381, 0, 383, 0, 385, 115, 387, 116, 389, 117, 391, 0, 393, 0, 395, 118, 397, 119, 399, 120, 401, 0, 403, 0, 405, 0, 407, 0, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 35, 2, 0, 68, 68, 100, 100, 2, 0, 73, 73, 105, 105, 2, 0, 83, 83, 115, 115, 2, 0, 69, 69, 101, 101, 2, 0, 67, 67, 99, 99, 2, 0, 84, 84, 116, 116, 2, 0, 82, 82, 114, 114, 2, 0, 79, 79, 111, 111, 2, 0, 80, 80, 112, 112, 2, 0, 78, 78, 110, 110, 2, 0, 72, 72, 104, 104, 2, 0, 86, 86, 118, 118, 2, 0, 65, 65, 97, 97, 2, 0, 76, 76, 108, 108, 2, 0, 88, 88, 120, 120, 2, 0, 70, 70, 102, 102, 2, 0, 77, 77, 109, 109, 2, 0, 71, 71, 103, 103, 2, 0, 75, 75, 107, 107, 2, 0, 87, 87, 119, 119, 2, 0, 85, 85, 117, 117, 6, 0, 9, 10, 13, 13, 32, 32, 47, 47, 91, 91, 93, 93, 2, 0, 10, 10, 13, 13, 3, 0, 9, 10, 13, 13, 32, 32, 1, 0, 48, 57, 2, 0, 65, 90, 97, 122, 8, 0, 34, 34, 78, 78, 82, 82, 84, 84, 92, 92, 110, 110, 114, 114, 116, 116, 4, 0, 10, 10, 13, 13, 34, 34, 92, 92, 2, 0, 43, 43, 45, 45, 1, 0, 96, 96, 2, 0, 66, 66, 98, 98, 2, 0, 89, 89, 121, 121, 11, 0, 9, 10, 13, 13, 32, 32, 34, 34, 44, 44, 47, 47, 58, 58, 61, 61, 91, 91, 93, 93, 124, 124, 2, 0, 42, 42, 47, 47, 11, 0, 9, 10, 13, 13, 32, 32, 34, 35, 44, 44, 47, 47, 58, 58, 60, 60, 62, 63, 92, 92, 124, 124, 1494, 0, 15, 1, 0, 0, 0, 0, 17, 1, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 23, 1, 0, 0, 0, 0, 25, 1, 0, 0, 0, 0, 27, 1, 0, 0, 0, 0, 29, 1, 0, 0, 0, 0, 31, 1, 0, 0, 0, 0, 33, 1, 0, 0, 0, 0, 35, 1, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 39, 1, 0, 0, 0, 0, 41, 1, 0, 0, 0, 0, 43, 1, 0, 0, 0, 0, 45, 1, 0, 0, 0, 0, 47, 1, 0, 0, 0, 0, 49, 1, 0, 0, 0, 0, 51, 1, 0, 0, 0, 0, 53, 1, 0, 0, 0, 0, 55, 1, 0, 0, 0, 0, 57, 1, 0, 0, 0, 0, 59, 1, 0, 0, 0, 1, 61, 1, 0, 0, 0, 1, 83, 1, 0, 0, 0, 1, 85, 1, 0, 0, 0, 1, 87, 1, 0, 0, 0, 1, 89, 1, 0, 0, 0, 1, 91, 1, 0, 0, 0, 1, 93, 1, 0, 0, 0, 1, 95, 1, 0, 0, 0, 1, 97, 1, 0, 0, 0, 1, 99, 1, 0, 0, 0, 1, 101, 1, 0, 0, 0, 1, 103, 1, 0, 0, 0, 1, 105, 1, 0, 0, 0, 1, 107, 1, 0, 0, 0, 1, 109, 1, 0, 0, 0, 1, 111, 1, 0, 0, 0, 1, 113, 1, 0, 0, 0, 1, 115, 1, 0, 0, 0, 1, 117, 1, 0, 0, 0, 1, 119, 1, 0, 0, 0, 1, 121, 1, 0, 0, 0, 1, 123, 1, 0, 0, 0, 1, 125, 1, 0, 0, 0, 1, 127, 1, 0, 0, 0, 1, 129, 1, 0, 0, 0, 1, 131, 1, 0, 0, 0, 1, 133, 1, 0, 0, 0, 1, 135, 1, 0, 0, 0, 1, 137, 1, 0, 0, 0, 1, 139, 1, 0, 0, 0, 1, 141, 1, 0, 0, 0, 1, 143, 1, 0, 0, 0, 1, 145, 1, 0, 0, 0, 1, 147, 1, 0, 0, 0, 1, 149, 1, 0, 0, 0, 1, 151, 1, 0, 0, 0, 1, 153, 1, 0, 0, 0, 1, 155, 1, 0, 0, 0, 1, 157, 1, 0, 0, 0, 1, 159, 1, 0, 0, 0, 1, 161, 1, 0, 0, 0, 1, 163, 1, 0, 0, 0, 1, 165, 1, 0, 0, 0, 1, 167, 1, 0, 0, 0, 1, 171, 1, 0, 0, 0, 1, 173, 1, 0, 0, 0, 1, 175, 1, 0, 0, 0, 1, 177, 1, 0, 0, 0, 2, 179, 1, 0, 0, 0, 2, 181, 1, 0, 0, 0, 2, 183, 1, 0, 0, 0, 2, 185, 1, 0, 0, 0, 2, 187, 1, 0, 0, 0, 3, 189, 1, 0, 0, 0, 3, 191, 1, 0, 0, 0, 3, 193, 1, 0, 0, 0, 3, 195, 1, 0, 0, 0, 3, 197, 1, 0, 0, 0, 3, 199, 1, 0, 0, 0, 3, 201, 1, 0, 0, 0, 3, 205, 1, 0, 0, 0, 3, 207, 1, 0, 0, 0, 3, 209, 1, 0, 0, 0, 3, 211, 1, 0, 0, 0, 3, 213, 1, 0, 0, 0, 3, 215, 1, 0, 0, 0, 4, 217, 1, 0, 0, 0, 4, 219, 1, 0, 0, 0, 4, 221, 1, 0, 0, 0, 4, 223, 1, 0, 0, 0, 4, 225, 1, 0, 0, 0, 4, 231, 1, 0, 0, 0, 4, 233, 1, 0, 0, 0, 4, 235, 1, 0, 0, 0, 4, 237, 1, 0, 0, 0, 5, 239, 1, 0, 0, 0, 5, 241, 1, 0, 0, 0, 5, 243, 1, 0, 0, 0, 5, 245, 1, 0, 0, 0, 5, 247, 1, 0, 0, 0, 5, 249, 1, 0, 0, 0, 5, 251, 1, 0, 0, 0, 5, 253, 1, 0, 0, 0, 5, 255, 1, 0, 0, 0, 5, 257, 1, 0, 0, 0, 5, 259, 1, 0, 0, 0, 6, 261, 1, 0, 0, 0, 6, 263, 1, 0, 0, 0, 6, 265, 1, 0, 0, 0, 6, 267, 1, 0, 0, 0, 6, 271, 1, 0, 0, 0, 6, 273, 1, 0, 0, 0, 6, 275, 1, 0, 0, 0, 6, 277, 1, 0, 0, 0, 6, 279, 1, 0, 0, 0, 7, 281, 1, 0, 0, 0, 7, 283, 1, 0, 0, 0, 7, 285, 1, 0, 0, 0, 7, 287, 1, 0, 0, 0, 7, 289, 1, 0, 0, 0, 7, 291, 1, 0, 0, 0, 7, 293, 1, 0, 0, 0, 7, 295, 1, 0, 0, 0, 7, 297, 1, 0, 0, 0, 7, 299, 1, 0, 0, 0, 7, 301, 1, 0, 0, 0, 7, 303, 1, 0, 0, 0, 8, 305, 1, 0, 0, 0, 8, 307, 1, 0, 0, 0, 8, 309, 1, 0, 0, 0, 8, 311, 1, 0, 0, 0, 8, 313, 1, 0, 0, 0, 8, 315, 1, 0, 0, 0, 8, 317, 1, 0, 0, 0, 8, 319, 1, 0, 0, 0, 8, 321, 1, 0, 0, 0, 9, 323, 1, 0, 0, 0, 9, 325, 1, 0, 0, 0, 9, 327, 1, 0, 0, 0, 9, 329, 1, 0, 0, 0, 9, 331, 1, 0, 0, 0, 10, 333, 1, 0, 0, 0, 10, 335, 1, 0, 0, 0, 10, 337, 1, 0, 0, 0, 10, 339, 1, 0, 0, 0, 10, 341, 1, 0, 0, 0, 10, 343, 1, 0, 0, 0, 11, 345, 1, 0, 0, 0, 11, 347, 1, 0, 0, 0, 11, 349, 1, 0, 0, 0, 11, 351, 1, 0, 0, 0, 11, 353, 1, 0, 0, 0, 11, 355, 1, 0, 0, 0, 11, 357, 1, 0, 0, 0, 11, 359, 1, 0, 0, 0, 11, 361, 1, 0, 0, 0, 11, 363, 1, 0, 0, 0, 12, 365, 1, 0, 0, 0, 12, 367, 1, 0, 0, 0, 12, 369, 1, 0, 0, 0, 12, 371, 1, 0, 0, 0, 12, 373, 1, 0, 0, 0, 12, 375, 1, 0, 0, 0, 12, 377, 1, 0, 0, 0, 13, 379, 1, 0, 0, 0, 13, 381, 1, 0, 0, 0, 13, 383, 1, 0, 0, 0, 13, 385, 1, 0, 0, 0, 13, 387, 1, 0, 0, 0, 13, 389, 1, 0, 0, 0, 14, 391, 1, 0, 0, 0, 14, 393, 1, 0, 0, 0, 14, 395, 1, 0, 0, 0, 14, 397, 1, 0, 0, 0, 14, 399, 1, 0, 0, 0, 14, 401, 1, 0, 0, 0, 14, 403, 1, 0, 0, 0, 14, 405, 1, 0, 0, 0, 14, 407, 1, 0, 0, 0, 15, 409, 1, 0, 0, 0, 17, 419, 1, 0, 0, 0, 19, 426, 1, 0, 0, 0, 21, 435, 1, 0, 0, 0, 23, 442, 1, 0, 0, 0, 25, 452, 1, 0, 0, 0, 27, 459, 1, 0, 0, 0, 29, 466, 1, 0, 0, 0, 31, 473, 1, 0, 0, 0, 33, 481, 1, 0, 0, 0, 35, 493, 1, 0, 0, 0, 37, 502, 1, 0, 0, 0, 39, 508, 1, 0, 0, 0, 41, 515, 1, 0, 0, 0, 43, 522, 1, 0, 0, 0, 45, 530, 1, 0, 0, 0, 47, 538, 1, 0, 0, 0, 49, 553, 1, 0, 0, 0, 51, 563, 1, 0, 0, 0, 53, 575, 1, 0, 0, 0, 55, 581, 1, 0, 0, 0, 57, 598, 1, 0, 0, 0, 59, 614, 1, 0, 0, 0, 61, 620, 1, 0, 0, 0, 63, 624, 1, 0, 0, 0, 65, 626, 1, 0, 0, 0, 67, 628, 1, 0, 0, 0, 69, 631, 1, 0, 0, 0, 71, 633, 1, 0, 0, 0, 73, 642, 1, 0, 0, 0, 75, 644, 1, 0, 0, 0, 77, 649, 1, 0, 0, 0, 79, 651, 1, 0, 0, 0, 81, 656, 1, 0, 0, 0, 83, 687, 1, 0, 0, 0, 85, 690, 1, 0, 0, 0, 87, 736, 1, 0, 0, 0, 89, 738, 1, 0, 0, 0, 91, 741, 1, 0, 0, 0, 93, 745, 1, 0, 0, 0, 95, 749, 1, 0, 0, 0, 97, 751, 1, 0, 0, 0, 99, 754, 1, 0, 0, 0, 101, 756, 1, 0, 0, 0, 103, 761, 1, 0, 0, 0, 105, 763, 1, 0, 0, 0, 107, 769, 1, 0, 0, 0, 109, 775, 1, 0, 0, 0, 111, 778, 1, 0, 0, 0, 113, 781, 1, 0, 0, 0, 115, 786, 1, 0, 0, 0, 117, 791, 1, 0, 0, 0, 119, 793, 1, 0, 0, 0, 121, 797, 1, 0, 0, 0, 123, 802, 1, 0, 0, 0, 125, 808, 1, 0, 0, 0, 127, 811, 1, 0, 0, 0, 129, 813, 1, 0, 0, 0, 131, 819, 1, 0, 0, 0, 133, 821, 1, 0, 0, 0, 135, 826, 1, 0, 0, 0, 137, 829, 1, 0, 0, 0, 139, 832, 1, 0, 0, 0, 141, 835, 1, 0, 0, 0, 143, 837, 1, 0, 0, 0, 145, 840, 1, 0, 0, 0, 147, 842, 1, 0, 0, 0, 149, 845, 1, 0, 0, 0, 151, 847, 1, 0, 0, 0, 153, 849, 1, 0, 0, 0, 155, 851, 1, 0, 0, 0, 157, 853, 1, 0, 0, 0, 159, 855, 1, 0, 0, 0, 161, 879, 1, 0, 0, 0, 163, 881, 1, 0, 0, 0, 165, 886, 1, 0, 0, 0, 167, 907, 1, 0, 0, 0, 169, 909, 1, 0, 0, 0, 171, 917, 1, 0, 0, 0, 173, 919, 1, 0, 0, 0, 175, 923, 1, 0, 0, 0, 177, 927, 1, 0, 0, 0, 179, 931, 1, 0, 0, 0, 181, 936, 1, 0, 0, 0, 183, 941, 1, 0, 0, 0, 185, 945, 1, 0, 0, 0, 187, 949, 1, 0, 0, 0, 189, 953, 1, 0, 0, 0, 191, 958, 1, 0, 0, 0, 193, 962, 1, 0, 0, 0, 195, 966, 1, 0, 0, 0, 197, 970, 1, 0, 0, 0, 199, 974, 1, 0, 0, 0, 201, 978, 1, 0, 0, 0, 203, 990, 1, 0, 0, 0, 205, 993, 1, 0, 0, 0, 207, 997, 1, 0, 0, 0, 209, 1001, 1, 0, 0, 0, 211, 1005, 1, 0, 0, 0, 213, 1009, 1, 0, 0, 0, 215, 1013, 1, 0, 0, 0, 217, 1017, 1, 0, 0, 0, 219, 1022, 1, 0, 0, 0, 221, 1026, 1, 0, 0, 0, 223, 1030, 1, 0, 0, 0, 225, 1034, 1, 0, 0, 0, 227, 1042, 1, 0, 0, 0, 229, 1063, 1, 0, 0, 0, 231, 1067, 1, 0, 0, 0, 233, 1071, 1, 0, 0, 0, 235, 1075, 1, 0, 0, 0, 237, 1079, 1, 0, 0, 0, 239, 1083, 1, 0, 0, 0, 241, 1088, 1, 0, 0, 0, 243, 1092, 1, 0, 0, 0, 245, 1096, 1, 0, 0, 0, 247, 1100, 1, 0, 0, 0, 249, 1104, 1, 0, 0, 0, 251, 1108, 1, 0, 0, 0, 253, 1111, 1, 0, 0, 0, 255, 1115, 1, 0, 0, 0, 257, 1119, 1, 0, 0, 0, 259, 1123, 1, 0, 0, 0, 261, 1127, 1, 0, 0, 0, 263, 1132, 1, 0, 0, 0, 265, 1137, 1, 0, 0, 0, 267, 1142, 1, 0, 0, 0, 269, 1149, 1, 0, 0, 0, 271, 1158, 1, 0, 0, 0, 273, 1165, 1, 0, 0, 0, 275, 1169, 1, 0, 0, 0, 277, 1173, 1, 0, 0, 0, 279, 1177, 1, 0, 0, 0, 281, 1181, 1, 0, 0, 0, 283, 1187, 1, 0, 0, 0, 285, 1191, 1, 0, 0, 0, 287, 1195, 1, 0, 0, 0, 289, 1199, 1, 0, 0, 0, 291, 1203, 1, 0, 0, 0, 293, 1207, 1, 0, 0, 0, 295, 1211, 1, 0, 0, 0, 297, 1215, 1, 0, 0, 0, 299, 1219, 1, 0, 0, 0, 301, 1223, 1, 0, 0, 0, 303, 1227, 1, 0, 0, 0, 305, 1231, 1, 0, 0, 0, 307, 1236, 1, 0, 0, 0, 309, 1240, 1, 0, 0, 0, 311, 1244, 1, 0, 0, 0, 313, 1248, 1, 0, 0, 0, 315, 1252, 1, 0, 0, 0, 317, 1256, 1, 0, 0, 0, 319, 1260, 1, 0, 0, 0, 321, 1264, 1, 0, 0, 0, 323, 1268, 1, 0, 0, 0, 325, 1273, 1, 0, 0, 0, 327, 1278, 1, 0, 0, 0, 329, 1282, 1, 0, 0, 0, 331, 1286, 1, 0, 0, 0, 333, 1290, 1, 0, 0, 0, 335, 1295, 1, 0, 0, 0, 337, 1302, 1, 0, 0, 0, 339, 1306, 1, 0, 0, 0, 341, 1310, 1, 0, 0, 0, 343, 1314, 1, 0, 0, 0, 345, 1318, 1, 0, 0, 0, 347, 1323, 1, 0, 0, 0, 349, 1327, 1, 0, 0, 0, 351, 1331, 1, 0, 0, 0, 353, 1335, 1, 0, 0, 0, 355, 1340, 1, 0, 0, 0, 357, 1344, 1, 0, 0, 0, 359, 1348, 1, 0, 0, 0, 361, 1352, 1, 0, 0, 0, 363, 1356, 1, 0, 0, 0, 365, 1360, 1, 0, 0, 0, 367, 1366, 1, 0, 0, 0, 369, 1370, 1, 0, 0, 0, 371, 1374, 1, 0, 0, 0, 373, 1378, 1, 0, 0, 0, 375, 1382, 1, 0, 0, 0, 377, 1386, 1, 0, 0, 0, 379, 1390, 1, 0, 0, 0, 381, 1395, 1, 0, 0, 0, 383, 1401, 1, 0, 0, 0, 385, 1407, 1, 0, 0, 0, 387, 1411, 1, 0, 0, 0, 389, 1415, 1, 0, 0, 0, 391, 1419, 1, 0, 0, 0, 393, 1425, 1, 0, 0, 0, 395, 1431, 1, 0, 0, 0, 397, 1435, 1, 0, 0, 0, 399, 1439, 1, 0, 0, 0, 401, 1443, 1, 0, 0, 0, 403, 1449, 1, 0, 0, 0, 405, 1455, 1, 0, 0, 0, 407, 1461, 1, 0, 0, 0, 409, 410, 7, 0, 0, 0, 410, 411, 7, 1, 0, 0, 411, 412, 7, 2, 0, 0, 412, 413, 7, 2, 0, 0, 413, 414, 7, 3, 0, 0, 414, 415, 7, 4, 0, 0, 415, 416, 7, 5, 0, 0, 416, 417, 1, 0, 0, 0, 417, 418, 6, 0, 0, 0, 418, 16, 1, 0, 0, 0, 419, 420, 7, 0, 0, 0, 420, 421, 7, 6, 0, 0, 421, 422, 7, 7, 0, 0, 422, 423, 7, 8, 0, 0, 423, 424, 1, 0, 0, 0, 424, 425, 6, 1, 1, 0, 425, 18, 1, 0, 0, 0, 426, 427, 7, 3, 0, 0, 427, 428, 7, 9, 0, 0, 428, 429, 7, 6, 0, 0, 429, 430, 7, 1, 0, 0, 430, 431, 7, 4, 0, 0, 431, 432, 7, 10, 0, 0, 432, 433, 1, 0, 0, 0, 433, 434, 6, 2, 2, 0, 434, 20, 1, 0, 0, 0, 435, 436, 7, 3, 0, 0, 436, 437, 7, 11, 0, 0, 437, 438, 7, 12, 0, 0, 438, 439, 7, 13, 0, 0, 439, 440, 1, 0, 0, 0, 440, 441, 6, 3, 0, 0, 441, 22, 1, 0, 0, 0, 442, 443, 7, 3, 0, 0, 443, 444, 7, 14, 0, 0, 444, 445, 7, 8, 0, 0, 445, 446, 7, 13, 0, 0, 446, 447, 7, 12, 0, 0, 447, 448, 7, 1, 0, 0, 448, 449, 7, 9, 0, 0, 449, 450, 1, 0, 0, 0, 450, 451, 6, 4, 3, 0, 451, 24, 1, 0, 0, 0, 452, 453, 7, 15, 0, 0, 453, 454, 7, 6, 0, 0, 454, 455, 7, 7, 0, 0, 455, 456, 7, 16, 0, 0, 456, 457, 1, 0, 0, 0, 457, 458, 6, 5, 4, 0, 458, 26, 1, 0, 0, 0, 459, 460, 7, 17, 0, 0, 460, 461, 7, 6, 0, 0, 461, 462, 7, 7, 0, 0, 462, 463, 7, 18, 0, 0, 463, 464, 1, 0, 0, 0, 464, 465, 6, 6, 0, 0, 465, 28, 1, 0, 0, 0, 466, 467, 7, 18, 0, 0, 467, 468, 7, 3, 0, 0, 468, 469, 7, 3, 0, 0, 469, 470, 7, 8, 0, 0, 470, 471, 1, 0, 0, 0, 471, 472, 6, 7, 1, 0, 472, 30, 1, 0, 0, 0, 473, 474, 7, 13, 0, 0, 474, 475, 7, 1, 0, 0, 475, 476, 7, 16, 0, 0, 476, 477, 7, 1, 0, 0, 477, 478, 7, 5, 0, 0, 478, 479, 1, 0, 0, 0, 479, 480, 6, 8, 0, 0, 480, 32, 1, 0, 0, 0, 481, 482, 7, 16, 0, 0, 482, 483, 7, 11, 0, 0, 483, 484, 5, 95, 0, 0, 484, 485, 7, 3, 0, 0, 485, 486, 7, 14, 0, 0, 486, 487, 7, 8, 0, 0, 487, 488, 7, 12, 0, 0, 488, 489, 7, 9, 0, 0, 489, 490, 7, 0, 0, 0, 490, 491, 1, 0, 0, 0, 491, 492, 6, 9, 5, 0, 492, 34, 1, 0, 0, 0, 493, 494, 7, 6, 0, 0, 494, 495, 7, 3, 0, 0, 495, 496, 7, 9, 0, 0, 496, 497, 7, 12, 0, 0, 497, 498, 7, 16, 0, 0, 498, 499, 7, 3, 0, 0, 499, 500, 1, 0, 0, 0, 500, 501, 6, 10, 6, 0, 501, 36, 1, 0, 0, 0, 502, 503, 7, 6, 0, 0, 503, 504, 7, 7, 0, 0, 504, 505, 7, 19, 0, 0, 505, 506, 1, 0, 0, 0, 506, 507, 6, 11, 0, 0, 507, 38, 1, 0, 0, 0, 508, 509, 7, 2, 0, 0, 509, 510, 7, 10, 0, 0, 510, 511, 7, 7, 0, 0, 511, 512, 7, 19, 0, 0, 512, 513, 1, 0, 0, 0, 513, 514, 6, 12, 7, 0, 514, 40, 1, 0, 0, 0, 515, 516, 7, 2, 0, 0, 516, 517, 7, 7, 0, 0, 517, 518, 7, 6, 0, 0, 518, 519, 7, 5, 0, 0, 519, 520, 1, 0, 0, 0, 520, 521, 6, 13, 0, 0, 521, 42, 1, 0, 0, 0, 522, 523, 7, 2, 0, 0, 523, 524, 7, 5, 0, 0, 524, 525, 7, 12, 0, 0, 525, 526, 7, 5, 0, 0, 526, 527, 7, 2, 0, 0, 527, 528, 1, 0, 0, 0, 528, 529, 6, 14, 0, 0, 529, 44, 1, 0, 0, 0, 530, 531, 7, 19, 0, 0, 531, 532, 7, 10, 0, 0, 532, 533, 7, 3, 0, 0, 533, 534, 7, 6, 0, 0, 534, 535, 7, 3, 0, 0, 535, 536, 1, 0, 0, 0, 536, 537, 6, 15, 0, 0, 537, 46, 1, 0, 0, 0, 538, 539, 4, 16, 0, 0, 539, 540, 7, 1, 0, 0, 540, 541, 7, 9, 0, 0, 541, 542, 7, 13, 0, 0, 542, 543, 7, 1, 0, 0, 543, 544, 7, 9, 0, 0, 544, 545, 7, 3, 0, 0, 545, 546, 7, 2, 0, 0, 546, 547, 7, 5, 0, 0, 547, 548, 7, 12, 0, 0, 548, 549, 7, 5, 0, 0, 549, 550, 7, 2, 0, 0, 550, 551, 1, 0, 0, 0, 551, 552, 6, 16, 0, 0, 552, 48, 1, 0, 0, 0, 553, 554, 4, 17, 1, 0, 554, 555, 7, 13, 0, 0, 555, 556, 7, 7, 0, 0, 556, 557, 7, 7, 0, 0, 557, 558, 7, 18, 0, 0, 558, 559, 7, 20, 0, 0, 559, 560, 7, 8, 0, 0, 560, 561, 1, 0, 0, 0, 561, 562, 6, 17, 8, 0, 562, 50, 1, 0, 0, 0, 563, 564, 4, 18, 2, 0, 564, 565, 7, 16, 0, 0, 565, 566, 7, 3, 0, 0, 566, 567, 7, 5, 0, 0, 567, 568, 7, 6, 0, 0, 568, 569, 7, 1, 0, 0, 569, 570, 7, 4, 0, 0, 570, 571, 7, 2, 0, 0, 571, 572, 1, 0, 0, 0, 572, 573, 6, 18, 9, 0, 573, 52, 1, 0, 0, 0, 574, 576, 8, 21, 0, 0, 575, 574, 1, 0, 0, 0, 576, 577, 1, 0, 0, 0, 577, 575, 1, 0, 0, 0, 577, 578, 1, 0, 0, 0, 578, 579, 1, 0, 0, 0, 579, 580, 6, 19, 0, 0, 580, 54, 1, 0, 0, 0, 581, 582, 5, 47, 0, 0, 582, 583, 5, 47, 0, 0, 583, 587, 1, 0, 0, 0, 584, 586, 8, 22, 0, 0, 585, 584, 1, 0, 0, 0, 586, 589, 1, 0, 0, 0, 587, 585, 1, 0, 0, 0, 587, 588, 1, 0, 0, 0, 588, 591, 1, 0, 0, 0, 589, 587, 1, 0, 0, 0, 590, 592, 5, 13, 0, 0, 591, 590, 1, 0, 0, 0, 591, 592, 1, 0, 0, 0, 592, 594, 1, 0, 0, 0, 593, 595, 5, 10, 0, 0, 594, 593, 1, 0, 0, 0, 594, 595, 1, 0, 0, 0, 595, 596, 1, 0, 0, 0, 596, 597, 6, 20, 10, 0, 597, 56, 1, 0, 0, 0, 598, 599, 5, 47, 0, 0, 599, 600, 5, 42, 0, 0, 600, 605, 1, 0, 0, 0, 601, 604, 3, 57, 21, 0, 602, 604, 9, 0, 0, 0, 603, 601, 1, 0, 0, 0, 603, 602, 1, 0, 0, 0, 604, 607, 1, 0, 0, 0, 605, 606, 1, 0, 0, 0, 605, 603, 1, 0, 0, 0, 606, 608, 1, 0, 0, 0, 607, 605, 1, 0, 0, 0, 608, 609, 5, 42, 0, 0, 609, 610, 5, 47, 0, 0, 610, 611, 1, 0, 0, 0, 611, 612, 6, 21, 10, 0, 612, 58, 1, 0, 0, 0, 613, 615, 7, 23, 0, 0, 614, 613, 1, 0, 0, 0, 615, 616, 1, 0, 0, 0, 616, 614, 1, 0, 0, 0, 616, 617, 1, 0, 0, 0, 617, 618, 1, 0, 0, 0, 618, 619, 6, 22, 10, 0, 619, 60, 1, 0, 0, 0, 620, 621, 5, 124, 0, 0, 621, 622, 1, 0, 0, 0, 622, 623, 6, 23, 11, 0, 623, 62, 1, 0, 0, 0, 624, 625, 7, 24, 0, 0, 625, 64, 1, 0, 0, 0, 626, 627, 7, 25, 0, 0, 627, 66, 1, 0, 0, 0, 628, 629, 5, 92, 0, 0, 629, 630, 7, 26, 0, 0, 630, 68, 1, 0, 0, 0, 631, 632, 8, 27, 0, 0, 632, 70, 1, 0, 0, 0, 633, 635, 7, 3, 0, 0, 634, 636, 7, 28, 0, 0, 635, 634, 1, 0, 0, 0, 635, 636, 1, 0, 0, 0, 636, 638, 1, 0, 0, 0, 637, 639, 3, 63, 24, 0, 638, 637, 1, 0, 0, 0, 639, 640, 1, 0, 0, 0, 640, 638, 1, 0, 0, 0, 640, 641, 1, 0, 0, 0, 641, 72, 1, 0, 0, 0, 642, 643, 5, 64, 0, 0, 643, 74, 1, 0, 0, 0, 644, 645, 5, 96, 0, 0, 645, 76, 1, 0, 0, 0, 646, 650, 8, 29, 0, 0, 647, 648, 5, 96, 0, 0, 648, 650, 5, 96, 0, 0, 649, 646, 1, 0, 0, 0, 649, 647, 1, 0, 0, 0, 650, 78, 1, 0, 0, 0, 651, 652, 5, 95, 0, 0, 652, 80, 1, 0, 0, 0, 653, 657, 3, 65, 25, 0, 654, 657, 3, 63, 24, 0, 655, 657, 3, 79, 32, 0, 656, 653, 1, 0, 0, 0, 656, 654, 1, 0, 0, 0, 656, 655, 1, 0, 0, 0, 657, 82, 1, 0, 0, 0, 658, 663, 5, 34, 0, 0, 659, 662, 3, 67, 26, 0, 660, 662, 3, 69, 27, 0, 661, 659, 1, 0, 0, 0, 661, 660, 1, 0, 0, 0, 662, 665, 1, 0, 0, 0, 663, 661, 1, 0, 0, 0, 663, 664, 1, 0, 0, 0, 664, 666, 1, 0, 0, 0, 665, 663, 1, 0, 0, 0, 666, 688, 5, 34, 0, 0, 667, 668, 5, 34, 0, 0, 668, 669, 5, 34, 0, 0, 669, 670, 5, 34, 0, 0, 670, 674, 1, 0, 0, 0, 671, 673, 8, 22, 0, 0, 672, 671, 1, 0, 0, 0, 673, 676, 1, 0, 0, 0, 674, 675, 1, 0, 0, 0, 674, 672, 1, 0, 0, 0, 675, 677, 1, 0, 0, 0, 676, 674, 1, 0, 0, 0, 677, 678, 5, 34, 0, 0, 678, 679, 5, 34, 0, 0, 679, 680, 5, 34, 0, 0, 680, 682, 1, 0, 0, 0, 681, 683, 5, 34, 0, 0, 682, 681, 1, 0, 0, 0, 682, 683, 1, 0, 0, 0, 683, 685, 1, 0, 0, 0, 684, 686, 5, 34, 0, 0, 685, 684, 1, 0, 0, 0, 685, 686, 1, 0, 0, 0, 686, 688, 1, 0, 0, 0, 687, 658, 1, 0, 0, 0, 687, 667, 1, 0, 0, 0, 688, 84, 1, 0, 0, 0, 689, 691, 3, 63, 24, 0, 690, 689, 1, 0, 0, 0, 691, 692, 1, 0, 0, 0, 692, 690, 1, 0, 0, 0, 692, 693, 1, 0, 0, 0, 693, 86, 1, 0, 0, 0, 694, 696, 3, 63, 24, 0, 695, 694, 1, 0, 0, 0, 696, 697, 1, 0, 0, 0, 697, 695, 1, 0, 0, 0, 697, 698, 1, 0, 0, 0, 698, 699, 1, 0, 0, 0, 699, 703, 3, 103, 44, 0, 700, 702, 3, 63, 24, 0, 701, 700, 1, 0, 0, 0, 702, 705, 1, 0, 0, 0, 703, 701, 1, 0, 0, 0, 703, 704, 1, 0, 0, 0, 704, 737, 1, 0, 0, 0, 705, 703, 1, 0, 0, 0, 706, 708, 3, 103, 44, 0, 707, 709, 3, 63, 24, 0, 708, 707, 1, 0, 0, 0, 709, 710, 1, 0, 0, 0, 710, 708, 1, 0, 0, 0, 710, 711, 1, 0, 0, 0, 711, 737, 1, 0, 0, 0, 712, 714, 3, 63, 24, 0, 713, 712, 1, 0, 0, 0, 714, 715, 1, 0, 0, 0, 715, 713, 1, 0, 0, 0, 715, 716, 1, 0, 0, 0, 716, 724, 1, 0, 0, 0, 717, 721, 3, 103, 44, 0, 718, 720, 3, 63, 24, 0, 719, 718, 1, 0, 0, 0, 720, 723, 1, 0, 0, 0, 721, 719, 1, 0, 0, 0, 721, 722, 1, 0, 0, 0, 722, 725, 1, 0, 0, 0, 723, 721, 1, 0, 0, 0, 724, 717, 1, 0, 0, 0, 724, 725, 1, 0, 0, 0, 725, 726, 1, 0, 0, 0, 726, 727, 3, 71, 28, 0, 727, 737, 1, 0, 0, 0, 728, 730, 3, 103, 44, 0, 729, 731, 3, 63, 24, 0, 730, 729, 1, 0, 0, 0, 731, 732, 1, 0, 0, 0, 732, 730, 1, 0, 0, 0, 732, 733, 1, 0, 0, 0, 733, 734, 1, 0, 0, 0, 734, 735, 3, 71, 28, 0, 735, 737, 1, 0, 0, 0, 736, 695, 1, 0, 0, 0, 736, 706, 1, 0, 0, 0, 736, 713, 1, 0, 0, 0, 736, 728, 1, 0, 0, 0, 737, 88, 1, 0, 0, 0, 738, 739, 7, 30, 0, 0, 739, 740, 7, 31, 0, 0, 740, 90, 1, 0, 0, 0, 741, 742, 7, 12, 0, 0, 742, 743, 7, 9, 0, 0, 743, 744, 7, 0, 0, 0, 744, 92, 1, 0, 0, 0, 745, 746, 7, 12, 0, 0, 746, 747, 7, 2, 0, 0, 747, 748, 7, 4, 0, 0, 748, 94, 1, 0, 0, 0, 749, 750, 5, 61, 0, 0, 750, 96, 1, 0, 0, 0, 751, 752, 5, 58, 0, 0, 752, 753, 5, 58, 0, 0, 753, 98, 1, 0, 0, 0, 754, 755, 5, 44, 0, 0, 755, 100, 1, 0, 0, 0, 756, 757, 7, 0, 0, 0, 757, 758, 7, 3, 0, 0, 758, 759, 7, 2, 0, 0, 759, 760, 7, 4, 0, 0, 760, 102, 1, 0, 0, 0, 761, 762, 5, 46, 0, 0, 762, 104, 1, 0, 0, 0, 763, 764, 7, 15, 0, 0, 764, 765, 7, 12, 0, 0, 765, 766, 7, 13, 0, 0, 766, 767, 7, 2, 0, 0, 767, 768, 7, 3, 0, 0, 768, 106, 1, 0, 0, 0, 769, 770, 7, 15, 0, 0, 770, 771, 7, 1, 0, 0, 771, 772, 7, 6, 0, 0, 772, 773, 7, 2, 0, 0, 773, 774, 7, 5, 0, 0, 774, 108, 1, 0, 0, 0, 775, 776, 7, 1, 0, 0, 776, 777, 7, 9, 0, 0, 777, 110, 1, 0, 0, 0, 778, 779, 7, 1, 0, 0, 779, 780, 7, 2, 0, 0, 780, 112, 1, 0, 0, 0, 781, 782, 7, 13, 0, 0, 782, 783, 7, 12, 0, 0, 783, 784, 7, 2, 0, 0, 784, 785, 7, 5, 0, 0, 785, 114, 1, 0, 0, 0, 786, 787, 7, 13, 0, 0, 787, 788, 7, 1, 0, 0, 788, 789, 7, 18, 0, 0, 789, 790, 7, 3, 0, 0, 790, 116, 1, 0, 0, 0, 791, 792, 5, 40, 0, 0, 792, 118, 1, 0, 0, 0, 793, 794, 7, 9, 0, 0, 794, 795, 7, 7, 0, 0, 795, 796, 7, 5, 0, 0, 796, 120, 1, 0, 0, 0, 797, 798, 7, 9, 0, 0, 798, 799, 7, 20, 0, 0, 799, 800, 7, 13, 0, 0, 800, 801, 7, 13, 0, 0, 801, 122, 1, 0, 0, 0, 802, 803, 7, 9, 0, 0, 803, 804, 7, 20, 0, 0, 804, 805, 7, 13, 0, 0, 805, 806, 7, 13, 0, 0, 806, 807, 7, 2, 0, 0, 807, 124, 1, 0, 0, 0, 808, 809, 7, 7, 0, 0, 809, 810, 7, 6, 0, 0, 810, 126, 1, 0, 0, 0, 811, 812, 5, 63, 0, 0, 812, 128, 1, 0, 0, 0, 813, 814, 7, 6, 0, 0, 814, 815, 7, 13, 0, 0, 815, 816, 7, 1, 0, 0, 816, 817, 7, 18, 0, 0, 817, 818, 7, 3, 0, 0, 818, 130, 1, 0, 0, 0, 819, 820, 5, 41, 0, 0, 820, 132, 1, 0, 0, 0, 821, 822, 7, 5, 0, 0, 822, 823, 7, 6, 0, 0, 823, 824, 7, 20, 0, 0, 824, 825, 7, 3, 0, 0, 825, 134, 1, 0, 0, 0, 826, 827, 5, 61, 0, 0, 827, 828, 5, 61, 0, 0, 828, 136, 1, 0, 0, 0, 829, 830, 5, 61, 0, 0, 830, 831, 5, 126, 0, 0, 831, 138, 1, 0, 0, 0, 832, 833, 5, 33, 0, 0, 833, 834, 5, 61, 0, 0, 834, 140, 1, 0, 0, 0, 835, 836, 5, 60, 0, 0, 836, 142, 1, 0, 0, 0, 837, 838, 5, 60, 0, 0, 838, 839, 5, 61, 0, 0, 839, 144, 1, 0, 0, 0, 840, 841, 5, 62, 0, 0, 841, 146, 1, 0, 0, 0, 842, 843, 5, 62, 0, 0, 843, 844, 5, 61, 0, 0, 844, 148, 1, 0, 0, 0, 845, 846, 5, 43, 0, 0, 846, 150, 1, 0, 0, 0, 847, 848, 5, 45, 0, 0, 848, 152, 1, 0, 0, 0, 849, 850, 5, 42, 0, 0, 850, 154, 1, 0, 0, 0, 851, 852, 5, 47, 0, 0, 852, 156, 1, 0, 0, 0, 853, 854, 5, 37, 0, 0, 854, 158, 1, 0, 0, 0, 855, 856, 4, 72, 3, 0, 856, 857, 7, 16, 0, 0, 857, 858, 7, 12, 0, 0, 858, 859, 7, 5, 0, 0, 859, 860, 7, 4, 0, 0, 860, 861, 7, 10, 0, 0, 861, 160, 1, 0, 0, 0, 862, 865, 3, 127, 56, 0, 863, 866, 3, 65, 25, 0, 864, 866, 3, 79, 32, 0, 865, 863, 1, 0, 0, 0, 865, 864, 1, 0, 0, 0, 866, 870, 1, 0, 0, 0, 867, 869, 3, 81, 33, 0, 868, 867, 1, 0, 0, 0, 869, 872, 1, 0, 0, 0, 870, 868, 1, 0, 0, 0, 870, 871, 1, 0, 0, 0, 871, 880, 1, 0, 0, 0, 872, 870, 1, 0, 0, 0, 873, 875, 3, 127, 56, 0, 874, 876, 3, 63, 24, 0, 875, 874, 1, 0, 0, 0, 876, 877, 1, 0, 0, 0, 877, 875, 1, 0, 0, 0, 877, 878, 1, 0, 0, 0, 878, 880, 1, 0, 0, 0, 879, 862, 1, 0, 0, 0, 879, 873, 1, 0, 0, 0, 880, 162, 1, 0, 0, 0, 881, 882, 5, 91, 0, 0, 882, 883, 1, 0, 0, 0, 883, 884, 6, 74, 0, 0, 884, 885, 6, 74, 0, 0, 885, 164, 1, 0, 0, 0, 886, 887, 5, 93, 0, 0, 887, 888, 1, 0, 0, 0, 888, 889, 6, 75, 11, 0, 889, 890, 6, 75, 11, 0, 890, 166, 1, 0, 0, 0, 891, 895, 3, 65, 25, 0, 892, 894, 3, 81, 33, 0, 893, 892, 1, 0, 0, 0, 894, 897, 1, 0, 0, 0, 895, 893, 1, 0, 0, 0, 895, 896, 1, 0, 0, 0, 896, 908, 1, 0, 0, 0, 897, 895, 1, 0, 0, 0, 898, 901, 3, 79, 32, 0, 899, 901, 3, 73, 29, 0, 900, 898, 1, 0, 0, 0, 900, 899, 1, 0, 0, 0, 901, 903, 1, 0, 0, 0, 902, 904, 3, 81, 33, 0, 903, 902, 1, 0, 0, 0, 904, 905, 1, 0, 0, 0, 905, 903, 1, 0, 0, 0, 905, 906, 1, 0, 0, 0, 906, 908, 1, 0, 0, 0, 907, 891, 1, 0, 0, 0, 907, 900, 1, 0, 0, 0, 908, 168, 1, 0, 0, 0, 909, 911, 3, 75, 30, 0, 910, 912, 3, 77, 31, 0, 911, 910, 1, 0, 0, 0, 912, 913, 1, 0, 0, 0, 913, 911, 1, 0, 0, 0, 913, 914, 1, 0, 0, 0, 914, 915, 1, 0, 0, 0, 915, 916, 3, 75, 30, 0, 916, 170, 1, 0, 0, 0, 917, 918, 3, 169, 77, 0, 918, 172, 1, 0, 0, 0, 919, 920, 3, 55, 20, 0, 920, 921, 1, 0, 0, 0, 921, 922, 6, 79, 10, 0, 922, 174, 1, 0, 0, 0, 923, 924, 3, 57, 21, 0, 924, 925, 1, 0, 0, 0, 925, 926, 6, 80, 10, 0, 926, 176, 1, 0, 0, 0, 927, 928, 3, 59, 22, 0, 928, 929, 1, 0, 0, 0, 929, 930, 6, 81, 10, 0, 930, 178, 1, 0, 0, 0, 931, 932, 3, 163, 74, 0, 932, 933, 1, 0, 0, 0, 933, 934, 6, 82, 12, 0, 934, 935, 6, 82, 13, 0, 935, 180, 1, 0, 0, 0, 936, 937, 3, 61, 23, 0, 937, 938, 1, 0, 0, 0, 938, 939, 6, 83, 14, 0, 939, 940, 6, 83, 11, 0, 940, 182, 1, 0, 0, 0, 941, 942, 3, 59, 22, 0, 942, 943, 1, 0, 0, 0, 943, 944, 6, 84, 10, 0, 944, 184, 1, 0, 0, 0, 945, 946, 3, 55, 20, 0, 946, 947, 1, 0, 0, 0, 947, 948, 6, 85, 10, 0, 948, 186, 1, 0, 0, 0, 949, 950, 3, 57, 21, 0, 950, 951, 1, 0, 0, 0, 951, 952, 6, 86, 10, 0, 952, 188, 1, 0, 0, 0, 953, 954, 3, 61, 23, 0, 954, 955, 1, 0, 0, 0, 955, 956, 6, 87, 14, 0, 956, 957, 6, 87, 11, 0, 957, 190, 1, 0, 0, 0, 958, 959, 3, 163, 74, 0, 959, 960, 1, 0, 0, 0, 960, 961, 6, 88, 12, 0, 961, 192, 1, 0, 0, 0, 962, 963, 3, 165, 75, 0, 963, 964, 1, 0, 0, 0, 964, 965, 6, 89, 15, 0, 965, 194, 1, 0, 0, 0, 966, 967, 3, 335, 160, 0, 967, 968, 1, 0, 0, 0, 968, 969, 6, 90, 16, 0, 969, 196, 1, 0, 0, 0, 970, 971, 3, 99, 42, 0, 971, 972, 1, 0, 0, 0, 972, 973, 6, 91, 17, 0, 973, 198, 1, 0, 0, 0, 974, 975, 3, 95, 40, 0, 975, 976, 1, 0, 0, 0, 976, 977, 6, 92, 18, 0, 977, 200, 1, 0, 0, 0, 978, 979, 7, 16, 0, 0, 979, 980, 7, 3, 0, 0, 980, 981, 7, 5, 0, 0, 981, 982, 7, 12, 0, 0, 982, 983, 7, 0, 0, 0, 983, 984, 7, 12, 0, 0, 984, 985, 7, 5, 0, 0, 985, 986, 7, 12, 0, 0, 986, 202, 1, 0, 0, 0, 987, 991, 8, 32, 0, 0, 988, 989, 5, 47, 0, 0, 989, 991, 8, 33, 0, 0, 990, 987, 1, 0, 0, 0, 990, 988, 1, 0, 0, 0, 991, 204, 1, 0, 0, 0, 992, 994, 3, 203, 94, 0, 993, 992, 1, 0, 0, 0, 994, 995, 1, 0, 0, 0, 995, 993, 1, 0, 0, 0, 995, 996, 1, 0, 0, 0, 996, 206, 1, 0, 0, 0, 997, 998, 3, 205, 95, 0, 998, 999, 1, 0, 0, 0, 999, 1000, 6, 96, 19, 0, 1000, 208, 1, 0, 0, 0, 1001, 1002, 3, 83, 34, 0, 1002, 1003, 1, 0, 0, 0, 1003, 1004, 6, 97, 20, 0, 1004, 210, 1, 0, 0, 0, 1005, 1006, 3, 55, 20, 0, 1006, 1007, 1, 0, 0, 0, 1007, 1008, 6, 98, 10, 0, 1008, 212, 1, 0, 0, 0, 1009, 1010, 3, 57, 21, 0, 1010, 1011, 1, 0, 0, 0, 1011, 1012, 6, 99, 10, 0, 1012, 214, 1, 0, 0, 0, 1013, 1014, 3, 59, 22, 0, 1014, 1015, 1, 0, 0, 0, 1015, 1016, 6, 100, 10, 0, 1016, 216, 1, 0, 0, 0, 1017, 1018, 3, 61, 23, 0, 1018, 1019, 1, 0, 0, 0, 1019, 1020, 6, 101, 14, 0, 1020, 1021, 6, 101, 11, 0, 1021, 218, 1, 0, 0, 0, 1022, 1023, 3, 103, 44, 0, 1023, 1024, 1, 0, 0, 0, 1024, 1025, 6, 102, 21, 0, 1025, 220, 1, 0, 0, 0, 1026, 1027, 3, 99, 42, 0, 1027, 1028, 1, 0, 0, 0, 1028, 1029, 6, 103, 17, 0, 1029, 222, 1, 0, 0, 0, 1030, 1031, 3, 127, 56, 0, 1031, 1032, 1, 0, 0, 0, 1032, 1033, 6, 104, 22, 0, 1033, 224, 1, 0, 0, 0, 1034, 1035, 3, 161, 73, 0, 1035, 1036, 1, 0, 0, 0, 1036, 1037, 6, 105, 23, 0, 1037, 226, 1, 0, 0, 0, 1038, 1043, 3, 65, 25, 0, 1039, 1043, 3, 63, 24, 0, 1040, 1043, 3, 79, 32, 0, 1041, 1043, 3, 153, 69, 0, 1042, 1038, 1, 0, 0, 0, 1042, 1039, 1, 0, 0, 0, 1042, 1040, 1, 0, 0, 0, 1042, 1041, 1, 0, 0, 0, 1043, 228, 1, 0, 0, 0, 1044, 1047, 3, 65, 25, 0, 1045, 1047, 3, 153, 69, 0, 1046, 1044, 1, 0, 0, 0, 1046, 1045, 1, 0, 0, 0, 1047, 1051, 1, 0, 0, 0, 1048, 1050, 3, 227, 106, 0, 1049, 1048, 1, 0, 0, 0, 1050, 1053, 1, 0, 0, 0, 1051, 1049, 1, 0, 0, 0, 1051, 1052, 1, 0, 0, 0, 1052, 1064, 1, 0, 0, 0, 1053, 1051, 1, 0, 0, 0, 1054, 1057, 3, 79, 32, 0, 1055, 1057, 3, 73, 29, 0, 1056, 1054, 1, 0, 0, 0, 1056, 1055, 1, 0, 0, 0, 1057, 1059, 1, 0, 0, 0, 1058, 1060, 3, 227, 106, 0, 1059, 1058, 1, 0, 0, 0, 1060, 1061, 1, 0, 0, 0, 1061, 1059, 1, 0, 0, 0, 1061, 1062, 1, 0, 0, 0, 1062, 1064, 1, 0, 0, 0, 1063, 1046, 1, 0, 0, 0, 1063, 1056, 1, 0, 0, 0, 1064, 230, 1, 0, 0, 0, 1065, 1068, 3, 229, 107, 0, 1066, 1068, 3, 169, 77, 0, 1067, 1065, 1, 0, 0, 0, 1067, 1066, 1, 0, 0, 0, 1068, 1069, 1, 0, 0, 0, 1069, 1067, 1, 0, 0, 0, 1069, 1070, 1, 0, 0, 0, 1070, 232, 1, 0, 0, 0, 1071, 1072, 3, 55, 20, 0, 1072, 1073, 1, 0, 0, 0, 1073, 1074, 6, 109, 10, 0, 1074, 234, 1, 0, 0, 0, 1075, 1076, 3, 57, 21, 0, 1076, 1077, 1, 0, 0, 0, 1077, 1078, 6, 110, 10, 0, 1078, 236, 1, 0, 0, 0, 1079, 1080, 3, 59, 22, 0, 1080, 1081, 1, 0, 0, 0, 1081, 1082, 6, 111, 10, 0, 1082, 238, 1, 0, 0, 0, 1083, 1084, 3, 61, 23, 0, 1084, 1085, 1, 0, 0, 0, 1085, 1086, 6, 112, 14, 0, 1086, 1087, 6, 112, 11, 0, 1087, 240, 1, 0, 0, 0, 1088, 1089, 3, 95, 40, 0, 1089, 1090, 1, 0, 0, 0, 1090, 1091, 6, 113, 18, 0, 1091, 242, 1, 0, 0, 0, 1092, 1093, 3, 99, 42, 0, 1093, 1094, 1, 0, 0, 0, 1094, 1095, 6, 114, 17, 0, 1095, 244, 1, 0, 0, 0, 1096, 1097, 3, 103, 44, 0, 1097, 1098, 1, 0, 0, 0, 1098, 1099, 6, 115, 21, 0, 1099, 246, 1, 0, 0, 0, 1100, 1101, 3, 127, 56, 0, 1101, 1102, 1, 0, 0, 0, 1102, 1103, 6, 116, 22, 0, 1103, 248, 1, 0, 0, 0, 1104, 1105, 3, 161, 73, 0, 1105, 1106, 1, 0, 0, 0, 1106, 1107, 6, 117, 23, 0, 1107, 250, 1, 0, 0, 0, 1108, 1109, 7, 12, 0, 0, 1109, 1110, 7, 2, 0, 0, 1110, 252, 1, 0, 0, 0, 1111, 1112, 3, 231, 108, 0, 1112, 1113, 1, 0, 0, 0, 1113, 1114, 6, 119, 24, 0, 1114, 254, 1, 0, 0, 0, 1115, 1116, 3, 55, 20, 0, 1116, 1117, 1, 0, 0, 0, 1117, 1118, 6, 120, 10, 0, 1118, 256, 1, 0, 0, 0, 1119, 1120, 3, 57, 21, 0, 1120, 1121, 1, 0, 0, 0, 1121, 1122, 6, 121, 10, 0, 1122, 258, 1, 0, 0, 0, 1123, 1124, 3, 59, 22, 0, 1124, 1125, 1, 0, 0, 0, 1125, 1126, 6, 122, 10, 0, 1126, 260, 1, 0, 0, 0, 1127, 1128, 3, 61, 23, 0, 1128, 1129, 1, 0, 0, 0, 1129, 1130, 6, 123, 14, 0, 1130, 1131, 6, 123, 11, 0, 1131, 262, 1, 0, 0, 0, 1132, 1133, 3, 163, 74, 0, 1133, 1134, 1, 0, 0, 0, 1134, 1135, 6, 124, 12, 0, 1135, 1136, 6, 124, 25, 0, 1136, 264, 1, 0, 0, 0, 1137, 1138, 7, 7, 0, 0, 1138, 1139, 7, 9, 0, 0, 1139, 1140, 1, 0, 0, 0, 1140, 1141, 6, 125, 26, 0, 1141, 266, 1, 0, 0, 0, 1142, 1143, 7, 19, 0, 0, 1143, 1144, 7, 1, 0, 0, 1144, 1145, 7, 5, 0, 0, 1145, 1146, 7, 10, 0, 0, 1146, 1147, 1, 0, 0, 0, 1147, 1148, 6, 126, 26, 0, 1148, 268, 1, 0, 0, 0, 1149, 1150, 8, 34, 0, 0, 1150, 270, 1, 0, 0, 0, 1151, 1153, 3, 269, 127, 0, 1152, 1151, 1, 0, 0, 0, 1153, 1154, 1, 0, 0, 0, 1154, 1152, 1, 0, 0, 0, 1154, 1155, 1, 0, 0, 0, 1155, 1156, 1, 0, 0, 0, 1156, 1157, 3, 335, 160, 0, 1157, 1159, 1, 0, 0, 0, 1158, 1152, 1, 0, 0, 0, 1158, 1159, 1, 0, 0, 0, 1159, 1161, 1, 0, 0, 0, 1160, 1162, 3, 269, 127, 0, 1161, 1160, 1, 0, 0, 0, 1162, 1163, 1, 0, 0, 0, 1163, 1161, 1, 0, 0, 0, 1163, 1164, 1, 0, 0, 0, 1164, 272, 1, 0, 0, 0, 1165, 1166, 3, 271, 128, 0, 1166, 1167, 1, 0, 0, 0, 1167, 1168, 6, 129, 27, 0, 1168, 274, 1, 0, 0, 0, 1169, 1170, 3, 55, 20, 0, 1170, 1171, 1, 0, 0, 0, 1171, 1172, 6, 130, 10, 0, 1172, 276, 1, 0, 0, 0, 1173, 1174, 3, 57, 21, 0, 1174, 1175, 1, 0, 0, 0, 1175, 1176, 6, 131, 10, 0, 1176, 278, 1, 0, 0, 0, 1177, 1178, 3, 59, 22, 0, 1178, 1179, 1, 0, 0, 0, 1179, 1180, 6, 132, 10, 0, 1180, 280, 1, 0, 0, 0, 1181, 1182, 3, 61, 23, 0, 1182, 1183, 1, 0, 0, 0, 1183, 1184, 6, 133, 14, 0, 1184, 1185, 6, 133, 11, 0, 1185, 1186, 6, 133, 11, 0, 1186, 282, 1, 0, 0, 0, 1187, 1188, 3, 95, 40, 0, 1188, 1189, 1, 0, 0, 0, 1189, 1190, 6, 134, 18, 0, 1190, 284, 1, 0, 0, 0, 1191, 1192, 3, 99, 42, 0, 1192, 1193, 1, 0, 0, 0, 1193, 1194, 6, 135, 17, 0, 1194, 286, 1, 0, 0, 0, 1195, 1196, 3, 103, 44, 0, 1196, 1197, 1, 0, 0, 0, 1197, 1198, 6, 136, 21, 0, 1198, 288, 1, 0, 0, 0, 1199, 1200, 3, 267, 126, 0, 1200, 1201, 1, 0, 0, 0, 1201, 1202, 6, 137, 28, 0, 1202, 290, 1, 0, 0, 0, 1203, 1204, 3, 231, 108, 0, 1204, 1205, 1, 0, 0, 0, 1205, 1206, 6, 138, 24, 0, 1206, 292, 1, 0, 0, 0, 1207, 1208, 3, 171, 78, 0, 1208, 1209, 1, 0, 0, 0, 1209, 1210, 6, 139, 29, 0, 1210, 294, 1, 0, 0, 0, 1211, 1212, 3, 127, 56, 0, 1212, 1213, 1, 0, 0, 0, 1213, 1214, 6, 140, 22, 0, 1214, 296, 1, 0, 0, 0, 1215, 1216, 3, 161, 73, 0, 1216, 1217, 1, 0, 0, 0, 1217, 1218, 6, 141, 23, 0, 1218, 298, 1, 0, 0, 0, 1219, 1220, 3, 55, 20, 0, 1220, 1221, 1, 0, 0, 0, 1221, 1222, 6, 142, 10, 0, 1222, 300, 1, 0, 0, 0, 1223, 1224, 3, 57, 21, 0, 1224, 1225, 1, 0, 0, 0, 1225, 1226, 6, 143, 10, 0, 1226, 302, 1, 0, 0, 0, 1227, 1228, 3, 59, 22, 0, 1228, 1229, 1, 0, 0, 0, 1229, 1230, 6, 144, 10, 0, 1230, 304, 1, 0, 0, 0, 1231, 1232, 3, 61, 23, 0, 1232, 1233, 1, 0, 0, 0, 1233, 1234, 6, 145, 14, 0, 1234, 1235, 6, 145, 11, 0, 1235, 306, 1, 0, 0, 0, 1236, 1237, 3, 103, 44, 0, 1237, 1238, 1, 0, 0, 0, 1238, 1239, 6, 146, 21, 0, 1239, 308, 1, 0, 0, 0, 1240, 1241, 3, 127, 56, 0, 1241, 1242, 1, 0, 0, 0, 1242, 1243, 6, 147, 22, 0, 1243, 310, 1, 0, 0, 0, 1244, 1245, 3, 161, 73, 0, 1245, 1246, 1, 0, 0, 0, 1246, 1247, 6, 148, 23, 0, 1247, 312, 1, 0, 0, 0, 1248, 1249, 3, 171, 78, 0, 1249, 1250, 1, 0, 0, 0, 1250, 1251, 6, 149, 29, 0, 1251, 314, 1, 0, 0, 0, 1252, 1253, 3, 167, 76, 0, 1253, 1254, 1, 0, 0, 0, 1254, 1255, 6, 150, 30, 0, 1255, 316, 1, 0, 0, 0, 1256, 1257, 3, 55, 20, 0, 1257, 1258, 1, 0, 0, 0, 1258, 1259, 6, 151, 10, 0, 1259, 318, 1, 0, 0, 0, 1260, 1261, 3, 57, 21, 0, 1261, 1262, 1, 0, 0, 0, 1262, 1263, 6, 152, 10, 0, 1263, 320, 1, 0, 0, 0, 1264, 1265, 3, 59, 22, 0, 1265, 1266, 1, 0, 0, 0, 1266, 1267, 6, 153, 10, 0, 1267, 322, 1, 0, 0, 0, 1268, 1269, 3, 61, 23, 0, 1269, 1270, 1, 0, 0, 0, 1270, 1271, 6, 154, 14, 0, 1271, 1272, 6, 154, 11, 0, 1272, 324, 1, 0, 0, 0, 1273, 1274, 7, 1, 0, 0, 1274, 1275, 7, 9, 0, 0, 1275, 1276, 7, 15, 0, 0, 1276, 1277, 7, 7, 0, 0, 1277, 326, 1, 0, 0, 0, 1278, 1279, 3, 55, 20, 0, 1279, 1280, 1, 0, 0, 0, 1280, 1281, 6, 156, 10, 0, 1281, 328, 1, 0, 0, 0, 1282, 1283, 3, 57, 21, 0, 1283, 1284, 1, 0, 0, 0, 1284, 1285, 6, 157, 10, 0, 1285, 330, 1, 0, 0, 0, 1286, 1287, 3, 59, 22, 0, 1287, 1288, 1, 0, 0, 0, 1288, 1289, 6, 158, 10, 0, 1289, 332, 1, 0, 0, 0, 1290, 1291, 3, 165, 75, 0, 1291, 1292, 1, 0, 0, 0, 1292, 1293, 6, 159, 15, 0, 1293, 1294, 6, 159, 11, 0, 1294, 334, 1, 0, 0, 0, 1295, 1296, 5, 58, 0, 0, 1296, 336, 1, 0, 0, 0, 1297, 1303, 3, 73, 29, 0, 1298, 1303, 3, 63, 24, 0, 1299, 1303, 3, 103, 44, 0, 1300, 1303, 3, 65, 25, 0, 1301, 1303, 3, 79, 32, 0, 1302, 1297, 1, 0, 0, 0, 1302, 1298, 1, 0, 0, 0, 1302, 1299, 1, 0, 0, 0, 1302, 1300, 1, 0, 0, 0, 1302, 1301, 1, 0, 0, 0, 1303, 1304, 1, 0, 0, 0, 1304, 1302, 1, 0, 0, 0, 1304, 1305, 1, 0, 0, 0, 1305, 338, 1, 0, 0, 0, 1306, 1307, 3, 55, 20, 0, 1307, 1308, 1, 0, 0, 0, 1308, 1309, 6, 162, 10, 0, 1309, 340, 1, 0, 0, 0, 1310, 1311, 3, 57, 21, 0, 1311, 1312, 1, 0, 0, 0, 1312, 1313, 6, 163, 10, 0, 1313, 342, 1, 0, 0, 0, 1314, 1315, 3, 59, 22, 0, 1315, 1316, 1, 0, 0, 0, 1316, 1317, 6, 164, 10, 0, 1317, 344, 1, 0, 0, 0, 1318, 1319, 3, 61, 23, 0, 1319, 1320, 1, 0, 0, 0, 1320, 1321, 6, 165, 14, 0, 1321, 1322, 6, 165, 11, 0, 1322, 346, 1, 0, 0, 0, 1323, 1324, 3, 335, 160, 0, 1324, 1325, 1, 0, 0, 0, 1325, 1326, 6, 166, 16, 0, 1326, 348, 1, 0, 0, 0, 1327, 1328, 3, 99, 42, 0, 1328, 1329, 1, 0, 0, 0, 1329, 1330, 6, 167, 17, 0, 1330, 350, 1, 0, 0, 0, 1331, 1332, 3, 103, 44, 0, 1332, 1333, 1, 0, 0, 0, 1333, 1334, 6, 168, 21, 0, 1334, 352, 1, 0, 0, 0, 1335, 1336, 3, 265, 125, 0, 1336, 1337, 1, 0, 0, 0, 1337, 1338, 6, 169, 31, 0, 1338, 1339, 6, 169, 32, 0, 1339, 354, 1, 0, 0, 0, 1340, 1341, 3, 205, 95, 0, 1341, 1342, 1, 0, 0, 0, 1342, 1343, 6, 170, 19, 0, 1343, 356, 1, 0, 0, 0, 1344, 1345, 3, 83, 34, 0, 1345, 1346, 1, 0, 0, 0, 1346, 1347, 6, 171, 20, 0, 1347, 358, 1, 0, 0, 0, 1348, 1349, 3, 55, 20, 0, 1349, 1350, 1, 0, 0, 0, 1350, 1351, 6, 172, 10, 0, 1351, 360, 1, 0, 0, 0, 1352, 1353, 3, 57, 21, 0, 1353, 1354, 1, 0, 0, 0, 1354, 1355, 6, 173, 10, 0, 1355, 362, 1, 0, 0, 0, 1356, 1357, 3, 59, 22, 0, 1357, 1358, 1, 0, 0, 0, 1358, 1359, 6, 174, 10, 0, 1359, 364, 1, 0, 0, 0, 1360, 1361, 3, 61, 23, 0, 1361, 1362, 1, 0, 0, 0, 1362, 1363, 6, 175, 14, 0, 1363, 1364, 6, 175, 11, 0, 1364, 1365, 6, 175, 11, 0, 1365, 366, 1, 0, 0, 0, 1366, 1367, 3, 99, 42, 0, 1367, 1368, 1, 0, 0, 0, 1368, 1369, 6, 176, 17, 0, 1369, 368, 1, 0, 0, 0, 1370, 1371, 3, 103, 44, 0, 1371, 1372, 1, 0, 0, 0, 1372, 1373, 6, 177, 21, 0, 1373, 370, 1, 0, 0, 0, 1374, 1375, 3, 231, 108, 0, 1375, 1376, 1, 0, 0, 0, 1376, 1377, 6, 178, 24, 0, 1377, 372, 1, 0, 0, 0, 1378, 1379, 3, 55, 20, 0, 1379, 1380, 1, 0, 0, 0, 1380, 1381, 6, 179, 10, 0, 1381, 374, 1, 0, 0, 0, 1382, 1383, 3, 57, 21, 0, 1383, 1384, 1, 0, 0, 0, 1384, 1385, 6, 180, 10, 0, 1385, 376, 1, 0, 0, 0, 1386, 1387, 3, 59, 22, 0, 1387, 1388, 1, 0, 0, 0, 1388, 1389, 6, 181, 10, 0, 1389, 378, 1, 0, 0, 0, 1390, 1391, 3, 61, 23, 0, 1391, 1392, 1, 0, 0, 0, 1392, 1393, 6, 182, 14, 0, 1393, 1394, 6, 182, 11, 0, 1394, 380, 1, 0, 0, 0, 1395, 1396, 3, 205, 95, 0, 1396, 1397, 1, 0, 0, 0, 1397, 1398, 6, 183, 19, 0, 1398, 1399, 6, 183, 11, 0, 1399, 1400, 6, 183, 33, 0, 1400, 382, 1, 0, 0, 0, 1401, 1402, 3, 83, 34, 0, 1402, 1403, 1, 0, 0, 0, 1403, 1404, 6, 184, 20, 0, 1404, 1405, 6, 184, 11, 0, 1405, 1406, 6, 184, 33, 0, 1406, 384, 1, 0, 0, 0, 1407, 1408, 3, 55, 20, 0, 1408, 1409, 1, 0, 0, 0, 1409, 1410, 6, 185, 10, 0, 1410, 386, 1, 0, 0, 0, 1411, 1412, 3, 57, 21, 0, 1412, 1413, 1, 0, 0, 0, 1413, 1414, 6, 186, 10, 0, 1414, 388, 1, 0, 0, 0, 1415, 1416, 3, 59, 22, 0, 1416, 1417, 1, 0, 0, 0, 1417, 1418, 6, 187, 10, 0, 1418, 390, 1, 0, 0, 0, 1419, 1420, 3, 335, 160, 0, 1420, 1421, 1, 0, 0, 0, 1421, 1422, 6, 188, 16, 0, 1422, 1423, 6, 188, 11, 0, 1423, 1424, 6, 188, 9, 0, 1424, 392, 1, 0, 0, 0, 1425, 1426, 3, 99, 42, 0, 1426, 1427, 1, 0, 0, 0, 1427, 1428, 6, 189, 17, 0, 1428, 1429, 6, 189, 11, 0, 1429, 1430, 6, 189, 9, 0, 1430, 394, 1, 0, 0, 0, 1431, 1432, 3, 55, 20, 0, 1432, 1433, 1, 0, 0, 0, 1433, 1434, 6, 190, 10, 0, 1434, 396, 1, 0, 0, 0, 1435, 1436, 3, 57, 21, 0, 1436, 1437, 1, 0, 0, 0, 1437, 1438, 6, 191, 10, 0, 1438, 398, 1, 0, 0, 0, 1439, 1440, 3, 59, 22, 0, 1440, 1441, 1, 0, 0, 0, 1441, 1442, 6, 192, 10, 0, 1442, 400, 1, 0, 0, 0, 1443, 1444, 3, 171, 78, 0, 1444, 1445, 1, 0, 0, 0, 1445, 1446, 6, 193, 11, 0, 1446, 1447, 6, 193, 0, 0, 1447, 1448, 6, 193, 29, 0, 1448, 402, 1, 0, 0, 0, 1449, 1450, 3, 167, 76, 0, 1450, 1451, 1, 0, 0, 0, 1451, 1452, 6, 194, 11, 0, 1452, 1453, 6, 194, 0, 0, 1453, 1454, 6, 194, 30, 0, 1454, 404, 1, 0, 0, 0, 1455, 1456, 3, 89, 37, 0, 1456, 1457, 1, 0, 0, 0, 1457, 1458, 6, 195, 11, 0, 1458, 1459, 6, 195, 0, 0, 1459, 1460, 6, 195, 34, 0, 1460, 406, 1, 0, 0, 0, 1461, 1462, 3, 61, 23, 0, 1462, 1463, 1, 0, 0, 0, 1463, 1464, 6, 196, 14, 0, 1464, 1465, 6, 196, 11, 0, 1465, 408, 1, 0, 0, 0, 65, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 577, 587, 591, 594, 603, 605, 616, 635, 640, 649, 656, 661, 663, 674, 682, 685, 687, 692, 697, 703, 710, 715, 721, 724, 732, 736, 865, 870, 877, 879, 895, 900, 905, 907, 913, 990, 995, 1042, 1046, 1051, 1056, 1061, 1063, 1067, 1069, 1154, 1158, 1163, 1302, 1304, 35, 5, 1, 0, 5, 4, 0, 5, 6, 0, 5, 2, 0, 5, 3, 0, 5, 8, 0, 5, 5, 0, 5, 9, 0, 5, 11, 0, 5, 13, 0, 0, 1, 0, 4, 0, 0, 7, 65, 0, 5, 0, 0, 7, 24, 0, 7, 66, 0, 7, 104, 0, 7, 33, 0, 7, 31, 0, 7, 76, 0, 7, 25, 0, 7, 35, 0, 7, 47, 0, 7, 64, 0, 7, 80, 0, 5, 10, 0, 5, 7, 0, 7, 90, 0, 7, 89, 0, 7, 68, 0, 7, 67, 0, 7, 88, 0, 5, 12, 0, 5, 14, 0, 7, 28, 0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java index f67daa29ab059..563e2418e7eff 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java @@ -8,14 +8,16 @@ * 2.0. */ -import org.antlr.v4.runtime.Lexer; import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.Token; -import org.antlr.v4.runtime.TokenStream; -import org.antlr.v4.runtime.*; -import org.antlr.v4.runtime.atn.*; +import org.antlr.v4.runtime.RuleContext; +import org.antlr.v4.runtime.RuntimeMetaData; +import org.antlr.v4.runtime.Vocabulary; +import org.antlr.v4.runtime.VocabularyImpl; +import org.antlr.v4.runtime.atn.ATN; +import org.antlr.v4.runtime.atn.ATNDeserializer; +import org.antlr.v4.runtime.atn.LexerATNSimulator; +import org.antlr.v4.runtime.atn.PredictionContextCache; import org.antlr.v4.runtime.dfa.DFA; -import org.antlr.v4.runtime.misc.*; @SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast", "CheckReturnValue", "this-escape"}) public class EsqlBaseLexer extends LexerConfig { @@ -25,90 +27,90 @@ public class EsqlBaseLexer extends LexerConfig { protected static final PredictionContextCache _sharedContextCache = new PredictionContextCache(); public static final int - DISSECT=1, DROP=2, ENRICH=3, EVAL=4, EXPLAIN=5, FROM=6, GROK=7, KEEP=8, - LIMIT=9, MV_EXPAND=10, RENAME=11, ROW=12, SHOW=13, SORT=14, STATS=15, - WHERE=16, DEV_INLINESTATS=17, DEV_LOOKUP=18, DEV_MATCH=19, DEV_METRICS=20, - UNKNOWN_CMD=21, LINE_COMMENT=22, MULTILINE_COMMENT=23, WS=24, PIPE=25, - QUOTED_STRING=26, INTEGER_LITERAL=27, DECIMAL_LITERAL=28, BY=29, AND=30, - ASC=31, ASSIGN=32, CAST_OP=33, COMMA=34, DESC=35, DOT=36, FALSE=37, FIRST=38, - IN=39, IS=40, LAST=41, LIKE=42, LP=43, NOT=44, NULL=45, NULLS=46, OR=47, - PARAM=48, RLIKE=49, RP=50, TRUE=51, EQ=52, CIEQ=53, NEQ=54, LT=55, LTE=56, - GT=57, GTE=58, PLUS=59, MINUS=60, ASTERISK=61, SLASH=62, PERCENT=63, NAMED_OR_POSITIONAL_PARAM=64, - OPENING_BRACKET=65, CLOSING_BRACKET=66, UNQUOTED_IDENTIFIER=67, QUOTED_IDENTIFIER=68, - EXPR_LINE_COMMENT=69, EXPR_MULTILINE_COMMENT=70, EXPR_WS=71, EXPLAIN_WS=72, - EXPLAIN_LINE_COMMENT=73, EXPLAIN_MULTILINE_COMMENT=74, METADATA=75, UNQUOTED_SOURCE=76, - FROM_LINE_COMMENT=77, FROM_MULTILINE_COMMENT=78, FROM_WS=79, ID_PATTERN=80, - PROJECT_LINE_COMMENT=81, PROJECT_MULTILINE_COMMENT=82, PROJECT_WS=83, - AS=84, RENAME_LINE_COMMENT=85, RENAME_MULTILINE_COMMENT=86, RENAME_WS=87, - ON=88, WITH=89, ENRICH_POLICY_NAME=90, ENRICH_LINE_COMMENT=91, ENRICH_MULTILINE_COMMENT=92, - ENRICH_WS=93, ENRICH_FIELD_LINE_COMMENT=94, ENRICH_FIELD_MULTILINE_COMMENT=95, - ENRICH_FIELD_WS=96, MVEXPAND_LINE_COMMENT=97, MVEXPAND_MULTILINE_COMMENT=98, - MVEXPAND_WS=99, INFO=100, SHOW_LINE_COMMENT=101, SHOW_MULTILINE_COMMENT=102, - SHOW_WS=103, COLON=104, SETTING=105, SETTING_LINE_COMMENT=106, SETTTING_MULTILINE_COMMENT=107, - SETTING_WS=108, LOOKUP_LINE_COMMENT=109, LOOKUP_MULTILINE_COMMENT=110, - LOOKUP_WS=111, LOOKUP_FIELD_LINE_COMMENT=112, LOOKUP_FIELD_MULTILINE_COMMENT=113, - LOOKUP_FIELD_WS=114, METRICS_LINE_COMMENT=115, METRICS_MULTILINE_COMMENT=116, - METRICS_WS=117, CLOSING_METRICS_LINE_COMMENT=118, CLOSING_METRICS_MULTILINE_COMMENT=119, + DISSECT=1, DROP=2, ENRICH=3, EVAL=4, EXPLAIN=5, FROM=6, GROK=7, KEEP=8, + LIMIT=9, MV_EXPAND=10, RENAME=11, ROW=12, SHOW=13, SORT=14, STATS=15, + WHERE=16, DEV_INLINESTATS=17, DEV_LOOKUP=18, DEV_METRICS=19, UNKNOWN_CMD=20, + LINE_COMMENT=21, MULTILINE_COMMENT=22, WS=23, PIPE=24, QUOTED_STRING=25, + INTEGER_LITERAL=26, DECIMAL_LITERAL=27, BY=28, AND=29, ASC=30, ASSIGN=31, + CAST_OP=32, COMMA=33, DESC=34, DOT=35, FALSE=36, FIRST=37, IN=38, IS=39, + LAST=40, LIKE=41, LP=42, NOT=43, NULL=44, NULLS=45, OR=46, PARAM=47, RLIKE=48, + RP=49, TRUE=50, EQ=51, CIEQ=52, NEQ=53, LT=54, LTE=55, GT=56, GTE=57, + PLUS=58, MINUS=59, ASTERISK=60, SLASH=61, PERCENT=62, DEV_MATCH=63, NAMED_OR_POSITIONAL_PARAM=64, + OPENING_BRACKET=65, CLOSING_BRACKET=66, UNQUOTED_IDENTIFIER=67, QUOTED_IDENTIFIER=68, + EXPR_LINE_COMMENT=69, EXPR_MULTILINE_COMMENT=70, EXPR_WS=71, EXPLAIN_WS=72, + EXPLAIN_LINE_COMMENT=73, EXPLAIN_MULTILINE_COMMENT=74, METADATA=75, UNQUOTED_SOURCE=76, + FROM_LINE_COMMENT=77, FROM_MULTILINE_COMMENT=78, FROM_WS=79, ID_PATTERN=80, + PROJECT_LINE_COMMENT=81, PROJECT_MULTILINE_COMMENT=82, PROJECT_WS=83, + AS=84, RENAME_LINE_COMMENT=85, RENAME_MULTILINE_COMMENT=86, RENAME_WS=87, + ON=88, WITH=89, ENRICH_POLICY_NAME=90, ENRICH_LINE_COMMENT=91, ENRICH_MULTILINE_COMMENT=92, + ENRICH_WS=93, ENRICH_FIELD_LINE_COMMENT=94, ENRICH_FIELD_MULTILINE_COMMENT=95, + ENRICH_FIELD_WS=96, MVEXPAND_LINE_COMMENT=97, MVEXPAND_MULTILINE_COMMENT=98, + MVEXPAND_WS=99, INFO=100, SHOW_LINE_COMMENT=101, SHOW_MULTILINE_COMMENT=102, + SHOW_WS=103, COLON=104, SETTING=105, SETTING_LINE_COMMENT=106, SETTTING_MULTILINE_COMMENT=107, + SETTING_WS=108, LOOKUP_LINE_COMMENT=109, LOOKUP_MULTILINE_COMMENT=110, + LOOKUP_WS=111, LOOKUP_FIELD_LINE_COMMENT=112, LOOKUP_FIELD_MULTILINE_COMMENT=113, + LOOKUP_FIELD_WS=114, METRICS_LINE_COMMENT=115, METRICS_MULTILINE_COMMENT=116, + METRICS_WS=117, CLOSING_METRICS_LINE_COMMENT=118, CLOSING_METRICS_MULTILINE_COMMENT=119, CLOSING_METRICS_WS=120; public static final int - EXPRESSION_MODE=1, EXPLAIN_MODE=2, FROM_MODE=3, PROJECT_MODE=4, RENAME_MODE=5, - ENRICH_MODE=6, ENRICH_FIELD_MODE=7, MVEXPAND_MODE=8, SHOW_MODE=9, SETTING_MODE=10, + EXPRESSION_MODE=1, EXPLAIN_MODE=2, FROM_MODE=3, PROJECT_MODE=4, RENAME_MODE=5, + ENRICH_MODE=6, ENRICH_FIELD_MODE=7, MVEXPAND_MODE=8, SHOW_MODE=9, SETTING_MODE=10, LOOKUP_MODE=11, LOOKUP_FIELD_MODE=12, METRICS_MODE=13, CLOSING_METRICS_MODE=14; public static String[] channelNames = { "DEFAULT_TOKEN_CHANNEL", "HIDDEN" }; public static String[] modeNames = { - "DEFAULT_MODE", "EXPRESSION_MODE", "EXPLAIN_MODE", "FROM_MODE", "PROJECT_MODE", - "RENAME_MODE", "ENRICH_MODE", "ENRICH_FIELD_MODE", "MVEXPAND_MODE", "SHOW_MODE", + "DEFAULT_MODE", "EXPRESSION_MODE", "EXPLAIN_MODE", "FROM_MODE", "PROJECT_MODE", + "RENAME_MODE", "ENRICH_MODE", "ENRICH_FIELD_MODE", "MVEXPAND_MODE", "SHOW_MODE", "SETTING_MODE", "LOOKUP_MODE", "LOOKUP_FIELD_MODE", "METRICS_MODE", "CLOSING_METRICS_MODE" }; private static String[] makeRuleNames() { return new String[] { - "DISSECT", "DROP", "ENRICH", "EVAL", "EXPLAIN", "FROM", "GROK", "KEEP", - "LIMIT", "MV_EXPAND", "RENAME", "ROW", "SHOW", "SORT", "STATS", "WHERE", - "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_MATCH", "DEV_METRICS", "UNKNOWN_CMD", - "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "DIGIT", "LETTER", - "ESCAPE_SEQUENCE", "UNESCAPED_CHARS", "EXPONENT", "ASPERAND", "BACKQUOTE", - "BACKQUOTE_BLOCK", "UNDERSCORE", "UNQUOTED_ID_BODY", "QUOTED_STRING", - "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", - "COMMA", "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", - "LP", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", - "CIEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", - "SLASH", "PERCENT", "DEV_MATCH_OP", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", - "CLOSING_BRACKET", "UNQUOTED_IDENTIFIER", "QUOTED_ID", "QUOTED_IDENTIFIER", - "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", "EXPR_WS", "EXPLAIN_OPENING_BRACKET", - "EXPLAIN_PIPE", "EXPLAIN_WS", "EXPLAIN_LINE_COMMENT", "EXPLAIN_MULTILINE_COMMENT", - "FROM_PIPE", "FROM_OPENING_BRACKET", "FROM_CLOSING_BRACKET", "FROM_COLON", - "FROM_COMMA", "FROM_ASSIGN", "METADATA", "UNQUOTED_SOURCE_PART", "UNQUOTED_SOURCE", - "FROM_UNQUOTED_SOURCE", "FROM_QUOTED_SOURCE", "FROM_LINE_COMMENT", "FROM_MULTILINE_COMMENT", - "FROM_WS", "PROJECT_PIPE", "PROJECT_DOT", "PROJECT_COMMA", "PROJECT_PARAM", - "PROJECT_NAMED_OR_POSITIONAL_PARAM", "UNQUOTED_ID_BODY_WITH_PATTERN", - "UNQUOTED_ID_PATTERN", "ID_PATTERN", "PROJECT_LINE_COMMENT", "PROJECT_MULTILINE_COMMENT", - "PROJECT_WS", "RENAME_PIPE", "RENAME_ASSIGN", "RENAME_COMMA", "RENAME_DOT", - "RENAME_PARAM", "RENAME_NAMED_OR_POSITIONAL_PARAM", "AS", "RENAME_ID_PATTERN", - "RENAME_LINE_COMMENT", "RENAME_MULTILINE_COMMENT", "RENAME_WS", "ENRICH_PIPE", - "ENRICH_OPENING_BRACKET", "ON", "WITH", "ENRICH_POLICY_NAME_BODY", "ENRICH_POLICY_NAME", - "ENRICH_MODE_UNQUOTED_VALUE", "ENRICH_LINE_COMMENT", "ENRICH_MULTILINE_COMMENT", - "ENRICH_WS", "ENRICH_FIELD_PIPE", "ENRICH_FIELD_ASSIGN", "ENRICH_FIELD_COMMA", - "ENRICH_FIELD_DOT", "ENRICH_FIELD_WITH", "ENRICH_FIELD_ID_PATTERN", "ENRICH_FIELD_QUOTED_IDENTIFIER", - "ENRICH_FIELD_PARAM", "ENRICH_FIELD_NAMED_OR_POSITIONAL_PARAM", "ENRICH_FIELD_LINE_COMMENT", - "ENRICH_FIELD_MULTILINE_COMMENT", "ENRICH_FIELD_WS", "MVEXPAND_PIPE", - "MVEXPAND_DOT", "MVEXPAND_PARAM", "MVEXPAND_NAMED_OR_POSITIONAL_PARAM", - "MVEXPAND_QUOTED_IDENTIFIER", "MVEXPAND_UNQUOTED_IDENTIFIER", "MVEXPAND_LINE_COMMENT", - "MVEXPAND_MULTILINE_COMMENT", "MVEXPAND_WS", "SHOW_PIPE", "INFO", "SHOW_LINE_COMMENT", - "SHOW_MULTILINE_COMMENT", "SHOW_WS", "SETTING_CLOSING_BRACKET", "COLON", - "SETTING", "SETTING_LINE_COMMENT", "SETTTING_MULTILINE_COMMENT", "SETTING_WS", - "LOOKUP_PIPE", "LOOKUP_COLON", "LOOKUP_COMMA", "LOOKUP_DOT", "LOOKUP_ON", - "LOOKUP_UNQUOTED_SOURCE", "LOOKUP_QUOTED_SOURCE", "LOOKUP_LINE_COMMENT", - "LOOKUP_MULTILINE_COMMENT", "LOOKUP_WS", "LOOKUP_FIELD_PIPE", "LOOKUP_FIELD_COMMA", - "LOOKUP_FIELD_DOT", "LOOKUP_FIELD_ID_PATTERN", "LOOKUP_FIELD_LINE_COMMENT", - "LOOKUP_FIELD_MULTILINE_COMMENT", "LOOKUP_FIELD_WS", "METRICS_PIPE", - "METRICS_UNQUOTED_SOURCE", "METRICS_QUOTED_SOURCE", "METRICS_LINE_COMMENT", - "METRICS_MULTILINE_COMMENT", "METRICS_WS", "CLOSING_METRICS_COLON", "CLOSING_METRICS_COMMA", - "CLOSING_METRICS_LINE_COMMENT", "CLOSING_METRICS_MULTILINE_COMMENT", - "CLOSING_METRICS_WS", "CLOSING_METRICS_QUOTED_IDENTIFIER", "CLOSING_METRICS_UNQUOTED_IDENTIFIER", + "DISSECT", "DROP", "ENRICH", "EVAL", "EXPLAIN", "FROM", "GROK", "KEEP", + "LIMIT", "MV_EXPAND", "RENAME", "ROW", "SHOW", "SORT", "STATS", "WHERE", + "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_METRICS", "UNKNOWN_CMD", "LINE_COMMENT", + "MULTILINE_COMMENT", "WS", "PIPE", "DIGIT", "LETTER", "ESCAPE_SEQUENCE", + "UNESCAPED_CHARS", "EXPONENT", "ASPERAND", "BACKQUOTE", "BACKQUOTE_BLOCK", + "UNDERSCORE", "UNQUOTED_ID_BODY", "QUOTED_STRING", "INTEGER_LITERAL", + "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", "COMMA", + "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", "LP", "NOT", + "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", "CIEQ", + "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", "SLASH", + "PERCENT", "DEV_MATCH", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", + "CLOSING_BRACKET", "UNQUOTED_IDENTIFIER", "QUOTED_ID", "QUOTED_IDENTIFIER", + "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", "EXPR_WS", "EXPLAIN_OPENING_BRACKET", + "EXPLAIN_PIPE", "EXPLAIN_WS", "EXPLAIN_LINE_COMMENT", "EXPLAIN_MULTILINE_COMMENT", + "FROM_PIPE", "FROM_OPENING_BRACKET", "FROM_CLOSING_BRACKET", "FROM_COLON", + "FROM_COMMA", "FROM_ASSIGN", "METADATA", "UNQUOTED_SOURCE_PART", "UNQUOTED_SOURCE", + "FROM_UNQUOTED_SOURCE", "FROM_QUOTED_SOURCE", "FROM_LINE_COMMENT", "FROM_MULTILINE_COMMENT", + "FROM_WS", "PROJECT_PIPE", "PROJECT_DOT", "PROJECT_COMMA", "PROJECT_PARAM", + "PROJECT_NAMED_OR_POSITIONAL_PARAM", "UNQUOTED_ID_BODY_WITH_PATTERN", + "UNQUOTED_ID_PATTERN", "ID_PATTERN", "PROJECT_LINE_COMMENT", "PROJECT_MULTILINE_COMMENT", + "PROJECT_WS", "RENAME_PIPE", "RENAME_ASSIGN", "RENAME_COMMA", "RENAME_DOT", + "RENAME_PARAM", "RENAME_NAMED_OR_POSITIONAL_PARAM", "AS", "RENAME_ID_PATTERN", + "RENAME_LINE_COMMENT", "RENAME_MULTILINE_COMMENT", "RENAME_WS", "ENRICH_PIPE", + "ENRICH_OPENING_BRACKET", "ON", "WITH", "ENRICH_POLICY_NAME_BODY", "ENRICH_POLICY_NAME", + "ENRICH_MODE_UNQUOTED_VALUE", "ENRICH_LINE_COMMENT", "ENRICH_MULTILINE_COMMENT", + "ENRICH_WS", "ENRICH_FIELD_PIPE", "ENRICH_FIELD_ASSIGN", "ENRICH_FIELD_COMMA", + "ENRICH_FIELD_DOT", "ENRICH_FIELD_WITH", "ENRICH_FIELD_ID_PATTERN", "ENRICH_FIELD_QUOTED_IDENTIFIER", + "ENRICH_FIELD_PARAM", "ENRICH_FIELD_NAMED_OR_POSITIONAL_PARAM", "ENRICH_FIELD_LINE_COMMENT", + "ENRICH_FIELD_MULTILINE_COMMENT", "ENRICH_FIELD_WS", "MVEXPAND_PIPE", + "MVEXPAND_DOT", "MVEXPAND_PARAM", "MVEXPAND_NAMED_OR_POSITIONAL_PARAM", + "MVEXPAND_QUOTED_IDENTIFIER", "MVEXPAND_UNQUOTED_IDENTIFIER", "MVEXPAND_LINE_COMMENT", + "MVEXPAND_MULTILINE_COMMENT", "MVEXPAND_WS", "SHOW_PIPE", "INFO", "SHOW_LINE_COMMENT", + "SHOW_MULTILINE_COMMENT", "SHOW_WS", "SETTING_CLOSING_BRACKET", "COLON", + "SETTING", "SETTING_LINE_COMMENT", "SETTTING_MULTILINE_COMMENT", "SETTING_WS", + "LOOKUP_PIPE", "LOOKUP_COLON", "LOOKUP_COMMA", "LOOKUP_DOT", "LOOKUP_ON", + "LOOKUP_UNQUOTED_SOURCE", "LOOKUP_QUOTED_SOURCE", "LOOKUP_LINE_COMMENT", + "LOOKUP_MULTILINE_COMMENT", "LOOKUP_WS", "LOOKUP_FIELD_PIPE", "LOOKUP_FIELD_COMMA", + "LOOKUP_FIELD_DOT", "LOOKUP_FIELD_ID_PATTERN", "LOOKUP_FIELD_LINE_COMMENT", + "LOOKUP_FIELD_MULTILINE_COMMENT", "LOOKUP_FIELD_WS", "METRICS_PIPE", + "METRICS_UNQUOTED_SOURCE", "METRICS_QUOTED_SOURCE", "METRICS_LINE_COMMENT", + "METRICS_MULTILINE_COMMENT", "METRICS_WS", "CLOSING_METRICS_COLON", "CLOSING_METRICS_COMMA", + "CLOSING_METRICS_LINE_COMMENT", "CLOSING_METRICS_MULTILINE_COMMENT", + "CLOSING_METRICS_WS", "CLOSING_METRICS_QUOTED_IDENTIFIER", "CLOSING_METRICS_UNQUOTED_IDENTIFIER", "CLOSING_METRICS_BY", "CLOSING_METRICS_PIPE" }; } @@ -116,46 +118,46 @@ private static String[] makeRuleNames() { private static String[] makeLiteralNames() { return new String[] { - null, "'dissect'", "'drop'", "'enrich'", "'eval'", "'explain'", "'from'", - "'grok'", "'keep'", "'limit'", "'mv_expand'", "'rename'", "'row'", "'show'", - "'sort'", "'stats'", "'where'", null, null, null, null, null, null, null, - null, "'|'", null, null, null, "'by'", "'and'", "'asc'", "'='", "'::'", - "','", "'desc'", "'.'", "'false'", "'first'", "'in'", "'is'", "'last'", - "'like'", "'('", "'not'", "'null'", "'nulls'", "'or'", "'?'", "'rlike'", - "')'", "'true'", "'=='", "'=~'", "'!='", "'<'", "'<='", "'>'", "'>='", - "'+'", "'-'", "'*'", "'/'", "'%'", null, null, "']'", null, null, null, - null, null, null, null, null, "'metadata'", null, null, null, null, null, - null, null, null, "'as'", null, null, null, "'on'", "'with'", null, null, - null, null, null, null, null, null, null, null, "'info'", null, null, + null, "'dissect'", "'drop'", "'enrich'", "'eval'", "'explain'", "'from'", + "'grok'", "'keep'", "'limit'", "'mv_expand'", "'rename'", "'row'", "'show'", + "'sort'", "'stats'", "'where'", null, null, null, null, null, null, null, + "'|'", null, null, null, "'by'", "'and'", "'asc'", "'='", "'::'", "','", + "'desc'", "'.'", "'false'", "'first'", "'in'", "'is'", "'last'", "'like'", + "'('", "'not'", "'null'", "'nulls'", "'or'", "'?'", "'rlike'", "')'", + "'true'", "'=='", "'=~'", "'!='", "'<'", "'<='", "'>'", "'>='", "'+'", + "'-'", "'*'", "'/'", "'%'", null, null, null, "']'", null, null, null, + null, null, null, null, null, "'metadata'", null, null, null, null, null, + null, null, null, "'as'", null, null, null, "'on'", "'with'", null, null, + null, null, null, null, null, null, null, null, "'info'", null, null, null, "':'" }; } private static final String[] _LITERAL_NAMES = makeLiteralNames(); private static String[] makeSymbolicNames() { return new String[] { - null, "DISSECT", "DROP", "ENRICH", "EVAL", "EXPLAIN", "FROM", "GROK", - "KEEP", "LIMIT", "MV_EXPAND", "RENAME", "ROW", "SHOW", "SORT", "STATS", - "WHERE", "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_MATCH", "DEV_METRICS", - "UNKNOWN_CMD", "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "QUOTED_STRING", - "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", - "COMMA", "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", - "LP", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", - "CIEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", - "SLASH", "PERCENT", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", "CLOSING_BRACKET", - "UNQUOTED_IDENTIFIER", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", - "EXPR_WS", "EXPLAIN_WS", "EXPLAIN_LINE_COMMENT", "EXPLAIN_MULTILINE_COMMENT", - "METADATA", "UNQUOTED_SOURCE", "FROM_LINE_COMMENT", "FROM_MULTILINE_COMMENT", - "FROM_WS", "ID_PATTERN", "PROJECT_LINE_COMMENT", "PROJECT_MULTILINE_COMMENT", - "PROJECT_WS", "AS", "RENAME_LINE_COMMENT", "RENAME_MULTILINE_COMMENT", - "RENAME_WS", "ON", "WITH", "ENRICH_POLICY_NAME", "ENRICH_LINE_COMMENT", - "ENRICH_MULTILINE_COMMENT", "ENRICH_WS", "ENRICH_FIELD_LINE_COMMENT", - "ENRICH_FIELD_MULTILINE_COMMENT", "ENRICH_FIELD_WS", "MVEXPAND_LINE_COMMENT", - "MVEXPAND_MULTILINE_COMMENT", "MVEXPAND_WS", "INFO", "SHOW_LINE_COMMENT", - "SHOW_MULTILINE_COMMENT", "SHOW_WS", "COLON", "SETTING", "SETTING_LINE_COMMENT", - "SETTTING_MULTILINE_COMMENT", "SETTING_WS", "LOOKUP_LINE_COMMENT", "LOOKUP_MULTILINE_COMMENT", - "LOOKUP_WS", "LOOKUP_FIELD_LINE_COMMENT", "LOOKUP_FIELD_MULTILINE_COMMENT", - "LOOKUP_FIELD_WS", "METRICS_LINE_COMMENT", "METRICS_MULTILINE_COMMENT", - "METRICS_WS", "CLOSING_METRICS_LINE_COMMENT", "CLOSING_METRICS_MULTILINE_COMMENT", + null, "DISSECT", "DROP", "ENRICH", "EVAL", "EXPLAIN", "FROM", "GROK", + "KEEP", "LIMIT", "MV_EXPAND", "RENAME", "ROW", "SHOW", "SORT", "STATS", + "WHERE", "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_METRICS", "UNKNOWN_CMD", + "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "QUOTED_STRING", "INTEGER_LITERAL", + "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", "COMMA", + "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", "LP", "NOT", + "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", "CIEQ", + "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", "SLASH", + "PERCENT", "DEV_MATCH", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", + "CLOSING_BRACKET", "UNQUOTED_IDENTIFIER", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", + "EXPR_MULTILINE_COMMENT", "EXPR_WS", "EXPLAIN_WS", "EXPLAIN_LINE_COMMENT", + "EXPLAIN_MULTILINE_COMMENT", "METADATA", "UNQUOTED_SOURCE", "FROM_LINE_COMMENT", + "FROM_MULTILINE_COMMENT", "FROM_WS", "ID_PATTERN", "PROJECT_LINE_COMMENT", + "PROJECT_MULTILINE_COMMENT", "PROJECT_WS", "AS", "RENAME_LINE_COMMENT", + "RENAME_MULTILINE_COMMENT", "RENAME_WS", "ON", "WITH", "ENRICH_POLICY_NAME", + "ENRICH_LINE_COMMENT", "ENRICH_MULTILINE_COMMENT", "ENRICH_WS", "ENRICH_FIELD_LINE_COMMENT", + "ENRICH_FIELD_MULTILINE_COMMENT", "ENRICH_FIELD_WS", "MVEXPAND_LINE_COMMENT", + "MVEXPAND_MULTILINE_COMMENT", "MVEXPAND_WS", "INFO", "SHOW_LINE_COMMENT", + "SHOW_MULTILINE_COMMENT", "SHOW_WS", "COLON", "SETTING", "SETTING_LINE_COMMENT", + "SETTTING_MULTILINE_COMMENT", "SETTING_WS", "LOOKUP_LINE_COMMENT", "LOOKUP_MULTILINE_COMMENT", + "LOOKUP_WS", "LOOKUP_FIELD_LINE_COMMENT", "LOOKUP_FIELD_MULTILINE_COMMENT", + "LOOKUP_FIELD_WS", "METRICS_LINE_COMMENT", "METRICS_MULTILINE_COMMENT", + "METRICS_WS", "CLOSING_METRICS_LINE_COMMENT", "CLOSING_METRICS_MULTILINE_COMMENT", "CLOSING_METRICS_WS" }; } @@ -226,11 +228,9 @@ public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) { case 17: return DEV_LOOKUP_sempred((RuleContext)_localctx, predIndex); case 18: - return DEV_MATCH_sempred((RuleContext)_localctx, predIndex); - case 19: return DEV_METRICS_sempred((RuleContext)_localctx, predIndex); - case 73: - return DEV_MATCH_OP_sempred((RuleContext)_localctx, predIndex); + case 72: + return DEV_MATCH_sempred((RuleContext)_localctx, predIndex); } return true; } @@ -248,30 +248,23 @@ private boolean DEV_LOOKUP_sempred(RuleContext _localctx, int predIndex) { } return true; } - private boolean DEV_MATCH_sempred(RuleContext _localctx, int predIndex) { + private boolean DEV_METRICS_sempred(RuleContext _localctx, int predIndex) { switch (predIndex) { case 2: return this.isDevVersion(); } return true; } - private boolean DEV_METRICS_sempred(RuleContext _localctx, int predIndex) { + private boolean DEV_MATCH_sempred(RuleContext _localctx, int predIndex) { switch (predIndex) { case 3: return this.isDevVersion(); } return true; } - private boolean DEV_MATCH_OP_sempred(RuleContext _localctx, int predIndex) { - switch (predIndex) { - case 4: - return this.isDevVersion(); - } - return true; - } public static final String _serializedATN = - "\u0004\u0000x\u05c3\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ + "\u0004\u0000x\u05ba\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ "\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ "\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ "\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ @@ -327,908 +320,902 @@ private boolean DEV_MATCH_OP_sempred(RuleContext _localctx, int predIndex) { "\u00ba\u0007\u00ba\u0002\u00bb\u0007\u00bb\u0002\u00bc\u0007\u00bc\u0002"+ "\u00bd\u0007\u00bd\u0002\u00be\u0007\u00be\u0002\u00bf\u0007\u00bf\u0002"+ "\u00c0\u0007\u00c0\u0002\u00c1\u0007\u00c1\u0002\u00c2\u0007\u00c2\u0002"+ - "\u00c3\u0007\u00c3\u0002\u00c4\u0007\u00c4\u0002\u00c5\u0007\u00c5\u0001"+ + "\u00c3\u0007\u00c3\u0002\u00c4\u0007\u00c4\u0001\u0000\u0001\u0000\u0001"+ "\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001"+ - "\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0001\u0001\u0001\u0001"+ - "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0002\u0001"+ + "\u0000\u0001\u0000\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+ + "\u0001\u0001\u0001\u0001\u0001\u0001\u0002\u0001\u0002\u0001\u0002\u0001"+ "\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001"+ - "\u0002\u0001\u0002\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+ - "\u0003\u0001\u0003\u0001\u0003\u0001\u0004\u0001\u0004\u0001\u0004\u0001"+ - "\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001"+ - "\u0004\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ - "\u0005\u0001\u0005\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001"+ - "\u0006\u0001\u0006\u0001\u0006\u0001\u0007\u0001\u0007\u0001\u0007\u0001"+ - "\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\b\u0001\b\u0001\b\u0001"+ - "\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\t\u0001\t\u0001\t\u0001\t\u0001"+ - "\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\n\u0001"+ - "\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\u000b"+ - "\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\f\u0001"+ - "\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\r\u0001\r\u0001\r\u0001"+ - "\r\u0001\r\u0001\r\u0001\r\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e"+ - "\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000f\u0001\u000f"+ - "\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f"+ + "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+ + "\u0003\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001"+ + "\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0005\u0001"+ + "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ + "\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001"+ + "\u0006\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001"+ + "\u0007\u0001\u0007\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001"+ + "\b\u0001\b\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001"+ + "\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001"+ + "\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\u000b\u0001\u000b\u0001\u000b"+ + "\u0001\u000b\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001\f\u0001\f\u0001"+ + "\f\u0001\f\u0001\f\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001"+ + "\r\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e"+ + "\u0001\u000e\u0001\u000e\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f"+ + "\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u0010\u0001\u0010"+ "\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010"+ "\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010"+ - "\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0011\u0001\u0011\u0001\u0011"+ - "\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011"+ - "\u0001\u0011\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012"+ - "\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0013\u0001\u0013"+ - "\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013"+ - "\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0014\u0004\u0014\u024b\b\u0014"+ - "\u000b\u0014\f\u0014\u024c\u0001\u0014\u0001\u0014\u0001\u0015\u0001\u0015"+ - "\u0001\u0015\u0001\u0015\u0005\u0015\u0255\b\u0015\n\u0015\f\u0015\u0258"+ - "\t\u0015\u0001\u0015\u0003\u0015\u025b\b\u0015\u0001\u0015\u0003\u0015"+ - "\u025e\b\u0015\u0001\u0015\u0001\u0015\u0001\u0016\u0001\u0016\u0001\u0016"+ - "\u0001\u0016\u0001\u0016\u0005\u0016\u0267\b\u0016\n\u0016\f\u0016\u026a"+ - "\t\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001"+ - "\u0017\u0004\u0017\u0272\b\u0017\u000b\u0017\f\u0017\u0273\u0001\u0017"+ - "\u0001\u0017\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0019"+ - "\u0001\u0019\u0001\u001a\u0001\u001a\u0001\u001b\u0001\u001b\u0001\u001b"+ - "\u0001\u001c\u0001\u001c\u0001\u001d\u0001\u001d\u0003\u001d\u0287\b\u001d"+ - "\u0001\u001d\u0004\u001d\u028a\b\u001d\u000b\u001d\f\u001d\u028b\u0001"+ - "\u001e\u0001\u001e\u0001\u001f\u0001\u001f\u0001 \u0001 \u0001 \u0003"+ - " \u0295\b \u0001!\u0001!\u0001\"\u0001\"\u0001\"\u0003\"\u029c\b\"\u0001"+ - "#\u0001#\u0001#\u0005#\u02a1\b#\n#\f#\u02a4\t#\u0001#\u0001#\u0001#\u0001"+ - "#\u0001#\u0001#\u0005#\u02ac\b#\n#\f#\u02af\t#\u0001#\u0001#\u0001#\u0001"+ - "#\u0001#\u0003#\u02b6\b#\u0001#\u0003#\u02b9\b#\u0003#\u02bb\b#\u0001"+ - "$\u0004$\u02be\b$\u000b$\f$\u02bf\u0001%\u0004%\u02c3\b%\u000b%\f%\u02c4"+ - "\u0001%\u0001%\u0005%\u02c9\b%\n%\f%\u02cc\t%\u0001%\u0001%\u0004%\u02d0"+ - "\b%\u000b%\f%\u02d1\u0001%\u0004%\u02d5\b%\u000b%\f%\u02d6\u0001%\u0001"+ - "%\u0005%\u02db\b%\n%\f%\u02de\t%\u0003%\u02e0\b%\u0001%\u0001%\u0001%"+ - "\u0001%\u0004%\u02e6\b%\u000b%\f%\u02e7\u0001%\u0001%\u0003%\u02ec\b%"+ - "\u0001&\u0001&\u0001&\u0001\'\u0001\'\u0001\'\u0001\'\u0001(\u0001(\u0001"+ - "(\u0001(\u0001)\u0001)\u0001*\u0001*\u0001*\u0001+\u0001+\u0001,\u0001"+ - ",\u0001,\u0001,\u0001,\u0001-\u0001-\u0001.\u0001.\u0001.\u0001.\u0001"+ - ".\u0001.\u0001/\u0001/\u0001/\u0001/\u0001/\u0001/\u00010\u00010\u0001"+ - "0\u00011\u00011\u00011\u00012\u00012\u00012\u00012\u00012\u00013\u0001"+ - "3\u00013\u00013\u00013\u00014\u00014\u00015\u00015\u00015\u00015\u0001"+ - "6\u00016\u00016\u00016\u00016\u00017\u00017\u00017\u00017\u00017\u0001"+ - "7\u00018\u00018\u00018\u00019\u00019\u0001:\u0001:\u0001:\u0001:\u0001"+ - ":\u0001:\u0001;\u0001;\u0001<\u0001<\u0001<\u0001<\u0001<\u0001=\u0001"+ - "=\u0001=\u0001>\u0001>\u0001>\u0001?\u0001?\u0001?\u0001@\u0001@\u0001"+ - "A\u0001A\u0001A\u0001B\u0001B\u0001C\u0001C\u0001C\u0001D\u0001D\u0001"+ - "E\u0001E\u0001F\u0001F\u0001G\u0001G\u0001H\u0001H\u0001I\u0001I\u0001"+ - "I\u0001I\u0001I\u0001J\u0001J\u0001J\u0003J\u036b\bJ\u0001J\u0005J\u036e"+ - "\bJ\nJ\fJ\u0371\tJ\u0001J\u0001J\u0004J\u0375\bJ\u000bJ\fJ\u0376\u0003"+ - "J\u0379\bJ\u0001K\u0001K\u0001K\u0001K\u0001K\u0001L\u0001L\u0001L\u0001"+ - "L\u0001L\u0001M\u0001M\u0005M\u0387\bM\nM\fM\u038a\tM\u0001M\u0001M\u0003"+ - "M\u038e\bM\u0001M\u0004M\u0391\bM\u000bM\fM\u0392\u0003M\u0395\bM\u0001"+ - "N\u0001N\u0004N\u0399\bN\u000bN\fN\u039a\u0001N\u0001N\u0001O\u0001O\u0001"+ - "P\u0001P\u0001P\u0001P\u0001Q\u0001Q\u0001Q\u0001Q\u0001R\u0001R\u0001"+ - "R\u0001R\u0001S\u0001S\u0001S\u0001S\u0001S\u0001T\u0001T\u0001T\u0001"+ - "T\u0001T\u0001U\u0001U\u0001U\u0001U\u0001V\u0001V\u0001V\u0001V\u0001"+ - "W\u0001W\u0001W\u0001W\u0001X\u0001X\u0001X\u0001X\u0001X\u0001Y\u0001"+ - "Y\u0001Y\u0001Y\u0001Z\u0001Z\u0001Z\u0001Z\u0001[\u0001[\u0001[\u0001"+ - "[\u0001\\\u0001\\\u0001\\\u0001\\\u0001]\u0001]\u0001]\u0001]\u0001^\u0001"+ - "^\u0001^\u0001^\u0001^\u0001^\u0001^\u0001^\u0001^\u0001_\u0001_\u0001"+ - "_\u0003_\u03e8\b_\u0001`\u0004`\u03eb\b`\u000b`\f`\u03ec\u0001a\u0001"+ - "a\u0001a\u0001a\u0001b\u0001b\u0001b\u0001b\u0001c\u0001c\u0001c\u0001"+ - "c\u0001d\u0001d\u0001d\u0001d\u0001e\u0001e\u0001e\u0001e\u0001f\u0001"+ - "f\u0001f\u0001f\u0001f\u0001g\u0001g\u0001g\u0001g\u0001h\u0001h\u0001"+ - "h\u0001h\u0001i\u0001i\u0001i\u0001i\u0001j\u0001j\u0001j\u0001j\u0001"+ - "k\u0001k\u0001k\u0001k\u0003k\u041c\bk\u0001l\u0001l\u0003l\u0420\bl\u0001"+ - "l\u0005l\u0423\bl\nl\fl\u0426\tl\u0001l\u0001l\u0003l\u042a\bl\u0001l"+ - "\u0004l\u042d\bl\u000bl\fl\u042e\u0003l\u0431\bl\u0001m\u0001m\u0004m"+ - "\u0435\bm\u000bm\fm\u0436\u0001n\u0001n\u0001n\u0001n\u0001o\u0001o\u0001"+ - "o\u0001o\u0001p\u0001p\u0001p\u0001p\u0001q\u0001q\u0001q\u0001q\u0001"+ - "q\u0001r\u0001r\u0001r\u0001r\u0001s\u0001s\u0001s\u0001s\u0001t\u0001"+ - "t\u0001t\u0001t\u0001u\u0001u\u0001u\u0001u\u0001v\u0001v\u0001v\u0001"+ - "v\u0001w\u0001w\u0001w\u0001x\u0001x\u0001x\u0001x\u0001y\u0001y\u0001"+ - "y\u0001y\u0001z\u0001z\u0001z\u0001z\u0001{\u0001{\u0001{\u0001{\u0001"+ - "|\u0001|\u0001|\u0001|\u0001|\u0001}\u0001}\u0001}\u0001}\u0001}\u0001"+ - "~\u0001~\u0001~\u0001~\u0001~\u0001\u007f\u0001\u007f\u0001\u007f\u0001"+ - "\u007f\u0001\u007f\u0001\u007f\u0001\u007f\u0001\u0080\u0001\u0080\u0001"+ - "\u0081\u0004\u0081\u048a\b\u0081\u000b\u0081\f\u0081\u048b\u0001\u0081"+ - "\u0001\u0081\u0003\u0081\u0490\b\u0081\u0001\u0081\u0004\u0081\u0493\b"+ - "\u0081\u000b\u0081\f\u0081\u0494\u0001\u0082\u0001\u0082\u0001\u0082\u0001"+ - "\u0082\u0001\u0083\u0001\u0083\u0001\u0083\u0001\u0083\u0001\u0084\u0001"+ - "\u0084\u0001\u0084\u0001\u0084\u0001\u0085\u0001\u0085\u0001\u0085\u0001"+ - "\u0085\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0086\u0001"+ - "\u0086\u0001\u0087\u0001\u0087\u0001\u0087\u0001\u0087\u0001\u0088\u0001"+ - "\u0088\u0001\u0088\u0001\u0088\u0001\u0089\u0001\u0089\u0001\u0089\u0001"+ - "\u0089\u0001\u008a\u0001\u008a\u0001\u008a\u0001\u008a\u0001\u008b\u0001"+ - "\u008b\u0001\u008b\u0001\u008b\u0001\u008c\u0001\u008c\u0001\u008c\u0001"+ - "\u008c\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008e\u0001"+ - "\u008e\u0001\u008e\u0001\u008e\u0001\u008f\u0001\u008f\u0001\u008f\u0001"+ - "\u008f\u0001\u0090\u0001\u0090\u0001\u0090\u0001\u0090\u0001\u0091\u0001"+ - "\u0091\u0001\u0091\u0001\u0091\u0001\u0092\u0001\u0092\u0001\u0092\u0001"+ - "\u0092\u0001\u0092\u0001\u0093\u0001\u0093\u0001\u0093\u0001\u0093\u0001"+ - "\u0094\u0001\u0094\u0001\u0094\u0001\u0094\u0001\u0095\u0001\u0095\u0001"+ - "\u0095\u0001\u0095\u0001\u0096\u0001\u0096\u0001\u0096\u0001\u0096\u0001"+ - "\u0097\u0001\u0097\u0001\u0097\u0001\u0097\u0001\u0098\u0001\u0098\u0001"+ - "\u0098\u0001\u0098\u0001\u0099\u0001\u0099\u0001\u0099\u0001\u0099\u0001"+ - "\u009a\u0001\u009a\u0001\u009a\u0001\u009a\u0001\u009b\u0001\u009b\u0001"+ - "\u009b\u0001\u009b\u0001\u009b\u0001\u009c\u0001\u009c\u0001\u009c\u0001"+ - "\u009c\u0001\u009c\u0001\u009d\u0001\u009d\u0001\u009d\u0001\u009d\u0001"+ - "\u009e\u0001\u009e\u0001\u009e\u0001\u009e\u0001\u009f\u0001\u009f\u0001"+ - "\u009f\u0001\u009f\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001"+ - "\u00a0\u0001\u00a1\u0001\u00a1\u0001\u00a2\u0001\u00a2\u0001\u00a2\u0001"+ - "\u00a2\u0001\u00a2\u0004\u00a2\u0520\b\u00a2\u000b\u00a2\f\u00a2\u0521"+ - "\u0001\u00a3\u0001\u00a3\u0001\u00a3\u0001\u00a3\u0001\u00a4\u0001\u00a4"+ - "\u0001\u00a4\u0001\u00a4\u0001\u00a5\u0001\u00a5\u0001\u00a5\u0001\u00a5"+ - "\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a7"+ - "\u0001\u00a7\u0001\u00a7\u0001\u00a7\u0001\u00a8\u0001\u00a8\u0001\u00a8"+ - "\u0001\u00a8\u0001\u00a9\u0001\u00a9\u0001\u00a9\u0001\u00a9\u0001\u00aa"+ - "\u0001\u00aa\u0001\u00aa\u0001\u00aa\u0001\u00aa\u0001\u00ab\u0001\u00ab"+ - "\u0001\u00ab\u0001\u00ab\u0001\u00ac\u0001\u00ac\u0001\u00ac\u0001\u00ac"+ - "\u0001\u00ad\u0001\u00ad\u0001\u00ad\u0001\u00ad\u0001\u00ae\u0001\u00ae"+ - "\u0001\u00ae\u0001\u00ae\u0001\u00af\u0001\u00af\u0001\u00af\u0001\u00af"+ - "\u0001\u00b0\u0001\u00b0\u0001\u00b0\u0001\u00b0\u0001\u00b0\u0001\u00b0"+ - "\u0001\u00b1\u0001\u00b1\u0001\u00b1\u0001\u00b1\u0001\u00b2\u0001\u00b2"+ - "\u0001\u00b2\u0001\u00b2\u0001\u00b3\u0001\u00b3\u0001\u00b3\u0001\u00b3"+ - "\u0001\u00b4\u0001\u00b4\u0001\u00b4\u0001\u00b4\u0001\u00b5\u0001\u00b5"+ - "\u0001\u00b5\u0001\u00b5\u0001\u00b6\u0001\u00b6\u0001\u00b6\u0001\u00b6"+ - "\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001\u00b8"+ - "\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001\u00b9"+ - "\u0001\u00b9\u0001\u00b9\u0001\u00b9\u0001\u00b9\u0001\u00b9\u0001\u00ba"+ - "\u0001\u00ba\u0001\u00ba\u0001\u00ba\u0001\u00bb\u0001\u00bb\u0001\u00bb"+ - "\u0001\u00bb\u0001\u00bc\u0001\u00bc\u0001\u00bc\u0001\u00bc\u0001\u00bd"+ - "\u0001\u00bd\u0001\u00bd\u0001\u00bd\u0001\u00bd\u0001\u00bd\u0001\u00be"+ - "\u0001\u00be\u0001\u00be\u0001\u00be\u0001\u00be\u0001\u00be\u0001\u00bf"+ - "\u0001\u00bf\u0001\u00bf\u0001\u00bf\u0001\u00c0\u0001\u00c0\u0001\u00c0"+ - "\u0001\u00c0\u0001\u00c1\u0001\u00c1\u0001\u00c1\u0001\u00c1\u0001\u00c2"+ - "\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001\u00c3"+ - "\u0001\u00c3\u0001\u00c3\u0001\u00c3\u0001\u00c3\u0001\u00c3\u0001\u00c4"+ - "\u0001\u00c4\u0001\u00c4\u0001\u00c4\u0001\u00c4\u0001\u00c4\u0001\u00c5"+ - "\u0001\u00c5\u0001\u00c5\u0001\u00c5\u0001\u00c5\u0002\u0268\u02ad\u0000"+ - "\u00c6\u000f\u0001\u0011\u0002\u0013\u0003\u0015\u0004\u0017\u0005\u0019"+ - "\u0006\u001b\u0007\u001d\b\u001f\t!\n#\u000b%\f\'\r)\u000e+\u000f-\u0010"+ - "/\u00111\u00123\u00135\u00147\u00159\u0016;\u0017=\u0018?\u0019A\u0000"+ - "C\u0000E\u0000G\u0000I\u0000K\u0000M\u0000O\u0000Q\u0000S\u0000U\u001a"+ - "W\u001bY\u001c[\u001d]\u001e_\u001fa c!e\"g#i$k%m&o\'q(s)u*w+y,{-}.\u007f"+ - "/\u00810\u00831\u00852\u00873\u00894\u008b5\u008d6\u008f7\u00918\u0093"+ - "9\u0095:\u0097;\u0099<\u009b=\u009d>\u009f?\u00a1\u0000\u00a3@\u00a5A"+ - "\u00a7B\u00a9C\u00ab\u0000\u00adD\u00afE\u00b1F\u00b3G\u00b5\u0000\u00b7"+ - "\u0000\u00b9H\u00bbI\u00bdJ\u00bf\u0000\u00c1\u0000\u00c3\u0000\u00c5"+ - "\u0000\u00c7\u0000\u00c9\u0000\u00cbK\u00cd\u0000\u00cfL\u00d1\u0000\u00d3"+ - "\u0000\u00d5M\u00d7N\u00d9O\u00db\u0000\u00dd\u0000\u00df\u0000\u00e1"+ - "\u0000\u00e3\u0000\u00e5\u0000\u00e7\u0000\u00e9P\u00ebQ\u00edR\u00ef"+ - "S\u00f1\u0000\u00f3\u0000\u00f5\u0000\u00f7\u0000\u00f9\u0000\u00fb\u0000"+ - "\u00fdT\u00ff\u0000\u0101U\u0103V\u0105W\u0107\u0000\u0109\u0000\u010b"+ - "X\u010dY\u010f\u0000\u0111Z\u0113\u0000\u0115[\u0117\\\u0119]\u011b\u0000"+ - "\u011d\u0000\u011f\u0000\u0121\u0000\u0123\u0000\u0125\u0000\u0127\u0000"+ - "\u0129\u0000\u012b\u0000\u012d^\u012f_\u0131`\u0133\u0000\u0135\u0000"+ - "\u0137\u0000\u0139\u0000\u013b\u0000\u013d\u0000\u013fa\u0141b\u0143c"+ - "\u0145\u0000\u0147d\u0149e\u014bf\u014dg\u014f\u0000\u0151h\u0153i\u0155"+ - "j\u0157k\u0159l\u015b\u0000\u015d\u0000\u015f\u0000\u0161\u0000\u0163"+ - "\u0000\u0165\u0000\u0167\u0000\u0169m\u016bn\u016do\u016f\u0000\u0171"+ - "\u0000\u0173\u0000\u0175\u0000\u0177p\u0179q\u017br\u017d\u0000\u017f"+ - "\u0000\u0181\u0000\u0183s\u0185t\u0187u\u0189\u0000\u018b\u0000\u018d"+ - "v\u018fw\u0191x\u0193\u0000\u0195\u0000\u0197\u0000\u0199\u0000\u000f"+ - "\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e"+ - "#\u0002\u0000DDdd\u0002\u0000IIii\u0002\u0000SSss\u0002\u0000EEee\u0002"+ - "\u0000CCcc\u0002\u0000TTtt\u0002\u0000RRrr\u0002\u0000OOoo\u0002\u0000"+ - "PPpp\u0002\u0000NNnn\u0002\u0000HHhh\u0002\u0000VVvv\u0002\u0000AAaa\u0002"+ - "\u0000LLll\u0002\u0000XXxx\u0002\u0000FFff\u0002\u0000MMmm\u0002\u0000"+ - "GGgg\u0002\u0000KKkk\u0002\u0000WWww\u0002\u0000UUuu\u0006\u0000\t\n\r"+ - "\r //[[]]\u0002\u0000\n\n\r\r\u0003\u0000\t\n\r\r \u0001\u000009\u0002"+ - "\u0000AZaz\b\u0000\"\"NNRRTT\\\\nnrrtt\u0004\u0000\n\n\r\r\"\"\\\\\u0002"+ - "\u0000++--\u0001\u0000``\u0002\u0000BBbb\u0002\u0000YYyy\u000b\u0000\t"+ - "\n\r\r \"\",,//::==[[]]||\u0002\u0000**//\u000b\u0000\t\n\r\r \"#,,"+ - "//::<<>?\\\\||\u05df\u0000\u000f\u0001\u0000\u0000\u0000\u0000\u0011\u0001"+ - "\u0000\u0000\u0000\u0000\u0013\u0001\u0000\u0000\u0000\u0000\u0015\u0001"+ - "\u0000\u0000\u0000\u0000\u0017\u0001\u0000\u0000\u0000\u0000\u0019\u0001"+ - "\u0000\u0000\u0000\u0000\u001b\u0001\u0000\u0000\u0000\u0000\u001d\u0001"+ - "\u0000\u0000\u0000\u0000\u001f\u0001\u0000\u0000\u0000\u0000!\u0001\u0000"+ - "\u0000\u0000\u0000#\u0001\u0000\u0000\u0000\u0000%\u0001\u0000\u0000\u0000"+ - "\u0000\'\u0001\u0000\u0000\u0000\u0000)\u0001\u0000\u0000\u0000\u0000"+ - "+\u0001\u0000\u0000\u0000\u0000-\u0001\u0000\u0000\u0000\u0000/\u0001"+ - "\u0000\u0000\u0000\u00001\u0001\u0000\u0000\u0000\u00003\u0001\u0000\u0000"+ - "\u0000\u00005\u0001\u0000\u0000\u0000\u00007\u0001\u0000\u0000\u0000\u0000"+ - "9\u0001\u0000\u0000\u0000\u0000;\u0001\u0000\u0000\u0000\u0000=\u0001"+ - "\u0000\u0000\u0000\u0001?\u0001\u0000\u0000\u0000\u0001U\u0001\u0000\u0000"+ - "\u0000\u0001W\u0001\u0000\u0000\u0000\u0001Y\u0001\u0000\u0000\u0000\u0001"+ - "[\u0001\u0000\u0000\u0000\u0001]\u0001\u0000\u0000\u0000\u0001_\u0001"+ - "\u0000\u0000\u0000\u0001a\u0001\u0000\u0000\u0000\u0001c\u0001\u0000\u0000"+ - "\u0000\u0001e\u0001\u0000\u0000\u0000\u0001g\u0001\u0000\u0000\u0000\u0001"+ - "i\u0001\u0000\u0000\u0000\u0001k\u0001\u0000\u0000\u0000\u0001m\u0001"+ - "\u0000\u0000\u0000\u0001o\u0001\u0000\u0000\u0000\u0001q\u0001\u0000\u0000"+ - "\u0000\u0001s\u0001\u0000\u0000\u0000\u0001u\u0001\u0000\u0000\u0000\u0001"+ - "w\u0001\u0000\u0000\u0000\u0001y\u0001\u0000\u0000\u0000\u0001{\u0001"+ - "\u0000\u0000\u0000\u0001}\u0001\u0000\u0000\u0000\u0001\u007f\u0001\u0000"+ - "\u0000\u0000\u0001\u0081\u0001\u0000\u0000\u0000\u0001\u0083\u0001\u0000"+ - "\u0000\u0000\u0001\u0085\u0001\u0000\u0000\u0000\u0001\u0087\u0001\u0000"+ - "\u0000\u0000\u0001\u0089\u0001\u0000\u0000\u0000\u0001\u008b\u0001\u0000"+ - "\u0000\u0000\u0001\u008d\u0001\u0000\u0000\u0000\u0001\u008f\u0001\u0000"+ - "\u0000\u0000\u0001\u0091\u0001\u0000\u0000\u0000\u0001\u0093\u0001\u0000"+ - "\u0000\u0000\u0001\u0095\u0001\u0000\u0000\u0000\u0001\u0097\u0001\u0000"+ - "\u0000\u0000\u0001\u0099\u0001\u0000\u0000\u0000\u0001\u009b\u0001\u0000"+ - "\u0000\u0000\u0001\u009d\u0001\u0000\u0000\u0000\u0001\u009f\u0001\u0000"+ - "\u0000\u0000\u0001\u00a1\u0001\u0000\u0000\u0000\u0001\u00a3\u0001\u0000"+ - "\u0000\u0000\u0001\u00a5\u0001\u0000\u0000\u0000\u0001\u00a7\u0001\u0000"+ - "\u0000\u0000\u0001\u00a9\u0001\u0000\u0000\u0000\u0001\u00ad\u0001\u0000"+ - "\u0000\u0000\u0001\u00af\u0001\u0000\u0000\u0000\u0001\u00b1\u0001\u0000"+ - "\u0000\u0000\u0001\u00b3\u0001\u0000\u0000\u0000\u0002\u00b5\u0001\u0000"+ - "\u0000\u0000\u0002\u00b7\u0001\u0000\u0000\u0000\u0002\u00b9\u0001\u0000"+ - "\u0000\u0000\u0002\u00bb\u0001\u0000\u0000\u0000\u0002\u00bd\u0001\u0000"+ - "\u0000\u0000\u0003\u00bf\u0001\u0000\u0000\u0000\u0003\u00c1\u0001\u0000"+ - "\u0000\u0000\u0003\u00c3\u0001\u0000\u0000\u0000\u0003\u00c5\u0001\u0000"+ - "\u0000\u0000\u0003\u00c7\u0001\u0000\u0000\u0000\u0003\u00c9\u0001\u0000"+ - "\u0000\u0000\u0003\u00cb\u0001\u0000\u0000\u0000\u0003\u00cf\u0001\u0000"+ - "\u0000\u0000\u0003\u00d1\u0001\u0000\u0000\u0000\u0003\u00d3\u0001\u0000"+ - "\u0000\u0000\u0003\u00d5\u0001\u0000\u0000\u0000\u0003\u00d7\u0001\u0000"+ - "\u0000\u0000\u0003\u00d9\u0001\u0000\u0000\u0000\u0004\u00db\u0001\u0000"+ - "\u0000\u0000\u0004\u00dd\u0001\u0000\u0000\u0000\u0004\u00df\u0001\u0000"+ - "\u0000\u0000\u0004\u00e1\u0001\u0000\u0000\u0000\u0004\u00e3\u0001\u0000"+ - "\u0000\u0000\u0004\u00e9\u0001\u0000\u0000\u0000\u0004\u00eb\u0001\u0000"+ - "\u0000\u0000\u0004\u00ed\u0001\u0000\u0000\u0000\u0004\u00ef\u0001\u0000"+ - "\u0000\u0000\u0005\u00f1\u0001\u0000\u0000\u0000\u0005\u00f3\u0001\u0000"+ - "\u0000\u0000\u0005\u00f5\u0001\u0000\u0000\u0000\u0005\u00f7\u0001\u0000"+ - "\u0000\u0000\u0005\u00f9\u0001\u0000\u0000\u0000\u0005\u00fb\u0001\u0000"+ - "\u0000\u0000\u0005\u00fd\u0001\u0000\u0000\u0000\u0005\u00ff\u0001\u0000"+ - "\u0000\u0000\u0005\u0101\u0001\u0000\u0000\u0000\u0005\u0103\u0001\u0000"+ - "\u0000\u0000\u0005\u0105\u0001\u0000\u0000\u0000\u0006\u0107\u0001\u0000"+ - "\u0000\u0000\u0006\u0109\u0001\u0000\u0000\u0000\u0006\u010b\u0001\u0000"+ - "\u0000\u0000\u0006\u010d\u0001\u0000\u0000\u0000\u0006\u0111\u0001\u0000"+ - "\u0000\u0000\u0006\u0113\u0001\u0000\u0000\u0000\u0006\u0115\u0001\u0000"+ - "\u0000\u0000\u0006\u0117\u0001\u0000\u0000\u0000\u0006\u0119\u0001\u0000"+ - "\u0000\u0000\u0007\u011b\u0001\u0000\u0000\u0000\u0007\u011d\u0001\u0000"+ - "\u0000\u0000\u0007\u011f\u0001\u0000\u0000\u0000\u0007\u0121\u0001\u0000"+ - "\u0000\u0000\u0007\u0123\u0001\u0000\u0000\u0000\u0007\u0125\u0001\u0000"+ - "\u0000\u0000\u0007\u0127\u0001\u0000\u0000\u0000\u0007\u0129\u0001\u0000"+ - "\u0000\u0000\u0007\u012b\u0001\u0000\u0000\u0000\u0007\u012d\u0001\u0000"+ - "\u0000\u0000\u0007\u012f\u0001\u0000\u0000\u0000\u0007\u0131\u0001\u0000"+ - "\u0000\u0000\b\u0133\u0001\u0000\u0000\u0000\b\u0135\u0001\u0000\u0000"+ - "\u0000\b\u0137\u0001\u0000\u0000\u0000\b\u0139\u0001\u0000\u0000\u0000"+ - "\b\u013b\u0001\u0000\u0000\u0000\b\u013d\u0001\u0000\u0000\u0000\b\u013f"+ - "\u0001\u0000\u0000\u0000\b\u0141\u0001\u0000\u0000\u0000\b\u0143\u0001"+ - "\u0000\u0000\u0000\t\u0145\u0001\u0000\u0000\u0000\t\u0147\u0001\u0000"+ - "\u0000\u0000\t\u0149\u0001\u0000\u0000\u0000\t\u014b\u0001\u0000\u0000"+ - "\u0000\t\u014d\u0001\u0000\u0000\u0000\n\u014f\u0001\u0000\u0000\u0000"+ - "\n\u0151\u0001\u0000\u0000\u0000\n\u0153\u0001\u0000\u0000\u0000\n\u0155"+ - "\u0001\u0000\u0000\u0000\n\u0157\u0001\u0000\u0000\u0000\n\u0159\u0001"+ - "\u0000\u0000\u0000\u000b\u015b\u0001\u0000\u0000\u0000\u000b\u015d\u0001"+ - "\u0000\u0000\u0000\u000b\u015f\u0001\u0000\u0000\u0000\u000b\u0161\u0001"+ - "\u0000\u0000\u0000\u000b\u0163\u0001\u0000\u0000\u0000\u000b\u0165\u0001"+ - "\u0000\u0000\u0000\u000b\u0167\u0001\u0000\u0000\u0000\u000b\u0169\u0001"+ - "\u0000\u0000\u0000\u000b\u016b\u0001\u0000\u0000\u0000\u000b\u016d\u0001"+ - "\u0000\u0000\u0000\f\u016f\u0001\u0000\u0000\u0000\f\u0171\u0001\u0000"+ - "\u0000\u0000\f\u0173\u0001\u0000\u0000\u0000\f\u0175\u0001\u0000\u0000"+ - "\u0000\f\u0177\u0001\u0000\u0000\u0000\f\u0179\u0001\u0000\u0000\u0000"+ - "\f\u017b\u0001\u0000\u0000\u0000\r\u017d\u0001\u0000\u0000\u0000\r\u017f"+ - "\u0001\u0000\u0000\u0000\r\u0181\u0001\u0000\u0000\u0000\r\u0183\u0001"+ - "\u0000\u0000\u0000\r\u0185\u0001\u0000\u0000\u0000\r\u0187\u0001\u0000"+ - "\u0000\u0000\u000e\u0189\u0001\u0000\u0000\u0000\u000e\u018b\u0001\u0000"+ - "\u0000\u0000\u000e\u018d\u0001\u0000\u0000\u0000\u000e\u018f\u0001\u0000"+ - "\u0000\u0000\u000e\u0191\u0001\u0000\u0000\u0000\u000e\u0193\u0001\u0000"+ - "\u0000\u0000\u000e\u0195\u0001\u0000\u0000\u0000\u000e\u0197\u0001\u0000"+ - "\u0000\u0000\u000e\u0199\u0001\u0000\u0000\u0000\u000f\u019b\u0001\u0000"+ - "\u0000\u0000\u0011\u01a5\u0001\u0000\u0000\u0000\u0013\u01ac\u0001\u0000"+ - "\u0000\u0000\u0015\u01b5\u0001\u0000\u0000\u0000\u0017\u01bc\u0001\u0000"+ - "\u0000\u0000\u0019\u01c6\u0001\u0000\u0000\u0000\u001b\u01cd\u0001\u0000"+ - "\u0000\u0000\u001d\u01d4\u0001\u0000\u0000\u0000\u001f\u01db\u0001\u0000"+ - "\u0000\u0000!\u01e3\u0001\u0000\u0000\u0000#\u01ef\u0001\u0000\u0000\u0000"+ - "%\u01f8\u0001\u0000\u0000\u0000\'\u01fe\u0001\u0000\u0000\u0000)\u0205"+ - "\u0001\u0000\u0000\u0000+\u020c\u0001\u0000\u0000\u0000-\u0214\u0001\u0000"+ - "\u0000\u0000/\u021c\u0001\u0000\u0000\u00001\u022b\u0001\u0000\u0000\u0000"+ - "3\u0235\u0001\u0000\u0000\u00005\u023e\u0001\u0000\u0000\u00007\u024a"+ - "\u0001\u0000\u0000\u00009\u0250\u0001\u0000\u0000\u0000;\u0261\u0001\u0000"+ - "\u0000\u0000=\u0271\u0001\u0000\u0000\u0000?\u0277\u0001\u0000\u0000\u0000"+ - "A\u027b\u0001\u0000\u0000\u0000C\u027d\u0001\u0000\u0000\u0000E\u027f"+ - "\u0001\u0000\u0000\u0000G\u0282\u0001\u0000\u0000\u0000I\u0284\u0001\u0000"+ - "\u0000\u0000K\u028d\u0001\u0000\u0000\u0000M\u028f\u0001\u0000\u0000\u0000"+ - "O\u0294\u0001\u0000\u0000\u0000Q\u0296\u0001\u0000\u0000\u0000S\u029b"+ - "\u0001\u0000\u0000\u0000U\u02ba\u0001\u0000\u0000\u0000W\u02bd\u0001\u0000"+ - "\u0000\u0000Y\u02eb\u0001\u0000\u0000\u0000[\u02ed\u0001\u0000\u0000\u0000"+ - "]\u02f0\u0001\u0000\u0000\u0000_\u02f4\u0001\u0000\u0000\u0000a\u02f8"+ - "\u0001\u0000\u0000\u0000c\u02fa\u0001\u0000\u0000\u0000e\u02fd\u0001\u0000"+ - "\u0000\u0000g\u02ff\u0001\u0000\u0000\u0000i\u0304\u0001\u0000\u0000\u0000"+ - "k\u0306\u0001\u0000\u0000\u0000m\u030c\u0001\u0000\u0000\u0000o\u0312"+ - "\u0001\u0000\u0000\u0000q\u0315\u0001\u0000\u0000\u0000s\u0318\u0001\u0000"+ - "\u0000\u0000u\u031d\u0001\u0000\u0000\u0000w\u0322\u0001\u0000\u0000\u0000"+ - "y\u0324\u0001\u0000\u0000\u0000{\u0328\u0001\u0000\u0000\u0000}\u032d"+ - "\u0001\u0000\u0000\u0000\u007f\u0333\u0001\u0000\u0000\u0000\u0081\u0336"+ - "\u0001\u0000\u0000\u0000\u0083\u0338\u0001\u0000\u0000\u0000\u0085\u033e"+ - "\u0001\u0000\u0000\u0000\u0087\u0340\u0001\u0000\u0000\u0000\u0089\u0345"+ - "\u0001\u0000\u0000\u0000\u008b\u0348\u0001\u0000\u0000\u0000\u008d\u034b"+ - "\u0001\u0000\u0000\u0000\u008f\u034e\u0001\u0000\u0000\u0000\u0091\u0350"+ - "\u0001\u0000\u0000\u0000\u0093\u0353\u0001\u0000\u0000\u0000\u0095\u0355"+ - "\u0001\u0000\u0000\u0000\u0097\u0358\u0001\u0000\u0000\u0000\u0099\u035a"+ - "\u0001\u0000\u0000\u0000\u009b\u035c\u0001\u0000\u0000\u0000\u009d\u035e"+ - "\u0001\u0000\u0000\u0000\u009f\u0360\u0001\u0000\u0000\u0000\u00a1\u0362"+ - "\u0001\u0000\u0000\u0000\u00a3\u0378\u0001\u0000\u0000\u0000\u00a5\u037a"+ - "\u0001\u0000\u0000\u0000\u00a7\u037f\u0001\u0000\u0000\u0000\u00a9\u0394"+ - "\u0001\u0000\u0000\u0000\u00ab\u0396\u0001\u0000\u0000\u0000\u00ad\u039e"+ - "\u0001\u0000\u0000\u0000\u00af\u03a0\u0001\u0000\u0000\u0000\u00b1\u03a4"+ - "\u0001\u0000\u0000\u0000\u00b3\u03a8\u0001\u0000\u0000\u0000\u00b5\u03ac"+ - "\u0001\u0000\u0000\u0000\u00b7\u03b1\u0001\u0000\u0000\u0000\u00b9\u03b6"+ - "\u0001\u0000\u0000\u0000\u00bb\u03ba\u0001\u0000\u0000\u0000\u00bd\u03be"+ - "\u0001\u0000\u0000\u0000\u00bf\u03c2\u0001\u0000\u0000\u0000\u00c1\u03c7"+ - "\u0001\u0000\u0000\u0000\u00c3\u03cb\u0001\u0000\u0000\u0000\u00c5\u03cf"+ - "\u0001\u0000\u0000\u0000\u00c7\u03d3\u0001\u0000\u0000\u0000\u00c9\u03d7"+ - "\u0001\u0000\u0000\u0000\u00cb\u03db\u0001\u0000\u0000\u0000\u00cd\u03e7"+ - "\u0001\u0000\u0000\u0000\u00cf\u03ea\u0001\u0000\u0000\u0000\u00d1\u03ee"+ - "\u0001\u0000\u0000\u0000\u00d3\u03f2\u0001\u0000\u0000\u0000\u00d5\u03f6"+ - "\u0001\u0000\u0000\u0000\u00d7\u03fa\u0001\u0000\u0000\u0000\u00d9\u03fe"+ - "\u0001\u0000\u0000\u0000\u00db\u0402\u0001\u0000\u0000\u0000\u00dd\u0407"+ - "\u0001\u0000\u0000\u0000\u00df\u040b\u0001\u0000\u0000\u0000\u00e1\u040f"+ - "\u0001\u0000\u0000\u0000\u00e3\u0413\u0001\u0000\u0000\u0000\u00e5\u041b"+ - "\u0001\u0000\u0000\u0000\u00e7\u0430\u0001\u0000\u0000\u0000\u00e9\u0434"+ - "\u0001\u0000\u0000\u0000\u00eb\u0438\u0001\u0000\u0000\u0000\u00ed\u043c"+ - "\u0001\u0000\u0000\u0000\u00ef\u0440\u0001\u0000\u0000\u0000\u00f1\u0444"+ - "\u0001\u0000\u0000\u0000\u00f3\u0449\u0001\u0000\u0000\u0000\u00f5\u044d"+ - "\u0001\u0000\u0000\u0000\u00f7\u0451\u0001\u0000\u0000\u0000\u00f9\u0455"+ - "\u0001\u0000\u0000\u0000\u00fb\u0459\u0001\u0000\u0000\u0000\u00fd\u045d"+ - "\u0001\u0000\u0000\u0000\u00ff\u0460\u0001\u0000\u0000\u0000\u0101\u0464"+ - "\u0001\u0000\u0000\u0000\u0103\u0468\u0001\u0000\u0000\u0000\u0105\u046c"+ - "\u0001\u0000\u0000\u0000\u0107\u0470\u0001\u0000\u0000\u0000\u0109\u0475"+ - "\u0001\u0000\u0000\u0000\u010b\u047a\u0001\u0000\u0000\u0000\u010d\u047f"+ - "\u0001\u0000\u0000\u0000\u010f\u0486\u0001\u0000\u0000\u0000\u0111\u048f"+ - "\u0001\u0000\u0000\u0000\u0113\u0496\u0001\u0000\u0000\u0000\u0115\u049a"+ - "\u0001\u0000\u0000\u0000\u0117\u049e\u0001\u0000\u0000\u0000\u0119\u04a2"+ - "\u0001\u0000\u0000\u0000\u011b\u04a6\u0001\u0000\u0000\u0000\u011d\u04ac"+ - "\u0001\u0000\u0000\u0000\u011f\u04b0\u0001\u0000\u0000\u0000\u0121\u04b4"+ - "\u0001\u0000\u0000\u0000\u0123\u04b8\u0001\u0000\u0000\u0000\u0125\u04bc"+ - "\u0001\u0000\u0000\u0000\u0127\u04c0\u0001\u0000\u0000\u0000\u0129\u04c4"+ - "\u0001\u0000\u0000\u0000\u012b\u04c8\u0001\u0000\u0000\u0000\u012d\u04cc"+ - "\u0001\u0000\u0000\u0000\u012f\u04d0\u0001\u0000\u0000\u0000\u0131\u04d4"+ - "\u0001\u0000\u0000\u0000\u0133\u04d8\u0001\u0000\u0000\u0000\u0135\u04dd"+ - "\u0001\u0000\u0000\u0000\u0137\u04e1\u0001\u0000\u0000\u0000\u0139\u04e5"+ - "\u0001\u0000\u0000\u0000\u013b\u04e9\u0001\u0000\u0000\u0000\u013d\u04ed"+ - "\u0001\u0000\u0000\u0000\u013f\u04f1\u0001\u0000\u0000\u0000\u0141\u04f5"+ - "\u0001\u0000\u0000\u0000\u0143\u04f9\u0001\u0000\u0000\u0000\u0145\u04fd"+ - "\u0001\u0000\u0000\u0000\u0147\u0502\u0001\u0000\u0000\u0000\u0149\u0507"+ - "\u0001\u0000\u0000\u0000\u014b\u050b\u0001\u0000\u0000\u0000\u014d\u050f"+ - "\u0001\u0000\u0000\u0000\u014f\u0513\u0001\u0000\u0000\u0000\u0151\u0518"+ - "\u0001\u0000\u0000\u0000\u0153\u051f\u0001\u0000\u0000\u0000\u0155\u0523"+ - "\u0001\u0000\u0000\u0000\u0157\u0527\u0001\u0000\u0000\u0000\u0159\u052b"+ - "\u0001\u0000\u0000\u0000\u015b\u052f\u0001\u0000\u0000\u0000\u015d\u0534"+ - "\u0001\u0000\u0000\u0000\u015f\u0538\u0001\u0000\u0000\u0000\u0161\u053c"+ - "\u0001\u0000\u0000\u0000\u0163\u0540\u0001\u0000\u0000\u0000\u0165\u0545"+ - "\u0001\u0000\u0000\u0000\u0167\u0549\u0001\u0000\u0000\u0000\u0169\u054d"+ - "\u0001\u0000\u0000\u0000\u016b\u0551\u0001\u0000\u0000\u0000\u016d\u0555"+ - "\u0001\u0000\u0000\u0000\u016f\u0559\u0001\u0000\u0000\u0000\u0171\u055f"+ - "\u0001\u0000\u0000\u0000\u0173\u0563\u0001\u0000\u0000\u0000\u0175\u0567"+ - "\u0001\u0000\u0000\u0000\u0177\u056b\u0001\u0000\u0000\u0000\u0179\u056f"+ - "\u0001\u0000\u0000\u0000\u017b\u0573\u0001\u0000\u0000\u0000\u017d\u0577"+ - "\u0001\u0000\u0000\u0000\u017f\u057c\u0001\u0000\u0000\u0000\u0181\u0582"+ - "\u0001\u0000\u0000\u0000\u0183\u0588\u0001\u0000\u0000\u0000\u0185\u058c"+ - "\u0001\u0000\u0000\u0000\u0187\u0590\u0001\u0000\u0000\u0000\u0189\u0594"+ - "\u0001\u0000\u0000\u0000\u018b\u059a\u0001\u0000\u0000\u0000\u018d\u05a0"+ - "\u0001\u0000\u0000\u0000\u018f\u05a4\u0001\u0000\u0000\u0000\u0191\u05a8"+ - "\u0001\u0000\u0000\u0000\u0193\u05ac\u0001\u0000\u0000\u0000\u0195\u05b2"+ - "\u0001\u0000\u0000\u0000\u0197\u05b8\u0001\u0000\u0000\u0000\u0199\u05be"+ - "\u0001\u0000\u0000\u0000\u019b\u019c\u0007\u0000\u0000\u0000\u019c\u019d"+ - "\u0007\u0001\u0000\u0000\u019d\u019e\u0007\u0002\u0000\u0000\u019e\u019f"+ - "\u0007\u0002\u0000\u0000\u019f\u01a0\u0007\u0003\u0000\u0000\u01a0\u01a1"+ - "\u0007\u0004\u0000\u0000\u01a1\u01a2\u0007\u0005\u0000\u0000\u01a2\u01a3"+ - "\u0001\u0000\u0000\u0000\u01a3\u01a4\u0006\u0000\u0000\u0000\u01a4\u0010"+ - "\u0001\u0000\u0000\u0000\u01a5\u01a6\u0007\u0000\u0000\u0000\u01a6\u01a7"+ - "\u0007\u0006\u0000\u0000\u01a7\u01a8\u0007\u0007\u0000\u0000\u01a8\u01a9"+ - "\u0007\b\u0000\u0000\u01a9\u01aa\u0001\u0000\u0000\u0000\u01aa\u01ab\u0006"+ - "\u0001\u0001\u0000\u01ab\u0012\u0001\u0000\u0000\u0000\u01ac\u01ad\u0007"+ - "\u0003\u0000\u0000\u01ad\u01ae\u0007\t\u0000\u0000\u01ae\u01af\u0007\u0006"+ - "\u0000\u0000\u01af\u01b0\u0007\u0001\u0000\u0000\u01b0\u01b1\u0007\u0004"+ - "\u0000\u0000\u01b1\u01b2\u0007\n\u0000\u0000\u01b2\u01b3\u0001\u0000\u0000"+ - "\u0000\u01b3\u01b4\u0006\u0002\u0002\u0000\u01b4\u0014\u0001\u0000\u0000"+ - "\u0000\u01b5\u01b6\u0007\u0003\u0000\u0000\u01b6\u01b7\u0007\u000b\u0000"+ - "\u0000\u01b7\u01b8\u0007\f\u0000\u0000\u01b8\u01b9\u0007\r\u0000\u0000"+ - "\u01b9\u01ba\u0001\u0000\u0000\u0000\u01ba\u01bb\u0006\u0003\u0000\u0000"+ - "\u01bb\u0016\u0001\u0000\u0000\u0000\u01bc\u01bd\u0007\u0003\u0000\u0000"+ - "\u01bd\u01be\u0007\u000e\u0000\u0000\u01be\u01bf\u0007\b\u0000\u0000\u01bf"+ - "\u01c0\u0007\r\u0000\u0000\u01c0\u01c1\u0007\f\u0000\u0000\u01c1\u01c2"+ - "\u0007\u0001\u0000\u0000\u01c2\u01c3\u0007\t\u0000\u0000\u01c3\u01c4\u0001"+ - "\u0000\u0000\u0000\u01c4\u01c5\u0006\u0004\u0003\u0000\u01c5\u0018\u0001"+ - "\u0000\u0000\u0000\u01c6\u01c7\u0007\u000f\u0000\u0000\u01c7\u01c8\u0007"+ - "\u0006\u0000\u0000\u01c8\u01c9\u0007\u0007\u0000\u0000\u01c9\u01ca\u0007"+ - "\u0010\u0000\u0000\u01ca\u01cb\u0001\u0000\u0000\u0000\u01cb\u01cc\u0006"+ - "\u0005\u0004\u0000\u01cc\u001a\u0001\u0000\u0000\u0000\u01cd\u01ce\u0007"+ - "\u0011\u0000\u0000\u01ce\u01cf\u0007\u0006\u0000\u0000\u01cf\u01d0\u0007"+ - "\u0007\u0000\u0000\u01d0\u01d1\u0007\u0012\u0000\u0000\u01d1\u01d2\u0001"+ - "\u0000\u0000\u0000\u01d2\u01d3\u0006\u0006\u0000\u0000\u01d3\u001c\u0001"+ - "\u0000\u0000\u0000\u01d4\u01d5\u0007\u0012\u0000\u0000\u01d5\u01d6\u0007"+ - "\u0003\u0000\u0000\u01d6\u01d7\u0007\u0003\u0000\u0000\u01d7\u01d8\u0007"+ - "\b\u0000\u0000\u01d8\u01d9\u0001\u0000\u0000\u0000\u01d9\u01da\u0006\u0007"+ - "\u0001\u0000\u01da\u001e\u0001\u0000\u0000\u0000\u01db\u01dc\u0007\r\u0000"+ - "\u0000\u01dc\u01dd\u0007\u0001\u0000\u0000\u01dd\u01de\u0007\u0010\u0000"+ - "\u0000\u01de\u01df\u0007\u0001\u0000\u0000\u01df\u01e0\u0007\u0005\u0000"+ - "\u0000\u01e0\u01e1\u0001\u0000\u0000\u0000\u01e1\u01e2\u0006\b\u0000\u0000"+ - "\u01e2 \u0001\u0000\u0000\u0000\u01e3\u01e4\u0007\u0010\u0000\u0000\u01e4"+ - "\u01e5\u0007\u000b\u0000\u0000\u01e5\u01e6\u0005_\u0000\u0000\u01e6\u01e7"+ - "\u0007\u0003\u0000\u0000\u01e7\u01e8\u0007\u000e\u0000\u0000\u01e8\u01e9"+ - "\u0007\b\u0000\u0000\u01e9\u01ea\u0007\f\u0000\u0000\u01ea\u01eb\u0007"+ - "\t\u0000\u0000\u01eb\u01ec\u0007\u0000\u0000\u0000\u01ec\u01ed\u0001\u0000"+ - "\u0000\u0000\u01ed\u01ee\u0006\t\u0005\u0000\u01ee\"\u0001\u0000\u0000"+ - "\u0000\u01ef\u01f0\u0007\u0006\u0000\u0000\u01f0\u01f1\u0007\u0003\u0000"+ - "\u0000\u01f1\u01f2\u0007\t\u0000\u0000\u01f2\u01f3\u0007\f\u0000\u0000"+ - "\u01f3\u01f4\u0007\u0010\u0000\u0000\u01f4\u01f5\u0007\u0003\u0000\u0000"+ - "\u01f5\u01f6\u0001\u0000\u0000\u0000\u01f6\u01f7\u0006\n\u0006\u0000\u01f7"+ - "$\u0001\u0000\u0000\u0000\u01f8\u01f9\u0007\u0006\u0000\u0000\u01f9\u01fa"+ - "\u0007\u0007\u0000\u0000\u01fa\u01fb\u0007\u0013\u0000\u0000\u01fb\u01fc"+ - "\u0001\u0000\u0000\u0000\u01fc\u01fd\u0006\u000b\u0000\u0000\u01fd&\u0001"+ - "\u0000\u0000\u0000\u01fe\u01ff\u0007\u0002\u0000\u0000\u01ff\u0200\u0007"+ - "\n\u0000\u0000\u0200\u0201\u0007\u0007\u0000\u0000\u0201\u0202\u0007\u0013"+ - "\u0000\u0000\u0202\u0203\u0001\u0000\u0000\u0000\u0203\u0204\u0006\f\u0007"+ - "\u0000\u0204(\u0001\u0000\u0000\u0000\u0205\u0206\u0007\u0002\u0000\u0000"+ - "\u0206\u0207\u0007\u0007\u0000\u0000\u0207\u0208\u0007\u0006\u0000\u0000"+ - "\u0208\u0209\u0007\u0005\u0000\u0000\u0209\u020a\u0001\u0000\u0000\u0000"+ - "\u020a\u020b\u0006\r\u0000\u0000\u020b*\u0001\u0000\u0000\u0000\u020c"+ - "\u020d\u0007\u0002\u0000\u0000\u020d\u020e\u0007\u0005\u0000\u0000\u020e"+ - "\u020f\u0007\f\u0000\u0000\u020f\u0210\u0007\u0005\u0000\u0000\u0210\u0211"+ - "\u0007\u0002\u0000\u0000\u0211\u0212\u0001\u0000\u0000\u0000\u0212\u0213"+ - "\u0006\u000e\u0000\u0000\u0213,\u0001\u0000\u0000\u0000\u0214\u0215\u0007"+ - "\u0013\u0000\u0000\u0215\u0216\u0007\n\u0000\u0000\u0216\u0217\u0007\u0003"+ - "\u0000\u0000\u0217\u0218\u0007\u0006\u0000\u0000\u0218\u0219\u0007\u0003"+ - "\u0000\u0000\u0219\u021a\u0001\u0000\u0000\u0000\u021a\u021b\u0006\u000f"+ - "\u0000\u0000\u021b.\u0001\u0000\u0000\u0000\u021c\u021d\u0004\u0010\u0000"+ - "\u0000\u021d\u021e\u0007\u0001\u0000\u0000\u021e\u021f\u0007\t\u0000\u0000"+ - "\u021f\u0220\u0007\r\u0000\u0000\u0220\u0221\u0007\u0001\u0000\u0000\u0221"+ - "\u0222\u0007\t\u0000\u0000\u0222\u0223\u0007\u0003\u0000\u0000\u0223\u0224"+ - "\u0007\u0002\u0000\u0000\u0224\u0225\u0007\u0005\u0000\u0000\u0225\u0226"+ - "\u0007\f\u0000\u0000\u0226\u0227\u0007\u0005\u0000\u0000\u0227\u0228\u0007"+ - "\u0002\u0000\u0000\u0228\u0229\u0001\u0000\u0000\u0000\u0229\u022a\u0006"+ - "\u0010\u0000\u0000\u022a0\u0001\u0000\u0000\u0000\u022b\u022c\u0004\u0011"+ - "\u0001\u0000\u022c\u022d\u0007\r\u0000\u0000\u022d\u022e\u0007\u0007\u0000"+ - "\u0000\u022e\u022f\u0007\u0007\u0000\u0000\u022f\u0230\u0007\u0012\u0000"+ - "\u0000\u0230\u0231\u0007\u0014\u0000\u0000\u0231\u0232\u0007\b\u0000\u0000"+ - "\u0232\u0233\u0001\u0000\u0000\u0000\u0233\u0234\u0006\u0011\b\u0000\u0234"+ - "2\u0001\u0000\u0000\u0000\u0235\u0236\u0004\u0012\u0002\u0000\u0236\u0237"+ - "\u0007\u0010\u0000\u0000\u0237\u0238\u0007\f\u0000\u0000\u0238\u0239\u0007"+ - "\u0005\u0000\u0000\u0239\u023a\u0007\u0004\u0000\u0000\u023a\u023b\u0007"+ - "\n\u0000\u0000\u023b\u023c\u0001\u0000\u0000\u0000\u023c\u023d\u0006\u0012"+ - "\u0000\u0000\u023d4\u0001\u0000\u0000\u0000\u023e\u023f\u0004\u0013\u0003"+ - "\u0000\u023f\u0240\u0007\u0010\u0000\u0000\u0240\u0241\u0007\u0003\u0000"+ - "\u0000\u0241\u0242\u0007\u0005\u0000\u0000\u0242\u0243\u0007\u0006\u0000"+ - "\u0000\u0243\u0244\u0007\u0001\u0000\u0000\u0244\u0245\u0007\u0004\u0000"+ - "\u0000\u0245\u0246\u0007\u0002\u0000\u0000\u0246\u0247\u0001\u0000\u0000"+ - "\u0000\u0247\u0248\u0006\u0013\t\u0000\u02486\u0001\u0000\u0000\u0000"+ - "\u0249\u024b\b\u0015\u0000\u0000\u024a\u0249\u0001\u0000\u0000\u0000\u024b"+ - "\u024c\u0001\u0000\u0000\u0000\u024c\u024a\u0001\u0000\u0000\u0000\u024c"+ - "\u024d\u0001\u0000\u0000\u0000\u024d\u024e\u0001\u0000\u0000\u0000\u024e"+ - "\u024f\u0006\u0014\u0000\u0000\u024f8\u0001\u0000\u0000\u0000\u0250\u0251"+ - "\u0005/\u0000\u0000\u0251\u0252\u0005/\u0000\u0000\u0252\u0256\u0001\u0000"+ - "\u0000\u0000\u0253\u0255\b\u0016\u0000\u0000\u0254\u0253\u0001\u0000\u0000"+ - "\u0000\u0255\u0258\u0001\u0000\u0000\u0000\u0256\u0254\u0001\u0000\u0000"+ - "\u0000\u0256\u0257\u0001\u0000\u0000\u0000\u0257\u025a\u0001\u0000\u0000"+ - "\u0000\u0258\u0256\u0001\u0000\u0000\u0000\u0259\u025b\u0005\r\u0000\u0000"+ - "\u025a\u0259\u0001\u0000\u0000\u0000\u025a\u025b\u0001\u0000\u0000\u0000"+ - "\u025b\u025d\u0001\u0000\u0000\u0000\u025c\u025e\u0005\n\u0000\u0000\u025d"+ - "\u025c\u0001\u0000\u0000\u0000\u025d\u025e\u0001\u0000\u0000\u0000\u025e"+ - "\u025f\u0001\u0000\u0000\u0000\u025f\u0260\u0006\u0015\n\u0000\u0260:"+ - "\u0001\u0000\u0000\u0000\u0261\u0262\u0005/\u0000\u0000\u0262\u0263\u0005"+ - "*\u0000\u0000\u0263\u0268\u0001\u0000\u0000\u0000\u0264\u0267\u0003;\u0016"+ - "\u0000\u0265\u0267\t\u0000\u0000\u0000\u0266\u0264\u0001\u0000\u0000\u0000"+ - "\u0266\u0265\u0001\u0000\u0000\u0000\u0267\u026a\u0001\u0000\u0000\u0000"+ - "\u0268\u0269\u0001\u0000\u0000\u0000\u0268\u0266\u0001\u0000\u0000\u0000"+ - "\u0269\u026b\u0001\u0000\u0000\u0000\u026a\u0268\u0001\u0000\u0000\u0000"+ - "\u026b\u026c\u0005*\u0000\u0000\u026c\u026d\u0005/\u0000\u0000\u026d\u026e"+ - "\u0001\u0000\u0000\u0000\u026e\u026f\u0006\u0016\n\u0000\u026f<\u0001"+ - "\u0000\u0000\u0000\u0270\u0272\u0007\u0017\u0000\u0000\u0271\u0270\u0001"+ - "\u0000\u0000\u0000\u0272\u0273\u0001\u0000\u0000\u0000\u0273\u0271\u0001"+ - "\u0000\u0000\u0000\u0273\u0274\u0001\u0000\u0000\u0000\u0274\u0275\u0001"+ - "\u0000\u0000\u0000\u0275\u0276\u0006\u0017\n\u0000\u0276>\u0001\u0000"+ - "\u0000\u0000\u0277\u0278\u0005|\u0000\u0000\u0278\u0279\u0001\u0000\u0000"+ - "\u0000\u0279\u027a\u0006\u0018\u000b\u0000\u027a@\u0001\u0000\u0000\u0000"+ - "\u027b\u027c\u0007\u0018\u0000\u0000\u027cB\u0001\u0000\u0000\u0000\u027d"+ - "\u027e\u0007\u0019\u0000\u0000\u027eD\u0001\u0000\u0000\u0000\u027f\u0280"+ - "\u0005\\\u0000\u0000\u0280\u0281\u0007\u001a\u0000\u0000\u0281F\u0001"+ - "\u0000\u0000\u0000\u0282\u0283\b\u001b\u0000\u0000\u0283H\u0001\u0000"+ - "\u0000\u0000\u0284\u0286\u0007\u0003\u0000\u0000\u0285\u0287\u0007\u001c"+ - "\u0000\u0000\u0286\u0285\u0001\u0000\u0000\u0000\u0286\u0287\u0001\u0000"+ - "\u0000\u0000\u0287\u0289\u0001\u0000\u0000\u0000\u0288\u028a\u0003A\u0019"+ - "\u0000\u0289\u0288\u0001\u0000\u0000\u0000\u028a\u028b\u0001\u0000\u0000"+ - "\u0000\u028b\u0289\u0001\u0000\u0000\u0000\u028b\u028c\u0001\u0000\u0000"+ - "\u0000\u028cJ\u0001\u0000\u0000\u0000\u028d\u028e\u0005@\u0000\u0000\u028e"+ - "L\u0001\u0000\u0000\u0000\u028f\u0290\u0005`\u0000\u0000\u0290N\u0001"+ - "\u0000\u0000\u0000\u0291\u0295\b\u001d\u0000\u0000\u0292\u0293\u0005`"+ - "\u0000\u0000\u0293\u0295\u0005`\u0000\u0000\u0294\u0291\u0001\u0000\u0000"+ - "\u0000\u0294\u0292\u0001\u0000\u0000\u0000\u0295P\u0001\u0000\u0000\u0000"+ - "\u0296\u0297\u0005_\u0000\u0000\u0297R\u0001\u0000\u0000\u0000\u0298\u029c"+ - "\u0003C\u001a\u0000\u0299\u029c\u0003A\u0019\u0000\u029a\u029c\u0003Q"+ - "!\u0000\u029b\u0298\u0001\u0000\u0000\u0000\u029b\u0299\u0001\u0000\u0000"+ - "\u0000\u029b\u029a\u0001\u0000\u0000\u0000\u029cT\u0001\u0000\u0000\u0000"+ - "\u029d\u02a2\u0005\"\u0000\u0000\u029e\u02a1\u0003E\u001b\u0000\u029f"+ - "\u02a1\u0003G\u001c\u0000\u02a0\u029e\u0001\u0000\u0000\u0000\u02a0\u029f"+ - "\u0001\u0000\u0000\u0000\u02a1\u02a4\u0001\u0000\u0000\u0000\u02a2\u02a0"+ - "\u0001\u0000\u0000\u0000\u02a2\u02a3\u0001\u0000\u0000\u0000\u02a3\u02a5"+ - "\u0001\u0000\u0000\u0000\u02a4\u02a2\u0001\u0000\u0000\u0000\u02a5\u02bb"+ - "\u0005\"\u0000\u0000\u02a6\u02a7\u0005\"\u0000\u0000\u02a7\u02a8\u0005"+ - "\"\u0000\u0000\u02a8\u02a9\u0005\"\u0000\u0000\u02a9\u02ad\u0001\u0000"+ - "\u0000\u0000\u02aa\u02ac\b\u0016\u0000\u0000\u02ab\u02aa\u0001\u0000\u0000"+ - "\u0000\u02ac\u02af\u0001\u0000\u0000\u0000\u02ad\u02ae\u0001\u0000\u0000"+ - "\u0000\u02ad\u02ab\u0001\u0000\u0000\u0000\u02ae\u02b0\u0001\u0000\u0000"+ - "\u0000\u02af\u02ad\u0001\u0000\u0000\u0000\u02b0\u02b1\u0005\"\u0000\u0000"+ - "\u02b1\u02b2\u0005\"\u0000\u0000\u02b2\u02b3\u0005\"\u0000\u0000\u02b3"+ - "\u02b5\u0001\u0000\u0000\u0000\u02b4\u02b6\u0005\"\u0000\u0000\u02b5\u02b4"+ - "\u0001\u0000\u0000\u0000\u02b5\u02b6\u0001\u0000\u0000\u0000\u02b6\u02b8"+ - "\u0001\u0000\u0000\u0000\u02b7\u02b9\u0005\"\u0000\u0000\u02b8\u02b7\u0001"+ - "\u0000\u0000\u0000\u02b8\u02b9\u0001\u0000\u0000\u0000\u02b9\u02bb\u0001"+ - "\u0000\u0000\u0000\u02ba\u029d\u0001\u0000\u0000\u0000\u02ba\u02a6\u0001"+ - "\u0000\u0000\u0000\u02bbV\u0001\u0000\u0000\u0000\u02bc\u02be\u0003A\u0019"+ - "\u0000\u02bd\u02bc\u0001\u0000\u0000\u0000\u02be\u02bf\u0001\u0000\u0000"+ - "\u0000\u02bf\u02bd\u0001\u0000\u0000\u0000\u02bf\u02c0\u0001\u0000\u0000"+ - "\u0000\u02c0X\u0001\u0000\u0000\u0000\u02c1\u02c3\u0003A\u0019\u0000\u02c2"+ - "\u02c1\u0001\u0000\u0000\u0000\u02c3\u02c4\u0001\u0000\u0000\u0000\u02c4"+ - "\u02c2\u0001\u0000\u0000\u0000\u02c4\u02c5\u0001\u0000\u0000\u0000\u02c5"+ - "\u02c6\u0001\u0000\u0000\u0000\u02c6\u02ca\u0003i-\u0000\u02c7\u02c9\u0003"+ - "A\u0019\u0000\u02c8\u02c7\u0001\u0000\u0000\u0000\u02c9\u02cc\u0001\u0000"+ - "\u0000\u0000\u02ca\u02c8\u0001\u0000\u0000\u0000\u02ca\u02cb\u0001\u0000"+ - "\u0000\u0000\u02cb\u02ec\u0001\u0000\u0000\u0000\u02cc\u02ca\u0001\u0000"+ - "\u0000\u0000\u02cd\u02cf\u0003i-\u0000\u02ce\u02d0\u0003A\u0019\u0000"+ - "\u02cf\u02ce\u0001\u0000\u0000\u0000\u02d0\u02d1\u0001\u0000\u0000\u0000"+ - "\u02d1\u02cf\u0001\u0000\u0000\u0000\u02d1\u02d2\u0001\u0000\u0000\u0000"+ - "\u02d2\u02ec\u0001\u0000\u0000\u0000\u02d3\u02d5\u0003A\u0019\u0000\u02d4"+ - "\u02d3\u0001\u0000\u0000\u0000\u02d5\u02d6\u0001\u0000\u0000\u0000\u02d6"+ - "\u02d4\u0001\u0000\u0000\u0000\u02d6\u02d7\u0001\u0000\u0000\u0000\u02d7"+ - "\u02df\u0001\u0000\u0000\u0000\u02d8\u02dc\u0003i-\u0000\u02d9\u02db\u0003"+ - "A\u0019\u0000\u02da\u02d9\u0001\u0000\u0000\u0000\u02db\u02de\u0001\u0000"+ - "\u0000\u0000\u02dc\u02da\u0001\u0000\u0000\u0000\u02dc\u02dd\u0001\u0000"+ - "\u0000\u0000\u02dd\u02e0\u0001\u0000\u0000\u0000\u02de\u02dc\u0001\u0000"+ - "\u0000\u0000\u02df\u02d8\u0001\u0000\u0000\u0000\u02df\u02e0\u0001\u0000"+ - "\u0000\u0000\u02e0\u02e1\u0001\u0000\u0000\u0000\u02e1\u02e2\u0003I\u001d"+ - "\u0000\u02e2\u02ec\u0001\u0000\u0000\u0000\u02e3\u02e5\u0003i-\u0000\u02e4"+ - "\u02e6\u0003A\u0019\u0000\u02e5\u02e4\u0001\u0000\u0000\u0000\u02e6\u02e7"+ - "\u0001\u0000\u0000\u0000\u02e7\u02e5\u0001\u0000\u0000\u0000\u02e7\u02e8"+ - "\u0001\u0000\u0000\u0000\u02e8\u02e9\u0001\u0000\u0000\u0000\u02e9\u02ea"+ - "\u0003I\u001d\u0000\u02ea\u02ec\u0001\u0000\u0000\u0000\u02eb\u02c2\u0001"+ - "\u0000\u0000\u0000\u02eb\u02cd\u0001\u0000\u0000\u0000\u02eb\u02d4\u0001"+ - "\u0000\u0000\u0000\u02eb\u02e3\u0001\u0000\u0000\u0000\u02ecZ\u0001\u0000"+ - "\u0000\u0000\u02ed\u02ee\u0007\u001e\u0000\u0000\u02ee\u02ef\u0007\u001f"+ - "\u0000\u0000\u02ef\\\u0001\u0000\u0000\u0000\u02f0\u02f1\u0007\f\u0000"+ - "\u0000\u02f1\u02f2\u0007\t\u0000\u0000\u02f2\u02f3\u0007\u0000\u0000\u0000"+ - "\u02f3^\u0001\u0000\u0000\u0000\u02f4\u02f5\u0007\f\u0000\u0000\u02f5"+ - "\u02f6\u0007\u0002\u0000\u0000\u02f6\u02f7\u0007\u0004\u0000\u0000\u02f7"+ - "`\u0001\u0000\u0000\u0000\u02f8\u02f9\u0005=\u0000\u0000\u02f9b\u0001"+ - "\u0000\u0000\u0000\u02fa\u02fb\u0005:\u0000\u0000\u02fb\u02fc\u0005:\u0000"+ - "\u0000\u02fcd\u0001\u0000\u0000\u0000\u02fd\u02fe\u0005,\u0000\u0000\u02fe"+ - "f\u0001\u0000\u0000\u0000\u02ff\u0300\u0007\u0000\u0000\u0000\u0300\u0301"+ - "\u0007\u0003\u0000\u0000\u0301\u0302\u0007\u0002\u0000\u0000\u0302\u0303"+ - "\u0007\u0004\u0000\u0000\u0303h\u0001\u0000\u0000\u0000\u0304\u0305\u0005"+ - ".\u0000\u0000\u0305j\u0001\u0000\u0000\u0000\u0306\u0307\u0007\u000f\u0000"+ - "\u0000\u0307\u0308\u0007\f\u0000\u0000\u0308\u0309\u0007\r\u0000\u0000"+ - "\u0309\u030a\u0007\u0002\u0000\u0000\u030a\u030b\u0007\u0003\u0000\u0000"+ - "\u030bl\u0001\u0000\u0000\u0000\u030c\u030d\u0007\u000f\u0000\u0000\u030d"+ - "\u030e\u0007\u0001\u0000\u0000\u030e\u030f\u0007\u0006\u0000\u0000\u030f"+ - "\u0310\u0007\u0002\u0000\u0000\u0310\u0311\u0007\u0005\u0000\u0000\u0311"+ - "n\u0001\u0000\u0000\u0000\u0312\u0313\u0007\u0001\u0000\u0000\u0313\u0314"+ - "\u0007\t\u0000\u0000\u0314p\u0001\u0000\u0000\u0000\u0315\u0316\u0007"+ - "\u0001\u0000\u0000\u0316\u0317\u0007\u0002\u0000\u0000\u0317r\u0001\u0000"+ - "\u0000\u0000\u0318\u0319\u0007\r\u0000\u0000\u0319\u031a\u0007\f\u0000"+ - "\u0000\u031a\u031b\u0007\u0002\u0000\u0000\u031b\u031c\u0007\u0005\u0000"+ - "\u0000\u031ct\u0001\u0000\u0000\u0000\u031d\u031e\u0007\r\u0000\u0000"+ - "\u031e\u031f\u0007\u0001\u0000\u0000\u031f\u0320\u0007\u0012\u0000\u0000"+ - "\u0320\u0321\u0007\u0003\u0000\u0000\u0321v\u0001\u0000\u0000\u0000\u0322"+ - "\u0323\u0005(\u0000\u0000\u0323x\u0001\u0000\u0000\u0000\u0324\u0325\u0007"+ - "\t\u0000\u0000\u0325\u0326\u0007\u0007\u0000\u0000\u0326\u0327\u0007\u0005"+ - "\u0000\u0000\u0327z\u0001\u0000\u0000\u0000\u0328\u0329\u0007\t\u0000"+ - "\u0000\u0329\u032a\u0007\u0014\u0000\u0000\u032a\u032b\u0007\r\u0000\u0000"+ - "\u032b\u032c\u0007\r\u0000\u0000\u032c|\u0001\u0000\u0000\u0000\u032d"+ - "\u032e\u0007\t\u0000\u0000\u032e\u032f\u0007\u0014\u0000\u0000\u032f\u0330"+ - "\u0007\r\u0000\u0000\u0330\u0331\u0007\r\u0000\u0000\u0331\u0332\u0007"+ - "\u0002\u0000\u0000\u0332~\u0001\u0000\u0000\u0000\u0333\u0334\u0007\u0007"+ - "\u0000\u0000\u0334\u0335\u0007\u0006\u0000\u0000\u0335\u0080\u0001\u0000"+ - "\u0000\u0000\u0336\u0337\u0005?\u0000\u0000\u0337\u0082\u0001\u0000\u0000"+ - "\u0000\u0338\u0339\u0007\u0006\u0000\u0000\u0339\u033a\u0007\r\u0000\u0000"+ - "\u033a\u033b\u0007\u0001\u0000\u0000\u033b\u033c\u0007\u0012\u0000\u0000"+ - "\u033c\u033d\u0007\u0003\u0000\u0000\u033d\u0084\u0001\u0000\u0000\u0000"+ - "\u033e\u033f\u0005)\u0000\u0000\u033f\u0086\u0001\u0000\u0000\u0000\u0340"+ - "\u0341\u0007\u0005\u0000\u0000\u0341\u0342\u0007\u0006\u0000\u0000\u0342"+ - "\u0343\u0007\u0014\u0000\u0000\u0343\u0344\u0007\u0003\u0000\u0000\u0344"+ - "\u0088\u0001\u0000\u0000\u0000\u0345\u0346\u0005=\u0000\u0000\u0346\u0347"+ - "\u0005=\u0000\u0000\u0347\u008a\u0001\u0000\u0000\u0000\u0348\u0349\u0005"+ - "=\u0000\u0000\u0349\u034a\u0005~\u0000\u0000\u034a\u008c\u0001\u0000\u0000"+ - "\u0000\u034b\u034c\u0005!\u0000\u0000\u034c\u034d\u0005=\u0000\u0000\u034d"+ - "\u008e\u0001\u0000\u0000\u0000\u034e\u034f\u0005<\u0000\u0000\u034f\u0090"+ - "\u0001\u0000\u0000\u0000\u0350\u0351\u0005<\u0000\u0000\u0351\u0352\u0005"+ - "=\u0000\u0000\u0352\u0092\u0001\u0000\u0000\u0000\u0353\u0354\u0005>\u0000"+ - "\u0000\u0354\u0094\u0001\u0000\u0000\u0000\u0355\u0356\u0005>\u0000\u0000"+ - "\u0356\u0357\u0005=\u0000\u0000\u0357\u0096\u0001\u0000\u0000\u0000\u0358"+ - "\u0359\u0005+\u0000\u0000\u0359\u0098\u0001\u0000\u0000\u0000\u035a\u035b"+ - "\u0005-\u0000\u0000\u035b\u009a\u0001\u0000\u0000\u0000\u035c\u035d\u0005"+ - "*\u0000\u0000\u035d\u009c\u0001\u0000\u0000\u0000\u035e\u035f\u0005/\u0000"+ - "\u0000\u035f\u009e\u0001\u0000\u0000\u0000\u0360\u0361\u0005%\u0000\u0000"+ - "\u0361\u00a0\u0001\u0000\u0000\u0000\u0362\u0363\u0004I\u0004\u0000\u0363"+ - "\u0364\u00033\u0012\u0000\u0364\u0365\u0001\u0000\u0000\u0000\u0365\u0366"+ - "\u0006I\f\u0000\u0366\u00a2\u0001\u0000\u0000\u0000\u0367\u036a\u0003"+ - "\u00819\u0000\u0368\u036b\u0003C\u001a\u0000\u0369\u036b\u0003Q!\u0000"+ - "\u036a\u0368\u0001\u0000\u0000\u0000\u036a\u0369\u0001\u0000\u0000\u0000"+ - "\u036b\u036f\u0001\u0000\u0000\u0000\u036c\u036e\u0003S\"\u0000\u036d"+ - "\u036c\u0001\u0000\u0000\u0000\u036e\u0371\u0001\u0000\u0000\u0000\u036f"+ - "\u036d\u0001\u0000\u0000\u0000\u036f\u0370\u0001\u0000\u0000\u0000\u0370"+ - "\u0379\u0001\u0000\u0000\u0000\u0371\u036f\u0001\u0000\u0000\u0000\u0372"+ - "\u0374\u0003\u00819\u0000\u0373\u0375\u0003A\u0019\u0000\u0374\u0373\u0001"+ - "\u0000\u0000\u0000\u0375\u0376\u0001\u0000\u0000\u0000\u0376\u0374\u0001"+ - "\u0000\u0000\u0000\u0376\u0377\u0001\u0000\u0000\u0000\u0377\u0379\u0001"+ - "\u0000\u0000\u0000\u0378\u0367\u0001\u0000\u0000\u0000\u0378\u0372\u0001"+ - "\u0000\u0000\u0000\u0379\u00a4\u0001\u0000\u0000\u0000\u037a\u037b\u0005"+ - "[\u0000\u0000\u037b\u037c\u0001\u0000\u0000\u0000\u037c\u037d\u0006K\u0000"+ - "\u0000\u037d\u037e\u0006K\u0000\u0000\u037e\u00a6\u0001\u0000\u0000\u0000"+ - "\u037f\u0380\u0005]\u0000\u0000\u0380\u0381\u0001\u0000\u0000\u0000\u0381"+ - "\u0382\u0006L\u000b\u0000\u0382\u0383\u0006L\u000b\u0000\u0383\u00a8\u0001"+ - "\u0000\u0000\u0000\u0384\u0388\u0003C\u001a\u0000\u0385\u0387\u0003S\""+ - "\u0000\u0386\u0385\u0001\u0000\u0000\u0000\u0387\u038a\u0001\u0000\u0000"+ - "\u0000\u0388\u0386\u0001\u0000\u0000\u0000\u0388\u0389\u0001\u0000\u0000"+ - "\u0000\u0389\u0395\u0001\u0000\u0000\u0000\u038a\u0388\u0001\u0000\u0000"+ - "\u0000\u038b\u038e\u0003Q!\u0000\u038c\u038e\u0003K\u001e\u0000\u038d"+ - "\u038b\u0001\u0000\u0000\u0000\u038d\u038c\u0001\u0000\u0000\u0000\u038e"+ - "\u0390\u0001\u0000\u0000\u0000\u038f\u0391\u0003S\"\u0000\u0390\u038f"+ - "\u0001\u0000\u0000\u0000\u0391\u0392\u0001\u0000\u0000\u0000\u0392\u0390"+ - "\u0001\u0000\u0000\u0000\u0392\u0393\u0001\u0000\u0000\u0000\u0393\u0395"+ - "\u0001\u0000\u0000\u0000\u0394\u0384\u0001\u0000\u0000\u0000\u0394\u038d"+ - "\u0001\u0000\u0000\u0000\u0395\u00aa\u0001\u0000\u0000\u0000\u0396\u0398"+ - "\u0003M\u001f\u0000\u0397\u0399\u0003O \u0000\u0398\u0397\u0001\u0000"+ - "\u0000\u0000\u0399\u039a\u0001\u0000\u0000\u0000\u039a\u0398\u0001\u0000"+ - "\u0000\u0000\u039a\u039b\u0001\u0000\u0000\u0000\u039b\u039c\u0001\u0000"+ - "\u0000\u0000\u039c\u039d\u0003M\u001f\u0000\u039d\u00ac\u0001\u0000\u0000"+ - "\u0000\u039e\u039f\u0003\u00abN\u0000\u039f\u00ae\u0001\u0000\u0000\u0000"+ - "\u03a0\u03a1\u00039\u0015\u0000\u03a1\u03a2\u0001\u0000\u0000\u0000\u03a2"+ - "\u03a3\u0006P\n\u0000\u03a3\u00b0\u0001\u0000\u0000\u0000\u03a4\u03a5"+ - "\u0003;\u0016\u0000\u03a5\u03a6\u0001\u0000\u0000\u0000\u03a6\u03a7\u0006"+ - "Q\n\u0000\u03a7\u00b2\u0001\u0000\u0000\u0000\u03a8\u03a9\u0003=\u0017"+ - "\u0000\u03a9\u03aa\u0001\u0000\u0000\u0000\u03aa\u03ab\u0006R\n\u0000"+ - "\u03ab\u00b4\u0001\u0000\u0000\u0000\u03ac\u03ad\u0003\u00a5K\u0000\u03ad"+ - "\u03ae\u0001\u0000\u0000\u0000\u03ae\u03af\u0006S\r\u0000\u03af\u03b0"+ - "\u0006S\u000e\u0000\u03b0\u00b6\u0001\u0000\u0000\u0000\u03b1\u03b2\u0003"+ - "?\u0018\u0000\u03b2\u03b3\u0001\u0000\u0000\u0000\u03b3\u03b4\u0006T\u000f"+ - "\u0000\u03b4\u03b5\u0006T\u000b\u0000\u03b5\u00b8\u0001\u0000\u0000\u0000"+ - "\u03b6\u03b7\u0003=\u0017\u0000\u03b7\u03b8\u0001\u0000\u0000\u0000\u03b8"+ - "\u03b9\u0006U\n\u0000\u03b9\u00ba\u0001\u0000\u0000\u0000\u03ba\u03bb"+ - "\u00039\u0015\u0000\u03bb\u03bc\u0001\u0000\u0000\u0000\u03bc\u03bd\u0006"+ - "V\n\u0000\u03bd\u00bc\u0001\u0000\u0000\u0000\u03be\u03bf\u0003;\u0016"+ - "\u0000\u03bf\u03c0\u0001\u0000\u0000\u0000\u03c0\u03c1\u0006W\n\u0000"+ - "\u03c1\u00be\u0001\u0000\u0000\u0000\u03c2\u03c3\u0003?\u0018\u0000\u03c3"+ - "\u03c4\u0001\u0000\u0000\u0000\u03c4\u03c5\u0006X\u000f\u0000\u03c5\u03c6"+ - "\u0006X\u000b\u0000\u03c6\u00c0\u0001\u0000\u0000\u0000\u03c7\u03c8\u0003"+ - "\u00a5K\u0000\u03c8\u03c9\u0001\u0000\u0000\u0000\u03c9\u03ca\u0006Y\r"+ - "\u0000\u03ca\u00c2\u0001\u0000\u0000\u0000\u03cb\u03cc\u0003\u00a7L\u0000"+ - "\u03cc\u03cd\u0001\u0000\u0000\u0000\u03cd\u03ce\u0006Z\u0010\u0000\u03ce"+ - "\u00c4\u0001\u0000\u0000\u0000\u03cf\u03d0\u0003\u0151\u00a1\u0000\u03d0"+ - "\u03d1\u0001\u0000\u0000\u0000\u03d1\u03d2\u0006[\u0011\u0000\u03d2\u00c6"+ - "\u0001\u0000\u0000\u0000\u03d3\u03d4\u0003e+\u0000\u03d4\u03d5\u0001\u0000"+ - "\u0000\u0000\u03d5\u03d6\u0006\\\u0012\u0000\u03d6\u00c8\u0001\u0000\u0000"+ - "\u0000\u03d7\u03d8\u0003a)\u0000\u03d8\u03d9\u0001\u0000\u0000\u0000\u03d9"+ - "\u03da\u0006]\u0013\u0000\u03da\u00ca\u0001\u0000\u0000\u0000\u03db\u03dc"+ - "\u0007\u0010\u0000\u0000\u03dc\u03dd\u0007\u0003\u0000\u0000\u03dd\u03de"+ - "\u0007\u0005\u0000\u0000\u03de\u03df\u0007\f\u0000\u0000\u03df\u03e0\u0007"+ - "\u0000\u0000\u0000\u03e0\u03e1\u0007\f\u0000\u0000\u03e1\u03e2\u0007\u0005"+ - "\u0000\u0000\u03e2\u03e3\u0007\f\u0000\u0000\u03e3\u00cc\u0001\u0000\u0000"+ - "\u0000\u03e4\u03e8\b \u0000\u0000\u03e5\u03e6\u0005/\u0000\u0000\u03e6"+ - "\u03e8\b!\u0000\u0000\u03e7\u03e4\u0001\u0000\u0000\u0000\u03e7\u03e5"+ - "\u0001\u0000\u0000\u0000\u03e8\u00ce\u0001\u0000\u0000\u0000\u03e9\u03eb"+ - "\u0003\u00cd_\u0000\u03ea\u03e9\u0001\u0000\u0000\u0000\u03eb\u03ec\u0001"+ - "\u0000\u0000\u0000\u03ec\u03ea\u0001\u0000\u0000\u0000\u03ec\u03ed\u0001"+ - "\u0000\u0000\u0000\u03ed\u00d0\u0001\u0000\u0000\u0000\u03ee\u03ef\u0003"+ - "\u00cf`\u0000\u03ef\u03f0\u0001\u0000\u0000\u0000\u03f0\u03f1\u0006a\u0014"+ - "\u0000\u03f1\u00d2\u0001\u0000\u0000\u0000\u03f2\u03f3\u0003U#\u0000\u03f3"+ - "\u03f4\u0001\u0000\u0000\u0000\u03f4\u03f5\u0006b\u0015\u0000\u03f5\u00d4"+ - "\u0001\u0000\u0000\u0000\u03f6\u03f7\u00039\u0015\u0000\u03f7\u03f8\u0001"+ - "\u0000\u0000\u0000\u03f8\u03f9\u0006c\n\u0000\u03f9\u00d6\u0001\u0000"+ - "\u0000\u0000\u03fa\u03fb\u0003;\u0016\u0000\u03fb\u03fc\u0001\u0000\u0000"+ - "\u0000\u03fc\u03fd\u0006d\n\u0000\u03fd\u00d8\u0001\u0000\u0000\u0000"+ - "\u03fe\u03ff\u0003=\u0017\u0000\u03ff\u0400\u0001\u0000\u0000\u0000\u0400"+ - "\u0401\u0006e\n\u0000\u0401\u00da\u0001\u0000\u0000\u0000\u0402\u0403"+ - "\u0003?\u0018\u0000\u0403\u0404\u0001\u0000\u0000\u0000\u0404\u0405\u0006"+ - "f\u000f\u0000\u0405\u0406\u0006f\u000b\u0000\u0406\u00dc\u0001\u0000\u0000"+ - "\u0000\u0407\u0408\u0003i-\u0000\u0408\u0409\u0001\u0000\u0000\u0000\u0409"+ - "\u040a\u0006g\u0016\u0000\u040a\u00de\u0001\u0000\u0000\u0000\u040b\u040c"+ - "\u0003e+\u0000\u040c\u040d\u0001\u0000\u0000\u0000\u040d\u040e\u0006h"+ - "\u0012\u0000\u040e\u00e0\u0001\u0000\u0000\u0000\u040f\u0410\u0003\u0081"+ - "9\u0000\u0410\u0411\u0001\u0000\u0000\u0000\u0411\u0412\u0006i\u0017\u0000"+ - "\u0412\u00e2\u0001\u0000\u0000\u0000\u0413\u0414\u0003\u00a3J\u0000\u0414"+ - "\u0415\u0001\u0000\u0000\u0000\u0415\u0416\u0006j\u0018\u0000\u0416\u00e4"+ - "\u0001\u0000\u0000\u0000\u0417\u041c\u0003C\u001a\u0000\u0418\u041c\u0003"+ - "A\u0019\u0000\u0419\u041c\u0003Q!\u0000\u041a\u041c\u0003\u009bF\u0000"+ - "\u041b\u0417\u0001\u0000\u0000\u0000\u041b\u0418\u0001\u0000\u0000\u0000"+ - "\u041b\u0419\u0001\u0000\u0000\u0000\u041b\u041a\u0001\u0000\u0000\u0000"+ - "\u041c\u00e6\u0001\u0000\u0000\u0000\u041d\u0420\u0003C\u001a\u0000\u041e"+ - "\u0420\u0003\u009bF\u0000\u041f\u041d\u0001\u0000\u0000\u0000\u041f\u041e"+ - "\u0001\u0000\u0000\u0000\u0420\u0424\u0001\u0000\u0000\u0000\u0421\u0423"+ - "\u0003\u00e5k\u0000\u0422\u0421\u0001\u0000\u0000\u0000\u0423\u0426\u0001"+ - "\u0000\u0000\u0000\u0424\u0422\u0001\u0000\u0000\u0000\u0424\u0425\u0001"+ - "\u0000\u0000\u0000\u0425\u0431\u0001\u0000\u0000\u0000\u0426\u0424\u0001"+ - "\u0000\u0000\u0000\u0427\u042a\u0003Q!\u0000\u0428\u042a\u0003K\u001e"+ - "\u0000\u0429\u0427\u0001\u0000\u0000\u0000\u0429\u0428\u0001\u0000\u0000"+ - "\u0000\u042a\u042c\u0001\u0000\u0000\u0000\u042b\u042d\u0003\u00e5k\u0000"+ - "\u042c\u042b\u0001\u0000\u0000\u0000\u042d\u042e\u0001\u0000\u0000\u0000"+ - "\u042e\u042c\u0001\u0000\u0000\u0000\u042e\u042f\u0001\u0000\u0000\u0000"+ - "\u042f\u0431\u0001\u0000\u0000\u0000\u0430\u041f\u0001\u0000\u0000\u0000"+ - "\u0430\u0429\u0001\u0000\u0000\u0000\u0431\u00e8\u0001\u0000\u0000\u0000"+ - "\u0432\u0435\u0003\u00e7l\u0000\u0433\u0435\u0003\u00abN\u0000\u0434\u0432"+ - "\u0001\u0000\u0000\u0000\u0434\u0433\u0001\u0000\u0000\u0000\u0435\u0436"+ - "\u0001\u0000\u0000\u0000\u0436\u0434\u0001\u0000\u0000\u0000\u0436\u0437"+ - "\u0001\u0000\u0000\u0000\u0437\u00ea\u0001\u0000\u0000\u0000\u0438\u0439"+ - "\u00039\u0015\u0000\u0439\u043a\u0001\u0000\u0000\u0000\u043a\u043b\u0006"+ - "n\n\u0000\u043b\u00ec\u0001\u0000\u0000\u0000\u043c\u043d\u0003;\u0016"+ - "\u0000\u043d\u043e\u0001\u0000\u0000\u0000\u043e\u043f\u0006o\n\u0000"+ - "\u043f\u00ee\u0001\u0000\u0000\u0000\u0440\u0441\u0003=\u0017\u0000\u0441"+ - "\u0442\u0001\u0000\u0000\u0000\u0442\u0443\u0006p\n\u0000\u0443\u00f0"+ - "\u0001\u0000\u0000\u0000\u0444\u0445\u0003?\u0018\u0000\u0445\u0446\u0001"+ - "\u0000\u0000\u0000\u0446\u0447\u0006q\u000f\u0000\u0447\u0448\u0006q\u000b"+ - "\u0000\u0448\u00f2\u0001\u0000\u0000\u0000\u0449\u044a\u0003a)\u0000\u044a"+ - "\u044b\u0001\u0000\u0000\u0000\u044b\u044c\u0006r\u0013\u0000\u044c\u00f4"+ - "\u0001\u0000\u0000\u0000\u044d\u044e\u0003e+\u0000\u044e\u044f\u0001\u0000"+ - "\u0000\u0000\u044f\u0450\u0006s\u0012\u0000\u0450\u00f6\u0001\u0000\u0000"+ - "\u0000\u0451\u0452\u0003i-\u0000\u0452\u0453\u0001\u0000\u0000\u0000\u0453"+ - "\u0454\u0006t\u0016\u0000\u0454\u00f8\u0001\u0000\u0000\u0000\u0455\u0456"+ - "\u0003\u00819\u0000\u0456\u0457\u0001\u0000\u0000\u0000\u0457\u0458\u0006"+ - "u\u0017\u0000\u0458\u00fa\u0001\u0000\u0000\u0000\u0459\u045a\u0003\u00a3"+ - "J\u0000\u045a\u045b\u0001\u0000\u0000\u0000\u045b\u045c\u0006v\u0018\u0000"+ - "\u045c\u00fc\u0001\u0000\u0000\u0000\u045d\u045e\u0007\f\u0000\u0000\u045e"+ - "\u045f\u0007\u0002\u0000\u0000\u045f\u00fe\u0001\u0000\u0000\u0000\u0460"+ - "\u0461\u0003\u00e9m\u0000\u0461\u0462\u0001\u0000\u0000\u0000\u0462\u0463"+ - "\u0006x\u0019\u0000\u0463\u0100\u0001\u0000\u0000\u0000\u0464\u0465\u0003"+ - "9\u0015\u0000\u0465\u0466\u0001\u0000\u0000\u0000\u0466\u0467\u0006y\n"+ - "\u0000\u0467\u0102\u0001\u0000\u0000\u0000\u0468\u0469\u0003;\u0016\u0000"+ - "\u0469\u046a\u0001\u0000\u0000\u0000\u046a\u046b\u0006z\n\u0000\u046b"+ - "\u0104\u0001\u0000\u0000\u0000\u046c\u046d\u0003=\u0017\u0000\u046d\u046e"+ - "\u0001\u0000\u0000\u0000\u046e\u046f\u0006{\n\u0000\u046f\u0106\u0001"+ - "\u0000\u0000\u0000\u0470\u0471\u0003?\u0018\u0000\u0471\u0472\u0001\u0000"+ - "\u0000\u0000\u0472\u0473\u0006|\u000f\u0000\u0473\u0474\u0006|\u000b\u0000"+ - "\u0474\u0108\u0001\u0000\u0000\u0000\u0475\u0476\u0003\u00a5K\u0000\u0476"+ - "\u0477\u0001\u0000\u0000\u0000\u0477\u0478\u0006}\r\u0000\u0478\u0479"+ - "\u0006}\u001a\u0000\u0479\u010a\u0001\u0000\u0000\u0000\u047a\u047b\u0007"+ - "\u0007\u0000\u0000\u047b\u047c\u0007\t\u0000\u0000\u047c\u047d\u0001\u0000"+ - "\u0000\u0000\u047d\u047e\u0006~\u001b\u0000\u047e\u010c\u0001\u0000\u0000"+ - "\u0000\u047f\u0480\u0007\u0013\u0000\u0000\u0480\u0481\u0007\u0001\u0000"+ - "\u0000\u0481\u0482\u0007\u0005\u0000\u0000\u0482\u0483\u0007\n\u0000\u0000"+ - "\u0483\u0484\u0001\u0000\u0000\u0000\u0484\u0485\u0006\u007f\u001b\u0000"+ - "\u0485\u010e\u0001\u0000\u0000\u0000\u0486\u0487\b\"\u0000\u0000\u0487"+ - "\u0110\u0001\u0000\u0000\u0000\u0488\u048a\u0003\u010f\u0080\u0000\u0489"+ - "\u0488\u0001\u0000\u0000\u0000\u048a\u048b\u0001\u0000\u0000\u0000\u048b"+ - "\u0489\u0001\u0000\u0000\u0000\u048b\u048c\u0001\u0000\u0000\u0000\u048c"+ - "\u048d\u0001\u0000\u0000\u0000\u048d\u048e\u0003\u0151\u00a1\u0000\u048e"+ - "\u0490\u0001\u0000\u0000\u0000\u048f\u0489\u0001\u0000\u0000\u0000\u048f"+ - "\u0490\u0001\u0000\u0000\u0000\u0490\u0492\u0001\u0000\u0000\u0000\u0491"+ - "\u0493\u0003\u010f\u0080\u0000\u0492\u0491\u0001\u0000\u0000\u0000\u0493"+ - "\u0494\u0001\u0000\u0000\u0000\u0494\u0492\u0001\u0000\u0000\u0000\u0494"+ - "\u0495\u0001\u0000\u0000\u0000\u0495\u0112\u0001\u0000\u0000\u0000\u0496"+ - "\u0497\u0003\u0111\u0081\u0000\u0497\u0498\u0001\u0000\u0000\u0000\u0498"+ - "\u0499\u0006\u0082\u001c\u0000\u0499\u0114\u0001\u0000\u0000\u0000\u049a"+ - "\u049b\u00039\u0015\u0000\u049b\u049c\u0001\u0000\u0000\u0000\u049c\u049d"+ - "\u0006\u0083\n\u0000\u049d\u0116\u0001\u0000\u0000\u0000\u049e\u049f\u0003"+ - ";\u0016\u0000\u049f\u04a0\u0001\u0000\u0000\u0000\u04a0\u04a1\u0006\u0084"+ - "\n\u0000\u04a1\u0118\u0001\u0000\u0000\u0000\u04a2\u04a3\u0003=\u0017"+ - "\u0000\u04a3\u04a4\u0001\u0000\u0000\u0000\u04a4\u04a5\u0006\u0085\n\u0000"+ - "\u04a5\u011a\u0001\u0000\u0000\u0000\u04a6\u04a7\u0003?\u0018\u0000\u04a7"+ - "\u04a8\u0001\u0000\u0000\u0000\u04a8\u04a9\u0006\u0086\u000f\u0000\u04a9"+ - "\u04aa\u0006\u0086\u000b\u0000\u04aa\u04ab\u0006\u0086\u000b\u0000\u04ab"+ - "\u011c\u0001\u0000\u0000\u0000\u04ac\u04ad\u0003a)\u0000\u04ad\u04ae\u0001"+ - "\u0000\u0000\u0000\u04ae\u04af\u0006\u0087\u0013\u0000\u04af\u011e\u0001"+ - "\u0000\u0000\u0000\u04b0\u04b1\u0003e+\u0000\u04b1\u04b2\u0001\u0000\u0000"+ - "\u0000\u04b2\u04b3\u0006\u0088\u0012\u0000\u04b3\u0120\u0001\u0000\u0000"+ - "\u0000\u04b4\u04b5\u0003i-\u0000\u04b5\u04b6\u0001\u0000\u0000\u0000\u04b6"+ - "\u04b7\u0006\u0089\u0016\u0000\u04b7\u0122\u0001\u0000\u0000\u0000\u04b8"+ - "\u04b9\u0003\u010d\u007f\u0000\u04b9\u04ba\u0001\u0000\u0000\u0000\u04ba"+ - "\u04bb\u0006\u008a\u001d\u0000\u04bb\u0124\u0001\u0000\u0000\u0000\u04bc"+ - "\u04bd\u0003\u00e9m\u0000\u04bd\u04be\u0001\u0000\u0000\u0000\u04be\u04bf"+ - "\u0006\u008b\u0019\u0000\u04bf\u0126\u0001\u0000\u0000\u0000\u04c0\u04c1"+ - "\u0003\u00adO\u0000\u04c1\u04c2\u0001\u0000\u0000\u0000\u04c2\u04c3\u0006"+ - "\u008c\u001e\u0000\u04c3\u0128\u0001\u0000\u0000\u0000\u04c4\u04c5\u0003"+ - "\u00819\u0000\u04c5\u04c6\u0001\u0000\u0000\u0000\u04c6\u04c7\u0006\u008d"+ - "\u0017\u0000\u04c7\u012a\u0001\u0000\u0000\u0000\u04c8\u04c9\u0003\u00a3"+ - "J\u0000\u04c9\u04ca\u0001\u0000\u0000\u0000\u04ca\u04cb\u0006\u008e\u0018"+ - "\u0000\u04cb\u012c\u0001\u0000\u0000\u0000\u04cc\u04cd\u00039\u0015\u0000"+ - "\u04cd\u04ce\u0001\u0000\u0000\u0000\u04ce\u04cf\u0006\u008f\n\u0000\u04cf"+ - "\u012e\u0001\u0000\u0000\u0000\u04d0\u04d1\u0003;\u0016\u0000\u04d1\u04d2"+ - "\u0001\u0000\u0000\u0000\u04d2\u04d3\u0006\u0090\n\u0000\u04d3\u0130\u0001"+ - "\u0000\u0000\u0000\u04d4\u04d5\u0003=\u0017\u0000\u04d5\u04d6\u0001\u0000"+ - "\u0000\u0000\u04d6\u04d7\u0006\u0091\n\u0000\u04d7\u0132\u0001\u0000\u0000"+ - "\u0000\u04d8\u04d9\u0003?\u0018\u0000\u04d9\u04da\u0001\u0000\u0000\u0000"+ - "\u04da\u04db\u0006\u0092\u000f\u0000\u04db\u04dc\u0006\u0092\u000b\u0000"+ - "\u04dc\u0134\u0001\u0000\u0000\u0000\u04dd\u04de\u0003i-\u0000\u04de\u04df"+ - "\u0001\u0000\u0000\u0000\u04df\u04e0\u0006\u0093\u0016\u0000\u04e0\u0136"+ - "\u0001\u0000\u0000\u0000\u04e1\u04e2\u0003\u00819\u0000\u04e2\u04e3\u0001"+ - "\u0000\u0000\u0000\u04e3\u04e4\u0006\u0094\u0017\u0000\u04e4\u0138\u0001"+ - "\u0000\u0000\u0000\u04e5\u04e6\u0003\u00a3J\u0000\u04e6\u04e7\u0001\u0000"+ - "\u0000\u0000\u04e7\u04e8\u0006\u0095\u0018\u0000\u04e8\u013a\u0001\u0000"+ - "\u0000\u0000\u04e9\u04ea\u0003\u00adO\u0000\u04ea\u04eb\u0001\u0000\u0000"+ - "\u0000\u04eb\u04ec\u0006\u0096\u001e\u0000\u04ec\u013c\u0001\u0000\u0000"+ - "\u0000\u04ed\u04ee\u0003\u00a9M\u0000\u04ee\u04ef\u0001\u0000\u0000\u0000"+ - "\u04ef\u04f0\u0006\u0097\u001f\u0000\u04f0\u013e\u0001\u0000\u0000\u0000"+ - "\u04f1\u04f2\u00039\u0015\u0000\u04f2\u04f3\u0001\u0000\u0000\u0000\u04f3"+ - "\u04f4\u0006\u0098\n\u0000\u04f4\u0140\u0001\u0000\u0000\u0000\u04f5\u04f6"+ - "\u0003;\u0016\u0000\u04f6\u04f7\u0001\u0000\u0000\u0000\u04f7\u04f8\u0006"+ - "\u0099\n\u0000\u04f8\u0142\u0001\u0000\u0000\u0000\u04f9\u04fa\u0003="+ - "\u0017\u0000\u04fa\u04fb\u0001\u0000\u0000\u0000\u04fb\u04fc\u0006\u009a"+ - "\n\u0000\u04fc\u0144\u0001\u0000\u0000\u0000\u04fd\u04fe\u0003?\u0018"+ - "\u0000\u04fe\u04ff\u0001\u0000\u0000\u0000\u04ff\u0500\u0006\u009b\u000f"+ - "\u0000\u0500\u0501\u0006\u009b\u000b\u0000\u0501\u0146\u0001\u0000\u0000"+ - "\u0000\u0502\u0503\u0007\u0001\u0000\u0000\u0503\u0504\u0007\t\u0000\u0000"+ - "\u0504\u0505\u0007\u000f\u0000\u0000\u0505\u0506\u0007\u0007\u0000\u0000"+ - "\u0506\u0148\u0001\u0000\u0000\u0000\u0507\u0508\u00039\u0015\u0000\u0508"+ - "\u0509\u0001\u0000\u0000\u0000\u0509\u050a\u0006\u009d\n\u0000\u050a\u014a"+ - "\u0001\u0000\u0000\u0000\u050b\u050c\u0003;\u0016\u0000\u050c\u050d\u0001"+ - "\u0000\u0000\u0000\u050d\u050e\u0006\u009e\n\u0000\u050e\u014c\u0001\u0000"+ - "\u0000\u0000\u050f\u0510\u0003=\u0017\u0000\u0510\u0511\u0001\u0000\u0000"+ - "\u0000\u0511\u0512\u0006\u009f\n\u0000\u0512\u014e\u0001\u0000\u0000\u0000"+ - "\u0513\u0514\u0003\u00a7L\u0000\u0514\u0515\u0001\u0000\u0000\u0000\u0515"+ - "\u0516\u0006\u00a0\u0010\u0000\u0516\u0517\u0006\u00a0\u000b\u0000\u0517"+ - "\u0150\u0001\u0000\u0000\u0000\u0518\u0519\u0005:\u0000\u0000\u0519\u0152"+ - "\u0001\u0000\u0000\u0000\u051a\u0520\u0003K\u001e\u0000\u051b\u0520\u0003"+ - "A\u0019\u0000\u051c\u0520\u0003i-\u0000\u051d\u0520\u0003C\u001a\u0000"+ - "\u051e\u0520\u0003Q!\u0000\u051f\u051a\u0001\u0000\u0000\u0000\u051f\u051b"+ - "\u0001\u0000\u0000\u0000\u051f\u051c\u0001\u0000\u0000\u0000\u051f\u051d"+ - "\u0001\u0000\u0000\u0000\u051f\u051e\u0001\u0000\u0000\u0000\u0520\u0521"+ - "\u0001\u0000\u0000\u0000\u0521\u051f\u0001\u0000\u0000\u0000\u0521\u0522"+ - "\u0001\u0000\u0000\u0000\u0522\u0154\u0001\u0000\u0000\u0000\u0523\u0524"+ - "\u00039\u0015\u0000\u0524\u0525\u0001\u0000\u0000\u0000\u0525\u0526\u0006"+ - "\u00a3\n\u0000\u0526\u0156\u0001\u0000\u0000\u0000\u0527\u0528\u0003;"+ - "\u0016\u0000\u0528\u0529\u0001\u0000\u0000\u0000\u0529\u052a\u0006\u00a4"+ - "\n\u0000\u052a\u0158\u0001\u0000\u0000\u0000\u052b\u052c\u0003=\u0017"+ - "\u0000\u052c\u052d\u0001\u0000\u0000\u0000\u052d\u052e\u0006\u00a5\n\u0000"+ - "\u052e\u015a\u0001\u0000\u0000\u0000\u052f\u0530\u0003?\u0018\u0000\u0530"+ - "\u0531\u0001\u0000\u0000\u0000\u0531\u0532\u0006\u00a6\u000f\u0000\u0532"+ - "\u0533\u0006\u00a6\u000b\u0000\u0533\u015c\u0001\u0000\u0000\u0000\u0534"+ - "\u0535\u0003\u0151\u00a1\u0000\u0535\u0536\u0001\u0000\u0000\u0000\u0536"+ - "\u0537\u0006\u00a7\u0011\u0000\u0537\u015e\u0001\u0000\u0000\u0000\u0538"+ - "\u0539\u0003e+\u0000\u0539\u053a\u0001\u0000\u0000\u0000\u053a\u053b\u0006"+ - "\u00a8\u0012\u0000\u053b\u0160\u0001\u0000\u0000\u0000\u053c\u053d\u0003"+ - "i-\u0000\u053d\u053e\u0001\u0000\u0000\u0000\u053e\u053f\u0006\u00a9\u0016"+ - "\u0000\u053f\u0162\u0001\u0000\u0000\u0000\u0540\u0541\u0003\u010b~\u0000"+ - "\u0541\u0542\u0001\u0000\u0000\u0000\u0542\u0543\u0006\u00aa \u0000\u0543"+ - "\u0544\u0006\u00aa!\u0000\u0544\u0164\u0001\u0000\u0000\u0000\u0545\u0546"+ - "\u0003\u00cf`\u0000\u0546\u0547\u0001\u0000\u0000\u0000\u0547\u0548\u0006"+ - "\u00ab\u0014\u0000\u0548\u0166\u0001\u0000\u0000\u0000\u0549\u054a\u0003"+ - "U#\u0000\u054a\u054b\u0001\u0000\u0000\u0000\u054b\u054c\u0006\u00ac\u0015"+ - "\u0000\u054c\u0168\u0001\u0000\u0000\u0000\u054d\u054e\u00039\u0015\u0000"+ - "\u054e\u054f\u0001\u0000\u0000\u0000\u054f\u0550\u0006\u00ad\n\u0000\u0550"+ - "\u016a\u0001\u0000\u0000\u0000\u0551\u0552\u0003;\u0016\u0000\u0552\u0553"+ - "\u0001\u0000\u0000\u0000\u0553\u0554\u0006\u00ae\n\u0000\u0554\u016c\u0001"+ - "\u0000\u0000\u0000\u0555\u0556\u0003=\u0017\u0000\u0556\u0557\u0001\u0000"+ - "\u0000\u0000\u0557\u0558\u0006\u00af\n\u0000\u0558\u016e\u0001\u0000\u0000"+ - "\u0000\u0559\u055a\u0003?\u0018\u0000\u055a\u055b\u0001\u0000\u0000\u0000"+ - "\u055b\u055c\u0006\u00b0\u000f\u0000\u055c\u055d\u0006\u00b0\u000b\u0000"+ - "\u055d\u055e\u0006\u00b0\u000b\u0000\u055e\u0170\u0001\u0000\u0000\u0000"+ - "\u055f\u0560\u0003e+\u0000\u0560\u0561\u0001\u0000\u0000\u0000\u0561\u0562"+ - "\u0006\u00b1\u0012\u0000\u0562\u0172\u0001\u0000\u0000\u0000\u0563\u0564"+ - "\u0003i-\u0000\u0564\u0565\u0001\u0000\u0000\u0000\u0565\u0566\u0006\u00b2"+ - "\u0016\u0000\u0566\u0174\u0001\u0000\u0000\u0000\u0567\u0568\u0003\u00e9"+ - "m\u0000\u0568\u0569\u0001\u0000\u0000\u0000\u0569\u056a\u0006\u00b3\u0019"+ - "\u0000\u056a\u0176\u0001\u0000\u0000\u0000\u056b\u056c\u00039\u0015\u0000"+ - "\u056c\u056d\u0001\u0000\u0000\u0000\u056d\u056e\u0006\u00b4\n\u0000\u056e"+ - "\u0178\u0001\u0000\u0000\u0000\u056f\u0570\u0003;\u0016\u0000\u0570\u0571"+ - "\u0001\u0000\u0000\u0000\u0571\u0572\u0006\u00b5\n\u0000\u0572\u017a\u0001"+ - "\u0000\u0000\u0000\u0573\u0574\u0003=\u0017\u0000\u0574\u0575\u0001\u0000"+ - "\u0000\u0000\u0575\u0576\u0006\u00b6\n\u0000\u0576\u017c\u0001\u0000\u0000"+ - "\u0000\u0577\u0578\u0003?\u0018\u0000\u0578\u0579\u0001\u0000\u0000\u0000"+ - "\u0579\u057a\u0006\u00b7\u000f\u0000\u057a\u057b\u0006\u00b7\u000b\u0000"+ - "\u057b\u017e\u0001\u0000\u0000\u0000\u057c\u057d\u0003\u00cf`\u0000\u057d"+ - "\u057e\u0001\u0000\u0000\u0000\u057e\u057f\u0006\u00b8\u0014\u0000\u057f"+ - "\u0580\u0006\u00b8\u000b\u0000\u0580\u0581\u0006\u00b8\"\u0000\u0581\u0180"+ - "\u0001\u0000\u0000\u0000\u0582\u0583\u0003U#\u0000\u0583\u0584\u0001\u0000"+ - "\u0000\u0000\u0584\u0585\u0006\u00b9\u0015\u0000\u0585\u0586\u0006\u00b9"+ - "\u000b\u0000\u0586\u0587\u0006\u00b9\"\u0000\u0587\u0182\u0001\u0000\u0000"+ - "\u0000\u0588\u0589\u00039\u0015\u0000\u0589\u058a\u0001\u0000\u0000\u0000"+ - "\u058a\u058b\u0006\u00ba\n\u0000\u058b\u0184\u0001\u0000\u0000\u0000\u058c"+ - "\u058d\u0003;\u0016\u0000\u058d\u058e\u0001\u0000\u0000\u0000\u058e\u058f"+ - "\u0006\u00bb\n\u0000\u058f\u0186\u0001\u0000\u0000\u0000\u0590\u0591\u0003"+ - "=\u0017\u0000\u0591\u0592\u0001\u0000\u0000\u0000\u0592\u0593\u0006\u00bc"+ - "\n\u0000\u0593\u0188\u0001\u0000\u0000\u0000\u0594\u0595\u0003\u0151\u00a1"+ - "\u0000\u0595\u0596\u0001\u0000\u0000\u0000\u0596\u0597\u0006\u00bd\u0011"+ - "\u0000\u0597\u0598\u0006\u00bd\u000b\u0000\u0598\u0599\u0006\u00bd\t\u0000"+ - "\u0599\u018a\u0001\u0000\u0000\u0000\u059a\u059b\u0003e+\u0000\u059b\u059c"+ - "\u0001\u0000\u0000\u0000\u059c\u059d\u0006\u00be\u0012\u0000\u059d\u059e"+ - "\u0006\u00be\u000b\u0000\u059e\u059f\u0006\u00be\t\u0000\u059f\u018c\u0001"+ - "\u0000\u0000\u0000\u05a0\u05a1\u00039\u0015\u0000\u05a1\u05a2\u0001\u0000"+ - "\u0000\u0000\u05a2\u05a3\u0006\u00bf\n\u0000\u05a3\u018e\u0001\u0000\u0000"+ - "\u0000\u05a4\u05a5\u0003;\u0016\u0000\u05a5\u05a6\u0001\u0000\u0000\u0000"+ - "\u05a6\u05a7\u0006\u00c0\n\u0000\u05a7\u0190\u0001\u0000\u0000\u0000\u05a8"+ - "\u05a9\u0003=\u0017\u0000\u05a9\u05aa\u0001\u0000\u0000\u0000\u05aa\u05ab"+ - "\u0006\u00c1\n\u0000\u05ab\u0192\u0001\u0000\u0000\u0000\u05ac\u05ad\u0003"+ - "\u00adO\u0000\u05ad\u05ae\u0001\u0000\u0000\u0000\u05ae\u05af\u0006\u00c2"+ - "\u000b\u0000\u05af\u05b0\u0006\u00c2\u0000\u0000\u05b0\u05b1\u0006\u00c2"+ - "\u001e\u0000\u05b1\u0194\u0001\u0000\u0000\u0000\u05b2\u05b3\u0003\u00a9"+ - "M\u0000\u05b3\u05b4\u0001\u0000\u0000\u0000\u05b4\u05b5\u0006\u00c3\u000b"+ - "\u0000\u05b5\u05b6\u0006\u00c3\u0000\u0000\u05b6\u05b7\u0006\u00c3\u001f"+ - "\u0000\u05b7\u0196\u0001\u0000\u0000\u0000\u05b8\u05b9\u0003[&\u0000\u05b9"+ - "\u05ba\u0001\u0000\u0000\u0000\u05ba\u05bb\u0006\u00c4\u000b\u0000\u05bb"+ - "\u05bc\u0006\u00c4\u0000\u0000\u05bc\u05bd\u0006\u00c4#\u0000\u05bd\u0198"+ - "\u0001\u0000\u0000\u0000\u05be\u05bf\u0003?\u0018\u0000\u05bf\u05c0\u0001"+ - "\u0000\u0000\u0000\u05c0\u05c1\u0006\u00c5\u000f\u0000\u05c1\u05c2\u0006"+ - "\u00c5\u000b\u0000\u05c2\u019a\u0001\u0000\u0000\u0000A\u0000\u0001\u0002"+ - "\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e\u024c\u0256\u025a"+ - "\u025d\u0266\u0268\u0273\u0286\u028b\u0294\u029b\u02a0\u02a2\u02ad\u02b5"+ - "\u02b8\u02ba\u02bf\u02c4\u02ca\u02d1\u02d6\u02dc\u02df\u02e7\u02eb\u036a"+ - "\u036f\u0376\u0378\u0388\u038d\u0392\u0394\u039a\u03e7\u03ec\u041b\u041f"+ - "\u0424\u0429\u042e\u0430\u0434\u0436\u048b\u048f\u0494\u051f\u0521$\u0005"+ - "\u0001\u0000\u0005\u0004\u0000\u0005\u0006\u0000\u0005\u0002\u0000\u0005"+ - "\u0003\u0000\u0005\b\u0000\u0005\u0005\u0000\u0005\t\u0000\u0005\u000b"+ - "\u0000\u0005\r\u0000\u0000\u0001\u0000\u0004\u0000\u0000\u0007\u0013\u0000"+ - "\u0007A\u0000\u0005\u0000\u0000\u0007\u0019\u0000\u0007B\u0000\u0007h"+ - "\u0000\u0007\"\u0000\u0007 \u0000\u0007L\u0000\u0007\u001a\u0000\u0007"+ - "$\u0000\u00070\u0000\u0007@\u0000\u0007P\u0000\u0005\n\u0000\u0005\u0007"+ - "\u0000\u0007Z\u0000\u0007Y\u0000\u0007D\u0000\u0007C\u0000\u0007X\u0000"+ - "\u0005\f\u0000\u0005\u000e\u0000\u0007\u001d\u0000"; + "\u0001\u0010\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011"+ + "\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0012"+ + "\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012"+ + "\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0013\u0004\u0013"+ + "\u0240\b\u0013\u000b\u0013\f\u0013\u0241\u0001\u0013\u0001\u0013\u0001"+ + "\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0005\u0014\u024a\b\u0014\n"+ + "\u0014\f\u0014\u024d\t\u0014\u0001\u0014\u0003\u0014\u0250\b\u0014\u0001"+ + "\u0014\u0003\u0014\u0253\b\u0014\u0001\u0014\u0001\u0014\u0001\u0015\u0001"+ + "\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0005\u0015\u025c\b\u0015\n"+ + "\u0015\f\u0015\u025f\t\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0001"+ + "\u0015\u0001\u0015\u0001\u0016\u0004\u0016\u0267\b\u0016\u000b\u0016\f"+ + "\u0016\u0268\u0001\u0016\u0001\u0016\u0001\u0017\u0001\u0017\u0001\u0017"+ + "\u0001\u0017\u0001\u0018\u0001\u0018\u0001\u0019\u0001\u0019\u0001\u001a"+ + "\u0001\u001a\u0001\u001a\u0001\u001b\u0001\u001b\u0001\u001c\u0001\u001c"+ + "\u0003\u001c\u027c\b\u001c\u0001\u001c\u0004\u001c\u027f\b\u001c\u000b"+ + "\u001c\f\u001c\u0280\u0001\u001d\u0001\u001d\u0001\u001e\u0001\u001e\u0001"+ + "\u001f\u0001\u001f\u0001\u001f\u0003\u001f\u028a\b\u001f\u0001 \u0001"+ + " \u0001!\u0001!\u0001!\u0003!\u0291\b!\u0001\"\u0001\"\u0001\"\u0005\""+ + "\u0296\b\"\n\"\f\"\u0299\t\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001"+ + "\"\u0005\"\u02a1\b\"\n\"\f\"\u02a4\t\"\u0001\"\u0001\"\u0001\"\u0001\""+ + "\u0001\"\u0003\"\u02ab\b\"\u0001\"\u0003\"\u02ae\b\"\u0003\"\u02b0\b\""+ + "\u0001#\u0004#\u02b3\b#\u000b#\f#\u02b4\u0001$\u0004$\u02b8\b$\u000b$"+ + "\f$\u02b9\u0001$\u0001$\u0005$\u02be\b$\n$\f$\u02c1\t$\u0001$\u0001$\u0004"+ + "$\u02c5\b$\u000b$\f$\u02c6\u0001$\u0004$\u02ca\b$\u000b$\f$\u02cb\u0001"+ + "$\u0001$\u0005$\u02d0\b$\n$\f$\u02d3\t$\u0003$\u02d5\b$\u0001$\u0001$"+ + "\u0001$\u0001$\u0004$\u02db\b$\u000b$\f$\u02dc\u0001$\u0001$\u0003$\u02e1"+ + "\b$\u0001%\u0001%\u0001%\u0001&\u0001&\u0001&\u0001&\u0001\'\u0001\'\u0001"+ + "\'\u0001\'\u0001(\u0001(\u0001)\u0001)\u0001)\u0001*\u0001*\u0001+\u0001"+ + "+\u0001+\u0001+\u0001+\u0001,\u0001,\u0001-\u0001-\u0001-\u0001-\u0001"+ + "-\u0001-\u0001.\u0001.\u0001.\u0001.\u0001.\u0001.\u0001/\u0001/\u0001"+ + "/\u00010\u00010\u00010\u00011\u00011\u00011\u00011\u00011\u00012\u0001"+ + "2\u00012\u00012\u00012\u00013\u00013\u00014\u00014\u00014\u00014\u0001"+ + "5\u00015\u00015\u00015\u00015\u00016\u00016\u00016\u00016\u00016\u0001"+ + "6\u00017\u00017\u00017\u00018\u00018\u00019\u00019\u00019\u00019\u0001"+ + "9\u00019\u0001:\u0001:\u0001;\u0001;\u0001;\u0001;\u0001;\u0001<\u0001"+ + "<\u0001<\u0001=\u0001=\u0001=\u0001>\u0001>\u0001>\u0001?\u0001?\u0001"+ + "@\u0001@\u0001@\u0001A\u0001A\u0001B\u0001B\u0001B\u0001C\u0001C\u0001"+ + "D\u0001D\u0001E\u0001E\u0001F\u0001F\u0001G\u0001G\u0001H\u0001H\u0001"+ + "H\u0001H\u0001H\u0001H\u0001H\u0001I\u0001I\u0001I\u0003I\u0362\bI\u0001"+ + "I\u0005I\u0365\bI\nI\fI\u0368\tI\u0001I\u0001I\u0004I\u036c\bI\u000bI"+ + "\fI\u036d\u0003I\u0370\bI\u0001J\u0001J\u0001J\u0001J\u0001J\u0001K\u0001"+ + "K\u0001K\u0001K\u0001K\u0001L\u0001L\u0005L\u037e\bL\nL\fL\u0381\tL\u0001"+ + "L\u0001L\u0003L\u0385\bL\u0001L\u0004L\u0388\bL\u000bL\fL\u0389\u0003"+ + "L\u038c\bL\u0001M\u0001M\u0004M\u0390\bM\u000bM\fM\u0391\u0001M\u0001"+ + "M\u0001N\u0001N\u0001O\u0001O\u0001O\u0001O\u0001P\u0001P\u0001P\u0001"+ + "P\u0001Q\u0001Q\u0001Q\u0001Q\u0001R\u0001R\u0001R\u0001R\u0001R\u0001"+ + "S\u0001S\u0001S\u0001S\u0001S\u0001T\u0001T\u0001T\u0001T\u0001U\u0001"+ + "U\u0001U\u0001U\u0001V\u0001V\u0001V\u0001V\u0001W\u0001W\u0001W\u0001"+ + "W\u0001W\u0001X\u0001X\u0001X\u0001X\u0001Y\u0001Y\u0001Y\u0001Y\u0001"+ + "Z\u0001Z\u0001Z\u0001Z\u0001[\u0001[\u0001[\u0001[\u0001\\\u0001\\\u0001"+ + "\\\u0001\\\u0001]\u0001]\u0001]\u0001]\u0001]\u0001]\u0001]\u0001]\u0001"+ + "]\u0001^\u0001^\u0001^\u0003^\u03df\b^\u0001_\u0004_\u03e2\b_\u000b_\f"+ + "_\u03e3\u0001`\u0001`\u0001`\u0001`\u0001a\u0001a\u0001a\u0001a\u0001"+ + "b\u0001b\u0001b\u0001b\u0001c\u0001c\u0001c\u0001c\u0001d\u0001d\u0001"+ + "d\u0001d\u0001e\u0001e\u0001e\u0001e\u0001e\u0001f\u0001f\u0001f\u0001"+ + "f\u0001g\u0001g\u0001g\u0001g\u0001h\u0001h\u0001h\u0001h\u0001i\u0001"+ + "i\u0001i\u0001i\u0001j\u0001j\u0001j\u0001j\u0003j\u0413\bj\u0001k\u0001"+ + "k\u0003k\u0417\bk\u0001k\u0005k\u041a\bk\nk\fk\u041d\tk\u0001k\u0001k"+ + "\u0003k\u0421\bk\u0001k\u0004k\u0424\bk\u000bk\fk\u0425\u0003k\u0428\b"+ + "k\u0001l\u0001l\u0004l\u042c\bl\u000bl\fl\u042d\u0001m\u0001m\u0001m\u0001"+ + "m\u0001n\u0001n\u0001n\u0001n\u0001o\u0001o\u0001o\u0001o\u0001p\u0001"+ + "p\u0001p\u0001p\u0001p\u0001q\u0001q\u0001q\u0001q\u0001r\u0001r\u0001"+ + "r\u0001r\u0001s\u0001s\u0001s\u0001s\u0001t\u0001t\u0001t\u0001t\u0001"+ + "u\u0001u\u0001u\u0001u\u0001v\u0001v\u0001v\u0001w\u0001w\u0001w\u0001"+ + "w\u0001x\u0001x\u0001x\u0001x\u0001y\u0001y\u0001y\u0001y\u0001z\u0001"+ + "z\u0001z\u0001z\u0001{\u0001{\u0001{\u0001{\u0001{\u0001|\u0001|\u0001"+ + "|\u0001|\u0001|\u0001}\u0001}\u0001}\u0001}\u0001}\u0001~\u0001~\u0001"+ + "~\u0001~\u0001~\u0001~\u0001~\u0001\u007f\u0001\u007f\u0001\u0080\u0004"+ + "\u0080\u0481\b\u0080\u000b\u0080\f\u0080\u0482\u0001\u0080\u0001\u0080"+ + "\u0003\u0080\u0487\b\u0080\u0001\u0080\u0004\u0080\u048a\b\u0080\u000b"+ + "\u0080\f\u0080\u048b\u0001\u0081\u0001\u0081\u0001\u0081\u0001\u0081\u0001"+ + "\u0082\u0001\u0082\u0001\u0082\u0001\u0082\u0001\u0083\u0001\u0083\u0001"+ + "\u0083\u0001\u0083\u0001\u0084\u0001\u0084\u0001\u0084\u0001\u0084\u0001"+ + "\u0085\u0001\u0085\u0001\u0085\u0001\u0085\u0001\u0085\u0001\u0085\u0001"+ + "\u0086\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0087\u0001\u0087\u0001"+ + "\u0087\u0001\u0087\u0001\u0088\u0001\u0088\u0001\u0088\u0001\u0088\u0001"+ + "\u0089\u0001\u0089\u0001\u0089\u0001\u0089\u0001\u008a\u0001\u008a\u0001"+ + "\u008a\u0001\u008a\u0001\u008b\u0001\u008b\u0001\u008b\u0001\u008b\u0001"+ + "\u008c\u0001\u008c\u0001\u008c\u0001\u008c\u0001\u008d\u0001\u008d\u0001"+ + "\u008d\u0001\u008d\u0001\u008e\u0001\u008e\u0001\u008e\u0001\u008e\u0001"+ + "\u008f\u0001\u008f\u0001\u008f\u0001\u008f\u0001\u0090\u0001\u0090\u0001"+ + "\u0090\u0001\u0090\u0001\u0091\u0001\u0091\u0001\u0091\u0001\u0091\u0001"+ + "\u0091\u0001\u0092\u0001\u0092\u0001\u0092\u0001\u0092\u0001\u0093\u0001"+ + "\u0093\u0001\u0093\u0001\u0093\u0001\u0094\u0001\u0094\u0001\u0094\u0001"+ + "\u0094\u0001\u0095\u0001\u0095\u0001\u0095\u0001\u0095\u0001\u0096\u0001"+ + "\u0096\u0001\u0096\u0001\u0096\u0001\u0097\u0001\u0097\u0001\u0097\u0001"+ + "\u0097\u0001\u0098\u0001\u0098\u0001\u0098\u0001\u0098\u0001\u0099\u0001"+ + "\u0099\u0001\u0099\u0001\u0099\u0001\u009a\u0001\u009a\u0001\u009a\u0001"+ + "\u009a\u0001\u009a\u0001\u009b\u0001\u009b\u0001\u009b\u0001\u009b\u0001"+ + "\u009b\u0001\u009c\u0001\u009c\u0001\u009c\u0001\u009c\u0001\u009d\u0001"+ + "\u009d\u0001\u009d\u0001\u009d\u0001\u009e\u0001\u009e\u0001\u009e\u0001"+ + "\u009e\u0001\u009f\u0001\u009f\u0001\u009f\u0001\u009f\u0001\u009f\u0001"+ + "\u00a0\u0001\u00a0\u0001\u00a1\u0001\u00a1\u0001\u00a1\u0001\u00a1\u0001"+ + "\u00a1\u0004\u00a1\u0517\b\u00a1\u000b\u00a1\f\u00a1\u0518\u0001\u00a2"+ + "\u0001\u00a2\u0001\u00a2\u0001\u00a2\u0001\u00a3\u0001\u00a3\u0001\u00a3"+ + "\u0001\u00a3\u0001\u00a4\u0001\u00a4\u0001\u00a4\u0001\u00a4\u0001\u00a5"+ + "\u0001\u00a5\u0001\u00a5\u0001\u00a5\u0001\u00a5\u0001\u00a6\u0001\u00a6"+ + "\u0001\u00a6\u0001\u00a6\u0001\u00a7\u0001\u00a7\u0001\u00a7\u0001\u00a7"+ + "\u0001\u00a8\u0001\u00a8\u0001\u00a8\u0001\u00a8\u0001\u00a9\u0001\u00a9"+ + "\u0001\u00a9\u0001\u00a9\u0001\u00a9\u0001\u00aa\u0001\u00aa\u0001\u00aa"+ + "\u0001\u00aa\u0001\u00ab\u0001\u00ab\u0001\u00ab\u0001\u00ab\u0001\u00ac"+ + "\u0001\u00ac\u0001\u00ac\u0001\u00ac\u0001\u00ad\u0001\u00ad\u0001\u00ad"+ + "\u0001\u00ad\u0001\u00ae\u0001\u00ae\u0001\u00ae\u0001\u00ae\u0001\u00af"+ + "\u0001\u00af\u0001\u00af\u0001\u00af\u0001\u00af\u0001\u00af\u0001\u00b0"+ + "\u0001\u00b0\u0001\u00b0\u0001\u00b0\u0001\u00b1\u0001\u00b1\u0001\u00b1"+ + "\u0001\u00b1\u0001\u00b2\u0001\u00b2\u0001\u00b2\u0001\u00b2\u0001\u00b3"+ + "\u0001\u00b3\u0001\u00b3\u0001\u00b3\u0001\u00b4\u0001\u00b4\u0001\u00b4"+ + "\u0001\u00b4\u0001\u00b5\u0001\u00b5\u0001\u00b5\u0001\u00b5\u0001\u00b6"+ + "\u0001\u00b6\u0001\u00b6\u0001\u00b6\u0001\u00b6\u0001\u00b7\u0001\u00b7"+ + "\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001\u00b8\u0001\u00b8"+ + "\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001\u00b9\u0001\u00b9"+ + "\u0001\u00b9\u0001\u00b9\u0001\u00ba\u0001\u00ba\u0001\u00ba\u0001\u00ba"+ + "\u0001\u00bb\u0001\u00bb\u0001\u00bb\u0001\u00bb\u0001\u00bc\u0001\u00bc"+ + "\u0001\u00bc\u0001\u00bc\u0001\u00bc\u0001\u00bc\u0001\u00bd\u0001\u00bd"+ + "\u0001\u00bd\u0001\u00bd\u0001\u00bd\u0001\u00bd\u0001\u00be\u0001\u00be"+ + "\u0001\u00be\u0001\u00be\u0001\u00bf\u0001\u00bf\u0001\u00bf\u0001\u00bf"+ + "\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c1\u0001\u00c1"+ + "\u0001\u00c1\u0001\u00c1\u0001\u00c1\u0001\u00c1\u0001\u00c2\u0001\u00c2"+ + "\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001\u00c3\u0001\u00c3"+ + "\u0001\u00c3\u0001\u00c3\u0001\u00c3\u0001\u00c3\u0001\u00c4\u0001\u00c4"+ + "\u0001\u00c4\u0001\u00c4\u0001\u00c4\u0002\u025d\u02a2\u0000\u00c5\u000f"+ + "\u0001\u0011\u0002\u0013\u0003\u0015\u0004\u0017\u0005\u0019\u0006\u001b"+ + "\u0007\u001d\b\u001f\t!\n#\u000b%\f\'\r)\u000e+\u000f-\u0010/\u00111\u0012"+ + "3\u00135\u00147\u00159\u0016;\u0017=\u0018?\u0000A\u0000C\u0000E\u0000"+ + "G\u0000I\u0000K\u0000M\u0000O\u0000Q\u0000S\u0019U\u001aW\u001bY\u001c"+ + "[\u001d]\u001e_\u001fa c!e\"g#i$k%m&o\'q(s)u*w+y,{-}.\u007f/\u00810\u0083"+ + "1\u00852\u00873\u00894\u008b5\u008d6\u008f7\u00918\u00939\u0095:\u0097"+ + ";\u0099<\u009b=\u009d>\u009f?\u00a1@\u00a3A\u00a5B\u00a7C\u00a9\u0000"+ + "\u00abD\u00adE\u00afF\u00b1G\u00b3\u0000\u00b5\u0000\u00b7H\u00b9I\u00bb"+ + "J\u00bd\u0000\u00bf\u0000\u00c1\u0000\u00c3\u0000\u00c5\u0000\u00c7\u0000"+ + "\u00c9K\u00cb\u0000\u00cdL\u00cf\u0000\u00d1\u0000\u00d3M\u00d5N\u00d7"+ + "O\u00d9\u0000\u00db\u0000\u00dd\u0000\u00df\u0000\u00e1\u0000\u00e3\u0000"+ + "\u00e5\u0000\u00e7P\u00e9Q\u00ebR\u00edS\u00ef\u0000\u00f1\u0000\u00f3"+ + "\u0000\u00f5\u0000\u00f7\u0000\u00f9\u0000\u00fbT\u00fd\u0000\u00ffU\u0101"+ + "V\u0103W\u0105\u0000\u0107\u0000\u0109X\u010bY\u010d\u0000\u010fZ\u0111"+ + "\u0000\u0113[\u0115\\\u0117]\u0119\u0000\u011b\u0000\u011d\u0000\u011f"+ + "\u0000\u0121\u0000\u0123\u0000\u0125\u0000\u0127\u0000\u0129\u0000\u012b"+ + "^\u012d_\u012f`\u0131\u0000\u0133\u0000\u0135\u0000\u0137\u0000\u0139"+ + "\u0000\u013b\u0000\u013da\u013fb\u0141c\u0143\u0000\u0145d\u0147e\u0149"+ + "f\u014bg\u014d\u0000\u014fh\u0151i\u0153j\u0155k\u0157l\u0159\u0000\u015b"+ + "\u0000\u015d\u0000\u015f\u0000\u0161\u0000\u0163\u0000\u0165\u0000\u0167"+ + "m\u0169n\u016bo\u016d\u0000\u016f\u0000\u0171\u0000\u0173\u0000\u0175"+ + "p\u0177q\u0179r\u017b\u0000\u017d\u0000\u017f\u0000\u0181s\u0183t\u0185"+ + "u\u0187\u0000\u0189\u0000\u018bv\u018dw\u018fx\u0191\u0000\u0193\u0000"+ + "\u0195\u0000\u0197\u0000\u000f\u0000\u0001\u0002\u0003\u0004\u0005\u0006"+ + "\u0007\b\t\n\u000b\f\r\u000e#\u0002\u0000DDdd\u0002\u0000IIii\u0002\u0000"+ + "SSss\u0002\u0000EEee\u0002\u0000CCcc\u0002\u0000TTtt\u0002\u0000RRrr\u0002"+ + "\u0000OOoo\u0002\u0000PPpp\u0002\u0000NNnn\u0002\u0000HHhh\u0002\u0000"+ + "VVvv\u0002\u0000AAaa\u0002\u0000LLll\u0002\u0000XXxx\u0002\u0000FFff\u0002"+ + "\u0000MMmm\u0002\u0000GGgg\u0002\u0000KKkk\u0002\u0000WWww\u0002\u0000"+ + "UUuu\u0006\u0000\t\n\r\r //[[]]\u0002\u0000\n\n\r\r\u0003\u0000\t\n\r"+ + "\r \u0001\u000009\u0002\u0000AZaz\b\u0000\"\"NNRRTT\\\\nnrrtt\u0004\u0000"+ + "\n\n\r\r\"\"\\\\\u0002\u0000++--\u0001\u0000``\u0002\u0000BBbb\u0002\u0000"+ + "YYyy\u000b\u0000\t\n\r\r \"\",,//::==[[]]||\u0002\u0000**//\u000b\u0000"+ + "\t\n\r\r \"#,,//::<<>?\\\\||\u05d6\u0000\u000f\u0001\u0000\u0000\u0000"+ + "\u0000\u0011\u0001\u0000\u0000\u0000\u0000\u0013\u0001\u0000\u0000\u0000"+ + "\u0000\u0015\u0001\u0000\u0000\u0000\u0000\u0017\u0001\u0000\u0000\u0000"+ + "\u0000\u0019\u0001\u0000\u0000\u0000\u0000\u001b\u0001\u0000\u0000\u0000"+ + "\u0000\u001d\u0001\u0000\u0000\u0000\u0000\u001f\u0001\u0000\u0000\u0000"+ + "\u0000!\u0001\u0000\u0000\u0000\u0000#\u0001\u0000\u0000\u0000\u0000%"+ + "\u0001\u0000\u0000\u0000\u0000\'\u0001\u0000\u0000\u0000\u0000)\u0001"+ + "\u0000\u0000\u0000\u0000+\u0001\u0000\u0000\u0000\u0000-\u0001\u0000\u0000"+ + "\u0000\u0000/\u0001\u0000\u0000\u0000\u00001\u0001\u0000\u0000\u0000\u0000"+ + "3\u0001\u0000\u0000\u0000\u00005\u0001\u0000\u0000\u0000\u00007\u0001"+ + "\u0000\u0000\u0000\u00009\u0001\u0000\u0000\u0000\u0000;\u0001\u0000\u0000"+ + "\u0000\u0001=\u0001\u0000\u0000\u0000\u0001S\u0001\u0000\u0000\u0000\u0001"+ + "U\u0001\u0000\u0000\u0000\u0001W\u0001\u0000\u0000\u0000\u0001Y\u0001"+ + "\u0000\u0000\u0000\u0001[\u0001\u0000\u0000\u0000\u0001]\u0001\u0000\u0000"+ + "\u0000\u0001_\u0001\u0000\u0000\u0000\u0001a\u0001\u0000\u0000\u0000\u0001"+ + "c\u0001\u0000\u0000\u0000\u0001e\u0001\u0000\u0000\u0000\u0001g\u0001"+ + "\u0000\u0000\u0000\u0001i\u0001\u0000\u0000\u0000\u0001k\u0001\u0000\u0000"+ + "\u0000\u0001m\u0001\u0000\u0000\u0000\u0001o\u0001\u0000\u0000\u0000\u0001"+ + "q\u0001\u0000\u0000\u0000\u0001s\u0001\u0000\u0000\u0000\u0001u\u0001"+ + "\u0000\u0000\u0000\u0001w\u0001\u0000\u0000\u0000\u0001y\u0001\u0000\u0000"+ + "\u0000\u0001{\u0001\u0000\u0000\u0000\u0001}\u0001\u0000\u0000\u0000\u0001"+ + "\u007f\u0001\u0000\u0000\u0000\u0001\u0081\u0001\u0000\u0000\u0000\u0001"+ + "\u0083\u0001\u0000\u0000\u0000\u0001\u0085\u0001\u0000\u0000\u0000\u0001"+ + "\u0087\u0001\u0000\u0000\u0000\u0001\u0089\u0001\u0000\u0000\u0000\u0001"+ + "\u008b\u0001\u0000\u0000\u0000\u0001\u008d\u0001\u0000\u0000\u0000\u0001"+ + "\u008f\u0001\u0000\u0000\u0000\u0001\u0091\u0001\u0000\u0000\u0000\u0001"+ + "\u0093\u0001\u0000\u0000\u0000\u0001\u0095\u0001\u0000\u0000\u0000\u0001"+ + "\u0097\u0001\u0000\u0000\u0000\u0001\u0099\u0001\u0000\u0000\u0000\u0001"+ + "\u009b\u0001\u0000\u0000\u0000\u0001\u009d\u0001\u0000\u0000\u0000\u0001"+ + "\u009f\u0001\u0000\u0000\u0000\u0001\u00a1\u0001\u0000\u0000\u0000\u0001"+ + "\u00a3\u0001\u0000\u0000\u0000\u0001\u00a5\u0001\u0000\u0000\u0000\u0001"+ + "\u00a7\u0001\u0000\u0000\u0000\u0001\u00ab\u0001\u0000\u0000\u0000\u0001"+ + "\u00ad\u0001\u0000\u0000\u0000\u0001\u00af\u0001\u0000\u0000\u0000\u0001"+ + "\u00b1\u0001\u0000\u0000\u0000\u0002\u00b3\u0001\u0000\u0000\u0000\u0002"+ + "\u00b5\u0001\u0000\u0000\u0000\u0002\u00b7\u0001\u0000\u0000\u0000\u0002"+ + "\u00b9\u0001\u0000\u0000\u0000\u0002\u00bb\u0001\u0000\u0000\u0000\u0003"+ + "\u00bd\u0001\u0000\u0000\u0000\u0003\u00bf\u0001\u0000\u0000\u0000\u0003"+ + "\u00c1\u0001\u0000\u0000\u0000\u0003\u00c3\u0001\u0000\u0000\u0000\u0003"+ + "\u00c5\u0001\u0000\u0000\u0000\u0003\u00c7\u0001\u0000\u0000\u0000\u0003"+ + "\u00c9\u0001\u0000\u0000\u0000\u0003\u00cd\u0001\u0000\u0000\u0000\u0003"+ + "\u00cf\u0001\u0000\u0000\u0000\u0003\u00d1\u0001\u0000\u0000\u0000\u0003"+ + "\u00d3\u0001\u0000\u0000\u0000\u0003\u00d5\u0001\u0000\u0000\u0000\u0003"+ + "\u00d7\u0001\u0000\u0000\u0000\u0004\u00d9\u0001\u0000\u0000\u0000\u0004"+ + "\u00db\u0001\u0000\u0000\u0000\u0004\u00dd\u0001\u0000\u0000\u0000\u0004"+ + "\u00df\u0001\u0000\u0000\u0000\u0004\u00e1\u0001\u0000\u0000\u0000\u0004"+ + "\u00e7\u0001\u0000\u0000\u0000\u0004\u00e9\u0001\u0000\u0000\u0000\u0004"+ + "\u00eb\u0001\u0000\u0000\u0000\u0004\u00ed\u0001\u0000\u0000\u0000\u0005"+ + "\u00ef\u0001\u0000\u0000\u0000\u0005\u00f1\u0001\u0000\u0000\u0000\u0005"+ + "\u00f3\u0001\u0000\u0000\u0000\u0005\u00f5\u0001\u0000\u0000\u0000\u0005"+ + "\u00f7\u0001\u0000\u0000\u0000\u0005\u00f9\u0001\u0000\u0000\u0000\u0005"+ + "\u00fb\u0001\u0000\u0000\u0000\u0005\u00fd\u0001\u0000\u0000\u0000\u0005"+ + "\u00ff\u0001\u0000\u0000\u0000\u0005\u0101\u0001\u0000\u0000\u0000\u0005"+ + "\u0103\u0001\u0000\u0000\u0000\u0006\u0105\u0001\u0000\u0000\u0000\u0006"+ + "\u0107\u0001\u0000\u0000\u0000\u0006\u0109\u0001\u0000\u0000\u0000\u0006"+ + "\u010b\u0001\u0000\u0000\u0000\u0006\u010f\u0001\u0000\u0000\u0000\u0006"+ + "\u0111\u0001\u0000\u0000\u0000\u0006\u0113\u0001\u0000\u0000\u0000\u0006"+ + "\u0115\u0001\u0000\u0000\u0000\u0006\u0117\u0001\u0000\u0000\u0000\u0007"+ + "\u0119\u0001\u0000\u0000\u0000\u0007\u011b\u0001\u0000\u0000\u0000\u0007"+ + "\u011d\u0001\u0000\u0000\u0000\u0007\u011f\u0001\u0000\u0000\u0000\u0007"+ + "\u0121\u0001\u0000\u0000\u0000\u0007\u0123\u0001\u0000\u0000\u0000\u0007"+ + "\u0125\u0001\u0000\u0000\u0000\u0007\u0127\u0001\u0000\u0000\u0000\u0007"+ + "\u0129\u0001\u0000\u0000\u0000\u0007\u012b\u0001\u0000\u0000\u0000\u0007"+ + "\u012d\u0001\u0000\u0000\u0000\u0007\u012f\u0001\u0000\u0000\u0000\b\u0131"+ + "\u0001\u0000\u0000\u0000\b\u0133\u0001\u0000\u0000\u0000\b\u0135\u0001"+ + "\u0000\u0000\u0000\b\u0137\u0001\u0000\u0000\u0000\b\u0139\u0001\u0000"+ + "\u0000\u0000\b\u013b\u0001\u0000\u0000\u0000\b\u013d\u0001\u0000\u0000"+ + "\u0000\b\u013f\u0001\u0000\u0000\u0000\b\u0141\u0001\u0000\u0000\u0000"+ + "\t\u0143\u0001\u0000\u0000\u0000\t\u0145\u0001\u0000\u0000\u0000\t\u0147"+ + "\u0001\u0000\u0000\u0000\t\u0149\u0001\u0000\u0000\u0000\t\u014b\u0001"+ + "\u0000\u0000\u0000\n\u014d\u0001\u0000\u0000\u0000\n\u014f\u0001\u0000"+ + "\u0000\u0000\n\u0151\u0001\u0000\u0000\u0000\n\u0153\u0001\u0000\u0000"+ + "\u0000\n\u0155\u0001\u0000\u0000\u0000\n\u0157\u0001\u0000\u0000\u0000"+ + "\u000b\u0159\u0001\u0000\u0000\u0000\u000b\u015b\u0001\u0000\u0000\u0000"+ + "\u000b\u015d\u0001\u0000\u0000\u0000\u000b\u015f\u0001\u0000\u0000\u0000"+ + "\u000b\u0161\u0001\u0000\u0000\u0000\u000b\u0163\u0001\u0000\u0000\u0000"+ + "\u000b\u0165\u0001\u0000\u0000\u0000\u000b\u0167\u0001\u0000\u0000\u0000"+ + "\u000b\u0169\u0001\u0000\u0000\u0000\u000b\u016b\u0001\u0000\u0000\u0000"+ + "\f\u016d\u0001\u0000\u0000\u0000\f\u016f\u0001\u0000\u0000\u0000\f\u0171"+ + "\u0001\u0000\u0000\u0000\f\u0173\u0001\u0000\u0000\u0000\f\u0175\u0001"+ + "\u0000\u0000\u0000\f\u0177\u0001\u0000\u0000\u0000\f\u0179\u0001\u0000"+ + "\u0000\u0000\r\u017b\u0001\u0000\u0000\u0000\r\u017d\u0001\u0000\u0000"+ + "\u0000\r\u017f\u0001\u0000\u0000\u0000\r\u0181\u0001\u0000\u0000\u0000"+ + "\r\u0183\u0001\u0000\u0000\u0000\r\u0185\u0001\u0000\u0000\u0000\u000e"+ + "\u0187\u0001\u0000\u0000\u0000\u000e\u0189\u0001\u0000\u0000\u0000\u000e"+ + "\u018b\u0001\u0000\u0000\u0000\u000e\u018d\u0001\u0000\u0000\u0000\u000e"+ + "\u018f\u0001\u0000\u0000\u0000\u000e\u0191\u0001\u0000\u0000\u0000\u000e"+ + "\u0193\u0001\u0000\u0000\u0000\u000e\u0195\u0001\u0000\u0000\u0000\u000e"+ + "\u0197\u0001\u0000\u0000\u0000\u000f\u0199\u0001\u0000\u0000\u0000\u0011"+ + "\u01a3\u0001\u0000\u0000\u0000\u0013\u01aa\u0001\u0000\u0000\u0000\u0015"+ + "\u01b3\u0001\u0000\u0000\u0000\u0017\u01ba\u0001\u0000\u0000\u0000\u0019"+ + "\u01c4\u0001\u0000\u0000\u0000\u001b\u01cb\u0001\u0000\u0000\u0000\u001d"+ + "\u01d2\u0001\u0000\u0000\u0000\u001f\u01d9\u0001\u0000\u0000\u0000!\u01e1"+ + "\u0001\u0000\u0000\u0000#\u01ed\u0001\u0000\u0000\u0000%\u01f6\u0001\u0000"+ + "\u0000\u0000\'\u01fc\u0001\u0000\u0000\u0000)\u0203\u0001\u0000\u0000"+ + "\u0000+\u020a\u0001\u0000\u0000\u0000-\u0212\u0001\u0000\u0000\u0000/"+ + "\u021a\u0001\u0000\u0000\u00001\u0229\u0001\u0000\u0000\u00003\u0233\u0001"+ + "\u0000\u0000\u00005\u023f\u0001\u0000\u0000\u00007\u0245\u0001\u0000\u0000"+ + "\u00009\u0256\u0001\u0000\u0000\u0000;\u0266\u0001\u0000\u0000\u0000="+ + "\u026c\u0001\u0000\u0000\u0000?\u0270\u0001\u0000\u0000\u0000A\u0272\u0001"+ + "\u0000\u0000\u0000C\u0274\u0001\u0000\u0000\u0000E\u0277\u0001\u0000\u0000"+ + "\u0000G\u0279\u0001\u0000\u0000\u0000I\u0282\u0001\u0000\u0000\u0000K"+ + "\u0284\u0001\u0000\u0000\u0000M\u0289\u0001\u0000\u0000\u0000O\u028b\u0001"+ + "\u0000\u0000\u0000Q\u0290\u0001\u0000\u0000\u0000S\u02af\u0001\u0000\u0000"+ + "\u0000U\u02b2\u0001\u0000\u0000\u0000W\u02e0\u0001\u0000\u0000\u0000Y"+ + "\u02e2\u0001\u0000\u0000\u0000[\u02e5\u0001\u0000\u0000\u0000]\u02e9\u0001"+ + "\u0000\u0000\u0000_\u02ed\u0001\u0000\u0000\u0000a\u02ef\u0001\u0000\u0000"+ + "\u0000c\u02f2\u0001\u0000\u0000\u0000e\u02f4\u0001\u0000\u0000\u0000g"+ + "\u02f9\u0001\u0000\u0000\u0000i\u02fb\u0001\u0000\u0000\u0000k\u0301\u0001"+ + "\u0000\u0000\u0000m\u0307\u0001\u0000\u0000\u0000o\u030a\u0001\u0000\u0000"+ + "\u0000q\u030d\u0001\u0000\u0000\u0000s\u0312\u0001\u0000\u0000\u0000u"+ + "\u0317\u0001\u0000\u0000\u0000w\u0319\u0001\u0000\u0000\u0000y\u031d\u0001"+ + "\u0000\u0000\u0000{\u0322\u0001\u0000\u0000\u0000}\u0328\u0001\u0000\u0000"+ + "\u0000\u007f\u032b\u0001\u0000\u0000\u0000\u0081\u032d\u0001\u0000\u0000"+ + "\u0000\u0083\u0333\u0001\u0000\u0000\u0000\u0085\u0335\u0001\u0000\u0000"+ + "\u0000\u0087\u033a\u0001\u0000\u0000\u0000\u0089\u033d\u0001\u0000\u0000"+ + "\u0000\u008b\u0340\u0001\u0000\u0000\u0000\u008d\u0343\u0001\u0000\u0000"+ + "\u0000\u008f\u0345\u0001\u0000\u0000\u0000\u0091\u0348\u0001\u0000\u0000"+ + "\u0000\u0093\u034a\u0001\u0000\u0000\u0000\u0095\u034d\u0001\u0000\u0000"+ + "\u0000\u0097\u034f\u0001\u0000\u0000\u0000\u0099\u0351\u0001\u0000\u0000"+ + "\u0000\u009b\u0353\u0001\u0000\u0000\u0000\u009d\u0355\u0001\u0000\u0000"+ + "\u0000\u009f\u0357\u0001\u0000\u0000\u0000\u00a1\u036f\u0001\u0000\u0000"+ + "\u0000\u00a3\u0371\u0001\u0000\u0000\u0000\u00a5\u0376\u0001\u0000\u0000"+ + "\u0000\u00a7\u038b\u0001\u0000\u0000\u0000\u00a9\u038d\u0001\u0000\u0000"+ + "\u0000\u00ab\u0395\u0001\u0000\u0000\u0000\u00ad\u0397\u0001\u0000\u0000"+ + "\u0000\u00af\u039b\u0001\u0000\u0000\u0000\u00b1\u039f\u0001\u0000\u0000"+ + "\u0000\u00b3\u03a3\u0001\u0000\u0000\u0000\u00b5\u03a8\u0001\u0000\u0000"+ + "\u0000\u00b7\u03ad\u0001\u0000\u0000\u0000\u00b9\u03b1\u0001\u0000\u0000"+ + "\u0000\u00bb\u03b5\u0001\u0000\u0000\u0000\u00bd\u03b9\u0001\u0000\u0000"+ + "\u0000\u00bf\u03be\u0001\u0000\u0000\u0000\u00c1\u03c2\u0001\u0000\u0000"+ + "\u0000\u00c3\u03c6\u0001\u0000\u0000\u0000\u00c5\u03ca\u0001\u0000\u0000"+ + "\u0000\u00c7\u03ce\u0001\u0000\u0000\u0000\u00c9\u03d2\u0001\u0000\u0000"+ + "\u0000\u00cb\u03de\u0001\u0000\u0000\u0000\u00cd\u03e1\u0001\u0000\u0000"+ + "\u0000\u00cf\u03e5\u0001\u0000\u0000\u0000\u00d1\u03e9\u0001\u0000\u0000"+ + "\u0000\u00d3\u03ed\u0001\u0000\u0000\u0000\u00d5\u03f1\u0001\u0000\u0000"+ + "\u0000\u00d7\u03f5\u0001\u0000\u0000\u0000\u00d9\u03f9\u0001\u0000\u0000"+ + "\u0000\u00db\u03fe\u0001\u0000\u0000\u0000\u00dd\u0402\u0001\u0000\u0000"+ + "\u0000\u00df\u0406\u0001\u0000\u0000\u0000\u00e1\u040a\u0001\u0000\u0000"+ + "\u0000\u00e3\u0412\u0001\u0000\u0000\u0000\u00e5\u0427\u0001\u0000\u0000"+ + "\u0000\u00e7\u042b\u0001\u0000\u0000\u0000\u00e9\u042f\u0001\u0000\u0000"+ + "\u0000\u00eb\u0433\u0001\u0000\u0000\u0000\u00ed\u0437\u0001\u0000\u0000"+ + "\u0000\u00ef\u043b\u0001\u0000\u0000\u0000\u00f1\u0440\u0001\u0000\u0000"+ + "\u0000\u00f3\u0444\u0001\u0000\u0000\u0000\u00f5\u0448\u0001\u0000\u0000"+ + "\u0000\u00f7\u044c\u0001\u0000\u0000\u0000\u00f9\u0450\u0001\u0000\u0000"+ + "\u0000\u00fb\u0454\u0001\u0000\u0000\u0000\u00fd\u0457\u0001\u0000\u0000"+ + "\u0000\u00ff\u045b\u0001\u0000\u0000\u0000\u0101\u045f\u0001\u0000\u0000"+ + "\u0000\u0103\u0463\u0001\u0000\u0000\u0000\u0105\u0467\u0001\u0000\u0000"+ + "\u0000\u0107\u046c\u0001\u0000\u0000\u0000\u0109\u0471\u0001\u0000\u0000"+ + "\u0000\u010b\u0476\u0001\u0000\u0000\u0000\u010d\u047d\u0001\u0000\u0000"+ + "\u0000\u010f\u0486\u0001\u0000\u0000\u0000\u0111\u048d\u0001\u0000\u0000"+ + "\u0000\u0113\u0491\u0001\u0000\u0000\u0000\u0115\u0495\u0001\u0000\u0000"+ + "\u0000\u0117\u0499\u0001\u0000\u0000\u0000\u0119\u049d\u0001\u0000\u0000"+ + "\u0000\u011b\u04a3\u0001\u0000\u0000\u0000\u011d\u04a7\u0001\u0000\u0000"+ + "\u0000\u011f\u04ab\u0001\u0000\u0000\u0000\u0121\u04af\u0001\u0000\u0000"+ + "\u0000\u0123\u04b3\u0001\u0000\u0000\u0000\u0125\u04b7\u0001\u0000\u0000"+ + "\u0000\u0127\u04bb\u0001\u0000\u0000\u0000\u0129\u04bf\u0001\u0000\u0000"+ + "\u0000\u012b\u04c3\u0001\u0000\u0000\u0000\u012d\u04c7\u0001\u0000\u0000"+ + "\u0000\u012f\u04cb\u0001\u0000\u0000\u0000\u0131\u04cf\u0001\u0000\u0000"+ + "\u0000\u0133\u04d4\u0001\u0000\u0000\u0000\u0135\u04d8\u0001\u0000\u0000"+ + "\u0000\u0137\u04dc\u0001\u0000\u0000\u0000\u0139\u04e0\u0001\u0000\u0000"+ + "\u0000\u013b\u04e4\u0001\u0000\u0000\u0000\u013d\u04e8\u0001\u0000\u0000"+ + "\u0000\u013f\u04ec\u0001\u0000\u0000\u0000\u0141\u04f0\u0001\u0000\u0000"+ + "\u0000\u0143\u04f4\u0001\u0000\u0000\u0000\u0145\u04f9\u0001\u0000\u0000"+ + "\u0000\u0147\u04fe\u0001\u0000\u0000\u0000\u0149\u0502\u0001\u0000\u0000"+ + "\u0000\u014b\u0506\u0001\u0000\u0000\u0000\u014d\u050a\u0001\u0000\u0000"+ + "\u0000\u014f\u050f\u0001\u0000\u0000\u0000\u0151\u0516\u0001\u0000\u0000"+ + "\u0000\u0153\u051a\u0001\u0000\u0000\u0000\u0155\u051e\u0001\u0000\u0000"+ + "\u0000\u0157\u0522\u0001\u0000\u0000\u0000\u0159\u0526\u0001\u0000\u0000"+ + "\u0000\u015b\u052b\u0001\u0000\u0000\u0000\u015d\u052f\u0001\u0000\u0000"+ + "\u0000\u015f\u0533\u0001\u0000\u0000\u0000\u0161\u0537\u0001\u0000\u0000"+ + "\u0000\u0163\u053c\u0001\u0000\u0000\u0000\u0165\u0540\u0001\u0000\u0000"+ + "\u0000\u0167\u0544\u0001\u0000\u0000\u0000\u0169\u0548\u0001\u0000\u0000"+ + "\u0000\u016b\u054c\u0001\u0000\u0000\u0000\u016d\u0550\u0001\u0000\u0000"+ + "\u0000\u016f\u0556\u0001\u0000\u0000\u0000\u0171\u055a\u0001\u0000\u0000"+ + "\u0000\u0173\u055e\u0001\u0000\u0000\u0000\u0175\u0562\u0001\u0000\u0000"+ + "\u0000\u0177\u0566\u0001\u0000\u0000\u0000\u0179\u056a\u0001\u0000\u0000"+ + "\u0000\u017b\u056e\u0001\u0000\u0000\u0000\u017d\u0573\u0001\u0000\u0000"+ + "\u0000\u017f\u0579\u0001\u0000\u0000\u0000\u0181\u057f\u0001\u0000\u0000"+ + "\u0000\u0183\u0583\u0001\u0000\u0000\u0000\u0185\u0587\u0001\u0000\u0000"+ + "\u0000\u0187\u058b\u0001\u0000\u0000\u0000\u0189\u0591\u0001\u0000\u0000"+ + "\u0000\u018b\u0597\u0001\u0000\u0000\u0000\u018d\u059b\u0001\u0000\u0000"+ + "\u0000\u018f\u059f\u0001\u0000\u0000\u0000\u0191\u05a3\u0001\u0000\u0000"+ + "\u0000\u0193\u05a9\u0001\u0000\u0000\u0000\u0195\u05af\u0001\u0000\u0000"+ + "\u0000\u0197\u05b5\u0001\u0000\u0000\u0000\u0199\u019a\u0007\u0000\u0000"+ + "\u0000\u019a\u019b\u0007\u0001\u0000\u0000\u019b\u019c\u0007\u0002\u0000"+ + "\u0000\u019c\u019d\u0007\u0002\u0000\u0000\u019d\u019e\u0007\u0003\u0000"+ + "\u0000\u019e\u019f\u0007\u0004\u0000\u0000\u019f\u01a0\u0007\u0005\u0000"+ + "\u0000\u01a0\u01a1\u0001\u0000\u0000\u0000\u01a1\u01a2\u0006\u0000\u0000"+ + "\u0000\u01a2\u0010\u0001\u0000\u0000\u0000\u01a3\u01a4\u0007\u0000\u0000"+ + "\u0000\u01a4\u01a5\u0007\u0006\u0000\u0000\u01a5\u01a6\u0007\u0007\u0000"+ + "\u0000\u01a6\u01a7\u0007\b\u0000\u0000\u01a7\u01a8\u0001\u0000\u0000\u0000"+ + "\u01a8\u01a9\u0006\u0001\u0001\u0000\u01a9\u0012\u0001\u0000\u0000\u0000"+ + "\u01aa\u01ab\u0007\u0003\u0000\u0000\u01ab\u01ac\u0007\t\u0000\u0000\u01ac"+ + "\u01ad\u0007\u0006\u0000\u0000\u01ad\u01ae\u0007\u0001\u0000\u0000\u01ae"+ + "\u01af\u0007\u0004\u0000\u0000\u01af\u01b0\u0007\n\u0000\u0000\u01b0\u01b1"+ + "\u0001\u0000\u0000\u0000\u01b1\u01b2\u0006\u0002\u0002\u0000\u01b2\u0014"+ + "\u0001\u0000\u0000\u0000\u01b3\u01b4\u0007\u0003\u0000\u0000\u01b4\u01b5"+ + "\u0007\u000b\u0000\u0000\u01b5\u01b6\u0007\f\u0000\u0000\u01b6\u01b7\u0007"+ + "\r\u0000\u0000\u01b7\u01b8\u0001\u0000\u0000\u0000\u01b8\u01b9\u0006\u0003"+ + "\u0000\u0000\u01b9\u0016\u0001\u0000\u0000\u0000\u01ba\u01bb\u0007\u0003"+ + "\u0000\u0000\u01bb\u01bc\u0007\u000e\u0000\u0000\u01bc\u01bd\u0007\b\u0000"+ + "\u0000\u01bd\u01be\u0007\r\u0000\u0000\u01be\u01bf\u0007\f\u0000\u0000"+ + "\u01bf\u01c0\u0007\u0001\u0000\u0000\u01c0\u01c1\u0007\t\u0000\u0000\u01c1"+ + "\u01c2\u0001\u0000\u0000\u0000\u01c2\u01c3\u0006\u0004\u0003\u0000\u01c3"+ + "\u0018\u0001\u0000\u0000\u0000\u01c4\u01c5\u0007\u000f\u0000\u0000\u01c5"+ + "\u01c6\u0007\u0006\u0000\u0000\u01c6\u01c7\u0007\u0007\u0000\u0000\u01c7"+ + "\u01c8\u0007\u0010\u0000\u0000\u01c8\u01c9\u0001\u0000\u0000\u0000\u01c9"+ + "\u01ca\u0006\u0005\u0004\u0000\u01ca\u001a\u0001\u0000\u0000\u0000\u01cb"+ + "\u01cc\u0007\u0011\u0000\u0000\u01cc\u01cd\u0007\u0006\u0000\u0000\u01cd"+ + "\u01ce\u0007\u0007\u0000\u0000\u01ce\u01cf\u0007\u0012\u0000\u0000\u01cf"+ + "\u01d0\u0001\u0000\u0000\u0000\u01d0\u01d1\u0006\u0006\u0000\u0000\u01d1"+ + "\u001c\u0001\u0000\u0000\u0000\u01d2\u01d3\u0007\u0012\u0000\u0000\u01d3"+ + "\u01d4\u0007\u0003\u0000\u0000\u01d4\u01d5\u0007\u0003\u0000\u0000\u01d5"+ + "\u01d6\u0007\b\u0000\u0000\u01d6\u01d7\u0001\u0000\u0000\u0000\u01d7\u01d8"+ + "\u0006\u0007\u0001\u0000\u01d8\u001e\u0001\u0000\u0000\u0000\u01d9\u01da"+ + "\u0007\r\u0000\u0000\u01da\u01db\u0007\u0001\u0000\u0000\u01db\u01dc\u0007"+ + "\u0010\u0000\u0000\u01dc\u01dd\u0007\u0001\u0000\u0000\u01dd\u01de\u0007"+ + "\u0005\u0000\u0000\u01de\u01df\u0001\u0000\u0000\u0000\u01df\u01e0\u0006"+ + "\b\u0000\u0000\u01e0 \u0001\u0000\u0000\u0000\u01e1\u01e2\u0007\u0010"+ + "\u0000\u0000\u01e2\u01e3\u0007\u000b\u0000\u0000\u01e3\u01e4\u0005_\u0000"+ + "\u0000\u01e4\u01e5\u0007\u0003\u0000\u0000\u01e5\u01e6\u0007\u000e\u0000"+ + "\u0000\u01e6\u01e7\u0007\b\u0000\u0000\u01e7\u01e8\u0007\f\u0000\u0000"+ + "\u01e8\u01e9\u0007\t\u0000\u0000\u01e9\u01ea\u0007\u0000\u0000\u0000\u01ea"+ + "\u01eb\u0001\u0000\u0000\u0000\u01eb\u01ec\u0006\t\u0005\u0000\u01ec\""+ + "\u0001\u0000\u0000\u0000\u01ed\u01ee\u0007\u0006\u0000\u0000\u01ee\u01ef"+ + "\u0007\u0003\u0000\u0000\u01ef\u01f0\u0007\t\u0000\u0000\u01f0\u01f1\u0007"+ + "\f\u0000\u0000\u01f1\u01f2\u0007\u0010\u0000\u0000\u01f2\u01f3\u0007\u0003"+ + "\u0000\u0000\u01f3\u01f4\u0001\u0000\u0000\u0000\u01f4\u01f5\u0006\n\u0006"+ + "\u0000\u01f5$\u0001\u0000\u0000\u0000\u01f6\u01f7\u0007\u0006\u0000\u0000"+ + "\u01f7\u01f8\u0007\u0007\u0000\u0000\u01f8\u01f9\u0007\u0013\u0000\u0000"+ + "\u01f9\u01fa\u0001\u0000\u0000\u0000\u01fa\u01fb\u0006\u000b\u0000\u0000"+ + "\u01fb&\u0001\u0000\u0000\u0000\u01fc\u01fd\u0007\u0002\u0000\u0000\u01fd"+ + "\u01fe\u0007\n\u0000\u0000\u01fe\u01ff\u0007\u0007\u0000\u0000\u01ff\u0200"+ + "\u0007\u0013\u0000\u0000\u0200\u0201\u0001\u0000\u0000\u0000\u0201\u0202"+ + "\u0006\f\u0007\u0000\u0202(\u0001\u0000\u0000\u0000\u0203\u0204\u0007"+ + "\u0002\u0000\u0000\u0204\u0205\u0007\u0007\u0000\u0000\u0205\u0206\u0007"+ + "\u0006\u0000\u0000\u0206\u0207\u0007\u0005\u0000\u0000\u0207\u0208\u0001"+ + "\u0000\u0000\u0000\u0208\u0209\u0006\r\u0000\u0000\u0209*\u0001\u0000"+ + "\u0000\u0000\u020a\u020b\u0007\u0002\u0000\u0000\u020b\u020c\u0007\u0005"+ + "\u0000\u0000\u020c\u020d\u0007\f\u0000\u0000\u020d\u020e\u0007\u0005\u0000"+ + "\u0000\u020e\u020f\u0007\u0002\u0000\u0000\u020f\u0210\u0001\u0000\u0000"+ + "\u0000\u0210\u0211\u0006\u000e\u0000\u0000\u0211,\u0001\u0000\u0000\u0000"+ + "\u0212\u0213\u0007\u0013\u0000\u0000\u0213\u0214\u0007\n\u0000\u0000\u0214"+ + "\u0215\u0007\u0003\u0000\u0000\u0215\u0216\u0007\u0006\u0000\u0000\u0216"+ + "\u0217\u0007\u0003\u0000\u0000\u0217\u0218\u0001\u0000\u0000\u0000\u0218"+ + "\u0219\u0006\u000f\u0000\u0000\u0219.\u0001\u0000\u0000\u0000\u021a\u021b"+ + "\u0004\u0010\u0000\u0000\u021b\u021c\u0007\u0001\u0000\u0000\u021c\u021d"+ + "\u0007\t\u0000\u0000\u021d\u021e\u0007\r\u0000\u0000\u021e\u021f\u0007"+ + "\u0001\u0000\u0000\u021f\u0220\u0007\t\u0000\u0000\u0220\u0221\u0007\u0003"+ + "\u0000\u0000\u0221\u0222\u0007\u0002\u0000\u0000\u0222\u0223\u0007\u0005"+ + "\u0000\u0000\u0223\u0224\u0007\f\u0000\u0000\u0224\u0225\u0007\u0005\u0000"+ + "\u0000\u0225\u0226\u0007\u0002\u0000\u0000\u0226\u0227\u0001\u0000\u0000"+ + "\u0000\u0227\u0228\u0006\u0010\u0000\u0000\u02280\u0001\u0000\u0000\u0000"+ + "\u0229\u022a\u0004\u0011\u0001\u0000\u022a\u022b\u0007\r\u0000\u0000\u022b"+ + "\u022c\u0007\u0007\u0000\u0000\u022c\u022d\u0007\u0007\u0000\u0000\u022d"+ + "\u022e\u0007\u0012\u0000\u0000\u022e\u022f\u0007\u0014\u0000\u0000\u022f"+ + "\u0230\u0007\b\u0000\u0000\u0230\u0231\u0001\u0000\u0000\u0000\u0231\u0232"+ + "\u0006\u0011\b\u0000\u02322\u0001\u0000\u0000\u0000\u0233\u0234\u0004"+ + "\u0012\u0002\u0000\u0234\u0235\u0007\u0010\u0000\u0000\u0235\u0236\u0007"+ + "\u0003\u0000\u0000\u0236\u0237\u0007\u0005\u0000\u0000\u0237\u0238\u0007"+ + "\u0006\u0000\u0000\u0238\u0239\u0007\u0001\u0000\u0000\u0239\u023a\u0007"+ + "\u0004\u0000\u0000\u023a\u023b\u0007\u0002\u0000\u0000\u023b\u023c\u0001"+ + "\u0000\u0000\u0000\u023c\u023d\u0006\u0012\t\u0000\u023d4\u0001\u0000"+ + "\u0000\u0000\u023e\u0240\b\u0015\u0000\u0000\u023f\u023e\u0001\u0000\u0000"+ + "\u0000\u0240\u0241\u0001\u0000\u0000\u0000\u0241\u023f\u0001\u0000\u0000"+ + "\u0000\u0241\u0242\u0001\u0000\u0000\u0000\u0242\u0243\u0001\u0000\u0000"+ + "\u0000\u0243\u0244\u0006\u0013\u0000\u0000\u02446\u0001\u0000\u0000\u0000"+ + "\u0245\u0246\u0005/\u0000\u0000\u0246\u0247\u0005/\u0000\u0000\u0247\u024b"+ + "\u0001\u0000\u0000\u0000\u0248\u024a\b\u0016\u0000\u0000\u0249\u0248\u0001"+ + "\u0000\u0000\u0000\u024a\u024d\u0001\u0000\u0000\u0000\u024b\u0249\u0001"+ + "\u0000\u0000\u0000\u024b\u024c\u0001\u0000\u0000\u0000\u024c\u024f\u0001"+ + "\u0000\u0000\u0000\u024d\u024b\u0001\u0000\u0000\u0000\u024e\u0250\u0005"+ + "\r\u0000\u0000\u024f\u024e\u0001\u0000\u0000\u0000\u024f\u0250\u0001\u0000"+ + "\u0000\u0000\u0250\u0252\u0001\u0000\u0000\u0000\u0251\u0253\u0005\n\u0000"+ + "\u0000\u0252\u0251\u0001\u0000\u0000\u0000\u0252\u0253\u0001\u0000\u0000"+ + "\u0000\u0253\u0254\u0001\u0000\u0000\u0000\u0254\u0255\u0006\u0014\n\u0000"+ + "\u02558\u0001\u0000\u0000\u0000\u0256\u0257\u0005/\u0000\u0000\u0257\u0258"+ + "\u0005*\u0000\u0000\u0258\u025d\u0001\u0000\u0000\u0000\u0259\u025c\u0003"+ + "9\u0015\u0000\u025a\u025c\t\u0000\u0000\u0000\u025b\u0259\u0001\u0000"+ + "\u0000\u0000\u025b\u025a\u0001\u0000\u0000\u0000\u025c\u025f\u0001\u0000"+ + "\u0000\u0000\u025d\u025e\u0001\u0000\u0000\u0000\u025d\u025b\u0001\u0000"+ + "\u0000\u0000\u025e\u0260\u0001\u0000\u0000\u0000\u025f\u025d\u0001\u0000"+ + "\u0000\u0000\u0260\u0261\u0005*\u0000\u0000\u0261\u0262\u0005/\u0000\u0000"+ + "\u0262\u0263\u0001\u0000\u0000\u0000\u0263\u0264\u0006\u0015\n\u0000\u0264"+ + ":\u0001\u0000\u0000\u0000\u0265\u0267\u0007\u0017\u0000\u0000\u0266\u0265"+ + "\u0001\u0000\u0000\u0000\u0267\u0268\u0001\u0000\u0000\u0000\u0268\u0266"+ + "\u0001\u0000\u0000\u0000\u0268\u0269\u0001\u0000\u0000\u0000\u0269\u026a"+ + "\u0001\u0000\u0000\u0000\u026a\u026b\u0006\u0016\n\u0000\u026b<\u0001"+ + "\u0000\u0000\u0000\u026c\u026d\u0005|\u0000\u0000\u026d\u026e\u0001\u0000"+ + "\u0000\u0000\u026e\u026f\u0006\u0017\u000b\u0000\u026f>\u0001\u0000\u0000"+ + "\u0000\u0270\u0271\u0007\u0018\u0000\u0000\u0271@\u0001\u0000\u0000\u0000"+ + "\u0272\u0273\u0007\u0019\u0000\u0000\u0273B\u0001\u0000\u0000\u0000\u0274"+ + "\u0275\u0005\\\u0000\u0000\u0275\u0276\u0007\u001a\u0000\u0000\u0276D"+ + "\u0001\u0000\u0000\u0000\u0277\u0278\b\u001b\u0000\u0000\u0278F\u0001"+ + "\u0000\u0000\u0000\u0279\u027b\u0007\u0003\u0000\u0000\u027a\u027c\u0007"+ + "\u001c\u0000\u0000\u027b\u027a\u0001\u0000\u0000\u0000\u027b\u027c\u0001"+ + "\u0000\u0000\u0000\u027c\u027e\u0001\u0000\u0000\u0000\u027d\u027f\u0003"+ + "?\u0018\u0000\u027e\u027d\u0001\u0000\u0000\u0000\u027f\u0280\u0001\u0000"+ + "\u0000\u0000\u0280\u027e\u0001\u0000\u0000\u0000\u0280\u0281\u0001\u0000"+ + "\u0000\u0000\u0281H\u0001\u0000\u0000\u0000\u0282\u0283\u0005@\u0000\u0000"+ + "\u0283J\u0001\u0000\u0000\u0000\u0284\u0285\u0005`\u0000\u0000\u0285L"+ + "\u0001\u0000\u0000\u0000\u0286\u028a\b\u001d\u0000\u0000\u0287\u0288\u0005"+ + "`\u0000\u0000\u0288\u028a\u0005`\u0000\u0000\u0289\u0286\u0001\u0000\u0000"+ + "\u0000\u0289\u0287\u0001\u0000\u0000\u0000\u028aN\u0001\u0000\u0000\u0000"+ + "\u028b\u028c\u0005_\u0000\u0000\u028cP\u0001\u0000\u0000\u0000\u028d\u0291"+ + "\u0003A\u0019\u0000\u028e\u0291\u0003?\u0018\u0000\u028f\u0291\u0003O"+ + " \u0000\u0290\u028d\u0001\u0000\u0000\u0000\u0290\u028e\u0001\u0000\u0000"+ + "\u0000\u0290\u028f\u0001\u0000\u0000\u0000\u0291R\u0001\u0000\u0000\u0000"+ + "\u0292\u0297\u0005\"\u0000\u0000\u0293\u0296\u0003C\u001a\u0000\u0294"+ + "\u0296\u0003E\u001b\u0000\u0295\u0293\u0001\u0000\u0000\u0000\u0295\u0294"+ + "\u0001\u0000\u0000\u0000\u0296\u0299\u0001\u0000\u0000\u0000\u0297\u0295"+ + "\u0001\u0000\u0000\u0000\u0297\u0298\u0001\u0000\u0000\u0000\u0298\u029a"+ + "\u0001\u0000\u0000\u0000\u0299\u0297\u0001\u0000\u0000\u0000\u029a\u02b0"+ + "\u0005\"\u0000\u0000\u029b\u029c\u0005\"\u0000\u0000\u029c\u029d\u0005"+ + "\"\u0000\u0000\u029d\u029e\u0005\"\u0000\u0000\u029e\u02a2\u0001\u0000"+ + "\u0000\u0000\u029f\u02a1\b\u0016\u0000\u0000\u02a0\u029f\u0001\u0000\u0000"+ + "\u0000\u02a1\u02a4\u0001\u0000\u0000\u0000\u02a2\u02a3\u0001\u0000\u0000"+ + "\u0000\u02a2\u02a0\u0001\u0000\u0000\u0000\u02a3\u02a5\u0001\u0000\u0000"+ + "\u0000\u02a4\u02a2\u0001\u0000\u0000\u0000\u02a5\u02a6\u0005\"\u0000\u0000"+ + "\u02a6\u02a7\u0005\"\u0000\u0000\u02a7\u02a8\u0005\"\u0000\u0000\u02a8"+ + "\u02aa\u0001\u0000\u0000\u0000\u02a9\u02ab\u0005\"\u0000\u0000\u02aa\u02a9"+ + "\u0001\u0000\u0000\u0000\u02aa\u02ab\u0001\u0000\u0000\u0000\u02ab\u02ad"+ + "\u0001\u0000\u0000\u0000\u02ac\u02ae\u0005\"\u0000\u0000\u02ad\u02ac\u0001"+ + "\u0000\u0000\u0000\u02ad\u02ae\u0001\u0000\u0000\u0000\u02ae\u02b0\u0001"+ + "\u0000\u0000\u0000\u02af\u0292\u0001\u0000\u0000\u0000\u02af\u029b\u0001"+ + "\u0000\u0000\u0000\u02b0T\u0001\u0000\u0000\u0000\u02b1\u02b3\u0003?\u0018"+ + "\u0000\u02b2\u02b1\u0001\u0000\u0000\u0000\u02b3\u02b4\u0001\u0000\u0000"+ + "\u0000\u02b4\u02b2\u0001\u0000\u0000\u0000\u02b4\u02b5\u0001\u0000\u0000"+ + "\u0000\u02b5V\u0001\u0000\u0000\u0000\u02b6\u02b8\u0003?\u0018\u0000\u02b7"+ + "\u02b6\u0001\u0000\u0000\u0000\u02b8\u02b9\u0001\u0000\u0000\u0000\u02b9"+ + "\u02b7\u0001\u0000\u0000\u0000\u02b9\u02ba\u0001\u0000\u0000\u0000\u02ba"+ + "\u02bb\u0001\u0000\u0000\u0000\u02bb\u02bf\u0003g,\u0000\u02bc\u02be\u0003"+ + "?\u0018\u0000\u02bd\u02bc\u0001\u0000\u0000\u0000\u02be\u02c1\u0001\u0000"+ + "\u0000\u0000\u02bf\u02bd\u0001\u0000\u0000\u0000\u02bf\u02c0\u0001\u0000"+ + "\u0000\u0000\u02c0\u02e1\u0001\u0000\u0000\u0000\u02c1\u02bf\u0001\u0000"+ + "\u0000\u0000\u02c2\u02c4\u0003g,\u0000\u02c3\u02c5\u0003?\u0018\u0000"+ + "\u02c4\u02c3\u0001\u0000\u0000\u0000\u02c5\u02c6\u0001\u0000\u0000\u0000"+ + "\u02c6\u02c4\u0001\u0000\u0000\u0000\u02c6\u02c7\u0001\u0000\u0000\u0000"+ + "\u02c7\u02e1\u0001\u0000\u0000\u0000\u02c8\u02ca\u0003?\u0018\u0000\u02c9"+ + "\u02c8\u0001\u0000\u0000\u0000\u02ca\u02cb\u0001\u0000\u0000\u0000\u02cb"+ + "\u02c9\u0001\u0000\u0000\u0000\u02cb\u02cc\u0001\u0000\u0000\u0000\u02cc"+ + "\u02d4\u0001\u0000\u0000\u0000\u02cd\u02d1\u0003g,\u0000\u02ce\u02d0\u0003"+ + "?\u0018\u0000\u02cf\u02ce\u0001\u0000\u0000\u0000\u02d0\u02d3\u0001\u0000"+ + "\u0000\u0000\u02d1\u02cf\u0001\u0000\u0000\u0000\u02d1\u02d2\u0001\u0000"+ + "\u0000\u0000\u02d2\u02d5\u0001\u0000\u0000\u0000\u02d3\u02d1\u0001\u0000"+ + "\u0000\u0000\u02d4\u02cd\u0001\u0000\u0000\u0000\u02d4\u02d5\u0001\u0000"+ + "\u0000\u0000\u02d5\u02d6\u0001\u0000\u0000\u0000\u02d6\u02d7\u0003G\u001c"+ + "\u0000\u02d7\u02e1\u0001\u0000\u0000\u0000\u02d8\u02da\u0003g,\u0000\u02d9"+ + "\u02db\u0003?\u0018\u0000\u02da\u02d9\u0001\u0000\u0000\u0000\u02db\u02dc"+ + "\u0001\u0000\u0000\u0000\u02dc\u02da\u0001\u0000\u0000\u0000\u02dc\u02dd"+ + "\u0001\u0000\u0000\u0000\u02dd\u02de\u0001\u0000\u0000\u0000\u02de\u02df"+ + "\u0003G\u001c\u0000\u02df\u02e1\u0001\u0000\u0000\u0000\u02e0\u02b7\u0001"+ + "\u0000\u0000\u0000\u02e0\u02c2\u0001\u0000\u0000\u0000\u02e0\u02c9\u0001"+ + "\u0000\u0000\u0000\u02e0\u02d8\u0001\u0000\u0000\u0000\u02e1X\u0001\u0000"+ + "\u0000\u0000\u02e2\u02e3\u0007\u001e\u0000\u0000\u02e3\u02e4\u0007\u001f"+ + "\u0000\u0000\u02e4Z\u0001\u0000\u0000\u0000\u02e5\u02e6\u0007\f\u0000"+ + "\u0000\u02e6\u02e7\u0007\t\u0000\u0000\u02e7\u02e8\u0007\u0000\u0000\u0000"+ + "\u02e8\\\u0001\u0000\u0000\u0000\u02e9\u02ea\u0007\f\u0000\u0000\u02ea"+ + "\u02eb\u0007\u0002\u0000\u0000\u02eb\u02ec\u0007\u0004\u0000\u0000\u02ec"+ + "^\u0001\u0000\u0000\u0000\u02ed\u02ee\u0005=\u0000\u0000\u02ee`\u0001"+ + "\u0000\u0000\u0000\u02ef\u02f0\u0005:\u0000\u0000\u02f0\u02f1\u0005:\u0000"+ + "\u0000\u02f1b\u0001\u0000\u0000\u0000\u02f2\u02f3\u0005,\u0000\u0000\u02f3"+ + "d\u0001\u0000\u0000\u0000\u02f4\u02f5\u0007\u0000\u0000\u0000\u02f5\u02f6"+ + "\u0007\u0003\u0000\u0000\u02f6\u02f7\u0007\u0002\u0000\u0000\u02f7\u02f8"+ + "\u0007\u0004\u0000\u0000\u02f8f\u0001\u0000\u0000\u0000\u02f9\u02fa\u0005"+ + ".\u0000\u0000\u02fah\u0001\u0000\u0000\u0000\u02fb\u02fc\u0007\u000f\u0000"+ + "\u0000\u02fc\u02fd\u0007\f\u0000\u0000\u02fd\u02fe\u0007\r\u0000\u0000"+ + "\u02fe\u02ff\u0007\u0002\u0000\u0000\u02ff\u0300\u0007\u0003\u0000\u0000"+ + "\u0300j\u0001\u0000\u0000\u0000\u0301\u0302\u0007\u000f\u0000\u0000\u0302"+ + "\u0303\u0007\u0001\u0000\u0000\u0303\u0304\u0007\u0006\u0000\u0000\u0304"+ + "\u0305\u0007\u0002\u0000\u0000\u0305\u0306\u0007\u0005\u0000\u0000\u0306"+ + "l\u0001\u0000\u0000\u0000\u0307\u0308\u0007\u0001\u0000\u0000\u0308\u0309"+ + "\u0007\t\u0000\u0000\u0309n\u0001\u0000\u0000\u0000\u030a\u030b\u0007"+ + "\u0001\u0000\u0000\u030b\u030c\u0007\u0002\u0000\u0000\u030cp\u0001\u0000"+ + "\u0000\u0000\u030d\u030e\u0007\r\u0000\u0000\u030e\u030f\u0007\f\u0000"+ + "\u0000\u030f\u0310\u0007\u0002\u0000\u0000\u0310\u0311\u0007\u0005\u0000"+ + "\u0000\u0311r\u0001\u0000\u0000\u0000\u0312\u0313\u0007\r\u0000\u0000"+ + "\u0313\u0314\u0007\u0001\u0000\u0000\u0314\u0315\u0007\u0012\u0000\u0000"+ + "\u0315\u0316\u0007\u0003\u0000\u0000\u0316t\u0001\u0000\u0000\u0000\u0317"+ + "\u0318\u0005(\u0000\u0000\u0318v\u0001\u0000\u0000\u0000\u0319\u031a\u0007"+ + "\t\u0000\u0000\u031a\u031b\u0007\u0007\u0000\u0000\u031b\u031c\u0007\u0005"+ + "\u0000\u0000\u031cx\u0001\u0000\u0000\u0000\u031d\u031e\u0007\t\u0000"+ + "\u0000\u031e\u031f\u0007\u0014\u0000\u0000\u031f\u0320\u0007\r\u0000\u0000"+ + "\u0320\u0321\u0007\r\u0000\u0000\u0321z\u0001\u0000\u0000\u0000\u0322"+ + "\u0323\u0007\t\u0000\u0000\u0323\u0324\u0007\u0014\u0000\u0000\u0324\u0325"+ + "\u0007\r\u0000\u0000\u0325\u0326\u0007\r\u0000\u0000\u0326\u0327\u0007"+ + "\u0002\u0000\u0000\u0327|\u0001\u0000\u0000\u0000\u0328\u0329\u0007\u0007"+ + "\u0000\u0000\u0329\u032a\u0007\u0006\u0000\u0000\u032a~\u0001\u0000\u0000"+ + "\u0000\u032b\u032c\u0005?\u0000\u0000\u032c\u0080\u0001\u0000\u0000\u0000"+ + "\u032d\u032e\u0007\u0006\u0000\u0000\u032e\u032f\u0007\r\u0000\u0000\u032f"+ + "\u0330\u0007\u0001\u0000\u0000\u0330\u0331\u0007\u0012\u0000\u0000\u0331"+ + "\u0332\u0007\u0003\u0000\u0000\u0332\u0082\u0001\u0000\u0000\u0000\u0333"+ + "\u0334\u0005)\u0000\u0000\u0334\u0084\u0001\u0000\u0000\u0000\u0335\u0336"+ + "\u0007\u0005\u0000\u0000\u0336\u0337\u0007\u0006\u0000\u0000\u0337\u0338"+ + "\u0007\u0014\u0000\u0000\u0338\u0339\u0007\u0003\u0000\u0000\u0339\u0086"+ + "\u0001\u0000\u0000\u0000\u033a\u033b\u0005=\u0000\u0000\u033b\u033c\u0005"+ + "=\u0000\u0000\u033c\u0088\u0001\u0000\u0000\u0000\u033d\u033e\u0005=\u0000"+ + "\u0000\u033e\u033f\u0005~\u0000\u0000\u033f\u008a\u0001\u0000\u0000\u0000"+ + "\u0340\u0341\u0005!\u0000\u0000\u0341\u0342\u0005=\u0000\u0000\u0342\u008c"+ + "\u0001\u0000\u0000\u0000\u0343\u0344\u0005<\u0000\u0000\u0344\u008e\u0001"+ + "\u0000\u0000\u0000\u0345\u0346\u0005<\u0000\u0000\u0346\u0347\u0005=\u0000"+ + "\u0000\u0347\u0090\u0001\u0000\u0000\u0000\u0348\u0349\u0005>\u0000\u0000"+ + "\u0349\u0092\u0001\u0000\u0000\u0000\u034a\u034b\u0005>\u0000\u0000\u034b"+ + "\u034c\u0005=\u0000\u0000\u034c\u0094\u0001\u0000\u0000\u0000\u034d\u034e"+ + "\u0005+\u0000\u0000\u034e\u0096\u0001\u0000\u0000\u0000\u034f\u0350\u0005"+ + "-\u0000\u0000\u0350\u0098\u0001\u0000\u0000\u0000\u0351\u0352\u0005*\u0000"+ + "\u0000\u0352\u009a\u0001\u0000\u0000\u0000\u0353\u0354\u0005/\u0000\u0000"+ + "\u0354\u009c\u0001\u0000\u0000\u0000\u0355\u0356\u0005%\u0000\u0000\u0356"+ + "\u009e\u0001\u0000\u0000\u0000\u0357\u0358\u0004H\u0003\u0000\u0358\u0359"+ + "\u0007\u0010\u0000\u0000\u0359\u035a\u0007\f\u0000\u0000\u035a\u035b\u0007"+ + "\u0005\u0000\u0000\u035b\u035c\u0007\u0004\u0000\u0000\u035c\u035d\u0007"+ + "\n\u0000\u0000\u035d\u00a0\u0001\u0000\u0000\u0000\u035e\u0361\u0003\u007f"+ + "8\u0000\u035f\u0362\u0003A\u0019\u0000\u0360\u0362\u0003O \u0000\u0361"+ + "\u035f\u0001\u0000\u0000\u0000\u0361\u0360\u0001\u0000\u0000\u0000\u0362"+ + "\u0366\u0001\u0000\u0000\u0000\u0363\u0365\u0003Q!\u0000\u0364\u0363\u0001"+ + "\u0000\u0000\u0000\u0365\u0368\u0001\u0000\u0000\u0000\u0366\u0364\u0001"+ + "\u0000\u0000\u0000\u0366\u0367\u0001\u0000\u0000\u0000\u0367\u0370\u0001"+ + "\u0000\u0000\u0000\u0368\u0366\u0001\u0000\u0000\u0000\u0369\u036b\u0003"+ + "\u007f8\u0000\u036a\u036c\u0003?\u0018\u0000\u036b\u036a\u0001\u0000\u0000"+ + "\u0000\u036c\u036d\u0001\u0000\u0000\u0000\u036d\u036b\u0001\u0000\u0000"+ + "\u0000\u036d\u036e\u0001\u0000\u0000\u0000\u036e\u0370\u0001\u0000\u0000"+ + "\u0000\u036f\u035e\u0001\u0000\u0000\u0000\u036f\u0369\u0001\u0000\u0000"+ + "\u0000\u0370\u00a2\u0001\u0000\u0000\u0000\u0371\u0372\u0005[\u0000\u0000"+ + "\u0372\u0373\u0001\u0000\u0000\u0000\u0373\u0374\u0006J\u0000\u0000\u0374"+ + "\u0375\u0006J\u0000\u0000\u0375\u00a4\u0001\u0000\u0000\u0000\u0376\u0377"+ + "\u0005]\u0000\u0000\u0377\u0378\u0001\u0000\u0000\u0000\u0378\u0379\u0006"+ + "K\u000b\u0000\u0379\u037a\u0006K\u000b\u0000\u037a\u00a6\u0001\u0000\u0000"+ + "\u0000\u037b\u037f\u0003A\u0019\u0000\u037c\u037e\u0003Q!\u0000\u037d"+ + "\u037c\u0001\u0000\u0000\u0000\u037e\u0381\u0001\u0000\u0000\u0000\u037f"+ + "\u037d\u0001\u0000\u0000\u0000\u037f\u0380\u0001\u0000\u0000\u0000\u0380"+ + "\u038c\u0001\u0000\u0000\u0000\u0381\u037f\u0001\u0000\u0000\u0000\u0382"+ + "\u0385\u0003O \u0000\u0383\u0385\u0003I\u001d\u0000\u0384\u0382\u0001"+ + "\u0000\u0000\u0000\u0384\u0383\u0001\u0000\u0000\u0000\u0385\u0387\u0001"+ + "\u0000\u0000\u0000\u0386\u0388\u0003Q!\u0000\u0387\u0386\u0001\u0000\u0000"+ + "\u0000\u0388\u0389\u0001\u0000\u0000\u0000\u0389\u0387\u0001\u0000\u0000"+ + "\u0000\u0389\u038a\u0001\u0000\u0000\u0000\u038a\u038c\u0001\u0000\u0000"+ + "\u0000\u038b\u037b\u0001\u0000\u0000\u0000\u038b\u0384\u0001\u0000\u0000"+ + "\u0000\u038c\u00a8\u0001\u0000\u0000\u0000\u038d\u038f\u0003K\u001e\u0000"+ + "\u038e\u0390\u0003M\u001f\u0000\u038f\u038e\u0001\u0000\u0000\u0000\u0390"+ + "\u0391\u0001\u0000\u0000\u0000\u0391\u038f\u0001\u0000\u0000\u0000\u0391"+ + "\u0392\u0001\u0000\u0000\u0000\u0392\u0393\u0001\u0000\u0000\u0000\u0393"+ + "\u0394\u0003K\u001e\u0000\u0394\u00aa\u0001\u0000\u0000\u0000\u0395\u0396"+ + "\u0003\u00a9M\u0000\u0396\u00ac\u0001\u0000\u0000\u0000\u0397\u0398\u0003"+ + "7\u0014\u0000\u0398\u0399\u0001\u0000\u0000\u0000\u0399\u039a\u0006O\n"+ + "\u0000\u039a\u00ae\u0001\u0000\u0000\u0000\u039b\u039c\u00039\u0015\u0000"+ + "\u039c\u039d\u0001\u0000\u0000\u0000\u039d\u039e\u0006P\n\u0000\u039e"+ + "\u00b0\u0001\u0000\u0000\u0000\u039f\u03a0\u0003;\u0016\u0000\u03a0\u03a1"+ + "\u0001\u0000\u0000\u0000\u03a1\u03a2\u0006Q\n\u0000\u03a2\u00b2\u0001"+ + "\u0000\u0000\u0000\u03a3\u03a4\u0003\u00a3J\u0000\u03a4\u03a5\u0001\u0000"+ + "\u0000\u0000\u03a5\u03a6\u0006R\f\u0000\u03a6\u03a7\u0006R\r\u0000\u03a7"+ + "\u00b4\u0001\u0000\u0000\u0000\u03a8\u03a9\u0003=\u0017\u0000\u03a9\u03aa"+ + "\u0001\u0000\u0000\u0000\u03aa\u03ab\u0006S\u000e\u0000\u03ab\u03ac\u0006"+ + "S\u000b\u0000\u03ac\u00b6\u0001\u0000\u0000\u0000\u03ad\u03ae\u0003;\u0016"+ + "\u0000\u03ae\u03af\u0001\u0000\u0000\u0000\u03af\u03b0\u0006T\n\u0000"+ + "\u03b0\u00b8\u0001\u0000\u0000\u0000\u03b1\u03b2\u00037\u0014\u0000\u03b2"+ + "\u03b3\u0001\u0000\u0000\u0000\u03b3\u03b4\u0006U\n\u0000\u03b4\u00ba"+ + "\u0001\u0000\u0000\u0000\u03b5\u03b6\u00039\u0015\u0000\u03b6\u03b7\u0001"+ + "\u0000\u0000\u0000\u03b7\u03b8\u0006V\n\u0000\u03b8\u00bc\u0001\u0000"+ + "\u0000\u0000\u03b9\u03ba\u0003=\u0017\u0000\u03ba\u03bb\u0001\u0000\u0000"+ + "\u0000\u03bb\u03bc\u0006W\u000e\u0000\u03bc\u03bd\u0006W\u000b\u0000\u03bd"+ + "\u00be\u0001\u0000\u0000\u0000\u03be\u03bf\u0003\u00a3J\u0000\u03bf\u03c0"+ + "\u0001\u0000\u0000\u0000\u03c0\u03c1\u0006X\f\u0000\u03c1\u00c0\u0001"+ + "\u0000\u0000\u0000\u03c2\u03c3\u0003\u00a5K\u0000\u03c3\u03c4\u0001\u0000"+ + "\u0000\u0000\u03c4\u03c5\u0006Y\u000f\u0000\u03c5\u00c2\u0001\u0000\u0000"+ + "\u0000\u03c6\u03c7\u0003\u014f\u00a0\u0000\u03c7\u03c8\u0001\u0000\u0000"+ + "\u0000\u03c8\u03c9\u0006Z\u0010\u0000\u03c9\u00c4\u0001\u0000\u0000\u0000"+ + "\u03ca\u03cb\u0003c*\u0000\u03cb\u03cc\u0001\u0000\u0000\u0000\u03cc\u03cd"+ + "\u0006[\u0011\u0000\u03cd\u00c6\u0001\u0000\u0000\u0000\u03ce\u03cf\u0003"+ + "_(\u0000\u03cf\u03d0\u0001\u0000\u0000\u0000\u03d0\u03d1\u0006\\\u0012"+ + "\u0000\u03d1\u00c8\u0001\u0000\u0000\u0000\u03d2\u03d3\u0007\u0010\u0000"+ + "\u0000\u03d3\u03d4\u0007\u0003\u0000\u0000\u03d4\u03d5\u0007\u0005\u0000"+ + "\u0000\u03d5\u03d6\u0007\f\u0000\u0000\u03d6\u03d7\u0007\u0000\u0000\u0000"+ + "\u03d7\u03d8\u0007\f\u0000\u0000\u03d8\u03d9\u0007\u0005\u0000\u0000\u03d9"+ + "\u03da\u0007\f\u0000\u0000\u03da\u00ca\u0001\u0000\u0000\u0000\u03db\u03df"+ + "\b \u0000\u0000\u03dc\u03dd\u0005/\u0000\u0000\u03dd\u03df\b!\u0000\u0000"+ + "\u03de\u03db\u0001\u0000\u0000\u0000\u03de\u03dc\u0001\u0000\u0000\u0000"+ + "\u03df\u00cc\u0001\u0000\u0000\u0000\u03e0\u03e2\u0003\u00cb^\u0000\u03e1"+ + "\u03e0\u0001\u0000\u0000\u0000\u03e2\u03e3\u0001\u0000\u0000\u0000\u03e3"+ + "\u03e1\u0001\u0000\u0000\u0000\u03e3\u03e4\u0001\u0000\u0000\u0000\u03e4"+ + "\u00ce\u0001\u0000\u0000\u0000\u03e5\u03e6\u0003\u00cd_\u0000\u03e6\u03e7"+ + "\u0001\u0000\u0000\u0000\u03e7\u03e8\u0006`\u0013\u0000\u03e8\u00d0\u0001"+ + "\u0000\u0000\u0000\u03e9\u03ea\u0003S\"\u0000\u03ea\u03eb\u0001\u0000"+ + "\u0000\u0000\u03eb\u03ec\u0006a\u0014\u0000\u03ec\u00d2\u0001\u0000\u0000"+ + "\u0000\u03ed\u03ee\u00037\u0014\u0000\u03ee\u03ef\u0001\u0000\u0000\u0000"+ + "\u03ef\u03f0\u0006b\n\u0000\u03f0\u00d4\u0001\u0000\u0000\u0000\u03f1"+ + "\u03f2\u00039\u0015\u0000\u03f2\u03f3\u0001\u0000\u0000\u0000\u03f3\u03f4"+ + "\u0006c\n\u0000\u03f4\u00d6\u0001\u0000\u0000\u0000\u03f5\u03f6\u0003"+ + ";\u0016\u0000\u03f6\u03f7\u0001\u0000\u0000\u0000\u03f7\u03f8\u0006d\n"+ + "\u0000\u03f8\u00d8\u0001\u0000\u0000\u0000\u03f9\u03fa\u0003=\u0017\u0000"+ + "\u03fa\u03fb\u0001\u0000\u0000\u0000\u03fb\u03fc\u0006e\u000e\u0000\u03fc"+ + "\u03fd\u0006e\u000b\u0000\u03fd\u00da\u0001\u0000\u0000\u0000\u03fe\u03ff"+ + "\u0003g,\u0000\u03ff\u0400\u0001\u0000\u0000\u0000\u0400\u0401\u0006f"+ + "\u0015\u0000\u0401\u00dc\u0001\u0000\u0000\u0000\u0402\u0403\u0003c*\u0000"+ + "\u0403\u0404\u0001\u0000\u0000\u0000\u0404\u0405\u0006g\u0011\u0000\u0405"+ + "\u00de\u0001\u0000\u0000\u0000\u0406\u0407\u0003\u007f8\u0000\u0407\u0408"+ + "\u0001\u0000\u0000\u0000\u0408\u0409\u0006h\u0016\u0000\u0409\u00e0\u0001"+ + "\u0000\u0000\u0000\u040a\u040b\u0003\u00a1I\u0000\u040b\u040c\u0001\u0000"+ + "\u0000\u0000\u040c\u040d\u0006i\u0017\u0000\u040d\u00e2\u0001\u0000\u0000"+ + "\u0000\u040e\u0413\u0003A\u0019\u0000\u040f\u0413\u0003?\u0018\u0000\u0410"+ + "\u0413\u0003O \u0000\u0411\u0413\u0003\u0099E\u0000\u0412\u040e\u0001"+ + "\u0000\u0000\u0000\u0412\u040f\u0001\u0000\u0000\u0000\u0412\u0410\u0001"+ + "\u0000\u0000\u0000\u0412\u0411\u0001\u0000\u0000\u0000\u0413\u00e4\u0001"+ + "\u0000\u0000\u0000\u0414\u0417\u0003A\u0019\u0000\u0415\u0417\u0003\u0099"+ + "E\u0000\u0416\u0414\u0001\u0000\u0000\u0000\u0416\u0415\u0001\u0000\u0000"+ + "\u0000\u0417\u041b\u0001\u0000\u0000\u0000\u0418\u041a\u0003\u00e3j\u0000"+ + "\u0419\u0418\u0001\u0000\u0000\u0000\u041a\u041d\u0001\u0000\u0000\u0000"+ + "\u041b\u0419\u0001\u0000\u0000\u0000\u041b\u041c\u0001\u0000\u0000\u0000"+ + "\u041c\u0428\u0001\u0000\u0000\u0000\u041d\u041b\u0001\u0000\u0000\u0000"+ + "\u041e\u0421\u0003O \u0000\u041f\u0421\u0003I\u001d\u0000\u0420\u041e"+ + "\u0001\u0000\u0000\u0000\u0420\u041f\u0001\u0000\u0000\u0000\u0421\u0423"+ + "\u0001\u0000\u0000\u0000\u0422\u0424\u0003\u00e3j\u0000\u0423\u0422\u0001"+ + "\u0000\u0000\u0000\u0424\u0425\u0001\u0000\u0000\u0000\u0425\u0423\u0001"+ + "\u0000\u0000\u0000\u0425\u0426\u0001\u0000\u0000\u0000\u0426\u0428\u0001"+ + "\u0000\u0000\u0000\u0427\u0416\u0001\u0000\u0000\u0000\u0427\u0420\u0001"+ + "\u0000\u0000\u0000\u0428\u00e6\u0001\u0000\u0000\u0000\u0429\u042c\u0003"+ + "\u00e5k\u0000\u042a\u042c\u0003\u00a9M\u0000\u042b\u0429\u0001\u0000\u0000"+ + "\u0000\u042b\u042a\u0001\u0000\u0000\u0000\u042c\u042d\u0001\u0000\u0000"+ + "\u0000\u042d\u042b\u0001\u0000\u0000\u0000\u042d\u042e\u0001\u0000\u0000"+ + "\u0000\u042e\u00e8\u0001\u0000\u0000\u0000\u042f\u0430\u00037\u0014\u0000"+ + "\u0430\u0431\u0001\u0000\u0000\u0000\u0431\u0432\u0006m\n\u0000\u0432"+ + "\u00ea\u0001\u0000\u0000\u0000\u0433\u0434\u00039\u0015\u0000\u0434\u0435"+ + "\u0001\u0000\u0000\u0000\u0435\u0436\u0006n\n\u0000\u0436\u00ec\u0001"+ + "\u0000\u0000\u0000\u0437\u0438\u0003;\u0016\u0000\u0438\u0439\u0001\u0000"+ + "\u0000\u0000\u0439\u043a\u0006o\n\u0000\u043a\u00ee\u0001\u0000\u0000"+ + "\u0000\u043b\u043c\u0003=\u0017\u0000\u043c\u043d\u0001\u0000\u0000\u0000"+ + "\u043d\u043e\u0006p\u000e\u0000\u043e\u043f\u0006p\u000b\u0000\u043f\u00f0"+ + "\u0001\u0000\u0000\u0000\u0440\u0441\u0003_(\u0000\u0441\u0442\u0001\u0000"+ + "\u0000\u0000\u0442\u0443\u0006q\u0012\u0000\u0443\u00f2\u0001\u0000\u0000"+ + "\u0000\u0444\u0445\u0003c*\u0000\u0445\u0446\u0001\u0000\u0000\u0000\u0446"+ + "\u0447\u0006r\u0011\u0000\u0447\u00f4\u0001\u0000\u0000\u0000\u0448\u0449"+ + "\u0003g,\u0000\u0449\u044a\u0001\u0000\u0000\u0000\u044a\u044b\u0006s"+ + "\u0015\u0000\u044b\u00f6\u0001\u0000\u0000\u0000\u044c\u044d\u0003\u007f"+ + "8\u0000\u044d\u044e\u0001\u0000\u0000\u0000\u044e\u044f\u0006t\u0016\u0000"+ + "\u044f\u00f8\u0001\u0000\u0000\u0000\u0450\u0451\u0003\u00a1I\u0000\u0451"+ + "\u0452\u0001\u0000\u0000\u0000\u0452\u0453\u0006u\u0017\u0000\u0453\u00fa"+ + "\u0001\u0000\u0000\u0000\u0454\u0455\u0007\f\u0000\u0000\u0455\u0456\u0007"+ + "\u0002\u0000\u0000\u0456\u00fc\u0001\u0000\u0000\u0000\u0457\u0458\u0003"+ + "\u00e7l\u0000\u0458\u0459\u0001\u0000\u0000\u0000\u0459\u045a\u0006w\u0018"+ + "\u0000\u045a\u00fe\u0001\u0000\u0000\u0000\u045b\u045c\u00037\u0014\u0000"+ + "\u045c\u045d\u0001\u0000\u0000\u0000\u045d\u045e\u0006x\n\u0000\u045e"+ + "\u0100\u0001\u0000\u0000\u0000\u045f\u0460\u00039\u0015\u0000\u0460\u0461"+ + "\u0001\u0000\u0000\u0000\u0461\u0462\u0006y\n\u0000\u0462\u0102\u0001"+ + "\u0000\u0000\u0000\u0463\u0464\u0003;\u0016\u0000\u0464\u0465\u0001\u0000"+ + "\u0000\u0000\u0465\u0466\u0006z\n\u0000\u0466\u0104\u0001\u0000\u0000"+ + "\u0000\u0467\u0468\u0003=\u0017\u0000\u0468\u0469\u0001\u0000\u0000\u0000"+ + "\u0469\u046a\u0006{\u000e\u0000\u046a\u046b\u0006{\u000b\u0000\u046b\u0106"+ + "\u0001\u0000\u0000\u0000\u046c\u046d\u0003\u00a3J\u0000\u046d\u046e\u0001"+ + "\u0000\u0000\u0000\u046e\u046f\u0006|\f\u0000\u046f\u0470\u0006|\u0019"+ + "\u0000\u0470\u0108\u0001\u0000\u0000\u0000\u0471\u0472\u0007\u0007\u0000"+ + "\u0000\u0472\u0473\u0007\t\u0000\u0000\u0473\u0474\u0001\u0000\u0000\u0000"+ + "\u0474\u0475\u0006}\u001a\u0000\u0475\u010a\u0001\u0000\u0000\u0000\u0476"+ + "\u0477\u0007\u0013\u0000\u0000\u0477\u0478\u0007\u0001\u0000\u0000\u0478"+ + "\u0479\u0007\u0005\u0000\u0000\u0479\u047a\u0007\n\u0000\u0000\u047a\u047b"+ + "\u0001\u0000\u0000\u0000\u047b\u047c\u0006~\u001a\u0000\u047c\u010c\u0001"+ + "\u0000\u0000\u0000\u047d\u047e\b\"\u0000\u0000\u047e\u010e\u0001\u0000"+ + "\u0000\u0000\u047f\u0481\u0003\u010d\u007f\u0000\u0480\u047f\u0001\u0000"+ + "\u0000\u0000\u0481\u0482\u0001\u0000\u0000\u0000\u0482\u0480\u0001\u0000"+ + "\u0000\u0000\u0482\u0483\u0001\u0000\u0000\u0000\u0483\u0484\u0001\u0000"+ + "\u0000\u0000\u0484\u0485\u0003\u014f\u00a0\u0000\u0485\u0487\u0001\u0000"+ + "\u0000\u0000\u0486\u0480\u0001\u0000\u0000\u0000\u0486\u0487\u0001\u0000"+ + "\u0000\u0000\u0487\u0489\u0001\u0000\u0000\u0000\u0488\u048a\u0003\u010d"+ + "\u007f\u0000\u0489\u0488\u0001\u0000\u0000\u0000\u048a\u048b\u0001\u0000"+ + "\u0000\u0000\u048b\u0489\u0001\u0000\u0000\u0000\u048b\u048c\u0001\u0000"+ + "\u0000\u0000\u048c\u0110\u0001\u0000\u0000\u0000\u048d\u048e\u0003\u010f"+ + "\u0080\u0000\u048e\u048f\u0001\u0000\u0000\u0000\u048f\u0490\u0006\u0081"+ + "\u001b\u0000\u0490\u0112\u0001\u0000\u0000\u0000\u0491\u0492\u00037\u0014"+ + "\u0000\u0492\u0493\u0001\u0000\u0000\u0000\u0493\u0494\u0006\u0082\n\u0000"+ + "\u0494\u0114\u0001\u0000\u0000\u0000\u0495\u0496\u00039\u0015\u0000\u0496"+ + "\u0497\u0001\u0000\u0000\u0000\u0497\u0498\u0006\u0083\n\u0000\u0498\u0116"+ + "\u0001\u0000\u0000\u0000\u0499\u049a\u0003;\u0016\u0000\u049a\u049b\u0001"+ + "\u0000\u0000\u0000\u049b\u049c\u0006\u0084\n\u0000\u049c\u0118\u0001\u0000"+ + "\u0000\u0000\u049d\u049e\u0003=\u0017\u0000\u049e\u049f\u0001\u0000\u0000"+ + "\u0000\u049f\u04a0\u0006\u0085\u000e\u0000\u04a0\u04a1\u0006\u0085\u000b"+ + "\u0000\u04a1\u04a2\u0006\u0085\u000b\u0000\u04a2\u011a\u0001\u0000\u0000"+ + "\u0000\u04a3\u04a4\u0003_(\u0000\u04a4\u04a5\u0001\u0000\u0000\u0000\u04a5"+ + "\u04a6\u0006\u0086\u0012\u0000\u04a6\u011c\u0001\u0000\u0000\u0000\u04a7"+ + "\u04a8\u0003c*\u0000\u04a8\u04a9\u0001\u0000\u0000\u0000\u04a9\u04aa\u0006"+ + "\u0087\u0011\u0000\u04aa\u011e\u0001\u0000\u0000\u0000\u04ab\u04ac\u0003"+ + "g,\u0000\u04ac\u04ad\u0001\u0000\u0000\u0000\u04ad\u04ae\u0006\u0088\u0015"+ + "\u0000\u04ae\u0120\u0001\u0000\u0000\u0000\u04af\u04b0\u0003\u010b~\u0000"+ + "\u04b0\u04b1\u0001\u0000\u0000\u0000\u04b1\u04b2\u0006\u0089\u001c\u0000"+ + "\u04b2\u0122\u0001\u0000\u0000\u0000\u04b3\u04b4\u0003\u00e7l\u0000\u04b4"+ + "\u04b5\u0001\u0000\u0000\u0000\u04b5\u04b6\u0006\u008a\u0018\u0000\u04b6"+ + "\u0124\u0001\u0000\u0000\u0000\u04b7\u04b8\u0003\u00abN\u0000\u04b8\u04b9"+ + "\u0001\u0000\u0000\u0000\u04b9\u04ba\u0006\u008b\u001d\u0000\u04ba\u0126"+ + "\u0001\u0000\u0000\u0000\u04bb\u04bc\u0003\u007f8\u0000\u04bc\u04bd\u0001"+ + "\u0000\u0000\u0000\u04bd\u04be\u0006\u008c\u0016\u0000\u04be\u0128\u0001"+ + "\u0000\u0000\u0000\u04bf\u04c0\u0003\u00a1I\u0000\u04c0\u04c1\u0001\u0000"+ + "\u0000\u0000\u04c1\u04c2\u0006\u008d\u0017\u0000\u04c2\u012a\u0001\u0000"+ + "\u0000\u0000\u04c3\u04c4\u00037\u0014\u0000\u04c4\u04c5\u0001\u0000\u0000"+ + "\u0000\u04c5\u04c6\u0006\u008e\n\u0000\u04c6\u012c\u0001\u0000\u0000\u0000"+ + "\u04c7\u04c8\u00039\u0015\u0000\u04c8\u04c9\u0001\u0000\u0000\u0000\u04c9"+ + "\u04ca\u0006\u008f\n\u0000\u04ca\u012e\u0001\u0000\u0000\u0000\u04cb\u04cc"+ + "\u0003;\u0016\u0000\u04cc\u04cd\u0001\u0000\u0000\u0000\u04cd\u04ce\u0006"+ + "\u0090\n\u0000\u04ce\u0130\u0001\u0000\u0000\u0000\u04cf\u04d0\u0003="+ + "\u0017\u0000\u04d0\u04d1\u0001\u0000\u0000\u0000\u04d1\u04d2\u0006\u0091"+ + "\u000e\u0000\u04d2\u04d3\u0006\u0091\u000b\u0000\u04d3\u0132\u0001\u0000"+ + "\u0000\u0000\u04d4\u04d5\u0003g,\u0000\u04d5\u04d6\u0001\u0000\u0000\u0000"+ + "\u04d6\u04d7\u0006\u0092\u0015\u0000\u04d7\u0134\u0001\u0000\u0000\u0000"+ + "\u04d8\u04d9\u0003\u007f8\u0000\u04d9\u04da\u0001\u0000\u0000\u0000\u04da"+ + "\u04db\u0006\u0093\u0016\u0000\u04db\u0136\u0001\u0000\u0000\u0000\u04dc"+ + "\u04dd\u0003\u00a1I\u0000\u04dd\u04de\u0001\u0000\u0000\u0000\u04de\u04df"+ + "\u0006\u0094\u0017\u0000\u04df\u0138\u0001\u0000\u0000\u0000\u04e0\u04e1"+ + "\u0003\u00abN\u0000\u04e1\u04e2\u0001\u0000\u0000\u0000\u04e2\u04e3\u0006"+ + "\u0095\u001d\u0000\u04e3\u013a\u0001\u0000\u0000\u0000\u04e4\u04e5\u0003"+ + "\u00a7L\u0000\u04e5\u04e6\u0001\u0000\u0000\u0000\u04e6\u04e7\u0006\u0096"+ + "\u001e\u0000\u04e7\u013c\u0001\u0000\u0000\u0000\u04e8\u04e9\u00037\u0014"+ + "\u0000\u04e9\u04ea\u0001\u0000\u0000\u0000\u04ea\u04eb\u0006\u0097\n\u0000"+ + "\u04eb\u013e\u0001\u0000\u0000\u0000\u04ec\u04ed\u00039\u0015\u0000\u04ed"+ + "\u04ee\u0001\u0000\u0000\u0000\u04ee\u04ef\u0006\u0098\n\u0000\u04ef\u0140"+ + "\u0001\u0000\u0000\u0000\u04f0\u04f1\u0003;\u0016\u0000\u04f1\u04f2\u0001"+ + "\u0000\u0000\u0000\u04f2\u04f3\u0006\u0099\n\u0000\u04f3\u0142\u0001\u0000"+ + "\u0000\u0000\u04f4\u04f5\u0003=\u0017\u0000\u04f5\u04f6\u0001\u0000\u0000"+ + "\u0000\u04f6\u04f7\u0006\u009a\u000e\u0000\u04f7\u04f8\u0006\u009a\u000b"+ + "\u0000\u04f8\u0144\u0001\u0000\u0000\u0000\u04f9\u04fa\u0007\u0001\u0000"+ + "\u0000\u04fa\u04fb\u0007\t\u0000\u0000\u04fb\u04fc\u0007\u000f\u0000\u0000"+ + "\u04fc\u04fd\u0007\u0007\u0000\u0000\u04fd\u0146\u0001\u0000\u0000\u0000"+ + "\u04fe\u04ff\u00037\u0014\u0000\u04ff\u0500\u0001\u0000\u0000\u0000\u0500"+ + "\u0501\u0006\u009c\n\u0000\u0501\u0148\u0001\u0000\u0000\u0000\u0502\u0503"+ + "\u00039\u0015\u0000\u0503\u0504\u0001\u0000\u0000\u0000\u0504\u0505\u0006"+ + "\u009d\n\u0000\u0505\u014a\u0001\u0000\u0000\u0000\u0506\u0507\u0003;"+ + "\u0016\u0000\u0507\u0508\u0001\u0000\u0000\u0000\u0508\u0509\u0006\u009e"+ + "\n\u0000\u0509\u014c\u0001\u0000\u0000\u0000\u050a\u050b\u0003\u00a5K"+ + "\u0000\u050b\u050c\u0001\u0000\u0000\u0000\u050c\u050d\u0006\u009f\u000f"+ + "\u0000\u050d\u050e\u0006\u009f\u000b\u0000\u050e\u014e\u0001\u0000\u0000"+ + "\u0000\u050f\u0510\u0005:\u0000\u0000\u0510\u0150\u0001\u0000\u0000\u0000"+ + "\u0511\u0517\u0003I\u001d\u0000\u0512\u0517\u0003?\u0018\u0000\u0513\u0517"+ + "\u0003g,\u0000\u0514\u0517\u0003A\u0019\u0000\u0515\u0517\u0003O \u0000"+ + "\u0516\u0511\u0001\u0000\u0000\u0000\u0516\u0512\u0001\u0000\u0000\u0000"+ + "\u0516\u0513\u0001\u0000\u0000\u0000\u0516\u0514\u0001\u0000\u0000\u0000"+ + "\u0516\u0515\u0001\u0000\u0000\u0000\u0517\u0518\u0001\u0000\u0000\u0000"+ + "\u0518\u0516\u0001\u0000\u0000\u0000\u0518\u0519\u0001\u0000\u0000\u0000"+ + "\u0519\u0152\u0001\u0000\u0000\u0000\u051a\u051b\u00037\u0014\u0000\u051b"+ + "\u051c\u0001\u0000\u0000\u0000\u051c\u051d\u0006\u00a2\n\u0000\u051d\u0154"+ + "\u0001\u0000\u0000\u0000\u051e\u051f\u00039\u0015\u0000\u051f\u0520\u0001"+ + "\u0000\u0000\u0000\u0520\u0521\u0006\u00a3\n\u0000\u0521\u0156\u0001\u0000"+ + "\u0000\u0000\u0522\u0523\u0003;\u0016\u0000\u0523\u0524\u0001\u0000\u0000"+ + "\u0000\u0524\u0525\u0006\u00a4\n\u0000\u0525\u0158\u0001\u0000\u0000\u0000"+ + "\u0526\u0527\u0003=\u0017\u0000\u0527\u0528\u0001\u0000\u0000\u0000\u0528"+ + "\u0529\u0006\u00a5\u000e\u0000\u0529\u052a\u0006\u00a5\u000b\u0000\u052a"+ + "\u015a\u0001\u0000\u0000\u0000\u052b\u052c\u0003\u014f\u00a0\u0000\u052c"+ + "\u052d\u0001\u0000\u0000\u0000\u052d\u052e\u0006\u00a6\u0010\u0000\u052e"+ + "\u015c\u0001\u0000\u0000\u0000\u052f\u0530\u0003c*\u0000\u0530\u0531\u0001"+ + "\u0000\u0000\u0000\u0531\u0532\u0006\u00a7\u0011\u0000\u0532\u015e\u0001"+ + "\u0000\u0000\u0000\u0533\u0534\u0003g,\u0000\u0534\u0535\u0001\u0000\u0000"+ + "\u0000\u0535\u0536\u0006\u00a8\u0015\u0000\u0536\u0160\u0001\u0000\u0000"+ + "\u0000\u0537\u0538\u0003\u0109}\u0000\u0538\u0539\u0001\u0000\u0000\u0000"+ + "\u0539\u053a\u0006\u00a9\u001f\u0000\u053a\u053b\u0006\u00a9 \u0000\u053b"+ + "\u0162\u0001\u0000\u0000\u0000\u053c\u053d\u0003\u00cd_\u0000\u053d\u053e"+ + "\u0001\u0000\u0000\u0000\u053e\u053f\u0006\u00aa\u0013\u0000\u053f\u0164"+ + "\u0001\u0000\u0000\u0000\u0540\u0541\u0003S\"\u0000\u0541\u0542\u0001"+ + "\u0000\u0000\u0000\u0542\u0543\u0006\u00ab\u0014\u0000\u0543\u0166\u0001"+ + "\u0000\u0000\u0000\u0544\u0545\u00037\u0014\u0000\u0545\u0546\u0001\u0000"+ + "\u0000\u0000\u0546\u0547\u0006\u00ac\n\u0000\u0547\u0168\u0001\u0000\u0000"+ + "\u0000\u0548\u0549\u00039\u0015\u0000\u0549\u054a\u0001\u0000\u0000\u0000"+ + "\u054a\u054b\u0006\u00ad\n\u0000\u054b\u016a\u0001\u0000\u0000\u0000\u054c"+ + "\u054d\u0003;\u0016\u0000\u054d\u054e\u0001\u0000\u0000\u0000\u054e\u054f"+ + "\u0006\u00ae\n\u0000\u054f\u016c\u0001\u0000\u0000\u0000\u0550\u0551\u0003"+ + "=\u0017\u0000\u0551\u0552\u0001\u0000\u0000\u0000\u0552\u0553\u0006\u00af"+ + "\u000e\u0000\u0553\u0554\u0006\u00af\u000b\u0000\u0554\u0555\u0006\u00af"+ + "\u000b\u0000\u0555\u016e\u0001\u0000\u0000\u0000\u0556\u0557\u0003c*\u0000"+ + "\u0557\u0558\u0001\u0000\u0000\u0000\u0558\u0559\u0006\u00b0\u0011\u0000"+ + "\u0559\u0170\u0001\u0000\u0000\u0000\u055a\u055b\u0003g,\u0000\u055b\u055c"+ + "\u0001\u0000\u0000\u0000\u055c\u055d\u0006\u00b1\u0015\u0000\u055d\u0172"+ + "\u0001\u0000\u0000\u0000\u055e\u055f\u0003\u00e7l\u0000\u055f\u0560\u0001"+ + "\u0000\u0000\u0000\u0560\u0561\u0006\u00b2\u0018\u0000\u0561\u0174\u0001"+ + "\u0000\u0000\u0000\u0562\u0563\u00037\u0014\u0000\u0563\u0564\u0001\u0000"+ + "\u0000\u0000\u0564\u0565\u0006\u00b3\n\u0000\u0565\u0176\u0001\u0000\u0000"+ + "\u0000\u0566\u0567\u00039\u0015\u0000\u0567\u0568\u0001\u0000\u0000\u0000"+ + "\u0568\u0569\u0006\u00b4\n\u0000\u0569\u0178\u0001\u0000\u0000\u0000\u056a"+ + "\u056b\u0003;\u0016\u0000\u056b\u056c\u0001\u0000\u0000\u0000\u056c\u056d"+ + "\u0006\u00b5\n\u0000\u056d\u017a\u0001\u0000\u0000\u0000\u056e\u056f\u0003"+ + "=\u0017\u0000\u056f\u0570\u0001\u0000\u0000\u0000\u0570\u0571\u0006\u00b6"+ + "\u000e\u0000\u0571\u0572\u0006\u00b6\u000b\u0000\u0572\u017c\u0001\u0000"+ + "\u0000\u0000\u0573\u0574\u0003\u00cd_\u0000\u0574\u0575\u0001\u0000\u0000"+ + "\u0000\u0575\u0576\u0006\u00b7\u0013\u0000\u0576\u0577\u0006\u00b7\u000b"+ + "\u0000\u0577\u0578\u0006\u00b7!\u0000\u0578\u017e\u0001\u0000\u0000\u0000"+ + "\u0579\u057a\u0003S\"\u0000\u057a\u057b\u0001\u0000\u0000\u0000\u057b"+ + "\u057c\u0006\u00b8\u0014\u0000\u057c\u057d\u0006\u00b8\u000b\u0000\u057d"+ + "\u057e\u0006\u00b8!\u0000\u057e\u0180\u0001\u0000\u0000\u0000\u057f\u0580"+ + "\u00037\u0014\u0000\u0580\u0581\u0001\u0000\u0000\u0000\u0581\u0582\u0006"+ + "\u00b9\n\u0000\u0582\u0182\u0001\u0000\u0000\u0000\u0583\u0584\u00039"+ + "\u0015\u0000\u0584\u0585\u0001\u0000\u0000\u0000\u0585\u0586\u0006\u00ba"+ + "\n\u0000\u0586\u0184\u0001\u0000\u0000\u0000\u0587\u0588\u0003;\u0016"+ + "\u0000\u0588\u0589\u0001\u0000\u0000\u0000\u0589\u058a\u0006\u00bb\n\u0000"+ + "\u058a\u0186\u0001\u0000\u0000\u0000\u058b\u058c\u0003\u014f\u00a0\u0000"+ + "\u058c\u058d\u0001\u0000\u0000\u0000\u058d\u058e\u0006\u00bc\u0010\u0000"+ + "\u058e\u058f\u0006\u00bc\u000b\u0000\u058f\u0590\u0006\u00bc\t\u0000\u0590"+ + "\u0188\u0001\u0000\u0000\u0000\u0591\u0592\u0003c*\u0000\u0592\u0593\u0001"+ + "\u0000\u0000\u0000\u0593\u0594\u0006\u00bd\u0011\u0000\u0594\u0595\u0006"+ + "\u00bd\u000b\u0000\u0595\u0596\u0006\u00bd\t\u0000\u0596\u018a\u0001\u0000"+ + "\u0000\u0000\u0597\u0598\u00037\u0014\u0000\u0598\u0599\u0001\u0000\u0000"+ + "\u0000\u0599\u059a\u0006\u00be\n\u0000\u059a\u018c\u0001\u0000\u0000\u0000"+ + "\u059b\u059c\u00039\u0015\u0000\u059c\u059d\u0001\u0000\u0000\u0000\u059d"+ + "\u059e\u0006\u00bf\n\u0000\u059e\u018e\u0001\u0000\u0000\u0000\u059f\u05a0"+ + "\u0003;\u0016\u0000\u05a0\u05a1\u0001\u0000\u0000\u0000\u05a1\u05a2\u0006"+ + "\u00c0\n\u0000\u05a2\u0190\u0001\u0000\u0000\u0000\u05a3\u05a4\u0003\u00ab"+ + "N\u0000\u05a4\u05a5\u0001\u0000\u0000\u0000\u05a5\u05a6\u0006\u00c1\u000b"+ + "\u0000\u05a6\u05a7\u0006\u00c1\u0000\u0000\u05a7\u05a8\u0006\u00c1\u001d"+ + "\u0000\u05a8\u0192\u0001\u0000\u0000\u0000\u05a9\u05aa\u0003\u00a7L\u0000"+ + "\u05aa\u05ab\u0001\u0000\u0000\u0000\u05ab\u05ac\u0006\u00c2\u000b\u0000"+ + "\u05ac\u05ad\u0006\u00c2\u0000\u0000\u05ad\u05ae\u0006\u00c2\u001e\u0000"+ + "\u05ae\u0194\u0001\u0000\u0000\u0000\u05af\u05b0\u0003Y%\u0000\u05b0\u05b1"+ + "\u0001\u0000\u0000\u0000\u05b1\u05b2\u0006\u00c3\u000b\u0000\u05b2\u05b3"+ + "\u0006\u00c3\u0000\u0000\u05b3\u05b4\u0006\u00c3\"\u0000\u05b4\u0196\u0001"+ + "\u0000\u0000\u0000\u05b5\u05b6\u0003=\u0017\u0000\u05b6\u05b7\u0001\u0000"+ + "\u0000\u0000\u05b7\u05b8\u0006\u00c4\u000e\u0000\u05b8\u05b9\u0006\u00c4"+ + "\u000b\u0000\u05b9\u0198\u0001\u0000\u0000\u0000A\u0000\u0001\u0002\u0003"+ + "\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e\u0241\u024b\u024f\u0252"+ + "\u025b\u025d\u0268\u027b\u0280\u0289\u0290\u0295\u0297\u02a2\u02aa\u02ad"+ + "\u02af\u02b4\u02b9\u02bf\u02c6\u02cb\u02d1\u02d4\u02dc\u02e0\u0361\u0366"+ + "\u036d\u036f\u037f\u0384\u0389\u038b\u0391\u03de\u03e3\u0412\u0416\u041b"+ + "\u0420\u0425\u0427\u042b\u042d\u0482\u0486\u048b\u0516\u0518#\u0005\u0001"+ + "\u0000\u0005\u0004\u0000\u0005\u0006\u0000\u0005\u0002\u0000\u0005\u0003"+ + "\u0000\u0005\b\u0000\u0005\u0005\u0000\u0005\t\u0000\u0005\u000b\u0000"+ + "\u0005\r\u0000\u0000\u0001\u0000\u0004\u0000\u0000\u0007A\u0000\u0005"+ + "\u0000\u0000\u0007\u0018\u0000\u0007B\u0000\u0007h\u0000\u0007!\u0000"+ + "\u0007\u001f\u0000\u0007L\u0000\u0007\u0019\u0000\u0007#\u0000\u0007/"+ + "\u0000\u0007@\u0000\u0007P\u0000\u0005\n\u0000\u0005\u0007\u0000\u0007"+ + "Z\u0000\u0007Y\u0000\u0007D\u0000\u0007C\u0000\u0007X\u0000\u0005\f\u0000"+ + "\u0005\u000e\u0000\u0007\u001c\u0000"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp index eb3c70385d628..5fdf80f24d9b0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp @@ -23,7 +23,6 @@ null null null null -null '|' null null @@ -65,6 +64,7 @@ null '%' null null +null ']' null null @@ -141,7 +141,6 @@ STATS WHERE DEV_INLINESTATS DEV_LOOKUP -DEV_MATCH DEV_METRICS UNKNOWN_CMD LINE_COMMENT @@ -186,6 +185,7 @@ MINUS ASTERISK SLASH PERCENT +DEV_MATCH NAMED_OR_POSITIONAL_PARAM OPENING_BRACKET CLOSING_BRACKET @@ -257,6 +257,7 @@ valueExpression operatorExpression primaryExpression functionExpression +functionName dataType rowCommand fields @@ -307,4 +308,4 @@ inlinestatsCommand atn: -[4, 1, 120, 580, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 128, 8, 1, 10, 1, 12, 1, 131, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 139, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 157, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 169, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 176, 8, 5, 10, 5, 12, 5, 179, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 186, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 192, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 200, 8, 5, 10, 5, 12, 5, 203, 9, 5, 1, 6, 1, 6, 3, 6, 207, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 214, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 219, 8, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 230, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 236, 8, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 5, 9, 244, 8, 9, 10, 9, 12, 9, 247, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 257, 8, 10, 1, 10, 1, 10, 1, 10, 5, 10, 262, 8, 10, 10, 10, 12, 10, 265, 9, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 5, 11, 273, 8, 11, 10, 11, 12, 11, 276, 9, 11, 3, 11, 278, 8, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 5, 14, 290, 8, 14, 10, 14, 12, 14, 293, 9, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 3, 15, 300, 8, 15, 1, 16, 1, 16, 1, 16, 1, 16, 5, 16, 306, 8, 16, 10, 16, 12, 16, 309, 9, 16, 1, 16, 3, 16, 312, 8, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 3, 17, 319, 8, 17, 1, 18, 1, 18, 1, 19, 1, 19, 1, 20, 1, 20, 3, 20, 327, 8, 20, 1, 21, 1, 21, 1, 21, 1, 21, 5, 21, 333, 8, 21, 10, 21, 12, 21, 336, 9, 21, 1, 22, 1, 22, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 5, 23, 346, 8, 23, 10, 23, 12, 23, 349, 9, 23, 1, 23, 3, 23, 352, 8, 23, 1, 23, 1, 23, 3, 23, 356, 8, 23, 1, 24, 1, 24, 1, 24, 1, 25, 1, 25, 3, 25, 363, 8, 25, 1, 25, 1, 25, 3, 25, 367, 8, 25, 1, 26, 1, 26, 1, 26, 5, 26, 372, 8, 26, 10, 26, 12, 26, 375, 9, 26, 1, 27, 1, 27, 1, 27, 5, 27, 380, 8, 27, 10, 27, 12, 27, 383, 9, 27, 1, 28, 1, 28, 1, 28, 5, 28, 388, 8, 28, 10, 28, 12, 28, 391, 9, 28, 1, 29, 1, 29, 1, 30, 1, 30, 3, 30, 397, 8, 30, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 5, 31, 412, 8, 31, 10, 31, 12, 31, 415, 9, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 5, 31, 423, 8, 31, 10, 31, 12, 31, 426, 9, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 5, 31, 434, 8, 31, 10, 31, 12, 31, 437, 9, 31, 1, 31, 1, 31, 3, 31, 441, 8, 31, 1, 32, 1, 32, 3, 32, 445, 8, 32, 1, 33, 1, 33, 3, 33, 449, 8, 33, 1, 34, 1, 34, 1, 34, 1, 35, 1, 35, 1, 35, 1, 35, 5, 35, 458, 8, 35, 10, 35, 12, 35, 461, 9, 35, 1, 36, 1, 36, 3, 36, 465, 8, 36, 1, 36, 1, 36, 3, 36, 469, 8, 36, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 1, 39, 1, 39, 1, 39, 1, 39, 5, 39, 481, 8, 39, 10, 39, 12, 39, 484, 9, 39, 1, 40, 1, 40, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 1, 41, 3, 41, 494, 8, 41, 1, 42, 1, 42, 1, 42, 1, 42, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 5, 44, 506, 8, 44, 10, 44, 12, 44, 509, 9, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 47, 1, 47, 3, 47, 519, 8, 47, 1, 48, 3, 48, 522, 8, 48, 1, 48, 1, 48, 1, 49, 3, 49, 527, 8, 49, 1, 49, 1, 49, 1, 50, 1, 50, 1, 51, 1, 51, 1, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 55, 3, 55, 549, 8, 55, 1, 55, 1, 55, 1, 55, 1, 55, 5, 55, 555, 8, 55, 10, 55, 12, 55, 558, 9, 55, 3, 55, 560, 8, 55, 1, 56, 1, 56, 1, 56, 3, 56, 565, 8, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 3, 58, 578, 8, 58, 1, 58, 0, 4, 2, 10, 18, 20, 59, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 0, 8, 1, 0, 59, 60, 1, 0, 61, 63, 2, 0, 26, 26, 76, 76, 1, 0, 67, 68, 2, 0, 31, 31, 35, 35, 2, 0, 38, 38, 41, 41, 2, 0, 37, 37, 51, 51, 2, 0, 52, 52, 54, 58, 606, 0, 118, 1, 0, 0, 0, 2, 121, 1, 0, 0, 0, 4, 138, 1, 0, 0, 0, 6, 156, 1, 0, 0, 0, 8, 158, 1, 0, 0, 0, 10, 191, 1, 0, 0, 0, 12, 218, 1, 0, 0, 0, 14, 220, 1, 0, 0, 0, 16, 229, 1, 0, 0, 0, 18, 235, 1, 0, 0, 0, 20, 256, 1, 0, 0, 0, 22, 266, 1, 0, 0, 0, 24, 281, 1, 0, 0, 0, 26, 283, 1, 0, 0, 0, 28, 286, 1, 0, 0, 0, 30, 299, 1, 0, 0, 0, 32, 301, 1, 0, 0, 0, 34, 318, 1, 0, 0, 0, 36, 320, 1, 0, 0, 0, 38, 322, 1, 0, 0, 0, 40, 326, 1, 0, 0, 0, 42, 328, 1, 0, 0, 0, 44, 337, 1, 0, 0, 0, 46, 341, 1, 0, 0, 0, 48, 357, 1, 0, 0, 0, 50, 360, 1, 0, 0, 0, 52, 368, 1, 0, 0, 0, 54, 376, 1, 0, 0, 0, 56, 384, 1, 0, 0, 0, 58, 392, 1, 0, 0, 0, 60, 396, 1, 0, 0, 0, 62, 440, 1, 0, 0, 0, 64, 444, 1, 0, 0, 0, 66, 448, 1, 0, 0, 0, 68, 450, 1, 0, 0, 0, 70, 453, 1, 0, 0, 0, 72, 462, 1, 0, 0, 0, 74, 470, 1, 0, 0, 0, 76, 473, 1, 0, 0, 0, 78, 476, 1, 0, 0, 0, 80, 485, 1, 0, 0, 0, 82, 489, 1, 0, 0, 0, 84, 495, 1, 0, 0, 0, 86, 499, 1, 0, 0, 0, 88, 502, 1, 0, 0, 0, 90, 510, 1, 0, 0, 0, 92, 514, 1, 0, 0, 0, 94, 518, 1, 0, 0, 0, 96, 521, 1, 0, 0, 0, 98, 526, 1, 0, 0, 0, 100, 530, 1, 0, 0, 0, 102, 532, 1, 0, 0, 0, 104, 534, 1, 0, 0, 0, 106, 537, 1, 0, 0, 0, 108, 541, 1, 0, 0, 0, 110, 544, 1, 0, 0, 0, 112, 564, 1, 0, 0, 0, 114, 568, 1, 0, 0, 0, 116, 573, 1, 0, 0, 0, 118, 119, 3, 2, 1, 0, 119, 120, 5, 0, 0, 1, 120, 1, 1, 0, 0, 0, 121, 122, 6, 1, -1, 0, 122, 123, 3, 4, 2, 0, 123, 129, 1, 0, 0, 0, 124, 125, 10, 1, 0, 0, 125, 126, 5, 25, 0, 0, 126, 128, 3, 6, 3, 0, 127, 124, 1, 0, 0, 0, 128, 131, 1, 0, 0, 0, 129, 127, 1, 0, 0, 0, 129, 130, 1, 0, 0, 0, 130, 3, 1, 0, 0, 0, 131, 129, 1, 0, 0, 0, 132, 139, 3, 104, 52, 0, 133, 139, 3, 32, 16, 0, 134, 139, 3, 26, 13, 0, 135, 139, 3, 108, 54, 0, 136, 137, 4, 2, 1, 0, 137, 139, 3, 46, 23, 0, 138, 132, 1, 0, 0, 0, 138, 133, 1, 0, 0, 0, 138, 134, 1, 0, 0, 0, 138, 135, 1, 0, 0, 0, 138, 136, 1, 0, 0, 0, 139, 5, 1, 0, 0, 0, 140, 157, 3, 48, 24, 0, 141, 157, 3, 8, 4, 0, 142, 157, 3, 74, 37, 0, 143, 157, 3, 68, 34, 0, 144, 157, 3, 50, 25, 0, 145, 157, 3, 70, 35, 0, 146, 157, 3, 76, 38, 0, 147, 157, 3, 78, 39, 0, 148, 157, 3, 82, 41, 0, 149, 157, 3, 84, 42, 0, 150, 157, 3, 110, 55, 0, 151, 157, 3, 86, 43, 0, 152, 153, 4, 3, 2, 0, 153, 157, 3, 116, 58, 0, 154, 155, 4, 3, 3, 0, 155, 157, 3, 114, 57, 0, 156, 140, 1, 0, 0, 0, 156, 141, 1, 0, 0, 0, 156, 142, 1, 0, 0, 0, 156, 143, 1, 0, 0, 0, 156, 144, 1, 0, 0, 0, 156, 145, 1, 0, 0, 0, 156, 146, 1, 0, 0, 0, 156, 147, 1, 0, 0, 0, 156, 148, 1, 0, 0, 0, 156, 149, 1, 0, 0, 0, 156, 150, 1, 0, 0, 0, 156, 151, 1, 0, 0, 0, 156, 152, 1, 0, 0, 0, 156, 154, 1, 0, 0, 0, 157, 7, 1, 0, 0, 0, 158, 159, 5, 16, 0, 0, 159, 160, 3, 10, 5, 0, 160, 9, 1, 0, 0, 0, 161, 162, 6, 5, -1, 0, 162, 163, 5, 44, 0, 0, 163, 192, 3, 10, 5, 8, 164, 192, 3, 16, 8, 0, 165, 192, 3, 12, 6, 0, 166, 168, 3, 16, 8, 0, 167, 169, 5, 44, 0, 0, 168, 167, 1, 0, 0, 0, 168, 169, 1, 0, 0, 0, 169, 170, 1, 0, 0, 0, 170, 171, 5, 39, 0, 0, 171, 172, 5, 43, 0, 0, 172, 177, 3, 16, 8, 0, 173, 174, 5, 34, 0, 0, 174, 176, 3, 16, 8, 0, 175, 173, 1, 0, 0, 0, 176, 179, 1, 0, 0, 0, 177, 175, 1, 0, 0, 0, 177, 178, 1, 0, 0, 0, 178, 180, 1, 0, 0, 0, 179, 177, 1, 0, 0, 0, 180, 181, 5, 50, 0, 0, 181, 192, 1, 0, 0, 0, 182, 183, 3, 16, 8, 0, 183, 185, 5, 40, 0, 0, 184, 186, 5, 44, 0, 0, 185, 184, 1, 0, 0, 0, 185, 186, 1, 0, 0, 0, 186, 187, 1, 0, 0, 0, 187, 188, 5, 45, 0, 0, 188, 192, 1, 0, 0, 0, 189, 190, 4, 5, 4, 0, 190, 192, 3, 14, 7, 0, 191, 161, 1, 0, 0, 0, 191, 164, 1, 0, 0, 0, 191, 165, 1, 0, 0, 0, 191, 166, 1, 0, 0, 0, 191, 182, 1, 0, 0, 0, 191, 189, 1, 0, 0, 0, 192, 201, 1, 0, 0, 0, 193, 194, 10, 5, 0, 0, 194, 195, 5, 30, 0, 0, 195, 200, 3, 10, 5, 6, 196, 197, 10, 4, 0, 0, 197, 198, 5, 47, 0, 0, 198, 200, 3, 10, 5, 5, 199, 193, 1, 0, 0, 0, 199, 196, 1, 0, 0, 0, 200, 203, 1, 0, 0, 0, 201, 199, 1, 0, 0, 0, 201, 202, 1, 0, 0, 0, 202, 11, 1, 0, 0, 0, 203, 201, 1, 0, 0, 0, 204, 206, 3, 16, 8, 0, 205, 207, 5, 44, 0, 0, 206, 205, 1, 0, 0, 0, 206, 207, 1, 0, 0, 0, 207, 208, 1, 0, 0, 0, 208, 209, 5, 42, 0, 0, 209, 210, 3, 100, 50, 0, 210, 219, 1, 0, 0, 0, 211, 213, 3, 16, 8, 0, 212, 214, 5, 44, 0, 0, 213, 212, 1, 0, 0, 0, 213, 214, 1, 0, 0, 0, 214, 215, 1, 0, 0, 0, 215, 216, 5, 49, 0, 0, 216, 217, 3, 100, 50, 0, 217, 219, 1, 0, 0, 0, 218, 204, 1, 0, 0, 0, 218, 211, 1, 0, 0, 0, 219, 13, 1, 0, 0, 0, 220, 221, 3, 16, 8, 0, 221, 222, 5, 19, 0, 0, 222, 223, 3, 100, 50, 0, 223, 15, 1, 0, 0, 0, 224, 230, 3, 18, 9, 0, 225, 226, 3, 18, 9, 0, 226, 227, 3, 102, 51, 0, 227, 228, 3, 18, 9, 0, 228, 230, 1, 0, 0, 0, 229, 224, 1, 0, 0, 0, 229, 225, 1, 0, 0, 0, 230, 17, 1, 0, 0, 0, 231, 232, 6, 9, -1, 0, 232, 236, 3, 20, 10, 0, 233, 234, 7, 0, 0, 0, 234, 236, 3, 18, 9, 3, 235, 231, 1, 0, 0, 0, 235, 233, 1, 0, 0, 0, 236, 245, 1, 0, 0, 0, 237, 238, 10, 2, 0, 0, 238, 239, 7, 1, 0, 0, 239, 244, 3, 18, 9, 3, 240, 241, 10, 1, 0, 0, 241, 242, 7, 0, 0, 0, 242, 244, 3, 18, 9, 2, 243, 237, 1, 0, 0, 0, 243, 240, 1, 0, 0, 0, 244, 247, 1, 0, 0, 0, 245, 243, 1, 0, 0, 0, 245, 246, 1, 0, 0, 0, 246, 19, 1, 0, 0, 0, 247, 245, 1, 0, 0, 0, 248, 249, 6, 10, -1, 0, 249, 257, 3, 62, 31, 0, 250, 257, 3, 52, 26, 0, 251, 257, 3, 22, 11, 0, 252, 253, 5, 43, 0, 0, 253, 254, 3, 10, 5, 0, 254, 255, 5, 50, 0, 0, 255, 257, 1, 0, 0, 0, 256, 248, 1, 0, 0, 0, 256, 250, 1, 0, 0, 0, 256, 251, 1, 0, 0, 0, 256, 252, 1, 0, 0, 0, 257, 263, 1, 0, 0, 0, 258, 259, 10, 1, 0, 0, 259, 260, 5, 33, 0, 0, 260, 262, 3, 24, 12, 0, 261, 258, 1, 0, 0, 0, 262, 265, 1, 0, 0, 0, 263, 261, 1, 0, 0, 0, 263, 264, 1, 0, 0, 0, 264, 21, 1, 0, 0, 0, 265, 263, 1, 0, 0, 0, 266, 267, 3, 66, 33, 0, 267, 277, 5, 43, 0, 0, 268, 278, 5, 61, 0, 0, 269, 274, 3, 10, 5, 0, 270, 271, 5, 34, 0, 0, 271, 273, 3, 10, 5, 0, 272, 270, 1, 0, 0, 0, 273, 276, 1, 0, 0, 0, 274, 272, 1, 0, 0, 0, 274, 275, 1, 0, 0, 0, 275, 278, 1, 0, 0, 0, 276, 274, 1, 0, 0, 0, 277, 268, 1, 0, 0, 0, 277, 269, 1, 0, 0, 0, 277, 278, 1, 0, 0, 0, 278, 279, 1, 0, 0, 0, 279, 280, 5, 50, 0, 0, 280, 23, 1, 0, 0, 0, 281, 282, 3, 58, 29, 0, 282, 25, 1, 0, 0, 0, 283, 284, 5, 12, 0, 0, 284, 285, 3, 28, 14, 0, 285, 27, 1, 0, 0, 0, 286, 291, 3, 30, 15, 0, 287, 288, 5, 34, 0, 0, 288, 290, 3, 30, 15, 0, 289, 287, 1, 0, 0, 0, 290, 293, 1, 0, 0, 0, 291, 289, 1, 0, 0, 0, 291, 292, 1, 0, 0, 0, 292, 29, 1, 0, 0, 0, 293, 291, 1, 0, 0, 0, 294, 300, 3, 10, 5, 0, 295, 296, 3, 52, 26, 0, 296, 297, 5, 32, 0, 0, 297, 298, 3, 10, 5, 0, 298, 300, 1, 0, 0, 0, 299, 294, 1, 0, 0, 0, 299, 295, 1, 0, 0, 0, 300, 31, 1, 0, 0, 0, 301, 302, 5, 6, 0, 0, 302, 307, 3, 34, 17, 0, 303, 304, 5, 34, 0, 0, 304, 306, 3, 34, 17, 0, 305, 303, 1, 0, 0, 0, 306, 309, 1, 0, 0, 0, 307, 305, 1, 0, 0, 0, 307, 308, 1, 0, 0, 0, 308, 311, 1, 0, 0, 0, 309, 307, 1, 0, 0, 0, 310, 312, 3, 40, 20, 0, 311, 310, 1, 0, 0, 0, 311, 312, 1, 0, 0, 0, 312, 33, 1, 0, 0, 0, 313, 314, 3, 36, 18, 0, 314, 315, 5, 104, 0, 0, 315, 316, 3, 38, 19, 0, 316, 319, 1, 0, 0, 0, 317, 319, 3, 38, 19, 0, 318, 313, 1, 0, 0, 0, 318, 317, 1, 0, 0, 0, 319, 35, 1, 0, 0, 0, 320, 321, 5, 76, 0, 0, 321, 37, 1, 0, 0, 0, 322, 323, 7, 2, 0, 0, 323, 39, 1, 0, 0, 0, 324, 327, 3, 42, 21, 0, 325, 327, 3, 44, 22, 0, 326, 324, 1, 0, 0, 0, 326, 325, 1, 0, 0, 0, 327, 41, 1, 0, 0, 0, 328, 329, 5, 75, 0, 0, 329, 334, 5, 76, 0, 0, 330, 331, 5, 34, 0, 0, 331, 333, 5, 76, 0, 0, 332, 330, 1, 0, 0, 0, 333, 336, 1, 0, 0, 0, 334, 332, 1, 0, 0, 0, 334, 335, 1, 0, 0, 0, 335, 43, 1, 0, 0, 0, 336, 334, 1, 0, 0, 0, 337, 338, 5, 65, 0, 0, 338, 339, 3, 42, 21, 0, 339, 340, 5, 66, 0, 0, 340, 45, 1, 0, 0, 0, 341, 342, 5, 20, 0, 0, 342, 347, 3, 34, 17, 0, 343, 344, 5, 34, 0, 0, 344, 346, 3, 34, 17, 0, 345, 343, 1, 0, 0, 0, 346, 349, 1, 0, 0, 0, 347, 345, 1, 0, 0, 0, 347, 348, 1, 0, 0, 0, 348, 351, 1, 0, 0, 0, 349, 347, 1, 0, 0, 0, 350, 352, 3, 28, 14, 0, 351, 350, 1, 0, 0, 0, 351, 352, 1, 0, 0, 0, 352, 355, 1, 0, 0, 0, 353, 354, 5, 29, 0, 0, 354, 356, 3, 28, 14, 0, 355, 353, 1, 0, 0, 0, 355, 356, 1, 0, 0, 0, 356, 47, 1, 0, 0, 0, 357, 358, 5, 4, 0, 0, 358, 359, 3, 28, 14, 0, 359, 49, 1, 0, 0, 0, 360, 362, 5, 15, 0, 0, 361, 363, 3, 28, 14, 0, 362, 361, 1, 0, 0, 0, 362, 363, 1, 0, 0, 0, 363, 366, 1, 0, 0, 0, 364, 365, 5, 29, 0, 0, 365, 367, 3, 28, 14, 0, 366, 364, 1, 0, 0, 0, 366, 367, 1, 0, 0, 0, 367, 51, 1, 0, 0, 0, 368, 373, 3, 66, 33, 0, 369, 370, 5, 36, 0, 0, 370, 372, 3, 66, 33, 0, 371, 369, 1, 0, 0, 0, 372, 375, 1, 0, 0, 0, 373, 371, 1, 0, 0, 0, 373, 374, 1, 0, 0, 0, 374, 53, 1, 0, 0, 0, 375, 373, 1, 0, 0, 0, 376, 381, 3, 60, 30, 0, 377, 378, 5, 36, 0, 0, 378, 380, 3, 60, 30, 0, 379, 377, 1, 0, 0, 0, 380, 383, 1, 0, 0, 0, 381, 379, 1, 0, 0, 0, 381, 382, 1, 0, 0, 0, 382, 55, 1, 0, 0, 0, 383, 381, 1, 0, 0, 0, 384, 389, 3, 54, 27, 0, 385, 386, 5, 34, 0, 0, 386, 388, 3, 54, 27, 0, 387, 385, 1, 0, 0, 0, 388, 391, 1, 0, 0, 0, 389, 387, 1, 0, 0, 0, 389, 390, 1, 0, 0, 0, 390, 57, 1, 0, 0, 0, 391, 389, 1, 0, 0, 0, 392, 393, 7, 3, 0, 0, 393, 59, 1, 0, 0, 0, 394, 397, 5, 80, 0, 0, 395, 397, 3, 64, 32, 0, 396, 394, 1, 0, 0, 0, 396, 395, 1, 0, 0, 0, 397, 61, 1, 0, 0, 0, 398, 441, 5, 45, 0, 0, 399, 400, 3, 98, 49, 0, 400, 401, 5, 67, 0, 0, 401, 441, 1, 0, 0, 0, 402, 441, 3, 96, 48, 0, 403, 441, 3, 98, 49, 0, 404, 441, 3, 92, 46, 0, 405, 441, 3, 64, 32, 0, 406, 441, 3, 100, 50, 0, 407, 408, 5, 65, 0, 0, 408, 413, 3, 94, 47, 0, 409, 410, 5, 34, 0, 0, 410, 412, 3, 94, 47, 0, 411, 409, 1, 0, 0, 0, 412, 415, 1, 0, 0, 0, 413, 411, 1, 0, 0, 0, 413, 414, 1, 0, 0, 0, 414, 416, 1, 0, 0, 0, 415, 413, 1, 0, 0, 0, 416, 417, 5, 66, 0, 0, 417, 441, 1, 0, 0, 0, 418, 419, 5, 65, 0, 0, 419, 424, 3, 92, 46, 0, 420, 421, 5, 34, 0, 0, 421, 423, 3, 92, 46, 0, 422, 420, 1, 0, 0, 0, 423, 426, 1, 0, 0, 0, 424, 422, 1, 0, 0, 0, 424, 425, 1, 0, 0, 0, 425, 427, 1, 0, 0, 0, 426, 424, 1, 0, 0, 0, 427, 428, 5, 66, 0, 0, 428, 441, 1, 0, 0, 0, 429, 430, 5, 65, 0, 0, 430, 435, 3, 100, 50, 0, 431, 432, 5, 34, 0, 0, 432, 434, 3, 100, 50, 0, 433, 431, 1, 0, 0, 0, 434, 437, 1, 0, 0, 0, 435, 433, 1, 0, 0, 0, 435, 436, 1, 0, 0, 0, 436, 438, 1, 0, 0, 0, 437, 435, 1, 0, 0, 0, 438, 439, 5, 66, 0, 0, 439, 441, 1, 0, 0, 0, 440, 398, 1, 0, 0, 0, 440, 399, 1, 0, 0, 0, 440, 402, 1, 0, 0, 0, 440, 403, 1, 0, 0, 0, 440, 404, 1, 0, 0, 0, 440, 405, 1, 0, 0, 0, 440, 406, 1, 0, 0, 0, 440, 407, 1, 0, 0, 0, 440, 418, 1, 0, 0, 0, 440, 429, 1, 0, 0, 0, 441, 63, 1, 0, 0, 0, 442, 445, 5, 48, 0, 0, 443, 445, 5, 64, 0, 0, 444, 442, 1, 0, 0, 0, 444, 443, 1, 0, 0, 0, 445, 65, 1, 0, 0, 0, 446, 449, 3, 58, 29, 0, 447, 449, 3, 64, 32, 0, 448, 446, 1, 0, 0, 0, 448, 447, 1, 0, 0, 0, 449, 67, 1, 0, 0, 0, 450, 451, 5, 9, 0, 0, 451, 452, 5, 27, 0, 0, 452, 69, 1, 0, 0, 0, 453, 454, 5, 14, 0, 0, 454, 459, 3, 72, 36, 0, 455, 456, 5, 34, 0, 0, 456, 458, 3, 72, 36, 0, 457, 455, 1, 0, 0, 0, 458, 461, 1, 0, 0, 0, 459, 457, 1, 0, 0, 0, 459, 460, 1, 0, 0, 0, 460, 71, 1, 0, 0, 0, 461, 459, 1, 0, 0, 0, 462, 464, 3, 10, 5, 0, 463, 465, 7, 4, 0, 0, 464, 463, 1, 0, 0, 0, 464, 465, 1, 0, 0, 0, 465, 468, 1, 0, 0, 0, 466, 467, 5, 46, 0, 0, 467, 469, 7, 5, 0, 0, 468, 466, 1, 0, 0, 0, 468, 469, 1, 0, 0, 0, 469, 73, 1, 0, 0, 0, 470, 471, 5, 8, 0, 0, 471, 472, 3, 56, 28, 0, 472, 75, 1, 0, 0, 0, 473, 474, 5, 2, 0, 0, 474, 475, 3, 56, 28, 0, 475, 77, 1, 0, 0, 0, 476, 477, 5, 11, 0, 0, 477, 482, 3, 80, 40, 0, 478, 479, 5, 34, 0, 0, 479, 481, 3, 80, 40, 0, 480, 478, 1, 0, 0, 0, 481, 484, 1, 0, 0, 0, 482, 480, 1, 0, 0, 0, 482, 483, 1, 0, 0, 0, 483, 79, 1, 0, 0, 0, 484, 482, 1, 0, 0, 0, 485, 486, 3, 54, 27, 0, 486, 487, 5, 84, 0, 0, 487, 488, 3, 54, 27, 0, 488, 81, 1, 0, 0, 0, 489, 490, 5, 1, 0, 0, 490, 491, 3, 20, 10, 0, 491, 493, 3, 100, 50, 0, 492, 494, 3, 88, 44, 0, 493, 492, 1, 0, 0, 0, 493, 494, 1, 0, 0, 0, 494, 83, 1, 0, 0, 0, 495, 496, 5, 7, 0, 0, 496, 497, 3, 20, 10, 0, 497, 498, 3, 100, 50, 0, 498, 85, 1, 0, 0, 0, 499, 500, 5, 10, 0, 0, 500, 501, 3, 52, 26, 0, 501, 87, 1, 0, 0, 0, 502, 507, 3, 90, 45, 0, 503, 504, 5, 34, 0, 0, 504, 506, 3, 90, 45, 0, 505, 503, 1, 0, 0, 0, 506, 509, 1, 0, 0, 0, 507, 505, 1, 0, 0, 0, 507, 508, 1, 0, 0, 0, 508, 89, 1, 0, 0, 0, 509, 507, 1, 0, 0, 0, 510, 511, 3, 58, 29, 0, 511, 512, 5, 32, 0, 0, 512, 513, 3, 62, 31, 0, 513, 91, 1, 0, 0, 0, 514, 515, 7, 6, 0, 0, 515, 93, 1, 0, 0, 0, 516, 519, 3, 96, 48, 0, 517, 519, 3, 98, 49, 0, 518, 516, 1, 0, 0, 0, 518, 517, 1, 0, 0, 0, 519, 95, 1, 0, 0, 0, 520, 522, 7, 0, 0, 0, 521, 520, 1, 0, 0, 0, 521, 522, 1, 0, 0, 0, 522, 523, 1, 0, 0, 0, 523, 524, 5, 28, 0, 0, 524, 97, 1, 0, 0, 0, 525, 527, 7, 0, 0, 0, 526, 525, 1, 0, 0, 0, 526, 527, 1, 0, 0, 0, 527, 528, 1, 0, 0, 0, 528, 529, 5, 27, 0, 0, 529, 99, 1, 0, 0, 0, 530, 531, 5, 26, 0, 0, 531, 101, 1, 0, 0, 0, 532, 533, 7, 7, 0, 0, 533, 103, 1, 0, 0, 0, 534, 535, 5, 5, 0, 0, 535, 536, 3, 106, 53, 0, 536, 105, 1, 0, 0, 0, 537, 538, 5, 65, 0, 0, 538, 539, 3, 2, 1, 0, 539, 540, 5, 66, 0, 0, 540, 107, 1, 0, 0, 0, 541, 542, 5, 13, 0, 0, 542, 543, 5, 100, 0, 0, 543, 109, 1, 0, 0, 0, 544, 545, 5, 3, 0, 0, 545, 548, 5, 90, 0, 0, 546, 547, 5, 88, 0, 0, 547, 549, 3, 54, 27, 0, 548, 546, 1, 0, 0, 0, 548, 549, 1, 0, 0, 0, 549, 559, 1, 0, 0, 0, 550, 551, 5, 89, 0, 0, 551, 556, 3, 112, 56, 0, 552, 553, 5, 34, 0, 0, 553, 555, 3, 112, 56, 0, 554, 552, 1, 0, 0, 0, 555, 558, 1, 0, 0, 0, 556, 554, 1, 0, 0, 0, 556, 557, 1, 0, 0, 0, 557, 560, 1, 0, 0, 0, 558, 556, 1, 0, 0, 0, 559, 550, 1, 0, 0, 0, 559, 560, 1, 0, 0, 0, 560, 111, 1, 0, 0, 0, 561, 562, 3, 54, 27, 0, 562, 563, 5, 32, 0, 0, 563, 565, 1, 0, 0, 0, 564, 561, 1, 0, 0, 0, 564, 565, 1, 0, 0, 0, 565, 566, 1, 0, 0, 0, 566, 567, 3, 54, 27, 0, 567, 113, 1, 0, 0, 0, 568, 569, 5, 18, 0, 0, 569, 570, 3, 34, 17, 0, 570, 571, 5, 88, 0, 0, 571, 572, 3, 56, 28, 0, 572, 115, 1, 0, 0, 0, 573, 574, 5, 17, 0, 0, 574, 577, 3, 28, 14, 0, 575, 576, 5, 29, 0, 0, 576, 578, 3, 28, 14, 0, 577, 575, 1, 0, 0, 0, 577, 578, 1, 0, 0, 0, 578, 117, 1, 0, 0, 0, 56, 129, 138, 156, 168, 177, 185, 191, 199, 201, 206, 213, 218, 229, 235, 243, 245, 256, 263, 274, 277, 291, 299, 307, 311, 318, 326, 334, 347, 351, 355, 362, 366, 373, 381, 389, 396, 413, 424, 435, 440, 444, 448, 459, 464, 468, 482, 493, 507, 518, 521, 526, 548, 556, 559, 564, 577] \ No newline at end of file +[4, 1, 120, 587, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 130, 8, 1, 10, 1, 12, 1, 133, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 141, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 159, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 171, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 178, 8, 5, 10, 5, 12, 5, 181, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 188, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 194, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 202, 8, 5, 10, 5, 12, 5, 205, 9, 5, 1, 6, 1, 6, 3, 6, 209, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 216, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 221, 8, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 232, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 238, 8, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 5, 9, 246, 8, 9, 10, 9, 12, 9, 249, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 259, 8, 10, 1, 10, 1, 10, 1, 10, 5, 10, 264, 8, 10, 10, 10, 12, 10, 267, 9, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 5, 11, 275, 8, 11, 10, 11, 12, 11, 278, 9, 11, 3, 11, 280, 8, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 3, 12, 287, 8, 12, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 5, 15, 297, 8, 15, 10, 15, 12, 15, 300, 9, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 3, 16, 307, 8, 16, 1, 17, 1, 17, 1, 17, 1, 17, 5, 17, 313, 8, 17, 10, 17, 12, 17, 316, 9, 17, 1, 17, 3, 17, 319, 8, 17, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 3, 18, 326, 8, 18, 1, 19, 1, 19, 1, 20, 1, 20, 1, 21, 1, 21, 3, 21, 334, 8, 21, 1, 22, 1, 22, 1, 22, 1, 22, 5, 22, 340, 8, 22, 10, 22, 12, 22, 343, 9, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 1, 24, 5, 24, 353, 8, 24, 10, 24, 12, 24, 356, 9, 24, 1, 24, 3, 24, 359, 8, 24, 1, 24, 1, 24, 3, 24, 363, 8, 24, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 3, 26, 370, 8, 26, 1, 26, 1, 26, 3, 26, 374, 8, 26, 1, 27, 1, 27, 1, 27, 5, 27, 379, 8, 27, 10, 27, 12, 27, 382, 9, 27, 1, 28, 1, 28, 1, 28, 5, 28, 387, 8, 28, 10, 28, 12, 28, 390, 9, 28, 1, 29, 1, 29, 1, 29, 5, 29, 395, 8, 29, 10, 29, 12, 29, 398, 9, 29, 1, 30, 1, 30, 1, 31, 1, 31, 3, 31, 404, 8, 31, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 5, 32, 419, 8, 32, 10, 32, 12, 32, 422, 9, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 5, 32, 430, 8, 32, 10, 32, 12, 32, 433, 9, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 5, 32, 441, 8, 32, 10, 32, 12, 32, 444, 9, 32, 1, 32, 1, 32, 3, 32, 448, 8, 32, 1, 33, 1, 33, 3, 33, 452, 8, 33, 1, 34, 1, 34, 3, 34, 456, 8, 34, 1, 35, 1, 35, 1, 35, 1, 36, 1, 36, 1, 36, 1, 36, 5, 36, 465, 8, 36, 10, 36, 12, 36, 468, 9, 36, 1, 37, 1, 37, 3, 37, 472, 8, 37, 1, 37, 1, 37, 3, 37, 476, 8, 37, 1, 38, 1, 38, 1, 38, 1, 39, 1, 39, 1, 39, 1, 40, 1, 40, 1, 40, 1, 40, 5, 40, 488, 8, 40, 10, 40, 12, 40, 491, 9, 40, 1, 41, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 42, 3, 42, 501, 8, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 45, 1, 45, 1, 45, 5, 45, 513, 8, 45, 10, 45, 12, 45, 516, 9, 45, 1, 46, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 48, 1, 48, 3, 48, 526, 8, 48, 1, 49, 3, 49, 529, 8, 49, 1, 49, 1, 49, 1, 50, 3, 50, 534, 8, 50, 1, 50, 1, 50, 1, 51, 1, 51, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 56, 3, 56, 556, 8, 56, 1, 56, 1, 56, 1, 56, 1, 56, 5, 56, 562, 8, 56, 10, 56, 12, 56, 565, 9, 56, 3, 56, 567, 8, 56, 1, 57, 1, 57, 1, 57, 3, 57, 572, 8, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 1, 58, 1, 59, 1, 59, 1, 59, 1, 59, 3, 59, 585, 8, 59, 1, 59, 0, 4, 2, 10, 18, 20, 60, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 0, 8, 1, 0, 58, 59, 1, 0, 60, 62, 2, 0, 25, 25, 76, 76, 1, 0, 67, 68, 2, 0, 30, 30, 34, 34, 2, 0, 37, 37, 40, 40, 2, 0, 36, 36, 50, 50, 2, 0, 51, 51, 53, 57, 613, 0, 120, 1, 0, 0, 0, 2, 123, 1, 0, 0, 0, 4, 140, 1, 0, 0, 0, 6, 158, 1, 0, 0, 0, 8, 160, 1, 0, 0, 0, 10, 193, 1, 0, 0, 0, 12, 220, 1, 0, 0, 0, 14, 222, 1, 0, 0, 0, 16, 231, 1, 0, 0, 0, 18, 237, 1, 0, 0, 0, 20, 258, 1, 0, 0, 0, 22, 268, 1, 0, 0, 0, 24, 286, 1, 0, 0, 0, 26, 288, 1, 0, 0, 0, 28, 290, 1, 0, 0, 0, 30, 293, 1, 0, 0, 0, 32, 306, 1, 0, 0, 0, 34, 308, 1, 0, 0, 0, 36, 325, 1, 0, 0, 0, 38, 327, 1, 0, 0, 0, 40, 329, 1, 0, 0, 0, 42, 333, 1, 0, 0, 0, 44, 335, 1, 0, 0, 0, 46, 344, 1, 0, 0, 0, 48, 348, 1, 0, 0, 0, 50, 364, 1, 0, 0, 0, 52, 367, 1, 0, 0, 0, 54, 375, 1, 0, 0, 0, 56, 383, 1, 0, 0, 0, 58, 391, 1, 0, 0, 0, 60, 399, 1, 0, 0, 0, 62, 403, 1, 0, 0, 0, 64, 447, 1, 0, 0, 0, 66, 451, 1, 0, 0, 0, 68, 455, 1, 0, 0, 0, 70, 457, 1, 0, 0, 0, 72, 460, 1, 0, 0, 0, 74, 469, 1, 0, 0, 0, 76, 477, 1, 0, 0, 0, 78, 480, 1, 0, 0, 0, 80, 483, 1, 0, 0, 0, 82, 492, 1, 0, 0, 0, 84, 496, 1, 0, 0, 0, 86, 502, 1, 0, 0, 0, 88, 506, 1, 0, 0, 0, 90, 509, 1, 0, 0, 0, 92, 517, 1, 0, 0, 0, 94, 521, 1, 0, 0, 0, 96, 525, 1, 0, 0, 0, 98, 528, 1, 0, 0, 0, 100, 533, 1, 0, 0, 0, 102, 537, 1, 0, 0, 0, 104, 539, 1, 0, 0, 0, 106, 541, 1, 0, 0, 0, 108, 544, 1, 0, 0, 0, 110, 548, 1, 0, 0, 0, 112, 551, 1, 0, 0, 0, 114, 571, 1, 0, 0, 0, 116, 575, 1, 0, 0, 0, 118, 580, 1, 0, 0, 0, 120, 121, 3, 2, 1, 0, 121, 122, 5, 0, 0, 1, 122, 1, 1, 0, 0, 0, 123, 124, 6, 1, -1, 0, 124, 125, 3, 4, 2, 0, 125, 131, 1, 0, 0, 0, 126, 127, 10, 1, 0, 0, 127, 128, 5, 24, 0, 0, 128, 130, 3, 6, 3, 0, 129, 126, 1, 0, 0, 0, 130, 133, 1, 0, 0, 0, 131, 129, 1, 0, 0, 0, 131, 132, 1, 0, 0, 0, 132, 3, 1, 0, 0, 0, 133, 131, 1, 0, 0, 0, 134, 141, 3, 106, 53, 0, 135, 141, 3, 34, 17, 0, 136, 141, 3, 28, 14, 0, 137, 141, 3, 110, 55, 0, 138, 139, 4, 2, 1, 0, 139, 141, 3, 48, 24, 0, 140, 134, 1, 0, 0, 0, 140, 135, 1, 0, 0, 0, 140, 136, 1, 0, 0, 0, 140, 137, 1, 0, 0, 0, 140, 138, 1, 0, 0, 0, 141, 5, 1, 0, 0, 0, 142, 159, 3, 50, 25, 0, 143, 159, 3, 8, 4, 0, 144, 159, 3, 76, 38, 0, 145, 159, 3, 70, 35, 0, 146, 159, 3, 52, 26, 0, 147, 159, 3, 72, 36, 0, 148, 159, 3, 78, 39, 0, 149, 159, 3, 80, 40, 0, 150, 159, 3, 84, 42, 0, 151, 159, 3, 86, 43, 0, 152, 159, 3, 112, 56, 0, 153, 159, 3, 88, 44, 0, 154, 155, 4, 3, 2, 0, 155, 159, 3, 118, 59, 0, 156, 157, 4, 3, 3, 0, 157, 159, 3, 116, 58, 0, 158, 142, 1, 0, 0, 0, 158, 143, 1, 0, 0, 0, 158, 144, 1, 0, 0, 0, 158, 145, 1, 0, 0, 0, 158, 146, 1, 0, 0, 0, 158, 147, 1, 0, 0, 0, 158, 148, 1, 0, 0, 0, 158, 149, 1, 0, 0, 0, 158, 150, 1, 0, 0, 0, 158, 151, 1, 0, 0, 0, 158, 152, 1, 0, 0, 0, 158, 153, 1, 0, 0, 0, 158, 154, 1, 0, 0, 0, 158, 156, 1, 0, 0, 0, 159, 7, 1, 0, 0, 0, 160, 161, 5, 16, 0, 0, 161, 162, 3, 10, 5, 0, 162, 9, 1, 0, 0, 0, 163, 164, 6, 5, -1, 0, 164, 165, 5, 43, 0, 0, 165, 194, 3, 10, 5, 8, 166, 194, 3, 16, 8, 0, 167, 194, 3, 12, 6, 0, 168, 170, 3, 16, 8, 0, 169, 171, 5, 43, 0, 0, 170, 169, 1, 0, 0, 0, 170, 171, 1, 0, 0, 0, 171, 172, 1, 0, 0, 0, 172, 173, 5, 38, 0, 0, 173, 174, 5, 42, 0, 0, 174, 179, 3, 16, 8, 0, 175, 176, 5, 33, 0, 0, 176, 178, 3, 16, 8, 0, 177, 175, 1, 0, 0, 0, 178, 181, 1, 0, 0, 0, 179, 177, 1, 0, 0, 0, 179, 180, 1, 0, 0, 0, 180, 182, 1, 0, 0, 0, 181, 179, 1, 0, 0, 0, 182, 183, 5, 49, 0, 0, 183, 194, 1, 0, 0, 0, 184, 185, 3, 16, 8, 0, 185, 187, 5, 39, 0, 0, 186, 188, 5, 43, 0, 0, 187, 186, 1, 0, 0, 0, 187, 188, 1, 0, 0, 0, 188, 189, 1, 0, 0, 0, 189, 190, 5, 44, 0, 0, 190, 194, 1, 0, 0, 0, 191, 192, 4, 5, 4, 0, 192, 194, 3, 14, 7, 0, 193, 163, 1, 0, 0, 0, 193, 166, 1, 0, 0, 0, 193, 167, 1, 0, 0, 0, 193, 168, 1, 0, 0, 0, 193, 184, 1, 0, 0, 0, 193, 191, 1, 0, 0, 0, 194, 203, 1, 0, 0, 0, 195, 196, 10, 5, 0, 0, 196, 197, 5, 29, 0, 0, 197, 202, 3, 10, 5, 6, 198, 199, 10, 4, 0, 0, 199, 200, 5, 46, 0, 0, 200, 202, 3, 10, 5, 5, 201, 195, 1, 0, 0, 0, 201, 198, 1, 0, 0, 0, 202, 205, 1, 0, 0, 0, 203, 201, 1, 0, 0, 0, 203, 204, 1, 0, 0, 0, 204, 11, 1, 0, 0, 0, 205, 203, 1, 0, 0, 0, 206, 208, 3, 16, 8, 0, 207, 209, 5, 43, 0, 0, 208, 207, 1, 0, 0, 0, 208, 209, 1, 0, 0, 0, 209, 210, 1, 0, 0, 0, 210, 211, 5, 41, 0, 0, 211, 212, 3, 102, 51, 0, 212, 221, 1, 0, 0, 0, 213, 215, 3, 16, 8, 0, 214, 216, 5, 43, 0, 0, 215, 214, 1, 0, 0, 0, 215, 216, 1, 0, 0, 0, 216, 217, 1, 0, 0, 0, 217, 218, 5, 48, 0, 0, 218, 219, 3, 102, 51, 0, 219, 221, 1, 0, 0, 0, 220, 206, 1, 0, 0, 0, 220, 213, 1, 0, 0, 0, 221, 13, 1, 0, 0, 0, 222, 223, 3, 16, 8, 0, 223, 224, 5, 63, 0, 0, 224, 225, 3, 102, 51, 0, 225, 15, 1, 0, 0, 0, 226, 232, 3, 18, 9, 0, 227, 228, 3, 18, 9, 0, 228, 229, 3, 104, 52, 0, 229, 230, 3, 18, 9, 0, 230, 232, 1, 0, 0, 0, 231, 226, 1, 0, 0, 0, 231, 227, 1, 0, 0, 0, 232, 17, 1, 0, 0, 0, 233, 234, 6, 9, -1, 0, 234, 238, 3, 20, 10, 0, 235, 236, 7, 0, 0, 0, 236, 238, 3, 18, 9, 3, 237, 233, 1, 0, 0, 0, 237, 235, 1, 0, 0, 0, 238, 247, 1, 0, 0, 0, 239, 240, 10, 2, 0, 0, 240, 241, 7, 1, 0, 0, 241, 246, 3, 18, 9, 3, 242, 243, 10, 1, 0, 0, 243, 244, 7, 0, 0, 0, 244, 246, 3, 18, 9, 2, 245, 239, 1, 0, 0, 0, 245, 242, 1, 0, 0, 0, 246, 249, 1, 0, 0, 0, 247, 245, 1, 0, 0, 0, 247, 248, 1, 0, 0, 0, 248, 19, 1, 0, 0, 0, 249, 247, 1, 0, 0, 0, 250, 251, 6, 10, -1, 0, 251, 259, 3, 64, 32, 0, 252, 259, 3, 54, 27, 0, 253, 259, 3, 22, 11, 0, 254, 255, 5, 42, 0, 0, 255, 256, 3, 10, 5, 0, 256, 257, 5, 49, 0, 0, 257, 259, 1, 0, 0, 0, 258, 250, 1, 0, 0, 0, 258, 252, 1, 0, 0, 0, 258, 253, 1, 0, 0, 0, 258, 254, 1, 0, 0, 0, 259, 265, 1, 0, 0, 0, 260, 261, 10, 1, 0, 0, 261, 262, 5, 32, 0, 0, 262, 264, 3, 26, 13, 0, 263, 260, 1, 0, 0, 0, 264, 267, 1, 0, 0, 0, 265, 263, 1, 0, 0, 0, 265, 266, 1, 0, 0, 0, 266, 21, 1, 0, 0, 0, 267, 265, 1, 0, 0, 0, 268, 269, 3, 24, 12, 0, 269, 279, 5, 42, 0, 0, 270, 280, 5, 60, 0, 0, 271, 276, 3, 10, 5, 0, 272, 273, 5, 33, 0, 0, 273, 275, 3, 10, 5, 0, 274, 272, 1, 0, 0, 0, 275, 278, 1, 0, 0, 0, 276, 274, 1, 0, 0, 0, 276, 277, 1, 0, 0, 0, 277, 280, 1, 0, 0, 0, 278, 276, 1, 0, 0, 0, 279, 270, 1, 0, 0, 0, 279, 271, 1, 0, 0, 0, 279, 280, 1, 0, 0, 0, 280, 281, 1, 0, 0, 0, 281, 282, 5, 49, 0, 0, 282, 23, 1, 0, 0, 0, 283, 284, 4, 12, 10, 0, 284, 287, 5, 63, 0, 0, 285, 287, 3, 68, 34, 0, 286, 283, 1, 0, 0, 0, 286, 285, 1, 0, 0, 0, 287, 25, 1, 0, 0, 0, 288, 289, 3, 60, 30, 0, 289, 27, 1, 0, 0, 0, 290, 291, 5, 12, 0, 0, 291, 292, 3, 30, 15, 0, 292, 29, 1, 0, 0, 0, 293, 298, 3, 32, 16, 0, 294, 295, 5, 33, 0, 0, 295, 297, 3, 32, 16, 0, 296, 294, 1, 0, 0, 0, 297, 300, 1, 0, 0, 0, 298, 296, 1, 0, 0, 0, 298, 299, 1, 0, 0, 0, 299, 31, 1, 0, 0, 0, 300, 298, 1, 0, 0, 0, 301, 307, 3, 10, 5, 0, 302, 303, 3, 54, 27, 0, 303, 304, 5, 31, 0, 0, 304, 305, 3, 10, 5, 0, 305, 307, 1, 0, 0, 0, 306, 301, 1, 0, 0, 0, 306, 302, 1, 0, 0, 0, 307, 33, 1, 0, 0, 0, 308, 309, 5, 6, 0, 0, 309, 314, 3, 36, 18, 0, 310, 311, 5, 33, 0, 0, 311, 313, 3, 36, 18, 0, 312, 310, 1, 0, 0, 0, 313, 316, 1, 0, 0, 0, 314, 312, 1, 0, 0, 0, 314, 315, 1, 0, 0, 0, 315, 318, 1, 0, 0, 0, 316, 314, 1, 0, 0, 0, 317, 319, 3, 42, 21, 0, 318, 317, 1, 0, 0, 0, 318, 319, 1, 0, 0, 0, 319, 35, 1, 0, 0, 0, 320, 321, 3, 38, 19, 0, 321, 322, 5, 104, 0, 0, 322, 323, 3, 40, 20, 0, 323, 326, 1, 0, 0, 0, 324, 326, 3, 40, 20, 0, 325, 320, 1, 0, 0, 0, 325, 324, 1, 0, 0, 0, 326, 37, 1, 0, 0, 0, 327, 328, 5, 76, 0, 0, 328, 39, 1, 0, 0, 0, 329, 330, 7, 2, 0, 0, 330, 41, 1, 0, 0, 0, 331, 334, 3, 44, 22, 0, 332, 334, 3, 46, 23, 0, 333, 331, 1, 0, 0, 0, 333, 332, 1, 0, 0, 0, 334, 43, 1, 0, 0, 0, 335, 336, 5, 75, 0, 0, 336, 341, 5, 76, 0, 0, 337, 338, 5, 33, 0, 0, 338, 340, 5, 76, 0, 0, 339, 337, 1, 0, 0, 0, 340, 343, 1, 0, 0, 0, 341, 339, 1, 0, 0, 0, 341, 342, 1, 0, 0, 0, 342, 45, 1, 0, 0, 0, 343, 341, 1, 0, 0, 0, 344, 345, 5, 65, 0, 0, 345, 346, 3, 44, 22, 0, 346, 347, 5, 66, 0, 0, 347, 47, 1, 0, 0, 0, 348, 349, 5, 19, 0, 0, 349, 354, 3, 36, 18, 0, 350, 351, 5, 33, 0, 0, 351, 353, 3, 36, 18, 0, 352, 350, 1, 0, 0, 0, 353, 356, 1, 0, 0, 0, 354, 352, 1, 0, 0, 0, 354, 355, 1, 0, 0, 0, 355, 358, 1, 0, 0, 0, 356, 354, 1, 0, 0, 0, 357, 359, 3, 30, 15, 0, 358, 357, 1, 0, 0, 0, 358, 359, 1, 0, 0, 0, 359, 362, 1, 0, 0, 0, 360, 361, 5, 28, 0, 0, 361, 363, 3, 30, 15, 0, 362, 360, 1, 0, 0, 0, 362, 363, 1, 0, 0, 0, 363, 49, 1, 0, 0, 0, 364, 365, 5, 4, 0, 0, 365, 366, 3, 30, 15, 0, 366, 51, 1, 0, 0, 0, 367, 369, 5, 15, 0, 0, 368, 370, 3, 30, 15, 0, 369, 368, 1, 0, 0, 0, 369, 370, 1, 0, 0, 0, 370, 373, 1, 0, 0, 0, 371, 372, 5, 28, 0, 0, 372, 374, 3, 30, 15, 0, 373, 371, 1, 0, 0, 0, 373, 374, 1, 0, 0, 0, 374, 53, 1, 0, 0, 0, 375, 380, 3, 68, 34, 0, 376, 377, 5, 35, 0, 0, 377, 379, 3, 68, 34, 0, 378, 376, 1, 0, 0, 0, 379, 382, 1, 0, 0, 0, 380, 378, 1, 0, 0, 0, 380, 381, 1, 0, 0, 0, 381, 55, 1, 0, 0, 0, 382, 380, 1, 0, 0, 0, 383, 388, 3, 62, 31, 0, 384, 385, 5, 35, 0, 0, 385, 387, 3, 62, 31, 0, 386, 384, 1, 0, 0, 0, 387, 390, 1, 0, 0, 0, 388, 386, 1, 0, 0, 0, 388, 389, 1, 0, 0, 0, 389, 57, 1, 0, 0, 0, 390, 388, 1, 0, 0, 0, 391, 396, 3, 56, 28, 0, 392, 393, 5, 33, 0, 0, 393, 395, 3, 56, 28, 0, 394, 392, 1, 0, 0, 0, 395, 398, 1, 0, 0, 0, 396, 394, 1, 0, 0, 0, 396, 397, 1, 0, 0, 0, 397, 59, 1, 0, 0, 0, 398, 396, 1, 0, 0, 0, 399, 400, 7, 3, 0, 0, 400, 61, 1, 0, 0, 0, 401, 404, 5, 80, 0, 0, 402, 404, 3, 66, 33, 0, 403, 401, 1, 0, 0, 0, 403, 402, 1, 0, 0, 0, 404, 63, 1, 0, 0, 0, 405, 448, 5, 44, 0, 0, 406, 407, 3, 100, 50, 0, 407, 408, 5, 67, 0, 0, 408, 448, 1, 0, 0, 0, 409, 448, 3, 98, 49, 0, 410, 448, 3, 100, 50, 0, 411, 448, 3, 94, 47, 0, 412, 448, 3, 66, 33, 0, 413, 448, 3, 102, 51, 0, 414, 415, 5, 65, 0, 0, 415, 420, 3, 96, 48, 0, 416, 417, 5, 33, 0, 0, 417, 419, 3, 96, 48, 0, 418, 416, 1, 0, 0, 0, 419, 422, 1, 0, 0, 0, 420, 418, 1, 0, 0, 0, 420, 421, 1, 0, 0, 0, 421, 423, 1, 0, 0, 0, 422, 420, 1, 0, 0, 0, 423, 424, 5, 66, 0, 0, 424, 448, 1, 0, 0, 0, 425, 426, 5, 65, 0, 0, 426, 431, 3, 94, 47, 0, 427, 428, 5, 33, 0, 0, 428, 430, 3, 94, 47, 0, 429, 427, 1, 0, 0, 0, 430, 433, 1, 0, 0, 0, 431, 429, 1, 0, 0, 0, 431, 432, 1, 0, 0, 0, 432, 434, 1, 0, 0, 0, 433, 431, 1, 0, 0, 0, 434, 435, 5, 66, 0, 0, 435, 448, 1, 0, 0, 0, 436, 437, 5, 65, 0, 0, 437, 442, 3, 102, 51, 0, 438, 439, 5, 33, 0, 0, 439, 441, 3, 102, 51, 0, 440, 438, 1, 0, 0, 0, 441, 444, 1, 0, 0, 0, 442, 440, 1, 0, 0, 0, 442, 443, 1, 0, 0, 0, 443, 445, 1, 0, 0, 0, 444, 442, 1, 0, 0, 0, 445, 446, 5, 66, 0, 0, 446, 448, 1, 0, 0, 0, 447, 405, 1, 0, 0, 0, 447, 406, 1, 0, 0, 0, 447, 409, 1, 0, 0, 0, 447, 410, 1, 0, 0, 0, 447, 411, 1, 0, 0, 0, 447, 412, 1, 0, 0, 0, 447, 413, 1, 0, 0, 0, 447, 414, 1, 0, 0, 0, 447, 425, 1, 0, 0, 0, 447, 436, 1, 0, 0, 0, 448, 65, 1, 0, 0, 0, 449, 452, 5, 47, 0, 0, 450, 452, 5, 64, 0, 0, 451, 449, 1, 0, 0, 0, 451, 450, 1, 0, 0, 0, 452, 67, 1, 0, 0, 0, 453, 456, 3, 60, 30, 0, 454, 456, 3, 66, 33, 0, 455, 453, 1, 0, 0, 0, 455, 454, 1, 0, 0, 0, 456, 69, 1, 0, 0, 0, 457, 458, 5, 9, 0, 0, 458, 459, 5, 26, 0, 0, 459, 71, 1, 0, 0, 0, 460, 461, 5, 14, 0, 0, 461, 466, 3, 74, 37, 0, 462, 463, 5, 33, 0, 0, 463, 465, 3, 74, 37, 0, 464, 462, 1, 0, 0, 0, 465, 468, 1, 0, 0, 0, 466, 464, 1, 0, 0, 0, 466, 467, 1, 0, 0, 0, 467, 73, 1, 0, 0, 0, 468, 466, 1, 0, 0, 0, 469, 471, 3, 10, 5, 0, 470, 472, 7, 4, 0, 0, 471, 470, 1, 0, 0, 0, 471, 472, 1, 0, 0, 0, 472, 475, 1, 0, 0, 0, 473, 474, 5, 45, 0, 0, 474, 476, 7, 5, 0, 0, 475, 473, 1, 0, 0, 0, 475, 476, 1, 0, 0, 0, 476, 75, 1, 0, 0, 0, 477, 478, 5, 8, 0, 0, 478, 479, 3, 58, 29, 0, 479, 77, 1, 0, 0, 0, 480, 481, 5, 2, 0, 0, 481, 482, 3, 58, 29, 0, 482, 79, 1, 0, 0, 0, 483, 484, 5, 11, 0, 0, 484, 489, 3, 82, 41, 0, 485, 486, 5, 33, 0, 0, 486, 488, 3, 82, 41, 0, 487, 485, 1, 0, 0, 0, 488, 491, 1, 0, 0, 0, 489, 487, 1, 0, 0, 0, 489, 490, 1, 0, 0, 0, 490, 81, 1, 0, 0, 0, 491, 489, 1, 0, 0, 0, 492, 493, 3, 56, 28, 0, 493, 494, 5, 84, 0, 0, 494, 495, 3, 56, 28, 0, 495, 83, 1, 0, 0, 0, 496, 497, 5, 1, 0, 0, 497, 498, 3, 20, 10, 0, 498, 500, 3, 102, 51, 0, 499, 501, 3, 90, 45, 0, 500, 499, 1, 0, 0, 0, 500, 501, 1, 0, 0, 0, 501, 85, 1, 0, 0, 0, 502, 503, 5, 7, 0, 0, 503, 504, 3, 20, 10, 0, 504, 505, 3, 102, 51, 0, 505, 87, 1, 0, 0, 0, 506, 507, 5, 10, 0, 0, 507, 508, 3, 54, 27, 0, 508, 89, 1, 0, 0, 0, 509, 514, 3, 92, 46, 0, 510, 511, 5, 33, 0, 0, 511, 513, 3, 92, 46, 0, 512, 510, 1, 0, 0, 0, 513, 516, 1, 0, 0, 0, 514, 512, 1, 0, 0, 0, 514, 515, 1, 0, 0, 0, 515, 91, 1, 0, 0, 0, 516, 514, 1, 0, 0, 0, 517, 518, 3, 60, 30, 0, 518, 519, 5, 31, 0, 0, 519, 520, 3, 64, 32, 0, 520, 93, 1, 0, 0, 0, 521, 522, 7, 6, 0, 0, 522, 95, 1, 0, 0, 0, 523, 526, 3, 98, 49, 0, 524, 526, 3, 100, 50, 0, 525, 523, 1, 0, 0, 0, 525, 524, 1, 0, 0, 0, 526, 97, 1, 0, 0, 0, 527, 529, 7, 0, 0, 0, 528, 527, 1, 0, 0, 0, 528, 529, 1, 0, 0, 0, 529, 530, 1, 0, 0, 0, 530, 531, 5, 27, 0, 0, 531, 99, 1, 0, 0, 0, 532, 534, 7, 0, 0, 0, 533, 532, 1, 0, 0, 0, 533, 534, 1, 0, 0, 0, 534, 535, 1, 0, 0, 0, 535, 536, 5, 26, 0, 0, 536, 101, 1, 0, 0, 0, 537, 538, 5, 25, 0, 0, 538, 103, 1, 0, 0, 0, 539, 540, 7, 7, 0, 0, 540, 105, 1, 0, 0, 0, 541, 542, 5, 5, 0, 0, 542, 543, 3, 108, 54, 0, 543, 107, 1, 0, 0, 0, 544, 545, 5, 65, 0, 0, 545, 546, 3, 2, 1, 0, 546, 547, 5, 66, 0, 0, 547, 109, 1, 0, 0, 0, 548, 549, 5, 13, 0, 0, 549, 550, 5, 100, 0, 0, 550, 111, 1, 0, 0, 0, 551, 552, 5, 3, 0, 0, 552, 555, 5, 90, 0, 0, 553, 554, 5, 88, 0, 0, 554, 556, 3, 56, 28, 0, 555, 553, 1, 0, 0, 0, 555, 556, 1, 0, 0, 0, 556, 566, 1, 0, 0, 0, 557, 558, 5, 89, 0, 0, 558, 563, 3, 114, 57, 0, 559, 560, 5, 33, 0, 0, 560, 562, 3, 114, 57, 0, 561, 559, 1, 0, 0, 0, 562, 565, 1, 0, 0, 0, 563, 561, 1, 0, 0, 0, 563, 564, 1, 0, 0, 0, 564, 567, 1, 0, 0, 0, 565, 563, 1, 0, 0, 0, 566, 557, 1, 0, 0, 0, 566, 567, 1, 0, 0, 0, 567, 113, 1, 0, 0, 0, 568, 569, 3, 56, 28, 0, 569, 570, 5, 31, 0, 0, 570, 572, 1, 0, 0, 0, 571, 568, 1, 0, 0, 0, 571, 572, 1, 0, 0, 0, 572, 573, 1, 0, 0, 0, 573, 574, 3, 56, 28, 0, 574, 115, 1, 0, 0, 0, 575, 576, 5, 18, 0, 0, 576, 577, 3, 36, 18, 0, 577, 578, 5, 88, 0, 0, 578, 579, 3, 58, 29, 0, 579, 117, 1, 0, 0, 0, 580, 581, 5, 17, 0, 0, 581, 584, 3, 30, 15, 0, 582, 583, 5, 28, 0, 0, 583, 585, 3, 30, 15, 0, 584, 582, 1, 0, 0, 0, 584, 585, 1, 0, 0, 0, 585, 119, 1, 0, 0, 0, 57, 131, 140, 158, 170, 179, 187, 193, 201, 203, 208, 215, 220, 231, 237, 245, 247, 258, 265, 276, 279, 286, 298, 306, 314, 318, 325, 333, 341, 354, 358, 362, 369, 373, 380, 388, 396, 403, 420, 431, 442, 447, 451, 455, 466, 471, 475, 489, 500, 514, 525, 528, 533, 555, 563, 566, 571, 584] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java index 14913849d1b51..522393fb42c4b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java @@ -8,14 +8,26 @@ * 2.0. */ -import org.antlr.v4.runtime.atn.*; +import org.antlr.v4.runtime.FailedPredicateException; +import org.antlr.v4.runtime.NoViableAltException; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.RuleContext; +import org.antlr.v4.runtime.RuntimeMetaData; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.TokenStream; +import org.antlr.v4.runtime.Vocabulary; +import org.antlr.v4.runtime.VocabularyImpl; +import org.antlr.v4.runtime.atn.ATN; +import org.antlr.v4.runtime.atn.ATNDeserializer; +import org.antlr.v4.runtime.atn.ParserATNSimulator; +import org.antlr.v4.runtime.atn.PredictionContextCache; import org.antlr.v4.runtime.dfa.DFA; -import org.antlr.v4.runtime.*; -import org.antlr.v4.runtime.misc.*; -import org.antlr.v4.runtime.tree.*; +import org.antlr.v4.runtime.tree.ParseTreeListener; +import org.antlr.v4.runtime.tree.ParseTreeVisitor; +import org.antlr.v4.runtime.tree.TerminalNode; + import java.util.List; -import java.util.Iterator; -import java.util.ArrayList; @SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast", "CheckReturnValue"}) public class EsqlBaseParser extends ParserConfig { @@ -25,66 +37,66 @@ public class EsqlBaseParser extends ParserConfig { protected static final PredictionContextCache _sharedContextCache = new PredictionContextCache(); public static final int - DISSECT=1, DROP=2, ENRICH=3, EVAL=4, EXPLAIN=5, FROM=6, GROK=7, KEEP=8, - LIMIT=9, MV_EXPAND=10, RENAME=11, ROW=12, SHOW=13, SORT=14, STATS=15, - WHERE=16, DEV_INLINESTATS=17, DEV_LOOKUP=18, DEV_MATCH=19, DEV_METRICS=20, - UNKNOWN_CMD=21, LINE_COMMENT=22, MULTILINE_COMMENT=23, WS=24, PIPE=25, - QUOTED_STRING=26, INTEGER_LITERAL=27, DECIMAL_LITERAL=28, BY=29, AND=30, - ASC=31, ASSIGN=32, CAST_OP=33, COMMA=34, DESC=35, DOT=36, FALSE=37, FIRST=38, - IN=39, IS=40, LAST=41, LIKE=42, LP=43, NOT=44, NULL=45, NULLS=46, OR=47, - PARAM=48, RLIKE=49, RP=50, TRUE=51, EQ=52, CIEQ=53, NEQ=54, LT=55, LTE=56, - GT=57, GTE=58, PLUS=59, MINUS=60, ASTERISK=61, SLASH=62, PERCENT=63, NAMED_OR_POSITIONAL_PARAM=64, - OPENING_BRACKET=65, CLOSING_BRACKET=66, UNQUOTED_IDENTIFIER=67, QUOTED_IDENTIFIER=68, - EXPR_LINE_COMMENT=69, EXPR_MULTILINE_COMMENT=70, EXPR_WS=71, EXPLAIN_WS=72, - EXPLAIN_LINE_COMMENT=73, EXPLAIN_MULTILINE_COMMENT=74, METADATA=75, UNQUOTED_SOURCE=76, - FROM_LINE_COMMENT=77, FROM_MULTILINE_COMMENT=78, FROM_WS=79, ID_PATTERN=80, - PROJECT_LINE_COMMENT=81, PROJECT_MULTILINE_COMMENT=82, PROJECT_WS=83, - AS=84, RENAME_LINE_COMMENT=85, RENAME_MULTILINE_COMMENT=86, RENAME_WS=87, - ON=88, WITH=89, ENRICH_POLICY_NAME=90, ENRICH_LINE_COMMENT=91, ENRICH_MULTILINE_COMMENT=92, - ENRICH_WS=93, ENRICH_FIELD_LINE_COMMENT=94, ENRICH_FIELD_MULTILINE_COMMENT=95, - ENRICH_FIELD_WS=96, MVEXPAND_LINE_COMMENT=97, MVEXPAND_MULTILINE_COMMENT=98, - MVEXPAND_WS=99, INFO=100, SHOW_LINE_COMMENT=101, SHOW_MULTILINE_COMMENT=102, - SHOW_WS=103, COLON=104, SETTING=105, SETTING_LINE_COMMENT=106, SETTTING_MULTILINE_COMMENT=107, - SETTING_WS=108, LOOKUP_LINE_COMMENT=109, LOOKUP_MULTILINE_COMMENT=110, - LOOKUP_WS=111, LOOKUP_FIELD_LINE_COMMENT=112, LOOKUP_FIELD_MULTILINE_COMMENT=113, - LOOKUP_FIELD_WS=114, METRICS_LINE_COMMENT=115, METRICS_MULTILINE_COMMENT=116, - METRICS_WS=117, CLOSING_METRICS_LINE_COMMENT=118, CLOSING_METRICS_MULTILINE_COMMENT=119, + DISSECT=1, DROP=2, ENRICH=3, EVAL=4, EXPLAIN=5, FROM=6, GROK=7, KEEP=8, + LIMIT=9, MV_EXPAND=10, RENAME=11, ROW=12, SHOW=13, SORT=14, STATS=15, + WHERE=16, DEV_INLINESTATS=17, DEV_LOOKUP=18, DEV_METRICS=19, UNKNOWN_CMD=20, + LINE_COMMENT=21, MULTILINE_COMMENT=22, WS=23, PIPE=24, QUOTED_STRING=25, + INTEGER_LITERAL=26, DECIMAL_LITERAL=27, BY=28, AND=29, ASC=30, ASSIGN=31, + CAST_OP=32, COMMA=33, DESC=34, DOT=35, FALSE=36, FIRST=37, IN=38, IS=39, + LAST=40, LIKE=41, LP=42, NOT=43, NULL=44, NULLS=45, OR=46, PARAM=47, RLIKE=48, + RP=49, TRUE=50, EQ=51, CIEQ=52, NEQ=53, LT=54, LTE=55, GT=56, GTE=57, + PLUS=58, MINUS=59, ASTERISK=60, SLASH=61, PERCENT=62, DEV_MATCH=63, NAMED_OR_POSITIONAL_PARAM=64, + OPENING_BRACKET=65, CLOSING_BRACKET=66, UNQUOTED_IDENTIFIER=67, QUOTED_IDENTIFIER=68, + EXPR_LINE_COMMENT=69, EXPR_MULTILINE_COMMENT=70, EXPR_WS=71, EXPLAIN_WS=72, + EXPLAIN_LINE_COMMENT=73, EXPLAIN_MULTILINE_COMMENT=74, METADATA=75, UNQUOTED_SOURCE=76, + FROM_LINE_COMMENT=77, FROM_MULTILINE_COMMENT=78, FROM_WS=79, ID_PATTERN=80, + PROJECT_LINE_COMMENT=81, PROJECT_MULTILINE_COMMENT=82, PROJECT_WS=83, + AS=84, RENAME_LINE_COMMENT=85, RENAME_MULTILINE_COMMENT=86, RENAME_WS=87, + ON=88, WITH=89, ENRICH_POLICY_NAME=90, ENRICH_LINE_COMMENT=91, ENRICH_MULTILINE_COMMENT=92, + ENRICH_WS=93, ENRICH_FIELD_LINE_COMMENT=94, ENRICH_FIELD_MULTILINE_COMMENT=95, + ENRICH_FIELD_WS=96, MVEXPAND_LINE_COMMENT=97, MVEXPAND_MULTILINE_COMMENT=98, + MVEXPAND_WS=99, INFO=100, SHOW_LINE_COMMENT=101, SHOW_MULTILINE_COMMENT=102, + SHOW_WS=103, COLON=104, SETTING=105, SETTING_LINE_COMMENT=106, SETTTING_MULTILINE_COMMENT=107, + SETTING_WS=108, LOOKUP_LINE_COMMENT=109, LOOKUP_MULTILINE_COMMENT=110, + LOOKUP_WS=111, LOOKUP_FIELD_LINE_COMMENT=112, LOOKUP_FIELD_MULTILINE_COMMENT=113, + LOOKUP_FIELD_WS=114, METRICS_LINE_COMMENT=115, METRICS_MULTILINE_COMMENT=116, + METRICS_WS=117, CLOSING_METRICS_LINE_COMMENT=118, CLOSING_METRICS_MULTILINE_COMMENT=119, CLOSING_METRICS_WS=120; public static final int - RULE_singleStatement = 0, RULE_query = 1, RULE_sourceCommand = 2, RULE_processingCommand = 3, - RULE_whereCommand = 4, RULE_booleanExpression = 5, RULE_regexBooleanExpression = 6, - RULE_matchBooleanExpression = 7, RULE_valueExpression = 8, RULE_operatorExpression = 9, - RULE_primaryExpression = 10, RULE_functionExpression = 11, RULE_dataType = 12, - RULE_rowCommand = 13, RULE_fields = 14, RULE_field = 15, RULE_fromCommand = 16, - RULE_indexPattern = 17, RULE_clusterString = 18, RULE_indexString = 19, - RULE_metadata = 20, RULE_metadataOption = 21, RULE_deprecated_metadata = 22, - RULE_metricsCommand = 23, RULE_evalCommand = 24, RULE_statsCommand = 25, - RULE_qualifiedName = 26, RULE_qualifiedNamePattern = 27, RULE_qualifiedNamePatterns = 28, - RULE_identifier = 29, RULE_identifierPattern = 30, RULE_constant = 31, - RULE_parameter = 32, RULE_identifierOrParameter = 33, RULE_limitCommand = 34, - RULE_sortCommand = 35, RULE_orderExpression = 36, RULE_keepCommand = 37, - RULE_dropCommand = 38, RULE_renameCommand = 39, RULE_renameClause = 40, - RULE_dissectCommand = 41, RULE_grokCommand = 42, RULE_mvExpandCommand = 43, - RULE_commandOptions = 44, RULE_commandOption = 45, RULE_booleanValue = 46, - RULE_numericValue = 47, RULE_decimalValue = 48, RULE_integerValue = 49, - RULE_string = 50, RULE_comparisonOperator = 51, RULE_explainCommand = 52, - RULE_subqueryExpression = 53, RULE_showCommand = 54, RULE_enrichCommand = 55, - RULE_enrichWithClause = 56, RULE_lookupCommand = 57, RULE_inlinestatsCommand = 58; + RULE_singleStatement = 0, RULE_query = 1, RULE_sourceCommand = 2, RULE_processingCommand = 3, + RULE_whereCommand = 4, RULE_booleanExpression = 5, RULE_regexBooleanExpression = 6, + RULE_matchBooleanExpression = 7, RULE_valueExpression = 8, RULE_operatorExpression = 9, + RULE_primaryExpression = 10, RULE_functionExpression = 11, RULE_functionName = 12, + RULE_dataType = 13, RULE_rowCommand = 14, RULE_fields = 15, RULE_field = 16, + RULE_fromCommand = 17, RULE_indexPattern = 18, RULE_clusterString = 19, + RULE_indexString = 20, RULE_metadata = 21, RULE_metadataOption = 22, RULE_deprecated_metadata = 23, + RULE_metricsCommand = 24, RULE_evalCommand = 25, RULE_statsCommand = 26, + RULE_qualifiedName = 27, RULE_qualifiedNamePattern = 28, RULE_qualifiedNamePatterns = 29, + RULE_identifier = 30, RULE_identifierPattern = 31, RULE_constant = 32, + RULE_parameter = 33, RULE_identifierOrParameter = 34, RULE_limitCommand = 35, + RULE_sortCommand = 36, RULE_orderExpression = 37, RULE_keepCommand = 38, + RULE_dropCommand = 39, RULE_renameCommand = 40, RULE_renameClause = 41, + RULE_dissectCommand = 42, RULE_grokCommand = 43, RULE_mvExpandCommand = 44, + RULE_commandOptions = 45, RULE_commandOption = 46, RULE_booleanValue = 47, + RULE_numericValue = 48, RULE_decimalValue = 49, RULE_integerValue = 50, + RULE_string = 51, RULE_comparisonOperator = 52, RULE_explainCommand = 53, + RULE_subqueryExpression = 54, RULE_showCommand = 55, RULE_enrichCommand = 56, + RULE_enrichWithClause = 57, RULE_lookupCommand = 58, RULE_inlinestatsCommand = 59; private static String[] makeRuleNames() { return new String[] { - "singleStatement", "query", "sourceCommand", "processingCommand", "whereCommand", - "booleanExpression", "regexBooleanExpression", "matchBooleanExpression", - "valueExpression", "operatorExpression", "primaryExpression", "functionExpression", - "dataType", "rowCommand", "fields", "field", "fromCommand", "indexPattern", - "clusterString", "indexString", "metadata", "metadataOption", "deprecated_metadata", - "metricsCommand", "evalCommand", "statsCommand", "qualifiedName", "qualifiedNamePattern", - "qualifiedNamePatterns", "identifier", "identifierPattern", "constant", - "parameter", "identifierOrParameter", "limitCommand", "sortCommand", - "orderExpression", "keepCommand", "dropCommand", "renameCommand", "renameClause", - "dissectCommand", "grokCommand", "mvExpandCommand", "commandOptions", - "commandOption", "booleanValue", "numericValue", "decimalValue", "integerValue", - "string", "comparisonOperator", "explainCommand", "subqueryExpression", - "showCommand", "enrichCommand", "enrichWithClause", "lookupCommand", + "singleStatement", "query", "sourceCommand", "processingCommand", "whereCommand", + "booleanExpression", "regexBooleanExpression", "matchBooleanExpression", + "valueExpression", "operatorExpression", "primaryExpression", "functionExpression", + "functionName", "dataType", "rowCommand", "fields", "field", "fromCommand", + "indexPattern", "clusterString", "indexString", "metadata", "metadataOption", + "deprecated_metadata", "metricsCommand", "evalCommand", "statsCommand", + "qualifiedName", "qualifiedNamePattern", "qualifiedNamePatterns", "identifier", + "identifierPattern", "constant", "parameter", "identifierOrParameter", + "limitCommand", "sortCommand", "orderExpression", "keepCommand", "dropCommand", + "renameCommand", "renameClause", "dissectCommand", "grokCommand", "mvExpandCommand", + "commandOptions", "commandOption", "booleanValue", "numericValue", "decimalValue", + "integerValue", "string", "comparisonOperator", "explainCommand", "subqueryExpression", + "showCommand", "enrichCommand", "enrichWithClause", "lookupCommand", "inlinestatsCommand" }; } @@ -92,46 +104,46 @@ private static String[] makeRuleNames() { private static String[] makeLiteralNames() { return new String[] { - null, "'dissect'", "'drop'", "'enrich'", "'eval'", "'explain'", "'from'", - "'grok'", "'keep'", "'limit'", "'mv_expand'", "'rename'", "'row'", "'show'", - "'sort'", "'stats'", "'where'", null, null, null, null, null, null, null, - null, "'|'", null, null, null, "'by'", "'and'", "'asc'", "'='", "'::'", - "','", "'desc'", "'.'", "'false'", "'first'", "'in'", "'is'", "'last'", - "'like'", "'('", "'not'", "'null'", "'nulls'", "'or'", "'?'", "'rlike'", - "')'", "'true'", "'=='", "'=~'", "'!='", "'<'", "'<='", "'>'", "'>='", - "'+'", "'-'", "'*'", "'/'", "'%'", null, null, "']'", null, null, null, - null, null, null, null, null, "'metadata'", null, null, null, null, null, - null, null, null, "'as'", null, null, null, "'on'", "'with'", null, null, - null, null, null, null, null, null, null, null, "'info'", null, null, + null, "'dissect'", "'drop'", "'enrich'", "'eval'", "'explain'", "'from'", + "'grok'", "'keep'", "'limit'", "'mv_expand'", "'rename'", "'row'", "'show'", + "'sort'", "'stats'", "'where'", null, null, null, null, null, null, null, + "'|'", null, null, null, "'by'", "'and'", "'asc'", "'='", "'::'", "','", + "'desc'", "'.'", "'false'", "'first'", "'in'", "'is'", "'last'", "'like'", + "'('", "'not'", "'null'", "'nulls'", "'or'", "'?'", "'rlike'", "')'", + "'true'", "'=='", "'=~'", "'!='", "'<'", "'<='", "'>'", "'>='", "'+'", + "'-'", "'*'", "'/'", "'%'", null, null, null, "']'", null, null, null, + null, null, null, null, null, "'metadata'", null, null, null, null, null, + null, null, null, "'as'", null, null, null, "'on'", "'with'", null, null, + null, null, null, null, null, null, null, null, "'info'", null, null, null, "':'" }; } private static final String[] _LITERAL_NAMES = makeLiteralNames(); private static String[] makeSymbolicNames() { return new String[] { - null, "DISSECT", "DROP", "ENRICH", "EVAL", "EXPLAIN", "FROM", "GROK", - "KEEP", "LIMIT", "MV_EXPAND", "RENAME", "ROW", "SHOW", "SORT", "STATS", - "WHERE", "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_MATCH", "DEV_METRICS", - "UNKNOWN_CMD", "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "QUOTED_STRING", - "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", - "COMMA", "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", - "LP", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", - "CIEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", - "SLASH", "PERCENT", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", "CLOSING_BRACKET", - "UNQUOTED_IDENTIFIER", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", - "EXPR_WS", "EXPLAIN_WS", "EXPLAIN_LINE_COMMENT", "EXPLAIN_MULTILINE_COMMENT", - "METADATA", "UNQUOTED_SOURCE", "FROM_LINE_COMMENT", "FROM_MULTILINE_COMMENT", - "FROM_WS", "ID_PATTERN", "PROJECT_LINE_COMMENT", "PROJECT_MULTILINE_COMMENT", - "PROJECT_WS", "AS", "RENAME_LINE_COMMENT", "RENAME_MULTILINE_COMMENT", - "RENAME_WS", "ON", "WITH", "ENRICH_POLICY_NAME", "ENRICH_LINE_COMMENT", - "ENRICH_MULTILINE_COMMENT", "ENRICH_WS", "ENRICH_FIELD_LINE_COMMENT", - "ENRICH_FIELD_MULTILINE_COMMENT", "ENRICH_FIELD_WS", "MVEXPAND_LINE_COMMENT", - "MVEXPAND_MULTILINE_COMMENT", "MVEXPAND_WS", "INFO", "SHOW_LINE_COMMENT", - "SHOW_MULTILINE_COMMENT", "SHOW_WS", "COLON", "SETTING", "SETTING_LINE_COMMENT", - "SETTTING_MULTILINE_COMMENT", "SETTING_WS", "LOOKUP_LINE_COMMENT", "LOOKUP_MULTILINE_COMMENT", - "LOOKUP_WS", "LOOKUP_FIELD_LINE_COMMENT", "LOOKUP_FIELD_MULTILINE_COMMENT", - "LOOKUP_FIELD_WS", "METRICS_LINE_COMMENT", "METRICS_MULTILINE_COMMENT", - "METRICS_WS", "CLOSING_METRICS_LINE_COMMENT", "CLOSING_METRICS_MULTILINE_COMMENT", + null, "DISSECT", "DROP", "ENRICH", "EVAL", "EXPLAIN", "FROM", "GROK", + "KEEP", "LIMIT", "MV_EXPAND", "RENAME", "ROW", "SHOW", "SORT", "STATS", + "WHERE", "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_METRICS", "UNKNOWN_CMD", + "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "QUOTED_STRING", "INTEGER_LITERAL", + "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", "COMMA", + "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", "LP", "NOT", + "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", "CIEQ", + "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", "SLASH", + "PERCENT", "DEV_MATCH", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", + "CLOSING_BRACKET", "UNQUOTED_IDENTIFIER", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", + "EXPR_MULTILINE_COMMENT", "EXPR_WS", "EXPLAIN_WS", "EXPLAIN_LINE_COMMENT", + "EXPLAIN_MULTILINE_COMMENT", "METADATA", "UNQUOTED_SOURCE", "FROM_LINE_COMMENT", + "FROM_MULTILINE_COMMENT", "FROM_WS", "ID_PATTERN", "PROJECT_LINE_COMMENT", + "PROJECT_MULTILINE_COMMENT", "PROJECT_WS", "AS", "RENAME_LINE_COMMENT", + "RENAME_MULTILINE_COMMENT", "RENAME_WS", "ON", "WITH", "ENRICH_POLICY_NAME", + "ENRICH_LINE_COMMENT", "ENRICH_MULTILINE_COMMENT", "ENRICH_WS", "ENRICH_FIELD_LINE_COMMENT", + "ENRICH_FIELD_MULTILINE_COMMENT", "ENRICH_FIELD_WS", "MVEXPAND_LINE_COMMENT", + "MVEXPAND_MULTILINE_COMMENT", "MVEXPAND_WS", "INFO", "SHOW_LINE_COMMENT", + "SHOW_MULTILINE_COMMENT", "SHOW_WS", "COLON", "SETTING", "SETTING_LINE_COMMENT", + "SETTTING_MULTILINE_COMMENT", "SETTING_WS", "LOOKUP_LINE_COMMENT", "LOOKUP_MULTILINE_COMMENT", + "LOOKUP_WS", "LOOKUP_FIELD_LINE_COMMENT", "LOOKUP_FIELD_MULTILINE_COMMENT", + "LOOKUP_FIELD_WS", "METRICS_LINE_COMMENT", "METRICS_MULTILINE_COMMENT", + "METRICS_WS", "CLOSING_METRICS_LINE_COMMENT", "CLOSING_METRICS_MULTILINE_COMMENT", "CLOSING_METRICS_WS" }; } @@ -219,9 +231,9 @@ public final SingleStatementContext singleStatement() throws RecognitionExceptio try { enterOuterAlt(_localctx, 1); { - setState(118); + setState(120); query(0); - setState(119); + setState(121); match(EOF); } } @@ -243,7 +255,7 @@ public QueryContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); } @Override public int getRuleIndex() { return RULE_query; } - + @SuppressWarnings("this-escape") public QueryContext() { } public void copyFrom(QueryContext ctx) { @@ -317,11 +329,11 @@ private QueryContext query(int _p) throws RecognitionException { _ctx = _localctx; _prevctx = _localctx; - setState(122); + setState(124); sourceCommand(); } _ctx.stop = _input.LT(-1); - setState(129); + setState(131); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,0,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -332,16 +344,16 @@ private QueryContext query(int _p) throws RecognitionException { { _localctx = new CompositeQueryContext(new QueryContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_query); - setState(124); + setState(126); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(125); + setState(127); match(PIPE); - setState(126); + setState(128); processingCommand(); } - } + } } - setState(131); + setState(133); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,0,_ctx); } @@ -399,43 +411,43 @@ public final SourceCommandContext sourceCommand() throws RecognitionException { SourceCommandContext _localctx = new SourceCommandContext(_ctx, getState()); enterRule(_localctx, 4, RULE_sourceCommand); try { - setState(138); + setState(140); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,1,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(132); + setState(134); explainCommand(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(133); + setState(135); fromCommand(); } break; case 3: enterOuterAlt(_localctx, 3); { - setState(134); + setState(136); rowCommand(); } break; case 4: enterOuterAlt(_localctx, 4); { - setState(135); + setState(137); showCommand(); } break; case 5: enterOuterAlt(_localctx, 5); { - setState(136); + setState(138); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(137); + setState(139); metricsCommand(); } break; @@ -520,108 +532,108 @@ public final ProcessingCommandContext processingCommand() throws RecognitionExce ProcessingCommandContext _localctx = new ProcessingCommandContext(_ctx, getState()); enterRule(_localctx, 6, RULE_processingCommand); try { - setState(156); + setState(158); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,2,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(140); + setState(142); evalCommand(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(141); + setState(143); whereCommand(); } break; case 3: enterOuterAlt(_localctx, 3); { - setState(142); + setState(144); keepCommand(); } break; case 4: enterOuterAlt(_localctx, 4); { - setState(143); + setState(145); limitCommand(); } break; case 5: enterOuterAlt(_localctx, 5); { - setState(144); + setState(146); statsCommand(); } break; case 6: enterOuterAlt(_localctx, 6); { - setState(145); + setState(147); sortCommand(); } break; case 7: enterOuterAlt(_localctx, 7); { - setState(146); + setState(148); dropCommand(); } break; case 8: enterOuterAlt(_localctx, 8); { - setState(147); + setState(149); renameCommand(); } break; case 9: enterOuterAlt(_localctx, 9); { - setState(148); + setState(150); dissectCommand(); } break; case 10: enterOuterAlt(_localctx, 10); { - setState(149); + setState(151); grokCommand(); } break; case 11: enterOuterAlt(_localctx, 11); { - setState(150); + setState(152); enrichCommand(); } break; case 12: enterOuterAlt(_localctx, 12); { - setState(151); + setState(153); mvExpandCommand(); } break; case 13: enterOuterAlt(_localctx, 13); { - setState(152); + setState(154); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(153); + setState(155); inlinestatsCommand(); } break; case 14: enterOuterAlt(_localctx, 14); { - setState(154); + setState(156); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(155); + setState(157); lookupCommand(); } break; @@ -670,9 +682,9 @@ public final WhereCommandContext whereCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(158); + setState(160); match(WHERE); - setState(159); + setState(161); booleanExpression(0); } } @@ -694,7 +706,7 @@ public BooleanExpressionContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); } @Override public int getRuleIndex() { return RULE_booleanExpression; } - + @SuppressWarnings("this-escape") public BooleanExpressionContext() { } public void copyFrom(BooleanExpressionContext ctx) { @@ -888,7 +900,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc int _alt; enterOuterAlt(_localctx, 1); { - setState(191); + setState(193); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,6,_ctx) ) { case 1: @@ -897,9 +909,9 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _ctx = _localctx; _prevctx = _localctx; - setState(162); + setState(164); match(NOT); - setState(163); + setState(165); booleanExpression(8); } break; @@ -908,7 +920,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new BooleanDefaultContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(164); + setState(166); valueExpression(); } break; @@ -917,7 +929,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new RegexExpressionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(165); + setState(167); regexBooleanExpression(); } break; @@ -926,41 +938,41 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalInContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(166); - valueExpression(); setState(168); + valueExpression(); + setState(170); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(167); + setState(169); match(NOT); } } - setState(170); + setState(172); match(IN); - setState(171); + setState(173); match(LP); - setState(172); + setState(174); valueExpression(); - setState(177); + setState(179); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(173); + setState(175); match(COMMA); - setState(174); + setState(176); valueExpression(); } } - setState(179); + setState(181); _errHandler.sync(this); _la = _input.LA(1); } - setState(180); + setState(182); match(RP); } break; @@ -969,21 +981,21 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new IsNullContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(182); + setState(184); valueExpression(); - setState(183); - match(IS); setState(185); + match(IS); + setState(187); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(184); + setState(186); match(NOT); } } - setState(187); + setState(189); match(NULL); } break; @@ -992,15 +1004,15 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new MatchExpressionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(189); + setState(191); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(190); + setState(192); matchBooleanExpression(); } break; } _ctx.stop = _input.LT(-1); - setState(201); + setState(203); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,8,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -1008,7 +1020,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc if ( _parseListeners!=null ) triggerExitRuleEvent(); _prevctx = _localctx; { - setState(199); + setState(201); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,7,_ctx) ) { case 1: @@ -1016,11 +1028,11 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState)); ((LogicalBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression); - setState(193); + setState(195); if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)"); - setState(194); + setState(196); ((LogicalBinaryContext)_localctx).operator = match(AND); - setState(195); + setState(197); ((LogicalBinaryContext)_localctx).right = booleanExpression(6); } break; @@ -1029,18 +1041,18 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState)); ((LogicalBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression); - setState(196); + setState(198); if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)"); - setState(197); + setState(199); ((LogicalBinaryContext)_localctx).operator = match(OR); - setState(198); + setState(200); ((LogicalBinaryContext)_localctx).right = booleanExpression(5); } break; } - } + } } - setState(203); + setState(205); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,8,_ctx); } @@ -1095,48 +1107,48 @@ public final RegexBooleanExpressionContext regexBooleanExpression() throws Recog enterRule(_localctx, 12, RULE_regexBooleanExpression); int _la; try { - setState(218); + setState(220); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,11,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(204); - valueExpression(); setState(206); + valueExpression(); + setState(208); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(205); + setState(207); match(NOT); } } - setState(208); + setState(210); ((RegexBooleanExpressionContext)_localctx).kind = match(LIKE); - setState(209); + setState(211); ((RegexBooleanExpressionContext)_localctx).pattern = string(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(211); - valueExpression(); setState(213); + valueExpression(); + setState(215); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(212); + setState(214); match(NOT); } } - setState(215); + setState(217); ((RegexBooleanExpressionContext)_localctx).kind = match(RLIKE); - setState(216); + setState(218); ((RegexBooleanExpressionContext)_localctx).pattern = string(); } break; @@ -1189,11 +1201,11 @@ public final MatchBooleanExpressionContext matchBooleanExpression() throws Recog try { enterOuterAlt(_localctx, 1); { - setState(220); + setState(222); valueExpression(); - setState(221); + setState(223); match(DEV_MATCH); - setState(222); + setState(224); ((MatchBooleanExpressionContext)_localctx).queryString = string(); } } @@ -1215,7 +1227,7 @@ public ValueExpressionContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); } @Override public int getRuleIndex() { return RULE_valueExpression; } - + @SuppressWarnings("this-escape") public ValueExpressionContext() { } public void copyFrom(ValueExpressionContext ctx) { @@ -1277,14 +1289,14 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio ValueExpressionContext _localctx = new ValueExpressionContext(_ctx, getState()); enterRule(_localctx, 16, RULE_valueExpression); try { - setState(229); + setState(231); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,12,_ctx) ) { case 1: _localctx = new ValueExpressionDefaultContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(224); + setState(226); operatorExpression(0); } break; @@ -1292,11 +1304,11 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio _localctx = new ComparisonContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(225); + setState(227); ((ComparisonContext)_localctx).left = operatorExpression(0); - setState(226); + setState(228); comparisonOperator(); - setState(227); + setState(229); ((ComparisonContext)_localctx).right = operatorExpression(0); } break; @@ -1320,7 +1332,7 @@ public OperatorExpressionContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); } @Override public int getRuleIndex() { return RULE_operatorExpression; } - + @SuppressWarnings("this-escape") public OperatorExpressionContext() { } public void copyFrom(OperatorExpressionContext ctx) { @@ -1421,7 +1433,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE int _alt; enterOuterAlt(_localctx, 1); { - setState(235); + setState(237); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,13,_ctx) ) { case 1: @@ -1430,7 +1442,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _ctx = _localctx; _prevctx = _localctx; - setState(232); + setState(234); primaryExpression(0); } break; @@ -1439,7 +1451,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticUnaryContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(233); + setState(235); ((ArithmeticUnaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { @@ -1450,13 +1462,13 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(234); + setState(236); operatorExpression(3); } break; } _ctx.stop = _input.LT(-1); - setState(245); + setState(247); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,15,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -1464,7 +1476,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE if ( _parseListeners!=null ) triggerExitRuleEvent(); _prevctx = _localctx; { - setState(243); + setState(245); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,14,_ctx) ) { case 1: @@ -1472,12 +1484,12 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState)); ((ArithmeticBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression); - setState(237); + setState(239); if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)"); - setState(238); + setState(240); ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); - if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & -2305843009213693952L) != 0)) ) { + if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 8070450532247928832L) != 0)) ) { ((ArithmeticBinaryContext)_localctx).operator = (Token)_errHandler.recoverInline(this); } else { @@ -1485,7 +1497,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(239); + setState(241); ((ArithmeticBinaryContext)_localctx).right = operatorExpression(3); } break; @@ -1494,9 +1506,9 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState)); ((ArithmeticBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression); - setState(240); + setState(242); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(241); + setState(243); ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { @@ -1507,14 +1519,14 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(242); + setState(244); ((ArithmeticBinaryContext)_localctx).right = operatorExpression(2); } break; } - } + } } - setState(247); + setState(249); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,15,_ctx); } @@ -1538,7 +1550,7 @@ public PrimaryExpressionContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); } @Override public int getRuleIndex() { return RULE_primaryExpression; } - + @SuppressWarnings("this-escape") public PrimaryExpressionContext() { } public void copyFrom(PrimaryExpressionContext ctx) { @@ -1672,7 +1684,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc int _alt; enterOuterAlt(_localctx, 1); { - setState(256); + setState(258); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,16,_ctx) ) { case 1: @@ -1681,7 +1693,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _ctx = _localctx; _prevctx = _localctx; - setState(249); + setState(251); constant(); } break; @@ -1690,7 +1702,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _localctx = new DereferenceContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(250); + setState(252); qualifiedName(); } break; @@ -1699,7 +1711,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _localctx = new FunctionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(251); + setState(253); functionExpression(); } break; @@ -1708,17 +1720,17 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _localctx = new ParenthesizedExpressionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(252); + setState(254); match(LP); - setState(253); + setState(255); booleanExpression(0); - setState(254); + setState(256); match(RP); } break; } _ctx.stop = _input.LT(-1); - setState(263); + setState(265); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,17,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -1729,16 +1741,16 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc { _localctx = new InlineCastContext(new PrimaryExpressionContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_primaryExpression); - setState(258); + setState(260); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(259); + setState(261); match(CAST_OP); - setState(260); + setState(262); dataType(); } - } + } } - setState(265); + setState(267); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,17,_ctx); } @@ -1757,8 +1769,8 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc @SuppressWarnings("CheckReturnValue") public static class FunctionExpressionContext extends ParserRuleContext { - public IdentifierOrParameterContext identifierOrParameter() { - return getRuleContext(IdentifierOrParameterContext.class,0); + public FunctionNameContext functionName() { + return getRuleContext(FunctionNameContext.class,0); } public TerminalNode LP() { return getToken(EsqlBaseParser.LP, 0); } public TerminalNode RP() { return getToken(EsqlBaseParser.RP, 0); } @@ -1800,37 +1812,37 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx try { enterOuterAlt(_localctx, 1); { - setState(266); - identifierOrParameter(); - setState(267); + setState(268); + functionName(); + setState(269); match(LP); - setState(277); + setState(279); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,19,_ctx) ) { case 1: { - setState(268); + setState(270); match(ASTERISK); } break; case 2: { { - setState(269); + setState(271); booleanExpression(0); - setState(274); + setState(276); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(270); + setState(272); match(COMMA); - setState(271); + setState(273); booleanExpression(0); } } - setState(276); + setState(278); _errHandler.sync(this); _la = _input.LA(1); } @@ -1838,7 +1850,7 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx } break; } - setState(279); + setState(281); match(RP); } } @@ -1853,6 +1865,68 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx return _localctx; } + @SuppressWarnings("CheckReturnValue") + public static class FunctionNameContext extends ParserRuleContext { + public TerminalNode DEV_MATCH() { return getToken(EsqlBaseParser.DEV_MATCH, 0); } + public IdentifierOrParameterContext identifierOrParameter() { + return getRuleContext(IdentifierOrParameterContext.class,0); + } + @SuppressWarnings("this-escape") + public FunctionNameContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_functionName; } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterFunctionName(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitFunctionName(this); + } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor)visitor).visitFunctionName(this); + else return visitor.visitChildren(this); + } + } + + public final FunctionNameContext functionName() throws RecognitionException { + FunctionNameContext _localctx = new FunctionNameContext(_ctx, getState()); + enterRule(_localctx, 24, RULE_functionName); + try { + setState(286); + _errHandler.sync(this); + switch ( getInterpreter().adaptivePredict(_input,20,_ctx) ) { + case 1: + enterOuterAlt(_localctx, 1); + { + setState(283); + if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); + setState(284); + match(DEV_MATCH); + } + break; + case 2: + enterOuterAlt(_localctx, 2); + { + setState(285); + identifierOrParameter(); + } + break; + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + @SuppressWarnings("CheckReturnValue") public static class DataTypeContext extends ParserRuleContext { @SuppressWarnings("this-escape") @@ -1860,7 +1934,7 @@ public DataTypeContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); } @Override public int getRuleIndex() { return RULE_dataType; } - + @SuppressWarnings("this-escape") public DataTypeContext() { } public void copyFrom(DataTypeContext ctx) { @@ -1891,12 +1965,12 @@ public T accept(ParseTreeVisitor visitor) { public final DataTypeContext dataType() throws RecognitionException { DataTypeContext _localctx = new DataTypeContext(_ctx, getState()); - enterRule(_localctx, 24, RULE_dataType); + enterRule(_localctx, 26, RULE_dataType); try { _localctx = new ToDataTypeContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(281); + setState(288); identifier(); } } @@ -1939,13 +2013,13 @@ public T accept(ParseTreeVisitor visitor) { public final RowCommandContext rowCommand() throws RecognitionException { RowCommandContext _localctx = new RowCommandContext(_ctx, getState()); - enterRule(_localctx, 26, RULE_rowCommand); + enterRule(_localctx, 28, RULE_rowCommand); try { enterOuterAlt(_localctx, 1); { - setState(283); + setState(290); match(ROW); - setState(284); + setState(291); fields(); } } @@ -1994,30 +2068,30 @@ public T accept(ParseTreeVisitor visitor) { public final FieldsContext fields() throws RecognitionException { FieldsContext _localctx = new FieldsContext(_ctx, getState()); - enterRule(_localctx, 28, RULE_fields); + enterRule(_localctx, 30, RULE_fields); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(286); + setState(293); field(); - setState(291); + setState(298); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,20,_ctx); + _alt = getInterpreter().adaptivePredict(_input,21,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(287); + setState(294); match(COMMA); - setState(288); + setState(295); field(); } - } + } } - setState(293); + setState(300); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,20,_ctx); + _alt = getInterpreter().adaptivePredict(_input,21,_ctx); } } } @@ -2063,26 +2137,26 @@ public T accept(ParseTreeVisitor visitor) { public final FieldContext field() throws RecognitionException { FieldContext _localctx = new FieldContext(_ctx, getState()); - enterRule(_localctx, 30, RULE_field); + enterRule(_localctx, 32, RULE_field); try { - setState(299); + setState(306); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,21,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,22,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(294); + setState(301); booleanExpression(0); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(295); + setState(302); qualifiedName(); - setState(296); + setState(303); match(ASSIGN); - setState(297); + setState(304); booleanExpression(0); } break; @@ -2137,39 +2211,39 @@ public T accept(ParseTreeVisitor visitor) { public final FromCommandContext fromCommand() throws RecognitionException { FromCommandContext _localctx = new FromCommandContext(_ctx, getState()); - enterRule(_localctx, 32, RULE_fromCommand); + enterRule(_localctx, 34, RULE_fromCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(301); + setState(308); match(FROM); - setState(302); + setState(309); indexPattern(); - setState(307); + setState(314); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,22,_ctx); + _alt = getInterpreter().adaptivePredict(_input,23,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(303); + setState(310); match(COMMA); - setState(304); + setState(311); indexPattern(); } - } + } } - setState(309); + setState(316); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,22,_ctx); + _alt = getInterpreter().adaptivePredict(_input,23,_ctx); } - setState(311); + setState(318); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,23,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) { case 1: { - setState(310); + setState(317); metadata(); } break; @@ -2218,26 +2292,26 @@ public T accept(ParseTreeVisitor visitor) { public final IndexPatternContext indexPattern() throws RecognitionException { IndexPatternContext _localctx = new IndexPatternContext(_ctx, getState()); - enterRule(_localctx, 34, RULE_indexPattern); + enterRule(_localctx, 36, RULE_indexPattern); try { - setState(318); + setState(325); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,25,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(313); + setState(320); clusterString(); - setState(314); + setState(321); match(COLON); - setState(315); + setState(322); indexString(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(317); + setState(324); indexString(); } break; @@ -2279,11 +2353,11 @@ public T accept(ParseTreeVisitor visitor) { public final ClusterStringContext clusterString() throws RecognitionException { ClusterStringContext _localctx = new ClusterStringContext(_ctx, getState()); - enterRule(_localctx, 36, RULE_clusterString); + enterRule(_localctx, 38, RULE_clusterString); try { enterOuterAlt(_localctx, 1); { - setState(320); + setState(327); match(UNQUOTED_SOURCE); } } @@ -2324,12 +2398,12 @@ public T accept(ParseTreeVisitor visitor) { public final IndexStringContext indexString() throws RecognitionException { IndexStringContext _localctx = new IndexStringContext(_ctx, getState()); - enterRule(_localctx, 38, RULE_indexString); + enterRule(_localctx, 40, RULE_indexString); int _la; try { enterOuterAlt(_localctx, 1); { - setState(322); + setState(329); _la = _input.LA(1); if ( !(_la==QUOTED_STRING || _la==UNQUOTED_SOURCE) ) { _errHandler.recoverInline(this); @@ -2382,22 +2456,22 @@ public T accept(ParseTreeVisitor visitor) { public final MetadataContext metadata() throws RecognitionException { MetadataContext _localctx = new MetadataContext(_ctx, getState()); - enterRule(_localctx, 40, RULE_metadata); + enterRule(_localctx, 42, RULE_metadata); try { - setState(326); + setState(333); _errHandler.sync(this); switch (_input.LA(1)) { case METADATA: enterOuterAlt(_localctx, 1); { - setState(324); + setState(331); metadataOption(); } break; case OPENING_BRACKET: enterOuterAlt(_localctx, 2); { - setState(325); + setState(332); deprecated_metadata(); } break; @@ -2449,32 +2523,32 @@ public T accept(ParseTreeVisitor visitor) { public final MetadataOptionContext metadataOption() throws RecognitionException { MetadataOptionContext _localctx = new MetadataOptionContext(_ctx, getState()); - enterRule(_localctx, 42, RULE_metadataOption); + enterRule(_localctx, 44, RULE_metadataOption); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(328); + setState(335); match(METADATA); - setState(329); + setState(336); match(UNQUOTED_SOURCE); - setState(334); + setState(341); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,26,_ctx); + _alt = getInterpreter().adaptivePredict(_input,27,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(330); + setState(337); match(COMMA); - setState(331); + setState(338); match(UNQUOTED_SOURCE); } - } + } } - setState(336); + setState(343); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,26,_ctx); + _alt = getInterpreter().adaptivePredict(_input,27,_ctx); } } } @@ -2517,15 +2591,15 @@ public T accept(ParseTreeVisitor visitor) { public final Deprecated_metadataContext deprecated_metadata() throws RecognitionException { Deprecated_metadataContext _localctx = new Deprecated_metadataContext(_ctx, getState()); - enterRule(_localctx, 44, RULE_deprecated_metadata); + enterRule(_localctx, 46, RULE_deprecated_metadata); try { enterOuterAlt(_localctx, 1); { - setState(337); + setState(344); match(OPENING_BRACKET); - setState(338); + setState(345); metadataOption(); - setState(339); + setState(346); match(CLOSING_BRACKET); } } @@ -2584,51 +2658,51 @@ public T accept(ParseTreeVisitor visitor) { public final MetricsCommandContext metricsCommand() throws RecognitionException { MetricsCommandContext _localctx = new MetricsCommandContext(_ctx, getState()); - enterRule(_localctx, 46, RULE_metricsCommand); + enterRule(_localctx, 48, RULE_metricsCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(341); + setState(348); match(DEV_METRICS); - setState(342); + setState(349); indexPattern(); - setState(347); + setState(354); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,27,_ctx); + _alt = getInterpreter().adaptivePredict(_input,28,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(343); + setState(350); match(COMMA); - setState(344); + setState(351); indexPattern(); } - } + } } - setState(349); + setState(356); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,27,_ctx); + _alt = getInterpreter().adaptivePredict(_input,28,_ctx); } - setState(351); + setState(358); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,28,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) { case 1: { - setState(350); + setState(357); ((MetricsCommandContext)_localctx).aggregates = fields(); } break; } - setState(355); + setState(362); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) { case 1: { - setState(353); + setState(360); match(BY); - setState(354); + setState(361); ((MetricsCommandContext)_localctx).grouping = fields(); } break; @@ -2674,13 +2748,13 @@ public T accept(ParseTreeVisitor visitor) { public final EvalCommandContext evalCommand() throws RecognitionException { EvalCommandContext _localctx = new EvalCommandContext(_ctx, getState()); - enterRule(_localctx, 48, RULE_evalCommand); + enterRule(_localctx, 50, RULE_evalCommand); try { enterOuterAlt(_localctx, 1); { - setState(357); + setState(364); match(EVAL); - setState(358); + setState(365); fields(); } } @@ -2729,30 +2803,30 @@ public T accept(ParseTreeVisitor visitor) { public final StatsCommandContext statsCommand() throws RecognitionException { StatsCommandContext _localctx = new StatsCommandContext(_ctx, getState()); - enterRule(_localctx, 50, RULE_statsCommand); + enterRule(_localctx, 52, RULE_statsCommand); try { enterOuterAlt(_localctx, 1); { - setState(360); + setState(367); match(STATS); - setState(362); + setState(369); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) { case 1: { - setState(361); + setState(368); ((StatsCommandContext)_localctx).stats = fields(); } break; } - setState(366); + setState(373); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,32,_ctx) ) { case 1: { - setState(364); + setState(371); match(BY); - setState(365); + setState(372); ((StatsCommandContext)_localctx).grouping = fields(); } break; @@ -2804,30 +2878,30 @@ public T accept(ParseTreeVisitor visitor) { public final QualifiedNameContext qualifiedName() throws RecognitionException { QualifiedNameContext _localctx = new QualifiedNameContext(_ctx, getState()); - enterRule(_localctx, 52, RULE_qualifiedName); + enterRule(_localctx, 54, RULE_qualifiedName); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(368); + setState(375); identifierOrParameter(); - setState(373); + setState(380); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,32,_ctx); + _alt = getInterpreter().adaptivePredict(_input,33,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(369); + setState(376); match(DOT); - setState(370); + setState(377); identifierOrParameter(); } - } + } } - setState(375); + setState(382); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,32,_ctx); + _alt = getInterpreter().adaptivePredict(_input,33,_ctx); } } } @@ -2876,30 +2950,30 @@ public T accept(ParseTreeVisitor visitor) { public final QualifiedNamePatternContext qualifiedNamePattern() throws RecognitionException { QualifiedNamePatternContext _localctx = new QualifiedNamePatternContext(_ctx, getState()); - enterRule(_localctx, 54, RULE_qualifiedNamePattern); + enterRule(_localctx, 56, RULE_qualifiedNamePattern); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(376); + setState(383); identifierPattern(); - setState(381); + setState(388); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,33,_ctx); + _alt = getInterpreter().adaptivePredict(_input,34,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(377); + setState(384); match(DOT); - setState(378); + setState(385); identifierPattern(); } - } + } } - setState(383); + setState(390); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,33,_ctx); + _alt = getInterpreter().adaptivePredict(_input,34,_ctx); } } } @@ -2948,30 +3022,30 @@ public T accept(ParseTreeVisitor visitor) { public final QualifiedNamePatternsContext qualifiedNamePatterns() throws RecognitionException { QualifiedNamePatternsContext _localctx = new QualifiedNamePatternsContext(_ctx, getState()); - enterRule(_localctx, 56, RULE_qualifiedNamePatterns); + enterRule(_localctx, 58, RULE_qualifiedNamePatterns); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(384); + setState(391); qualifiedNamePattern(); - setState(389); + setState(396); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,34,_ctx); + _alt = getInterpreter().adaptivePredict(_input,35,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(385); + setState(392); match(COMMA); - setState(386); + setState(393); qualifiedNamePattern(); } - } + } } - setState(391); + setState(398); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,34,_ctx); + _alt = getInterpreter().adaptivePredict(_input,35,_ctx); } } } @@ -3012,12 +3086,12 @@ public T accept(ParseTreeVisitor visitor) { public final IdentifierContext identifier() throws RecognitionException { IdentifierContext _localctx = new IdentifierContext(_ctx, getState()); - enterRule(_localctx, 58, RULE_identifier); + enterRule(_localctx, 60, RULE_identifier); int _la; try { enterOuterAlt(_localctx, 1); { - setState(392); + setState(399); _la = _input.LA(1); if ( !(_la==UNQUOTED_IDENTIFIER || _la==QUOTED_IDENTIFIER) ) { _errHandler.recoverInline(this); @@ -3068,15 +3142,15 @@ public T accept(ParseTreeVisitor visitor) { public final IdentifierPatternContext identifierPattern() throws RecognitionException { IdentifierPatternContext _localctx = new IdentifierPatternContext(_ctx, getState()); - enterRule(_localctx, 60, RULE_identifierPattern); + enterRule(_localctx, 62, RULE_identifierPattern); try { - setState(396); + setState(403); _errHandler.sync(this); switch (_input.LA(1)) { case ID_PATTERN: enterOuterAlt(_localctx, 1); { - setState(394); + setState(401); match(ID_PATTERN); } break; @@ -3084,7 +3158,7 @@ public final IdentifierPatternContext identifierPattern() throws RecognitionExce case NAMED_OR_POSITIONAL_PARAM: enterOuterAlt(_localctx, 2); { - setState(395); + setState(402); parameter(); } break; @@ -3110,7 +3184,7 @@ public ConstantContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); } @Override public int getRuleIndex() { return RULE_constant; } - + @SuppressWarnings("this-escape") public ConstantContext() { } public void copyFrom(ConstantContext ctx) { @@ -3356,17 +3430,17 @@ public T accept(ParseTreeVisitor visitor) { public final ConstantContext constant() throws RecognitionException { ConstantContext _localctx = new ConstantContext(_ctx, getState()); - enterRule(_localctx, 62, RULE_constant); + enterRule(_localctx, 64, RULE_constant); int _la; try { - setState(440); + setState(447); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,39,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,40,_ctx) ) { case 1: _localctx = new NullLiteralContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(398); + setState(405); match(NULL); } break; @@ -3374,9 +3448,9 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new QualifiedIntegerLiteralContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(399); + setState(406); integerValue(); - setState(400); + setState(407); match(UNQUOTED_IDENTIFIER); } break; @@ -3384,7 +3458,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new DecimalLiteralContext(_localctx); enterOuterAlt(_localctx, 3); { - setState(402); + setState(409); decimalValue(); } break; @@ -3392,7 +3466,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new IntegerLiteralContext(_localctx); enterOuterAlt(_localctx, 4); { - setState(403); + setState(410); integerValue(); } break; @@ -3400,7 +3474,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new BooleanLiteralContext(_localctx); enterOuterAlt(_localctx, 5); { - setState(404); + setState(411); booleanValue(); } break; @@ -3408,7 +3482,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new InputParameterContext(_localctx); enterOuterAlt(_localctx, 6); { - setState(405); + setState(412); parameter(); } break; @@ -3416,7 +3490,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new StringLiteralContext(_localctx); enterOuterAlt(_localctx, 7); { - setState(406); + setState(413); string(); } break; @@ -3424,27 +3498,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new NumericArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 8); { - setState(407); + setState(414); match(OPENING_BRACKET); - setState(408); + setState(415); numericValue(); - setState(413); + setState(420); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(409); + setState(416); match(COMMA); - setState(410); + setState(417); numericValue(); } } - setState(415); + setState(422); _errHandler.sync(this); _la = _input.LA(1); } - setState(416); + setState(423); match(CLOSING_BRACKET); } break; @@ -3452,27 +3526,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new BooleanArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 9); { - setState(418); + setState(425); match(OPENING_BRACKET); - setState(419); + setState(426); booleanValue(); - setState(424); + setState(431); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(420); + setState(427); match(COMMA); - setState(421); + setState(428); booleanValue(); } } - setState(426); + setState(433); _errHandler.sync(this); _la = _input.LA(1); } - setState(427); + setState(434); match(CLOSING_BRACKET); } break; @@ -3480,27 +3554,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new StringArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 10); { - setState(429); + setState(436); match(OPENING_BRACKET); - setState(430); + setState(437); string(); - setState(435); + setState(442); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(431); + setState(438); match(COMMA); - setState(432); + setState(439); string(); } } - setState(437); + setState(444); _errHandler.sync(this); _la = _input.LA(1); } - setState(438); + setState(445); match(CLOSING_BRACKET); } break; @@ -3524,7 +3598,7 @@ public ParameterContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); } @Override public int getRuleIndex() { return RULE_parameter; } - + @SuppressWarnings("this-escape") public ParameterContext() { } public void copyFrom(ParameterContext ctx) { @@ -3572,16 +3646,16 @@ public T accept(ParseTreeVisitor visitor) { public final ParameterContext parameter() throws RecognitionException { ParameterContext _localctx = new ParameterContext(_ctx, getState()); - enterRule(_localctx, 64, RULE_parameter); + enterRule(_localctx, 66, RULE_parameter); try { - setState(444); + setState(451); _errHandler.sync(this); switch (_input.LA(1)) { case PARAM: _localctx = new InputParamContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(442); + setState(449); match(PARAM); } break; @@ -3589,7 +3663,7 @@ public final ParameterContext parameter() throws RecognitionException { _localctx = new InputNamedOrPositionalParamContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(443); + setState(450); match(NAMED_OR_POSITIONAL_PARAM); } break; @@ -3638,16 +3712,16 @@ public T accept(ParseTreeVisitor visitor) { public final IdentifierOrParameterContext identifierOrParameter() throws RecognitionException { IdentifierOrParameterContext _localctx = new IdentifierOrParameterContext(_ctx, getState()); - enterRule(_localctx, 66, RULE_identifierOrParameter); + enterRule(_localctx, 68, RULE_identifierOrParameter); try { - setState(448); + setState(455); _errHandler.sync(this); switch (_input.LA(1)) { case UNQUOTED_IDENTIFIER: case QUOTED_IDENTIFIER: enterOuterAlt(_localctx, 1); { - setState(446); + setState(453); identifier(); } break; @@ -3655,7 +3729,7 @@ public final IdentifierOrParameterContext identifierOrParameter() throws Recogni case NAMED_OR_POSITIONAL_PARAM: enterOuterAlt(_localctx, 2); { - setState(447); + setState(454); parameter(); } break; @@ -3700,13 +3774,13 @@ public T accept(ParseTreeVisitor visitor) { public final LimitCommandContext limitCommand() throws RecognitionException { LimitCommandContext _localctx = new LimitCommandContext(_ctx, getState()); - enterRule(_localctx, 68, RULE_limitCommand); + enterRule(_localctx, 70, RULE_limitCommand); try { enterOuterAlt(_localctx, 1); { - setState(450); + setState(457); match(LIMIT); - setState(451); + setState(458); match(INTEGER_LITERAL); } } @@ -3756,32 +3830,32 @@ public T accept(ParseTreeVisitor visitor) { public final SortCommandContext sortCommand() throws RecognitionException { SortCommandContext _localctx = new SortCommandContext(_ctx, getState()); - enterRule(_localctx, 70, RULE_sortCommand); + enterRule(_localctx, 72, RULE_sortCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(453); + setState(460); match(SORT); - setState(454); + setState(461); orderExpression(); - setState(459); + setState(466); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,42,_ctx); + _alt = getInterpreter().adaptivePredict(_input,43,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(455); + setState(462); match(COMMA); - setState(456); + setState(463); orderExpression(); } - } + } } - setState(461); + setState(468); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,42,_ctx); + _alt = getInterpreter().adaptivePredict(_input,43,_ctx); } } } @@ -3830,19 +3904,19 @@ public T accept(ParseTreeVisitor visitor) { public final OrderExpressionContext orderExpression() throws RecognitionException { OrderExpressionContext _localctx = new OrderExpressionContext(_ctx, getState()); - enterRule(_localctx, 72, RULE_orderExpression); + enterRule(_localctx, 74, RULE_orderExpression); int _la; try { enterOuterAlt(_localctx, 1); { - setState(462); + setState(469); booleanExpression(0); - setState(464); + setState(471); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,43,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,44,_ctx) ) { case 1: { - setState(463); + setState(470); ((OrderExpressionContext)_localctx).ordering = _input.LT(1); _la = _input.LA(1); if ( !(_la==ASC || _la==DESC) ) { @@ -3856,14 +3930,14 @@ public final OrderExpressionContext orderExpression() throws RecognitionExceptio } break; } - setState(468); + setState(475); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,44,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,45,_ctx) ) { case 1: { - setState(466); + setState(473); match(NULLS); - setState(467); + setState(474); ((OrderExpressionContext)_localctx).nullOrdering = _input.LT(1); _la = _input.LA(1); if ( !(_la==FIRST || _la==LAST) ) { @@ -3918,13 +3992,13 @@ public T accept(ParseTreeVisitor visitor) { public final KeepCommandContext keepCommand() throws RecognitionException { KeepCommandContext _localctx = new KeepCommandContext(_ctx, getState()); - enterRule(_localctx, 74, RULE_keepCommand); + enterRule(_localctx, 76, RULE_keepCommand); try { enterOuterAlt(_localctx, 1); { - setState(470); + setState(477); match(KEEP); - setState(471); + setState(478); qualifiedNamePatterns(); } } @@ -3967,13 +4041,13 @@ public T accept(ParseTreeVisitor visitor) { public final DropCommandContext dropCommand() throws RecognitionException { DropCommandContext _localctx = new DropCommandContext(_ctx, getState()); - enterRule(_localctx, 76, RULE_dropCommand); + enterRule(_localctx, 78, RULE_dropCommand); try { enterOuterAlt(_localctx, 1); { - setState(473); + setState(480); match(DROP); - setState(474); + setState(481); qualifiedNamePatterns(); } } @@ -4023,32 +4097,32 @@ public T accept(ParseTreeVisitor visitor) { public final RenameCommandContext renameCommand() throws RecognitionException { RenameCommandContext _localctx = new RenameCommandContext(_ctx, getState()); - enterRule(_localctx, 78, RULE_renameCommand); + enterRule(_localctx, 80, RULE_renameCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(476); + setState(483); match(RENAME); - setState(477); + setState(484); renameClause(); - setState(482); + setState(489); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,45,_ctx); + _alt = getInterpreter().adaptivePredict(_input,46,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(478); + setState(485); match(COMMA); - setState(479); + setState(486); renameClause(); } - } + } } - setState(484); + setState(491); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,45,_ctx); + _alt = getInterpreter().adaptivePredict(_input,46,_ctx); } } } @@ -4096,15 +4170,15 @@ public T accept(ParseTreeVisitor visitor) { public final RenameClauseContext renameClause() throws RecognitionException { RenameClauseContext _localctx = new RenameClauseContext(_ctx, getState()); - enterRule(_localctx, 80, RULE_renameClause); + enterRule(_localctx, 82, RULE_renameClause); try { enterOuterAlt(_localctx, 1); { - setState(485); + setState(492); ((RenameClauseContext)_localctx).oldName = qualifiedNamePattern(); - setState(486); + setState(493); match(AS); - setState(487); + setState(494); ((RenameClauseContext)_localctx).newName = qualifiedNamePattern(); } } @@ -4153,22 +4227,22 @@ public T accept(ParseTreeVisitor visitor) { public final DissectCommandContext dissectCommand() throws RecognitionException { DissectCommandContext _localctx = new DissectCommandContext(_ctx, getState()); - enterRule(_localctx, 82, RULE_dissectCommand); + enterRule(_localctx, 84, RULE_dissectCommand); try { enterOuterAlt(_localctx, 1); { - setState(489); + setState(496); match(DISSECT); - setState(490); + setState(497); primaryExpression(0); - setState(491); + setState(498); string(); - setState(493); + setState(500); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,46,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,47,_ctx) ) { case 1: { - setState(492); + setState(499); commandOptions(); } break; @@ -4217,15 +4291,15 @@ public T accept(ParseTreeVisitor visitor) { public final GrokCommandContext grokCommand() throws RecognitionException { GrokCommandContext _localctx = new GrokCommandContext(_ctx, getState()); - enterRule(_localctx, 84, RULE_grokCommand); + enterRule(_localctx, 86, RULE_grokCommand); try { enterOuterAlt(_localctx, 1); { - setState(495); + setState(502); match(GROK); - setState(496); + setState(503); primaryExpression(0); - setState(497); + setState(504); string(); } } @@ -4268,13 +4342,13 @@ public T accept(ParseTreeVisitor visitor) { public final MvExpandCommandContext mvExpandCommand() throws RecognitionException { MvExpandCommandContext _localctx = new MvExpandCommandContext(_ctx, getState()); - enterRule(_localctx, 86, RULE_mvExpandCommand); + enterRule(_localctx, 88, RULE_mvExpandCommand); try { enterOuterAlt(_localctx, 1); { - setState(499); + setState(506); match(MV_EXPAND); - setState(500); + setState(507); qualifiedName(); } } @@ -4323,30 +4397,30 @@ public T accept(ParseTreeVisitor visitor) { public final CommandOptionsContext commandOptions() throws RecognitionException { CommandOptionsContext _localctx = new CommandOptionsContext(_ctx, getState()); - enterRule(_localctx, 88, RULE_commandOptions); + enterRule(_localctx, 90, RULE_commandOptions); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(502); + setState(509); commandOption(); - setState(507); + setState(514); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,47,_ctx); + _alt = getInterpreter().adaptivePredict(_input,48,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(503); + setState(510); match(COMMA); - setState(504); + setState(511); commandOption(); } - } + } } - setState(509); + setState(516); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,47,_ctx); + _alt = getInterpreter().adaptivePredict(_input,48,_ctx); } } } @@ -4392,15 +4466,15 @@ public T accept(ParseTreeVisitor visitor) { public final CommandOptionContext commandOption() throws RecognitionException { CommandOptionContext _localctx = new CommandOptionContext(_ctx, getState()); - enterRule(_localctx, 90, RULE_commandOption); + enterRule(_localctx, 92, RULE_commandOption); try { enterOuterAlt(_localctx, 1); { - setState(510); + setState(517); identifier(); - setState(511); + setState(518); match(ASSIGN); - setState(512); + setState(519); constant(); } } @@ -4441,12 +4515,12 @@ public T accept(ParseTreeVisitor visitor) { public final BooleanValueContext booleanValue() throws RecognitionException { BooleanValueContext _localctx = new BooleanValueContext(_ctx, getState()); - enterRule(_localctx, 92, RULE_booleanValue); + enterRule(_localctx, 94, RULE_booleanValue); int _la; try { enterOuterAlt(_localctx, 1); { - setState(514); + setState(521); _la = _input.LA(1); if ( !(_la==FALSE || _la==TRUE) ) { _errHandler.recoverInline(this); @@ -4499,22 +4573,22 @@ public T accept(ParseTreeVisitor visitor) { public final NumericValueContext numericValue() throws RecognitionException { NumericValueContext _localctx = new NumericValueContext(_ctx, getState()); - enterRule(_localctx, 94, RULE_numericValue); + enterRule(_localctx, 96, RULE_numericValue); try { - setState(518); + setState(525); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,48,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,49,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(516); + setState(523); decimalValue(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(517); + setState(524); integerValue(); } break; @@ -4558,17 +4632,17 @@ public T accept(ParseTreeVisitor visitor) { public final DecimalValueContext decimalValue() throws RecognitionException { DecimalValueContext _localctx = new DecimalValueContext(_ctx, getState()); - enterRule(_localctx, 96, RULE_decimalValue); + enterRule(_localctx, 98, RULE_decimalValue); int _la; try { enterOuterAlt(_localctx, 1); { - setState(521); + setState(528); _errHandler.sync(this); _la = _input.LA(1); if (_la==PLUS || _la==MINUS) { { - setState(520); + setState(527); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { _errHandler.recoverInline(this); @@ -4581,7 +4655,7 @@ public final DecimalValueContext decimalValue() throws RecognitionException { } } - setState(523); + setState(530); match(DECIMAL_LITERAL); } } @@ -4623,17 +4697,17 @@ public T accept(ParseTreeVisitor visitor) { public final IntegerValueContext integerValue() throws RecognitionException { IntegerValueContext _localctx = new IntegerValueContext(_ctx, getState()); - enterRule(_localctx, 98, RULE_integerValue); + enterRule(_localctx, 100, RULE_integerValue); int _la; try { enterOuterAlt(_localctx, 1); { - setState(526); + setState(533); _errHandler.sync(this); _la = _input.LA(1); if (_la==PLUS || _la==MINUS) { { - setState(525); + setState(532); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { _errHandler.recoverInline(this); @@ -4646,7 +4720,7 @@ public final IntegerValueContext integerValue() throws RecognitionException { } } - setState(528); + setState(535); match(INTEGER_LITERAL); } } @@ -4686,11 +4760,11 @@ public T accept(ParseTreeVisitor visitor) { public final StringContext string() throws RecognitionException { StringContext _localctx = new StringContext(_ctx, getState()); - enterRule(_localctx, 100, RULE_string); + enterRule(_localctx, 102, RULE_string); try { enterOuterAlt(_localctx, 1); { - setState(530); + setState(537); match(QUOTED_STRING); } } @@ -4735,14 +4809,14 @@ public T accept(ParseTreeVisitor visitor) { public final ComparisonOperatorContext comparisonOperator() throws RecognitionException { ComparisonOperatorContext _localctx = new ComparisonOperatorContext(_ctx, getState()); - enterRule(_localctx, 102, RULE_comparisonOperator); + enterRule(_localctx, 104, RULE_comparisonOperator); int _la; try { enterOuterAlt(_localctx, 1); { - setState(532); + setState(539); _la = _input.LA(1); - if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 562949953421312000L) != 0)) ) { + if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 281474976710656000L) != 0)) ) { _errHandler.recoverInline(this); } else { @@ -4791,13 +4865,13 @@ public T accept(ParseTreeVisitor visitor) { public final ExplainCommandContext explainCommand() throws RecognitionException { ExplainCommandContext _localctx = new ExplainCommandContext(_ctx, getState()); - enterRule(_localctx, 104, RULE_explainCommand); + enterRule(_localctx, 106, RULE_explainCommand); try { enterOuterAlt(_localctx, 1); { - setState(534); + setState(541); match(EXPLAIN); - setState(535); + setState(542); subqueryExpression(); } } @@ -4841,15 +4915,15 @@ public T accept(ParseTreeVisitor visitor) { public final SubqueryExpressionContext subqueryExpression() throws RecognitionException { SubqueryExpressionContext _localctx = new SubqueryExpressionContext(_ctx, getState()); - enterRule(_localctx, 106, RULE_subqueryExpression); + enterRule(_localctx, 108, RULE_subqueryExpression); try { enterOuterAlt(_localctx, 1); { - setState(537); + setState(544); match(OPENING_BRACKET); - setState(538); + setState(545); query(0); - setState(539); + setState(546); match(CLOSING_BRACKET); } } @@ -4871,7 +4945,7 @@ public ShowCommandContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); } @Override public int getRuleIndex() { return RULE_showCommand; } - + @SuppressWarnings("this-escape") public ShowCommandContext() { } public void copyFrom(ShowCommandContext ctx) { @@ -4901,14 +4975,14 @@ public T accept(ParseTreeVisitor visitor) { public final ShowCommandContext showCommand() throws RecognitionException { ShowCommandContext _localctx = new ShowCommandContext(_ctx, getState()); - enterRule(_localctx, 108, RULE_showCommand); + enterRule(_localctx, 110, RULE_showCommand); try { _localctx = new ShowInfoContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(541); + setState(548); match(SHOW); - setState(542); + setState(549); match(INFO); } } @@ -4966,53 +5040,53 @@ public T accept(ParseTreeVisitor visitor) { public final EnrichCommandContext enrichCommand() throws RecognitionException { EnrichCommandContext _localctx = new EnrichCommandContext(_ctx, getState()); - enterRule(_localctx, 110, RULE_enrichCommand); + enterRule(_localctx, 112, RULE_enrichCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(544); + setState(551); match(ENRICH); - setState(545); + setState(552); ((EnrichCommandContext)_localctx).policyName = match(ENRICH_POLICY_NAME); - setState(548); + setState(555); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,51,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,52,_ctx) ) { case 1: { - setState(546); + setState(553); match(ON); - setState(547); + setState(554); ((EnrichCommandContext)_localctx).matchField = qualifiedNamePattern(); } break; } - setState(559); + setState(566); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,53,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,54,_ctx) ) { case 1: { - setState(550); + setState(557); match(WITH); - setState(551); + setState(558); enrichWithClause(); - setState(556); + setState(563); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,52,_ctx); + _alt = getInterpreter().adaptivePredict(_input,53,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(552); + setState(559); match(COMMA); - setState(553); + setState(560); enrichWithClause(); } - } + } } - setState(558); + setState(565); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,52,_ctx); + _alt = getInterpreter().adaptivePredict(_input,53,_ctx); } } break; @@ -5063,23 +5137,23 @@ public T accept(ParseTreeVisitor visitor) { public final EnrichWithClauseContext enrichWithClause() throws RecognitionException { EnrichWithClauseContext _localctx = new EnrichWithClauseContext(_ctx, getState()); - enterRule(_localctx, 112, RULE_enrichWithClause); + enterRule(_localctx, 114, RULE_enrichWithClause); try { enterOuterAlt(_localctx, 1); { - setState(564); + setState(571); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,54,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,55,_ctx) ) { case 1: { - setState(561); + setState(568); ((EnrichWithClauseContext)_localctx).newName = qualifiedNamePattern(); - setState(562); + setState(569); match(ASSIGN); } break; } - setState(566); + setState(573); ((EnrichWithClauseContext)_localctx).enrichField = qualifiedNamePattern(); } } @@ -5128,17 +5202,17 @@ public T accept(ParseTreeVisitor visitor) { public final LookupCommandContext lookupCommand() throws RecognitionException { LookupCommandContext _localctx = new LookupCommandContext(_ctx, getState()); - enterRule(_localctx, 114, RULE_lookupCommand); + enterRule(_localctx, 116, RULE_lookupCommand); try { enterOuterAlt(_localctx, 1); { - setState(568); + setState(575); match(DEV_LOOKUP); - setState(569); + setState(576); ((LookupCommandContext)_localctx).tableName = indexPattern(); - setState(570); + setState(577); match(ON); - setState(571); + setState(578); ((LookupCommandContext)_localctx).matchFields = qualifiedNamePatterns(); } } @@ -5187,22 +5261,22 @@ public T accept(ParseTreeVisitor visitor) { public final InlinestatsCommandContext inlinestatsCommand() throws RecognitionException { InlinestatsCommandContext _localctx = new InlinestatsCommandContext(_ctx, getState()); - enterRule(_localctx, 116, RULE_inlinestatsCommand); + enterRule(_localctx, 118, RULE_inlinestatsCommand); try { enterOuterAlt(_localctx, 1); { - setState(573); + setState(580); match(DEV_INLINESTATS); - setState(574); + setState(581); ((InlinestatsCommandContext)_localctx).stats = fields(); - setState(577); + setState(584); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,55,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,56,_ctx) ) { case 1: { - setState(575); + setState(582); match(BY); - setState(576); + setState(583); ((InlinestatsCommandContext)_localctx).grouping = fields(); } break; @@ -5234,6 +5308,8 @@ public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) { return operatorExpression_sempred((OperatorExpressionContext)_localctx, predIndex); case 10: return primaryExpression_sempred((PrimaryExpressionContext)_localctx, predIndex); + case 12: + return functionName_sempred((FunctionNameContext)_localctx, predIndex); } return true; } @@ -5287,9 +5363,16 @@ private boolean primaryExpression_sempred(PrimaryExpressionContext _localctx, in } return true; } + private boolean functionName_sempred(FunctionNameContext _localctx, int predIndex) { + switch (predIndex) { + case 10: + return this.isDevVersion(); + } + return true; + } public static final String _serializedATN = - "\u0004\u0001x\u0244\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002"+ + "\u0004\u0001x\u024b\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002"+ "\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004\u0007\u0004\u0002"+ "\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002\u0007\u0007\u0007\u0002"+ "\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002\u000b\u0007\u000b\u0002"+ @@ -5304,360 +5387,361 @@ private boolean primaryExpression_sempred(PrimaryExpressionContext _localctx, in "(\u0007(\u0002)\u0007)\u0002*\u0007*\u0002+\u0007+\u0002,\u0007,\u0002"+ "-\u0007-\u0002.\u0007.\u0002/\u0007/\u00020\u00070\u00021\u00071\u0002"+ "2\u00072\u00023\u00073\u00024\u00074\u00025\u00075\u00026\u00076\u0002"+ - "7\u00077\u00028\u00078\u00029\u00079\u0002:\u0007:\u0001\u0000\u0001\u0000"+ - "\u0001\u0000\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+ - "\u0001\u0001\u0005\u0001\u0080\b\u0001\n\u0001\f\u0001\u0083\t\u0001\u0001"+ - "\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0003"+ - "\u0002\u008b\b\u0002\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+ - "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+ - "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0003"+ - "\u0003\u009d\b\u0003\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0005\u0001"+ - "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003"+ - "\u0005\u00a9\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ - "\u0005\u0005\u0005\u00b0\b\u0005\n\u0005\f\u0005\u00b3\t\u0005\u0001\u0005"+ - "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00ba\b\u0005"+ - "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00c0\b\u0005"+ + "7\u00077\u00028\u00078\u00029\u00079\u0002:\u0007:\u0002;\u0007;\u0001"+ + "\u0000\u0001\u0000\u0001\u0000\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+ + "\u0001\u0001\u0001\u0001\u0001\u0005\u0001\u0082\b\u0001\n\u0001\f\u0001"+ + "\u0085\t\u0001\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002"+ + "\u0001\u0002\u0003\u0002\u008d\b\u0002\u0001\u0003\u0001\u0003\u0001\u0003"+ + "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003"+ + "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003"+ + "\u0001\u0003\u0003\u0003\u009f\b\u0003\u0001\u0004\u0001\u0004\u0001\u0004"+ "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005"+ - "\u0005\u0005\u00c8\b\u0005\n\u0005\f\u0005\u00cb\t\u0005\u0001\u0006\u0001"+ - "\u0006\u0003\u0006\u00cf\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001"+ - "\u0006\u0001\u0006\u0003\u0006\u00d6\b\u0006\u0001\u0006\u0001\u0006\u0001"+ - "\u0006\u0003\u0006\u00db\b\u0006\u0001\u0007\u0001\u0007\u0001\u0007\u0001"+ - "\u0007\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0003\b\u00e6\b\b\u0001"+ - "\t\u0001\t\u0001\t\u0001\t\u0003\t\u00ec\b\t\u0001\t\u0001\t\u0001\t\u0001"+ - "\t\u0001\t\u0001\t\u0005\t\u00f4\b\t\n\t\f\t\u00f7\t\t\u0001\n\u0001\n"+ - "\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0003\n\u0101\b\n\u0001"+ - "\n\u0001\n\u0001\n\u0005\n\u0106\b\n\n\n\f\n\u0109\t\n\u0001\u000b\u0001"+ - "\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0005\u000b\u0111"+ - "\b\u000b\n\u000b\f\u000b\u0114\t\u000b\u0003\u000b\u0116\b\u000b\u0001"+ - "\u000b\u0001\u000b\u0001\f\u0001\f\u0001\r\u0001\r\u0001\r\u0001\u000e"+ - "\u0001\u000e\u0001\u000e\u0005\u000e\u0122\b\u000e\n\u000e\f\u000e\u0125"+ - "\t\u000e\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0003"+ - "\u000f\u012c\b\u000f\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0005"+ - "\u0010\u0132\b\u0010\n\u0010\f\u0010\u0135\t\u0010\u0001\u0010\u0003\u0010"+ - "\u0138\b\u0010\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011"+ - "\u0003\u0011\u013f\b\u0011\u0001\u0012\u0001\u0012\u0001\u0013\u0001\u0013"+ - "\u0001\u0014\u0001\u0014\u0003\u0014\u0147\b\u0014\u0001\u0015\u0001\u0015"+ - "\u0001\u0015\u0001\u0015\u0005\u0015\u014d\b\u0015\n\u0015\f\u0015\u0150"+ - "\t\u0015\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0017\u0001"+ - "\u0017\u0001\u0017\u0001\u0017\u0005\u0017\u015a\b\u0017\n\u0017\f\u0017"+ - "\u015d\t\u0017\u0001\u0017\u0003\u0017\u0160\b\u0017\u0001\u0017\u0001"+ - "\u0017\u0003\u0017\u0164\b\u0017\u0001\u0018\u0001\u0018\u0001\u0018\u0001"+ - "\u0019\u0001\u0019\u0003\u0019\u016b\b\u0019\u0001\u0019\u0001\u0019\u0003"+ - "\u0019\u016f\b\u0019\u0001\u001a\u0001\u001a\u0001\u001a\u0005\u001a\u0174"+ - "\b\u001a\n\u001a\f\u001a\u0177\t\u001a\u0001\u001b\u0001\u001b\u0001\u001b"+ - "\u0005\u001b\u017c\b\u001b\n\u001b\f\u001b\u017f\t\u001b\u0001\u001c\u0001"+ - "\u001c\u0001\u001c\u0005\u001c\u0184\b\u001c\n\u001c\f\u001c\u0187\t\u001c"+ - "\u0001\u001d\u0001\u001d\u0001\u001e\u0001\u001e\u0003\u001e\u018d\b\u001e"+ - "\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f"+ - "\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f"+ - "\u0001\u001f\u0005\u001f\u019c\b\u001f\n\u001f\f\u001f\u019f\t\u001f\u0001"+ - "\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0005"+ - "\u001f\u01a7\b\u001f\n\u001f\f\u001f\u01aa\t\u001f\u0001\u001f\u0001\u001f"+ - "\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0005\u001f\u01b2\b\u001f"+ - "\n\u001f\f\u001f\u01b5\t\u001f\u0001\u001f\u0001\u001f\u0003\u001f\u01b9"+ - "\b\u001f\u0001 \u0001 \u0003 \u01bd\b \u0001!\u0001!\u0003!\u01c1\b!\u0001"+ - "\"\u0001\"\u0001\"\u0001#\u0001#\u0001#\u0001#\u0005#\u01ca\b#\n#\f#\u01cd"+ - "\t#\u0001$\u0001$\u0003$\u01d1\b$\u0001$\u0001$\u0003$\u01d5\b$\u0001"+ - "%\u0001%\u0001%\u0001&\u0001&\u0001&\u0001\'\u0001\'\u0001\'\u0001\'\u0005"+ - "\'\u01e1\b\'\n\'\f\'\u01e4\t\'\u0001(\u0001(\u0001(\u0001(\u0001)\u0001"+ - ")\u0001)\u0001)\u0003)\u01ee\b)\u0001*\u0001*\u0001*\u0001*\u0001+\u0001"+ - "+\u0001+\u0001,\u0001,\u0001,\u0005,\u01fa\b,\n,\f,\u01fd\t,\u0001-\u0001"+ - "-\u0001-\u0001-\u0001.\u0001.\u0001/\u0001/\u0003/\u0207\b/\u00010\u0003"+ - "0\u020a\b0\u00010\u00010\u00011\u00031\u020f\b1\u00011\u00011\u00012\u0001"+ - "2\u00013\u00013\u00014\u00014\u00014\u00015\u00015\u00015\u00015\u0001"+ - "6\u00016\u00016\u00017\u00017\u00017\u00017\u00037\u0225\b7\u00017\u0001"+ - "7\u00017\u00017\u00057\u022b\b7\n7\f7\u022e\t7\u00037\u0230\b7\u00018"+ - "\u00018\u00018\u00038\u0235\b8\u00018\u00018\u00019\u00019\u00019\u0001"+ - "9\u00019\u0001:\u0001:\u0001:\u0001:\u0003:\u0242\b:\u0001:\u0000\u0004"+ - "\u0002\n\u0012\u0014;\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010\u0012"+ - "\u0014\u0016\u0018\u001a\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPRTVXZ\\"+ - "^`bdfhjlnprt\u0000\b\u0001\u0000;<\u0001\u0000=?\u0002\u0000\u001a\u001a"+ - "LL\u0001\u0000CD\u0002\u0000\u001f\u001f##\u0002\u0000&&))\u0002\u0000"+ - "%%33\u0002\u0000446:\u025e\u0000v\u0001\u0000\u0000\u0000\u0002y\u0001"+ - "\u0000\u0000\u0000\u0004\u008a\u0001\u0000\u0000\u0000\u0006\u009c\u0001"+ - "\u0000\u0000\u0000\b\u009e\u0001\u0000\u0000\u0000\n\u00bf\u0001\u0000"+ - "\u0000\u0000\f\u00da\u0001\u0000\u0000\u0000\u000e\u00dc\u0001\u0000\u0000"+ - "\u0000\u0010\u00e5\u0001\u0000\u0000\u0000\u0012\u00eb\u0001\u0000\u0000"+ - "\u0000\u0014\u0100\u0001\u0000\u0000\u0000\u0016\u010a\u0001\u0000\u0000"+ - "\u0000\u0018\u0119\u0001\u0000\u0000\u0000\u001a\u011b\u0001\u0000\u0000"+ - "\u0000\u001c\u011e\u0001\u0000\u0000\u0000\u001e\u012b\u0001\u0000\u0000"+ - "\u0000 \u012d\u0001\u0000\u0000\u0000\"\u013e\u0001\u0000\u0000\u0000"+ - "$\u0140\u0001\u0000\u0000\u0000&\u0142\u0001\u0000\u0000\u0000(\u0146"+ - "\u0001\u0000\u0000\u0000*\u0148\u0001\u0000\u0000\u0000,\u0151\u0001\u0000"+ - "\u0000\u0000.\u0155\u0001\u0000\u0000\u00000\u0165\u0001\u0000\u0000\u0000"+ - "2\u0168\u0001\u0000\u0000\u00004\u0170\u0001\u0000\u0000\u00006\u0178"+ - "\u0001\u0000\u0000\u00008\u0180\u0001\u0000\u0000\u0000:\u0188\u0001\u0000"+ - "\u0000\u0000<\u018c\u0001\u0000\u0000\u0000>\u01b8\u0001\u0000\u0000\u0000"+ - "@\u01bc\u0001\u0000\u0000\u0000B\u01c0\u0001\u0000\u0000\u0000D\u01c2"+ - "\u0001\u0000\u0000\u0000F\u01c5\u0001\u0000\u0000\u0000H\u01ce\u0001\u0000"+ - "\u0000\u0000J\u01d6\u0001\u0000\u0000\u0000L\u01d9\u0001\u0000\u0000\u0000"+ - "N\u01dc\u0001\u0000\u0000\u0000P\u01e5\u0001\u0000\u0000\u0000R\u01e9"+ - "\u0001\u0000\u0000\u0000T\u01ef\u0001\u0000\u0000\u0000V\u01f3\u0001\u0000"+ - "\u0000\u0000X\u01f6\u0001\u0000\u0000\u0000Z\u01fe\u0001\u0000\u0000\u0000"+ - "\\\u0202\u0001\u0000\u0000\u0000^\u0206\u0001\u0000\u0000\u0000`\u0209"+ - "\u0001\u0000\u0000\u0000b\u020e\u0001\u0000\u0000\u0000d\u0212\u0001\u0000"+ - "\u0000\u0000f\u0214\u0001\u0000\u0000\u0000h\u0216\u0001\u0000\u0000\u0000"+ - "j\u0219\u0001\u0000\u0000\u0000l\u021d\u0001\u0000\u0000\u0000n\u0220"+ - "\u0001\u0000\u0000\u0000p\u0234\u0001\u0000\u0000\u0000r\u0238\u0001\u0000"+ - "\u0000\u0000t\u023d\u0001\u0000\u0000\u0000vw\u0003\u0002\u0001\u0000"+ - "wx\u0005\u0000\u0000\u0001x\u0001\u0001\u0000\u0000\u0000yz\u0006\u0001"+ - "\uffff\uffff\u0000z{\u0003\u0004\u0002\u0000{\u0081\u0001\u0000\u0000"+ - "\u0000|}\n\u0001\u0000\u0000}~\u0005\u0019\u0000\u0000~\u0080\u0003\u0006"+ - "\u0003\u0000\u007f|\u0001\u0000\u0000\u0000\u0080\u0083\u0001\u0000\u0000"+ - "\u0000\u0081\u007f\u0001\u0000\u0000\u0000\u0081\u0082\u0001\u0000\u0000"+ - "\u0000\u0082\u0003\u0001\u0000\u0000\u0000\u0083\u0081\u0001\u0000\u0000"+ - "\u0000\u0084\u008b\u0003h4\u0000\u0085\u008b\u0003 \u0010\u0000\u0086"+ - "\u008b\u0003\u001a\r\u0000\u0087\u008b\u0003l6\u0000\u0088\u0089\u0004"+ - "\u0002\u0001\u0000\u0089\u008b\u0003.\u0017\u0000\u008a\u0084\u0001\u0000"+ - "\u0000\u0000\u008a\u0085\u0001\u0000\u0000\u0000\u008a\u0086\u0001\u0000"+ - "\u0000\u0000\u008a\u0087\u0001\u0000\u0000\u0000\u008a\u0088\u0001\u0000"+ - "\u0000\u0000\u008b\u0005\u0001\u0000\u0000\u0000\u008c\u009d\u00030\u0018"+ - "\u0000\u008d\u009d\u0003\b\u0004\u0000\u008e\u009d\u0003J%\u0000\u008f"+ - "\u009d\u0003D\"\u0000\u0090\u009d\u00032\u0019\u0000\u0091\u009d\u0003"+ - "F#\u0000\u0092\u009d\u0003L&\u0000\u0093\u009d\u0003N\'\u0000\u0094\u009d"+ - "\u0003R)\u0000\u0095\u009d\u0003T*\u0000\u0096\u009d\u0003n7\u0000\u0097"+ - "\u009d\u0003V+\u0000\u0098\u0099\u0004\u0003\u0002\u0000\u0099\u009d\u0003"+ - "t:\u0000\u009a\u009b\u0004\u0003\u0003\u0000\u009b\u009d\u0003r9\u0000"+ - "\u009c\u008c\u0001\u0000\u0000\u0000\u009c\u008d\u0001\u0000\u0000\u0000"+ - "\u009c\u008e\u0001\u0000\u0000\u0000\u009c\u008f\u0001\u0000\u0000\u0000"+ - "\u009c\u0090\u0001\u0000\u0000\u0000\u009c\u0091\u0001\u0000\u0000\u0000"+ - "\u009c\u0092\u0001\u0000\u0000\u0000\u009c\u0093\u0001\u0000\u0000\u0000"+ - "\u009c\u0094\u0001\u0000\u0000\u0000\u009c\u0095\u0001\u0000\u0000\u0000"+ - "\u009c\u0096\u0001\u0000\u0000\u0000\u009c\u0097\u0001\u0000\u0000\u0000"+ - "\u009c\u0098\u0001\u0000\u0000\u0000\u009c\u009a\u0001\u0000\u0000\u0000"+ - "\u009d\u0007\u0001\u0000\u0000\u0000\u009e\u009f\u0005\u0010\u0000\u0000"+ - "\u009f\u00a0\u0003\n\u0005\u0000\u00a0\t\u0001\u0000\u0000\u0000\u00a1"+ - "\u00a2\u0006\u0005\uffff\uffff\u0000\u00a2\u00a3\u0005,\u0000\u0000\u00a3"+ - "\u00c0\u0003\n\u0005\b\u00a4\u00c0\u0003\u0010\b\u0000\u00a5\u00c0\u0003"+ - "\f\u0006\u0000\u00a6\u00a8\u0003\u0010\b\u0000\u00a7\u00a9\u0005,\u0000"+ - "\u0000\u00a8\u00a7\u0001\u0000\u0000\u0000\u00a8\u00a9\u0001\u0000\u0000"+ - "\u0000\u00a9\u00aa\u0001\u0000\u0000\u0000\u00aa\u00ab\u0005\'\u0000\u0000"+ - "\u00ab\u00ac\u0005+\u0000\u0000\u00ac\u00b1\u0003\u0010\b\u0000\u00ad"+ - "\u00ae\u0005\"\u0000\u0000\u00ae\u00b0\u0003\u0010\b\u0000\u00af\u00ad"+ - "\u0001\u0000\u0000\u0000\u00b0\u00b3\u0001\u0000\u0000\u0000\u00b1\u00af"+ - "\u0001\u0000\u0000\u0000\u00b1\u00b2\u0001\u0000\u0000\u0000\u00b2\u00b4"+ - "\u0001\u0000\u0000\u0000\u00b3\u00b1\u0001\u0000\u0000\u0000\u00b4\u00b5"+ - "\u00052\u0000\u0000\u00b5\u00c0\u0001\u0000\u0000\u0000\u00b6\u00b7\u0003"+ - "\u0010\b\u0000\u00b7\u00b9\u0005(\u0000\u0000\u00b8\u00ba\u0005,\u0000"+ - "\u0000\u00b9\u00b8\u0001\u0000\u0000\u0000\u00b9\u00ba\u0001\u0000\u0000"+ - "\u0000\u00ba\u00bb\u0001\u0000\u0000\u0000\u00bb\u00bc\u0005-\u0000\u0000"+ - "\u00bc\u00c0\u0001\u0000\u0000\u0000\u00bd\u00be\u0004\u0005\u0004\u0000"+ - "\u00be\u00c0\u0003\u000e\u0007\u0000\u00bf\u00a1\u0001\u0000\u0000\u0000"+ - "\u00bf\u00a4\u0001\u0000\u0000\u0000\u00bf\u00a5\u0001\u0000\u0000\u0000"+ - "\u00bf\u00a6\u0001\u0000\u0000\u0000\u00bf\u00b6\u0001\u0000\u0000\u0000"+ - "\u00bf\u00bd\u0001\u0000\u0000\u0000\u00c0\u00c9\u0001\u0000\u0000\u0000"+ - "\u00c1\u00c2\n\u0005\u0000\u0000\u00c2\u00c3\u0005\u001e\u0000\u0000\u00c3"+ - "\u00c8\u0003\n\u0005\u0006\u00c4\u00c5\n\u0004\u0000\u0000\u00c5\u00c6"+ - "\u0005/\u0000\u0000\u00c6\u00c8\u0003\n\u0005\u0005\u00c7\u00c1\u0001"+ - "\u0000\u0000\u0000\u00c7\u00c4\u0001\u0000\u0000\u0000\u00c8\u00cb\u0001"+ - "\u0000\u0000\u0000\u00c9\u00c7\u0001\u0000\u0000\u0000\u00c9\u00ca\u0001"+ - "\u0000\u0000\u0000\u00ca\u000b\u0001\u0000\u0000\u0000\u00cb\u00c9\u0001"+ - "\u0000\u0000\u0000\u00cc\u00ce\u0003\u0010\b\u0000\u00cd\u00cf\u0005,"+ - "\u0000\u0000\u00ce\u00cd\u0001\u0000\u0000\u0000\u00ce\u00cf\u0001\u0000"+ - "\u0000\u0000\u00cf\u00d0\u0001\u0000\u0000\u0000\u00d0\u00d1\u0005*\u0000"+ - "\u0000\u00d1\u00d2\u0003d2\u0000\u00d2\u00db\u0001\u0000\u0000\u0000\u00d3"+ - "\u00d5\u0003\u0010\b\u0000\u00d4\u00d6\u0005,\u0000\u0000\u00d5\u00d4"+ - "\u0001\u0000\u0000\u0000\u00d5\u00d6\u0001\u0000\u0000\u0000\u00d6\u00d7"+ - "\u0001\u0000\u0000\u0000\u00d7\u00d8\u00051\u0000\u0000\u00d8\u00d9\u0003"+ - "d2\u0000\u00d9\u00db\u0001\u0000\u0000\u0000\u00da\u00cc\u0001\u0000\u0000"+ - "\u0000\u00da\u00d3\u0001\u0000\u0000\u0000\u00db\r\u0001\u0000\u0000\u0000"+ - "\u00dc\u00dd\u0003\u0010\b\u0000\u00dd\u00de\u0005\u0013\u0000\u0000\u00de"+ - "\u00df\u0003d2\u0000\u00df\u000f\u0001\u0000\u0000\u0000\u00e0\u00e6\u0003"+ - "\u0012\t\u0000\u00e1\u00e2\u0003\u0012\t\u0000\u00e2\u00e3\u0003f3\u0000"+ - "\u00e3\u00e4\u0003\u0012\t\u0000\u00e4\u00e6\u0001\u0000\u0000\u0000\u00e5"+ - "\u00e0\u0001\u0000\u0000\u0000\u00e5\u00e1\u0001\u0000\u0000\u0000\u00e6"+ - "\u0011\u0001\u0000\u0000\u0000\u00e7\u00e8\u0006\t\uffff\uffff\u0000\u00e8"+ - "\u00ec\u0003\u0014\n\u0000\u00e9\u00ea\u0007\u0000\u0000\u0000\u00ea\u00ec"+ - "\u0003\u0012\t\u0003\u00eb\u00e7\u0001\u0000\u0000\u0000\u00eb\u00e9\u0001"+ - "\u0000\u0000\u0000\u00ec\u00f5\u0001\u0000\u0000\u0000\u00ed\u00ee\n\u0002"+ - "\u0000\u0000\u00ee\u00ef\u0007\u0001\u0000\u0000\u00ef\u00f4\u0003\u0012"+ - "\t\u0003\u00f0\u00f1\n\u0001\u0000\u0000\u00f1\u00f2\u0007\u0000\u0000"+ - "\u0000\u00f2\u00f4\u0003\u0012\t\u0002\u00f3\u00ed\u0001\u0000\u0000\u0000"+ - "\u00f3\u00f0\u0001\u0000\u0000\u0000\u00f4\u00f7\u0001\u0000\u0000\u0000"+ - "\u00f5\u00f3\u0001\u0000\u0000\u0000\u00f5\u00f6\u0001\u0000\u0000\u0000"+ - "\u00f6\u0013\u0001\u0000\u0000\u0000\u00f7\u00f5\u0001\u0000\u0000\u0000"+ - "\u00f8\u00f9\u0006\n\uffff\uffff\u0000\u00f9\u0101\u0003>\u001f\u0000"+ - "\u00fa\u0101\u00034\u001a\u0000\u00fb\u0101\u0003\u0016\u000b\u0000\u00fc"+ - "\u00fd\u0005+\u0000\u0000\u00fd\u00fe\u0003\n\u0005\u0000\u00fe\u00ff"+ - "\u00052\u0000\u0000\u00ff\u0101\u0001\u0000\u0000\u0000\u0100\u00f8\u0001"+ - "\u0000\u0000\u0000\u0100\u00fa\u0001\u0000\u0000\u0000\u0100\u00fb\u0001"+ - "\u0000\u0000\u0000\u0100\u00fc\u0001\u0000\u0000\u0000\u0101\u0107\u0001"+ - "\u0000\u0000\u0000\u0102\u0103\n\u0001\u0000\u0000\u0103\u0104\u0005!"+ - "\u0000\u0000\u0104\u0106\u0003\u0018\f\u0000\u0105\u0102\u0001\u0000\u0000"+ - "\u0000\u0106\u0109\u0001\u0000\u0000\u0000\u0107\u0105\u0001\u0000\u0000"+ - "\u0000\u0107\u0108\u0001\u0000\u0000\u0000\u0108\u0015\u0001\u0000\u0000"+ - "\u0000\u0109\u0107\u0001\u0000\u0000\u0000\u010a\u010b\u0003B!\u0000\u010b"+ - "\u0115\u0005+\u0000\u0000\u010c\u0116\u0005=\u0000\u0000\u010d\u0112\u0003"+ - "\n\u0005\u0000\u010e\u010f\u0005\"\u0000\u0000\u010f\u0111\u0003\n\u0005"+ - "\u0000\u0110\u010e\u0001\u0000\u0000\u0000\u0111\u0114\u0001\u0000\u0000"+ - "\u0000\u0112\u0110\u0001\u0000\u0000\u0000\u0112\u0113\u0001\u0000\u0000"+ - "\u0000\u0113\u0116\u0001\u0000\u0000\u0000\u0114\u0112\u0001\u0000\u0000"+ - "\u0000\u0115\u010c\u0001\u0000\u0000\u0000\u0115\u010d\u0001\u0000\u0000"+ - "\u0000\u0115\u0116\u0001\u0000\u0000\u0000\u0116\u0117\u0001\u0000\u0000"+ - "\u0000\u0117\u0118\u00052\u0000\u0000\u0118\u0017\u0001\u0000\u0000\u0000"+ - "\u0119\u011a\u0003:\u001d\u0000\u011a\u0019\u0001\u0000\u0000\u0000\u011b"+ - "\u011c\u0005\f\u0000\u0000\u011c\u011d\u0003\u001c\u000e\u0000\u011d\u001b"+ - "\u0001\u0000\u0000\u0000\u011e\u0123\u0003\u001e\u000f\u0000\u011f\u0120"+ - "\u0005\"\u0000\u0000\u0120\u0122\u0003\u001e\u000f\u0000\u0121\u011f\u0001"+ - "\u0000\u0000\u0000\u0122\u0125\u0001\u0000\u0000\u0000\u0123\u0121\u0001"+ - "\u0000\u0000\u0000\u0123\u0124\u0001\u0000\u0000\u0000\u0124\u001d\u0001"+ - "\u0000\u0000\u0000\u0125\u0123\u0001\u0000\u0000\u0000\u0126\u012c\u0003"+ - "\n\u0005\u0000\u0127\u0128\u00034\u001a\u0000\u0128\u0129\u0005 \u0000"+ - "\u0000\u0129\u012a\u0003\n\u0005\u0000\u012a\u012c\u0001\u0000\u0000\u0000"+ - "\u012b\u0126\u0001\u0000\u0000\u0000\u012b\u0127\u0001\u0000\u0000\u0000"+ - "\u012c\u001f\u0001\u0000\u0000\u0000\u012d\u012e\u0005\u0006\u0000\u0000"+ - "\u012e\u0133\u0003\"\u0011\u0000\u012f\u0130\u0005\"\u0000\u0000\u0130"+ - "\u0132\u0003\"\u0011\u0000\u0131\u012f\u0001\u0000\u0000\u0000\u0132\u0135"+ - "\u0001\u0000\u0000\u0000\u0133\u0131\u0001\u0000\u0000\u0000\u0133\u0134"+ - "\u0001\u0000\u0000\u0000\u0134\u0137\u0001\u0000\u0000\u0000\u0135\u0133"+ - "\u0001\u0000\u0000\u0000\u0136\u0138\u0003(\u0014\u0000\u0137\u0136\u0001"+ - "\u0000\u0000\u0000\u0137\u0138\u0001\u0000\u0000\u0000\u0138!\u0001\u0000"+ - "\u0000\u0000\u0139\u013a\u0003$\u0012\u0000\u013a\u013b\u0005h\u0000\u0000"+ - "\u013b\u013c\u0003&\u0013\u0000\u013c\u013f\u0001\u0000\u0000\u0000\u013d"+ - "\u013f\u0003&\u0013\u0000\u013e\u0139\u0001\u0000\u0000\u0000\u013e\u013d"+ - "\u0001\u0000\u0000\u0000\u013f#\u0001\u0000\u0000\u0000\u0140\u0141\u0005"+ - "L\u0000\u0000\u0141%\u0001\u0000\u0000\u0000\u0142\u0143\u0007\u0002\u0000"+ - "\u0000\u0143\'\u0001\u0000\u0000\u0000\u0144\u0147\u0003*\u0015\u0000"+ - "\u0145\u0147\u0003,\u0016\u0000\u0146\u0144\u0001\u0000\u0000\u0000\u0146"+ - "\u0145\u0001\u0000\u0000\u0000\u0147)\u0001\u0000\u0000\u0000\u0148\u0149"+ - "\u0005K\u0000\u0000\u0149\u014e\u0005L\u0000\u0000\u014a\u014b\u0005\""+ - "\u0000\u0000\u014b\u014d\u0005L\u0000\u0000\u014c\u014a\u0001\u0000\u0000"+ - "\u0000\u014d\u0150\u0001\u0000\u0000\u0000\u014e\u014c\u0001\u0000\u0000"+ - "\u0000\u014e\u014f\u0001\u0000\u0000\u0000\u014f+\u0001\u0000\u0000\u0000"+ - "\u0150\u014e\u0001\u0000\u0000\u0000\u0151\u0152\u0005A\u0000\u0000\u0152"+ - "\u0153\u0003*\u0015\u0000\u0153\u0154\u0005B\u0000\u0000\u0154-\u0001"+ - "\u0000\u0000\u0000\u0155\u0156\u0005\u0014\u0000\u0000\u0156\u015b\u0003"+ - "\"\u0011\u0000\u0157\u0158\u0005\"\u0000\u0000\u0158\u015a\u0003\"\u0011"+ - "\u0000\u0159\u0157\u0001\u0000\u0000\u0000\u015a\u015d\u0001\u0000\u0000"+ - "\u0000\u015b\u0159\u0001\u0000\u0000\u0000\u015b\u015c\u0001\u0000\u0000"+ - "\u0000\u015c\u015f\u0001\u0000\u0000\u0000\u015d\u015b\u0001\u0000\u0000"+ - "\u0000\u015e\u0160\u0003\u001c\u000e\u0000\u015f\u015e\u0001\u0000\u0000"+ - "\u0000\u015f\u0160\u0001\u0000\u0000\u0000\u0160\u0163\u0001\u0000\u0000"+ - "\u0000\u0161\u0162\u0005\u001d\u0000\u0000\u0162\u0164\u0003\u001c\u000e"+ - "\u0000\u0163\u0161\u0001\u0000\u0000\u0000\u0163\u0164\u0001\u0000\u0000"+ - "\u0000\u0164/\u0001\u0000\u0000\u0000\u0165\u0166\u0005\u0004\u0000\u0000"+ - "\u0166\u0167\u0003\u001c\u000e\u0000\u01671\u0001\u0000\u0000\u0000\u0168"+ - "\u016a\u0005\u000f\u0000\u0000\u0169\u016b\u0003\u001c\u000e\u0000\u016a"+ - "\u0169\u0001\u0000\u0000\u0000\u016a\u016b\u0001\u0000\u0000\u0000\u016b"+ - "\u016e\u0001\u0000\u0000\u0000\u016c\u016d\u0005\u001d\u0000\u0000\u016d"+ - "\u016f\u0003\u001c\u000e\u0000\u016e\u016c\u0001\u0000\u0000\u0000\u016e"+ - "\u016f\u0001\u0000\u0000\u0000\u016f3\u0001\u0000\u0000\u0000\u0170\u0175"+ - "\u0003B!\u0000\u0171\u0172\u0005$\u0000\u0000\u0172\u0174\u0003B!\u0000"+ - "\u0173\u0171\u0001\u0000\u0000\u0000\u0174\u0177\u0001\u0000\u0000\u0000"+ - "\u0175\u0173\u0001\u0000\u0000\u0000\u0175\u0176\u0001\u0000\u0000\u0000"+ - "\u01765\u0001\u0000\u0000\u0000\u0177\u0175\u0001\u0000\u0000\u0000\u0178"+ - "\u017d\u0003<\u001e\u0000\u0179\u017a\u0005$\u0000\u0000\u017a\u017c\u0003"+ - "<\u001e\u0000\u017b\u0179\u0001\u0000\u0000\u0000\u017c\u017f\u0001\u0000"+ - "\u0000\u0000\u017d\u017b\u0001\u0000\u0000\u0000\u017d\u017e\u0001\u0000"+ - "\u0000\u0000\u017e7\u0001\u0000\u0000\u0000\u017f\u017d\u0001\u0000\u0000"+ - "\u0000\u0180\u0185\u00036\u001b\u0000\u0181\u0182\u0005\"\u0000\u0000"+ - "\u0182\u0184\u00036\u001b\u0000\u0183\u0181\u0001\u0000\u0000\u0000\u0184"+ - "\u0187\u0001\u0000\u0000\u0000\u0185\u0183\u0001\u0000\u0000\u0000\u0185"+ - "\u0186\u0001\u0000\u0000\u0000\u01869\u0001\u0000\u0000\u0000\u0187\u0185"+ - "\u0001\u0000\u0000\u0000\u0188\u0189\u0007\u0003\u0000\u0000\u0189;\u0001"+ - "\u0000\u0000\u0000\u018a\u018d\u0005P\u0000\u0000\u018b\u018d\u0003@ "+ - "\u0000\u018c\u018a\u0001\u0000\u0000\u0000\u018c\u018b\u0001\u0000\u0000"+ - "\u0000\u018d=\u0001\u0000\u0000\u0000\u018e\u01b9\u0005-\u0000\u0000\u018f"+ - "\u0190\u0003b1\u0000\u0190\u0191\u0005C\u0000\u0000\u0191\u01b9\u0001"+ - "\u0000\u0000\u0000\u0192\u01b9\u0003`0\u0000\u0193\u01b9\u0003b1\u0000"+ - "\u0194\u01b9\u0003\\.\u0000\u0195\u01b9\u0003@ \u0000\u0196\u01b9\u0003"+ - "d2\u0000\u0197\u0198\u0005A\u0000\u0000\u0198\u019d\u0003^/\u0000\u0199"+ - "\u019a\u0005\"\u0000\u0000\u019a\u019c\u0003^/\u0000\u019b\u0199\u0001"+ - "\u0000\u0000\u0000\u019c\u019f\u0001\u0000\u0000\u0000\u019d\u019b\u0001"+ - "\u0000\u0000\u0000\u019d\u019e\u0001\u0000\u0000\u0000\u019e\u01a0\u0001"+ - "\u0000\u0000\u0000\u019f\u019d\u0001\u0000\u0000\u0000\u01a0\u01a1\u0005"+ - "B\u0000\u0000\u01a1\u01b9\u0001\u0000\u0000\u0000\u01a2\u01a3\u0005A\u0000"+ - "\u0000\u01a3\u01a8\u0003\\.\u0000\u01a4\u01a5\u0005\"\u0000\u0000\u01a5"+ - "\u01a7\u0003\\.\u0000\u01a6\u01a4\u0001\u0000\u0000\u0000\u01a7\u01aa"+ - "\u0001\u0000\u0000\u0000\u01a8\u01a6\u0001\u0000\u0000\u0000\u01a8\u01a9"+ - "\u0001\u0000\u0000\u0000\u01a9\u01ab\u0001\u0000\u0000\u0000\u01aa\u01a8"+ - "\u0001\u0000\u0000\u0000\u01ab\u01ac\u0005B\u0000\u0000\u01ac\u01b9\u0001"+ - "\u0000\u0000\u0000\u01ad\u01ae\u0005A\u0000\u0000\u01ae\u01b3\u0003d2"+ - "\u0000\u01af\u01b0\u0005\"\u0000\u0000\u01b0\u01b2\u0003d2\u0000\u01b1"+ - "\u01af\u0001\u0000\u0000\u0000\u01b2\u01b5\u0001\u0000\u0000\u0000\u01b3"+ - "\u01b1\u0001\u0000\u0000\u0000\u01b3\u01b4\u0001\u0000\u0000\u0000\u01b4"+ - "\u01b6\u0001\u0000\u0000\u0000\u01b5\u01b3\u0001\u0000\u0000\u0000\u01b6"+ - "\u01b7\u0005B\u0000\u0000\u01b7\u01b9\u0001\u0000\u0000\u0000\u01b8\u018e"+ - "\u0001\u0000\u0000\u0000\u01b8\u018f\u0001\u0000\u0000\u0000\u01b8\u0192"+ - "\u0001\u0000\u0000\u0000\u01b8\u0193\u0001\u0000\u0000\u0000\u01b8\u0194"+ - "\u0001\u0000\u0000\u0000\u01b8\u0195\u0001\u0000\u0000\u0000\u01b8\u0196"+ - "\u0001\u0000\u0000\u0000\u01b8\u0197\u0001\u0000\u0000\u0000\u01b8\u01a2"+ - "\u0001\u0000\u0000\u0000\u01b8\u01ad\u0001\u0000\u0000\u0000\u01b9?\u0001"+ - "\u0000\u0000\u0000\u01ba\u01bd\u00050\u0000\u0000\u01bb\u01bd\u0005@\u0000"+ - "\u0000\u01bc\u01ba\u0001\u0000\u0000\u0000\u01bc\u01bb\u0001\u0000\u0000"+ - "\u0000\u01bdA\u0001\u0000\u0000\u0000\u01be\u01c1\u0003:\u001d\u0000\u01bf"+ - "\u01c1\u0003@ \u0000\u01c0\u01be\u0001\u0000\u0000\u0000\u01c0\u01bf\u0001"+ - "\u0000\u0000\u0000\u01c1C\u0001\u0000\u0000\u0000\u01c2\u01c3\u0005\t"+ - "\u0000\u0000\u01c3\u01c4\u0005\u001b\u0000\u0000\u01c4E\u0001\u0000\u0000"+ - "\u0000\u01c5\u01c6\u0005\u000e\u0000\u0000\u01c6\u01cb\u0003H$\u0000\u01c7"+ - "\u01c8\u0005\"\u0000\u0000\u01c8\u01ca\u0003H$\u0000\u01c9\u01c7\u0001"+ - "\u0000\u0000\u0000\u01ca\u01cd\u0001\u0000\u0000\u0000\u01cb\u01c9\u0001"+ - "\u0000\u0000\u0000\u01cb\u01cc\u0001\u0000\u0000\u0000\u01ccG\u0001\u0000"+ - "\u0000\u0000\u01cd\u01cb\u0001\u0000\u0000\u0000\u01ce\u01d0\u0003\n\u0005"+ - "\u0000\u01cf\u01d1\u0007\u0004\u0000\u0000\u01d0\u01cf\u0001\u0000\u0000"+ - "\u0000\u01d0\u01d1\u0001\u0000\u0000\u0000\u01d1\u01d4\u0001\u0000\u0000"+ - "\u0000\u01d2\u01d3\u0005.\u0000\u0000\u01d3\u01d5\u0007\u0005\u0000\u0000"+ - "\u01d4\u01d2\u0001\u0000\u0000\u0000\u01d4\u01d5\u0001\u0000\u0000\u0000"+ - "\u01d5I\u0001\u0000\u0000\u0000\u01d6\u01d7\u0005\b\u0000\u0000\u01d7"+ - "\u01d8\u00038\u001c\u0000\u01d8K\u0001\u0000\u0000\u0000\u01d9\u01da\u0005"+ - "\u0002\u0000\u0000\u01da\u01db\u00038\u001c\u0000\u01dbM\u0001\u0000\u0000"+ - "\u0000\u01dc\u01dd\u0005\u000b\u0000\u0000\u01dd\u01e2\u0003P(\u0000\u01de"+ - "\u01df\u0005\"\u0000\u0000\u01df\u01e1\u0003P(\u0000\u01e0\u01de\u0001"+ - "\u0000\u0000\u0000\u01e1\u01e4\u0001\u0000\u0000\u0000\u01e2\u01e0\u0001"+ - "\u0000\u0000\u0000\u01e2\u01e3\u0001\u0000\u0000\u0000\u01e3O\u0001\u0000"+ - "\u0000\u0000\u01e4\u01e2\u0001\u0000\u0000\u0000\u01e5\u01e6\u00036\u001b"+ - "\u0000\u01e6\u01e7\u0005T\u0000\u0000\u01e7\u01e8\u00036\u001b\u0000\u01e8"+ - "Q\u0001\u0000\u0000\u0000\u01e9\u01ea\u0005\u0001\u0000\u0000\u01ea\u01eb"+ - "\u0003\u0014\n\u0000\u01eb\u01ed\u0003d2\u0000\u01ec\u01ee\u0003X,\u0000"+ - "\u01ed\u01ec\u0001\u0000\u0000\u0000\u01ed\u01ee\u0001\u0000\u0000\u0000"+ - "\u01eeS\u0001\u0000\u0000\u0000\u01ef\u01f0\u0005\u0007\u0000\u0000\u01f0"+ - "\u01f1\u0003\u0014\n\u0000\u01f1\u01f2\u0003d2\u0000\u01f2U\u0001\u0000"+ - "\u0000\u0000\u01f3\u01f4\u0005\n\u0000\u0000\u01f4\u01f5\u00034\u001a"+ - "\u0000\u01f5W\u0001\u0000\u0000\u0000\u01f6\u01fb\u0003Z-\u0000\u01f7"+ - "\u01f8\u0005\"\u0000\u0000\u01f8\u01fa\u0003Z-\u0000\u01f9\u01f7\u0001"+ - "\u0000\u0000\u0000\u01fa\u01fd\u0001\u0000\u0000\u0000\u01fb\u01f9\u0001"+ - "\u0000\u0000\u0000\u01fb\u01fc\u0001\u0000\u0000\u0000\u01fcY\u0001\u0000"+ - "\u0000\u0000\u01fd\u01fb\u0001\u0000\u0000\u0000\u01fe\u01ff\u0003:\u001d"+ - "\u0000\u01ff\u0200\u0005 \u0000\u0000\u0200\u0201\u0003>\u001f\u0000\u0201"+ - "[\u0001\u0000\u0000\u0000\u0202\u0203\u0007\u0006\u0000\u0000\u0203]\u0001"+ - "\u0000\u0000\u0000\u0204\u0207\u0003`0\u0000\u0205\u0207\u0003b1\u0000"+ - "\u0206\u0204\u0001\u0000\u0000\u0000\u0206\u0205\u0001\u0000\u0000\u0000"+ - "\u0207_\u0001\u0000\u0000\u0000\u0208\u020a\u0007\u0000\u0000\u0000\u0209"+ - "\u0208\u0001\u0000\u0000\u0000\u0209\u020a\u0001\u0000\u0000\u0000\u020a"+ - "\u020b\u0001\u0000\u0000\u0000\u020b\u020c\u0005\u001c\u0000\u0000\u020c"+ - "a\u0001\u0000\u0000\u0000\u020d\u020f\u0007\u0000\u0000\u0000\u020e\u020d"+ - "\u0001\u0000\u0000\u0000\u020e\u020f\u0001\u0000\u0000\u0000\u020f\u0210"+ - "\u0001\u0000\u0000\u0000\u0210\u0211\u0005\u001b\u0000\u0000\u0211c\u0001"+ - "\u0000\u0000\u0000\u0212\u0213\u0005\u001a\u0000\u0000\u0213e\u0001\u0000"+ - "\u0000\u0000\u0214\u0215\u0007\u0007\u0000\u0000\u0215g\u0001\u0000\u0000"+ - "\u0000\u0216\u0217\u0005\u0005\u0000\u0000\u0217\u0218\u0003j5\u0000\u0218"+ - "i\u0001\u0000\u0000\u0000\u0219\u021a\u0005A\u0000\u0000\u021a\u021b\u0003"+ - "\u0002\u0001\u0000\u021b\u021c\u0005B\u0000\u0000\u021ck\u0001\u0000\u0000"+ - "\u0000\u021d\u021e\u0005\r\u0000\u0000\u021e\u021f\u0005d\u0000\u0000"+ - "\u021fm\u0001\u0000\u0000\u0000\u0220\u0221\u0005\u0003\u0000\u0000\u0221"+ - "\u0224\u0005Z\u0000\u0000\u0222\u0223\u0005X\u0000\u0000\u0223\u0225\u0003"+ - "6\u001b\u0000\u0224\u0222\u0001\u0000\u0000\u0000\u0224\u0225\u0001\u0000"+ - "\u0000\u0000\u0225\u022f\u0001\u0000\u0000\u0000\u0226\u0227\u0005Y\u0000"+ - "\u0000\u0227\u022c\u0003p8\u0000\u0228\u0229\u0005\"\u0000\u0000\u0229"+ - "\u022b\u0003p8\u0000\u022a\u0228\u0001\u0000\u0000\u0000\u022b\u022e\u0001"+ - "\u0000\u0000\u0000\u022c\u022a\u0001\u0000\u0000\u0000\u022c\u022d\u0001"+ - "\u0000\u0000\u0000\u022d\u0230\u0001\u0000\u0000\u0000\u022e\u022c\u0001"+ - "\u0000\u0000\u0000\u022f\u0226\u0001\u0000\u0000\u0000\u022f\u0230\u0001"+ - "\u0000\u0000\u0000\u0230o\u0001\u0000\u0000\u0000\u0231\u0232\u00036\u001b"+ - "\u0000\u0232\u0233\u0005 \u0000\u0000\u0233\u0235\u0001\u0000\u0000\u0000"+ - "\u0234\u0231\u0001\u0000\u0000\u0000\u0234\u0235\u0001\u0000\u0000\u0000"+ - "\u0235\u0236\u0001\u0000\u0000\u0000\u0236\u0237\u00036\u001b\u0000\u0237"+ - "q\u0001\u0000\u0000\u0000\u0238\u0239\u0005\u0012\u0000\u0000\u0239\u023a"+ - "\u0003\"\u0011\u0000\u023a\u023b\u0005X\u0000\u0000\u023b\u023c\u0003"+ - "8\u001c\u0000\u023cs\u0001\u0000\u0000\u0000\u023d\u023e\u0005\u0011\u0000"+ - "\u0000\u023e\u0241\u0003\u001c\u000e\u0000\u023f\u0240\u0005\u001d\u0000"+ - "\u0000\u0240\u0242\u0003\u001c\u000e\u0000\u0241\u023f\u0001\u0000\u0000"+ - "\u0000\u0241\u0242\u0001\u0000\u0000\u0000\u0242u\u0001\u0000\u0000\u0000"+ - "8\u0081\u008a\u009c\u00a8\u00b1\u00b9\u00bf\u00c7\u00c9\u00ce\u00d5\u00da"+ - "\u00e5\u00eb\u00f3\u00f5\u0100\u0107\u0112\u0115\u0123\u012b\u0133\u0137"+ - "\u013e\u0146\u014e\u015b\u015f\u0163\u016a\u016e\u0175\u017d\u0185\u018c"+ - "\u019d\u01a8\u01b3\u01b8\u01bc\u01c0\u01cb\u01d0\u01d4\u01e2\u01ed\u01fb"+ - "\u0206\u0209\u020e\u0224\u022c\u022f\u0234\u0241"; + "\u0001\u0005\u0003\u0005\u00ab\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005"+ + "\u0001\u0005\u0001\u0005\u0005\u0005\u00b2\b\u0005\n\u0005\f\u0005\u00b5"+ + "\t\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003"+ + "\u0005\u00bc\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003"+ + "\u0005\u00c2\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ + "\u0005\u0001\u0005\u0005\u0005\u00ca\b\u0005\n\u0005\f\u0005\u00cd\t\u0005"+ + "\u0001\u0006\u0001\u0006\u0003\u0006\u00d1\b\u0006\u0001\u0006\u0001\u0006"+ + "\u0001\u0006\u0001\u0006\u0001\u0006\u0003\u0006\u00d8\b\u0006\u0001\u0006"+ + "\u0001\u0006\u0001\u0006\u0003\u0006\u00dd\b\u0006\u0001\u0007\u0001\u0007"+ + "\u0001\u0007\u0001\u0007\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0003"+ + "\b\u00e8\b\b\u0001\t\u0001\t\u0001\t\u0001\t\u0003\t\u00ee\b\t\u0001\t"+ + "\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0005\t\u00f6\b\t\n\t\f\t\u00f9"+ + "\t\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0003"+ + "\n\u0103\b\n\u0001\n\u0001\n\u0001\n\u0005\n\u0108\b\n\n\n\f\n\u010b\t"+ + "\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b"+ + "\u0005\u000b\u0113\b\u000b\n\u000b\f\u000b\u0116\t\u000b\u0003\u000b\u0118"+ + "\b\u000b\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001\f\u0003\f\u011f"+ + "\b\f\u0001\r\u0001\r\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000f\u0001"+ + "\u000f\u0001\u000f\u0005\u000f\u0129\b\u000f\n\u000f\f\u000f\u012c\t\u000f"+ + "\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0003\u0010"+ + "\u0133\b\u0010\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0005\u0011"+ + "\u0139\b\u0011\n\u0011\f\u0011\u013c\t\u0011\u0001\u0011\u0003\u0011\u013f"+ + "\b\u0011\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0003"+ + "\u0012\u0146\b\u0012\u0001\u0013\u0001\u0013\u0001\u0014\u0001\u0014\u0001"+ + "\u0015\u0001\u0015\u0003\u0015\u014e\b\u0015\u0001\u0016\u0001\u0016\u0001"+ + "\u0016\u0001\u0016\u0005\u0016\u0154\b\u0016\n\u0016\f\u0016\u0157\t\u0016"+ + "\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0018\u0001\u0018"+ + "\u0001\u0018\u0001\u0018\u0005\u0018\u0161\b\u0018\n\u0018\f\u0018\u0164"+ + "\t\u0018\u0001\u0018\u0003\u0018\u0167\b\u0018\u0001\u0018\u0001\u0018"+ + "\u0003\u0018\u016b\b\u0018\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u001a"+ + "\u0001\u001a\u0003\u001a\u0172\b\u001a\u0001\u001a\u0001\u001a\u0003\u001a"+ + "\u0176\b\u001a\u0001\u001b\u0001\u001b\u0001\u001b\u0005\u001b\u017b\b"+ + "\u001b\n\u001b\f\u001b\u017e\t\u001b\u0001\u001c\u0001\u001c\u0001\u001c"+ + "\u0005\u001c\u0183\b\u001c\n\u001c\f\u001c\u0186\t\u001c\u0001\u001d\u0001"+ + "\u001d\u0001\u001d\u0005\u001d\u018b\b\u001d\n\u001d\f\u001d\u018e\t\u001d"+ + "\u0001\u001e\u0001\u001e\u0001\u001f\u0001\u001f\u0003\u001f\u0194\b\u001f"+ + "\u0001 \u0001 \u0001 \u0001 \u0001 \u0001 \u0001 \u0001 \u0001 \u0001"+ + " \u0001 \u0001 \u0001 \u0005 \u01a3\b \n \f \u01a6\t \u0001 \u0001 \u0001"+ + " \u0001 \u0001 \u0001 \u0005 \u01ae\b \n \f \u01b1\t \u0001 \u0001 \u0001"+ + " \u0001 \u0001 \u0001 \u0005 \u01b9\b \n \f \u01bc\t \u0001 \u0001 \u0003"+ + " \u01c0\b \u0001!\u0001!\u0003!\u01c4\b!\u0001\"\u0001\"\u0003\"\u01c8"+ + "\b\"\u0001#\u0001#\u0001#\u0001$\u0001$\u0001$\u0001$\u0005$\u01d1\b$"+ + "\n$\f$\u01d4\t$\u0001%\u0001%\u0003%\u01d8\b%\u0001%\u0001%\u0003%\u01dc"+ + "\b%\u0001&\u0001&\u0001&\u0001\'\u0001\'\u0001\'\u0001(\u0001(\u0001("+ + "\u0001(\u0005(\u01e8\b(\n(\f(\u01eb\t(\u0001)\u0001)\u0001)\u0001)\u0001"+ + "*\u0001*\u0001*\u0001*\u0003*\u01f5\b*\u0001+\u0001+\u0001+\u0001+\u0001"+ + ",\u0001,\u0001,\u0001-\u0001-\u0001-\u0005-\u0201\b-\n-\f-\u0204\t-\u0001"+ + ".\u0001.\u0001.\u0001.\u0001/\u0001/\u00010\u00010\u00030\u020e\b0\u0001"+ + "1\u00031\u0211\b1\u00011\u00011\u00012\u00032\u0216\b2\u00012\u00012\u0001"+ + "3\u00013\u00014\u00014\u00015\u00015\u00015\u00016\u00016\u00016\u0001"+ + "6\u00017\u00017\u00017\u00018\u00018\u00018\u00018\u00038\u022c\b8\u0001"+ + "8\u00018\u00018\u00018\u00058\u0232\b8\n8\f8\u0235\t8\u00038\u0237\b8"+ + "\u00019\u00019\u00019\u00039\u023c\b9\u00019\u00019\u0001:\u0001:\u0001"+ + ":\u0001:\u0001:\u0001;\u0001;\u0001;\u0001;\u0003;\u0249\b;\u0001;\u0000"+ + "\u0004\u0002\n\u0012\u0014<\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010"+ + "\u0012\u0014\u0016\u0018\u001a\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPR"+ + "TVXZ\\^`bdfhjlnprtv\u0000\b\u0001\u0000:;\u0001\u0000<>\u0002\u0000\u0019"+ + "\u0019LL\u0001\u0000CD\u0002\u0000\u001e\u001e\"\"\u0002\u0000%%((\u0002"+ + "\u0000$$22\u0002\u00003359\u0265\u0000x\u0001\u0000\u0000\u0000\u0002"+ + "{\u0001\u0000\u0000\u0000\u0004\u008c\u0001\u0000\u0000\u0000\u0006\u009e"+ + "\u0001\u0000\u0000\u0000\b\u00a0\u0001\u0000\u0000\u0000\n\u00c1\u0001"+ + "\u0000\u0000\u0000\f\u00dc\u0001\u0000\u0000\u0000\u000e\u00de\u0001\u0000"+ + "\u0000\u0000\u0010\u00e7\u0001\u0000\u0000\u0000\u0012\u00ed\u0001\u0000"+ + "\u0000\u0000\u0014\u0102\u0001\u0000\u0000\u0000\u0016\u010c\u0001\u0000"+ + "\u0000\u0000\u0018\u011e\u0001\u0000\u0000\u0000\u001a\u0120\u0001\u0000"+ + "\u0000\u0000\u001c\u0122\u0001\u0000\u0000\u0000\u001e\u0125\u0001\u0000"+ + "\u0000\u0000 \u0132\u0001\u0000\u0000\u0000\"\u0134\u0001\u0000\u0000"+ + "\u0000$\u0145\u0001\u0000\u0000\u0000&\u0147\u0001\u0000\u0000\u0000("+ + "\u0149\u0001\u0000\u0000\u0000*\u014d\u0001\u0000\u0000\u0000,\u014f\u0001"+ + "\u0000\u0000\u0000.\u0158\u0001\u0000\u0000\u00000\u015c\u0001\u0000\u0000"+ + "\u00002\u016c\u0001\u0000\u0000\u00004\u016f\u0001\u0000\u0000\u00006"+ + "\u0177\u0001\u0000\u0000\u00008\u017f\u0001\u0000\u0000\u0000:\u0187\u0001"+ + "\u0000\u0000\u0000<\u018f\u0001\u0000\u0000\u0000>\u0193\u0001\u0000\u0000"+ + "\u0000@\u01bf\u0001\u0000\u0000\u0000B\u01c3\u0001\u0000\u0000\u0000D"+ + "\u01c7\u0001\u0000\u0000\u0000F\u01c9\u0001\u0000\u0000\u0000H\u01cc\u0001"+ + "\u0000\u0000\u0000J\u01d5\u0001\u0000\u0000\u0000L\u01dd\u0001\u0000\u0000"+ + "\u0000N\u01e0\u0001\u0000\u0000\u0000P\u01e3\u0001\u0000\u0000\u0000R"+ + "\u01ec\u0001\u0000\u0000\u0000T\u01f0\u0001\u0000\u0000\u0000V\u01f6\u0001"+ + "\u0000\u0000\u0000X\u01fa\u0001\u0000\u0000\u0000Z\u01fd\u0001\u0000\u0000"+ + "\u0000\\\u0205\u0001\u0000\u0000\u0000^\u0209\u0001\u0000\u0000\u0000"+ + "`\u020d\u0001\u0000\u0000\u0000b\u0210\u0001\u0000\u0000\u0000d\u0215"+ + "\u0001\u0000\u0000\u0000f\u0219\u0001\u0000\u0000\u0000h\u021b\u0001\u0000"+ + "\u0000\u0000j\u021d\u0001\u0000\u0000\u0000l\u0220\u0001\u0000\u0000\u0000"+ + "n\u0224\u0001\u0000\u0000\u0000p\u0227\u0001\u0000\u0000\u0000r\u023b"+ + "\u0001\u0000\u0000\u0000t\u023f\u0001\u0000\u0000\u0000v\u0244\u0001\u0000"+ + "\u0000\u0000xy\u0003\u0002\u0001\u0000yz\u0005\u0000\u0000\u0001z\u0001"+ + "\u0001\u0000\u0000\u0000{|\u0006\u0001\uffff\uffff\u0000|}\u0003\u0004"+ + "\u0002\u0000}\u0083\u0001\u0000\u0000\u0000~\u007f\n\u0001\u0000\u0000"+ + "\u007f\u0080\u0005\u0018\u0000\u0000\u0080\u0082\u0003\u0006\u0003\u0000"+ + "\u0081~\u0001\u0000\u0000\u0000\u0082\u0085\u0001\u0000\u0000\u0000\u0083"+ + "\u0081\u0001\u0000\u0000\u0000\u0083\u0084\u0001\u0000\u0000\u0000\u0084"+ + "\u0003\u0001\u0000\u0000\u0000\u0085\u0083\u0001\u0000\u0000\u0000\u0086"+ + "\u008d\u0003j5\u0000\u0087\u008d\u0003\"\u0011\u0000\u0088\u008d\u0003"+ + "\u001c\u000e\u0000\u0089\u008d\u0003n7\u0000\u008a\u008b\u0004\u0002\u0001"+ + "\u0000\u008b\u008d\u00030\u0018\u0000\u008c\u0086\u0001\u0000\u0000\u0000"+ + "\u008c\u0087\u0001\u0000\u0000\u0000\u008c\u0088\u0001\u0000\u0000\u0000"+ + "\u008c\u0089\u0001\u0000\u0000\u0000\u008c\u008a\u0001\u0000\u0000\u0000"+ + "\u008d\u0005\u0001\u0000\u0000\u0000\u008e\u009f\u00032\u0019\u0000\u008f"+ + "\u009f\u0003\b\u0004\u0000\u0090\u009f\u0003L&\u0000\u0091\u009f\u0003"+ + "F#\u0000\u0092\u009f\u00034\u001a\u0000\u0093\u009f\u0003H$\u0000\u0094"+ + "\u009f\u0003N\'\u0000\u0095\u009f\u0003P(\u0000\u0096\u009f\u0003T*\u0000"+ + "\u0097\u009f\u0003V+\u0000\u0098\u009f\u0003p8\u0000\u0099\u009f\u0003"+ + "X,\u0000\u009a\u009b\u0004\u0003\u0002\u0000\u009b\u009f\u0003v;\u0000"+ + "\u009c\u009d\u0004\u0003\u0003\u0000\u009d\u009f\u0003t:\u0000\u009e\u008e"+ + "\u0001\u0000\u0000\u0000\u009e\u008f\u0001\u0000\u0000\u0000\u009e\u0090"+ + "\u0001\u0000\u0000\u0000\u009e\u0091\u0001\u0000\u0000\u0000\u009e\u0092"+ + "\u0001\u0000\u0000\u0000\u009e\u0093\u0001\u0000\u0000\u0000\u009e\u0094"+ + "\u0001\u0000\u0000\u0000\u009e\u0095\u0001\u0000\u0000\u0000\u009e\u0096"+ + "\u0001\u0000\u0000\u0000\u009e\u0097\u0001\u0000\u0000\u0000\u009e\u0098"+ + "\u0001\u0000\u0000\u0000\u009e\u0099\u0001\u0000\u0000\u0000\u009e\u009a"+ + "\u0001\u0000\u0000\u0000\u009e\u009c\u0001\u0000\u0000\u0000\u009f\u0007"+ + "\u0001\u0000\u0000\u0000\u00a0\u00a1\u0005\u0010\u0000\u0000\u00a1\u00a2"+ + "\u0003\n\u0005\u0000\u00a2\t\u0001\u0000\u0000\u0000\u00a3\u00a4\u0006"+ + "\u0005\uffff\uffff\u0000\u00a4\u00a5\u0005+\u0000\u0000\u00a5\u00c2\u0003"+ + "\n\u0005\b\u00a6\u00c2\u0003\u0010\b\u0000\u00a7\u00c2\u0003\f\u0006\u0000"+ + "\u00a8\u00aa\u0003\u0010\b\u0000\u00a9\u00ab\u0005+\u0000\u0000\u00aa"+ + "\u00a9\u0001\u0000\u0000\u0000\u00aa\u00ab\u0001\u0000\u0000\u0000\u00ab"+ + "\u00ac\u0001\u0000\u0000\u0000\u00ac\u00ad\u0005&\u0000\u0000\u00ad\u00ae"+ + "\u0005*\u0000\u0000\u00ae\u00b3\u0003\u0010\b\u0000\u00af\u00b0\u0005"+ + "!\u0000\u0000\u00b0\u00b2\u0003\u0010\b\u0000\u00b1\u00af\u0001\u0000"+ + "\u0000\u0000\u00b2\u00b5\u0001\u0000\u0000\u0000\u00b3\u00b1\u0001\u0000"+ + "\u0000\u0000\u00b3\u00b4\u0001\u0000\u0000\u0000\u00b4\u00b6\u0001\u0000"+ + "\u0000\u0000\u00b5\u00b3\u0001\u0000\u0000\u0000\u00b6\u00b7\u00051\u0000"+ + "\u0000\u00b7\u00c2\u0001\u0000\u0000\u0000\u00b8\u00b9\u0003\u0010\b\u0000"+ + "\u00b9\u00bb\u0005\'\u0000\u0000\u00ba\u00bc\u0005+\u0000\u0000\u00bb"+ + "\u00ba\u0001\u0000\u0000\u0000\u00bb\u00bc\u0001\u0000\u0000\u0000\u00bc"+ + "\u00bd\u0001\u0000\u0000\u0000\u00bd\u00be\u0005,\u0000\u0000\u00be\u00c2"+ + "\u0001\u0000\u0000\u0000\u00bf\u00c0\u0004\u0005\u0004\u0000\u00c0\u00c2"+ + "\u0003\u000e\u0007\u0000\u00c1\u00a3\u0001\u0000\u0000\u0000\u00c1\u00a6"+ + "\u0001\u0000\u0000\u0000\u00c1\u00a7\u0001\u0000\u0000\u0000\u00c1\u00a8"+ + "\u0001\u0000\u0000\u0000\u00c1\u00b8\u0001\u0000\u0000\u0000\u00c1\u00bf"+ + "\u0001\u0000\u0000\u0000\u00c2\u00cb\u0001\u0000\u0000\u0000\u00c3\u00c4"+ + "\n\u0005\u0000\u0000\u00c4\u00c5\u0005\u001d\u0000\u0000\u00c5\u00ca\u0003"+ + "\n\u0005\u0006\u00c6\u00c7\n\u0004\u0000\u0000\u00c7\u00c8\u0005.\u0000"+ + "\u0000\u00c8\u00ca\u0003\n\u0005\u0005\u00c9\u00c3\u0001\u0000\u0000\u0000"+ + "\u00c9\u00c6\u0001\u0000\u0000\u0000\u00ca\u00cd\u0001\u0000\u0000\u0000"+ + "\u00cb\u00c9\u0001\u0000\u0000\u0000\u00cb\u00cc\u0001\u0000\u0000\u0000"+ + "\u00cc\u000b\u0001\u0000\u0000\u0000\u00cd\u00cb\u0001\u0000\u0000\u0000"+ + "\u00ce\u00d0\u0003\u0010\b\u0000\u00cf\u00d1\u0005+\u0000\u0000\u00d0"+ + "\u00cf\u0001\u0000\u0000\u0000\u00d0\u00d1\u0001\u0000\u0000\u0000\u00d1"+ + "\u00d2\u0001\u0000\u0000\u0000\u00d2\u00d3\u0005)\u0000\u0000\u00d3\u00d4"+ + "\u0003f3\u0000\u00d4\u00dd\u0001\u0000\u0000\u0000\u00d5\u00d7\u0003\u0010"+ + "\b\u0000\u00d6\u00d8\u0005+\u0000\u0000\u00d7\u00d6\u0001\u0000\u0000"+ + "\u0000\u00d7\u00d8\u0001\u0000\u0000\u0000\u00d8\u00d9\u0001\u0000\u0000"+ + "\u0000\u00d9\u00da\u00050\u0000\u0000\u00da\u00db\u0003f3\u0000\u00db"+ + "\u00dd\u0001\u0000\u0000\u0000\u00dc\u00ce\u0001\u0000\u0000\u0000\u00dc"+ + "\u00d5\u0001\u0000\u0000\u0000\u00dd\r\u0001\u0000\u0000\u0000\u00de\u00df"+ + "\u0003\u0010\b\u0000\u00df\u00e0\u0005?\u0000\u0000\u00e0\u00e1\u0003"+ + "f3\u0000\u00e1\u000f\u0001\u0000\u0000\u0000\u00e2\u00e8\u0003\u0012\t"+ + "\u0000\u00e3\u00e4\u0003\u0012\t\u0000\u00e4\u00e5\u0003h4\u0000\u00e5"+ + "\u00e6\u0003\u0012\t\u0000\u00e6\u00e8\u0001\u0000\u0000\u0000\u00e7\u00e2"+ + "\u0001\u0000\u0000\u0000\u00e7\u00e3\u0001\u0000\u0000\u0000\u00e8\u0011"+ + "\u0001\u0000\u0000\u0000\u00e9\u00ea\u0006\t\uffff\uffff\u0000\u00ea\u00ee"+ + "\u0003\u0014\n\u0000\u00eb\u00ec\u0007\u0000\u0000\u0000\u00ec\u00ee\u0003"+ + "\u0012\t\u0003\u00ed\u00e9\u0001\u0000\u0000\u0000\u00ed\u00eb\u0001\u0000"+ + "\u0000\u0000\u00ee\u00f7\u0001\u0000\u0000\u0000\u00ef\u00f0\n\u0002\u0000"+ + "\u0000\u00f0\u00f1\u0007\u0001\u0000\u0000\u00f1\u00f6\u0003\u0012\t\u0003"+ + "\u00f2\u00f3\n\u0001\u0000\u0000\u00f3\u00f4\u0007\u0000\u0000\u0000\u00f4"+ + "\u00f6\u0003\u0012\t\u0002\u00f5\u00ef\u0001\u0000\u0000\u0000\u00f5\u00f2"+ + "\u0001\u0000\u0000\u0000\u00f6\u00f9\u0001\u0000\u0000\u0000\u00f7\u00f5"+ + "\u0001\u0000\u0000\u0000\u00f7\u00f8\u0001\u0000\u0000\u0000\u00f8\u0013"+ + "\u0001\u0000\u0000\u0000\u00f9\u00f7\u0001\u0000\u0000\u0000\u00fa\u00fb"+ + "\u0006\n\uffff\uffff\u0000\u00fb\u0103\u0003@ \u0000\u00fc\u0103\u0003"+ + "6\u001b\u0000\u00fd\u0103\u0003\u0016\u000b\u0000\u00fe\u00ff\u0005*\u0000"+ + "\u0000\u00ff\u0100\u0003\n\u0005\u0000\u0100\u0101\u00051\u0000\u0000"+ + "\u0101\u0103\u0001\u0000\u0000\u0000\u0102\u00fa\u0001\u0000\u0000\u0000"+ + "\u0102\u00fc\u0001\u0000\u0000\u0000\u0102\u00fd\u0001\u0000\u0000\u0000"+ + "\u0102\u00fe\u0001\u0000\u0000\u0000\u0103\u0109\u0001\u0000\u0000\u0000"+ + "\u0104\u0105\n\u0001\u0000\u0000\u0105\u0106\u0005 \u0000\u0000\u0106"+ + "\u0108\u0003\u001a\r\u0000\u0107\u0104\u0001\u0000\u0000\u0000\u0108\u010b"+ + "\u0001\u0000\u0000\u0000\u0109\u0107\u0001\u0000\u0000\u0000\u0109\u010a"+ + "\u0001\u0000\u0000\u0000\u010a\u0015\u0001\u0000\u0000\u0000\u010b\u0109"+ + "\u0001\u0000\u0000\u0000\u010c\u010d\u0003\u0018\f\u0000\u010d\u0117\u0005"+ + "*\u0000\u0000\u010e\u0118\u0005<\u0000\u0000\u010f\u0114\u0003\n\u0005"+ + "\u0000\u0110\u0111\u0005!\u0000\u0000\u0111\u0113\u0003\n\u0005\u0000"+ + "\u0112\u0110\u0001\u0000\u0000\u0000\u0113\u0116\u0001\u0000\u0000\u0000"+ + "\u0114\u0112\u0001\u0000\u0000\u0000\u0114\u0115\u0001\u0000\u0000\u0000"+ + "\u0115\u0118\u0001\u0000\u0000\u0000\u0116\u0114\u0001\u0000\u0000\u0000"+ + "\u0117\u010e\u0001\u0000\u0000\u0000\u0117\u010f\u0001\u0000\u0000\u0000"+ + "\u0117\u0118\u0001\u0000\u0000\u0000\u0118\u0119\u0001\u0000\u0000\u0000"+ + "\u0119\u011a\u00051\u0000\u0000\u011a\u0017\u0001\u0000\u0000\u0000\u011b"+ + "\u011c\u0004\f\n\u0000\u011c\u011f\u0005?\u0000\u0000\u011d\u011f\u0003"+ + "D\"\u0000\u011e\u011b\u0001\u0000\u0000\u0000\u011e\u011d\u0001\u0000"+ + "\u0000\u0000\u011f\u0019\u0001\u0000\u0000\u0000\u0120\u0121\u0003<\u001e"+ + "\u0000\u0121\u001b\u0001\u0000\u0000\u0000\u0122\u0123\u0005\f\u0000\u0000"+ + "\u0123\u0124\u0003\u001e\u000f\u0000\u0124\u001d\u0001\u0000\u0000\u0000"+ + "\u0125\u012a\u0003 \u0010\u0000\u0126\u0127\u0005!\u0000\u0000\u0127\u0129"+ + "\u0003 \u0010\u0000\u0128\u0126\u0001\u0000\u0000\u0000\u0129\u012c\u0001"+ + "\u0000\u0000\u0000\u012a\u0128\u0001\u0000\u0000\u0000\u012a\u012b\u0001"+ + "\u0000\u0000\u0000\u012b\u001f\u0001\u0000\u0000\u0000\u012c\u012a\u0001"+ + "\u0000\u0000\u0000\u012d\u0133\u0003\n\u0005\u0000\u012e\u012f\u00036"+ + "\u001b\u0000\u012f\u0130\u0005\u001f\u0000\u0000\u0130\u0131\u0003\n\u0005"+ + "\u0000\u0131\u0133\u0001\u0000\u0000\u0000\u0132\u012d\u0001\u0000\u0000"+ + "\u0000\u0132\u012e\u0001\u0000\u0000\u0000\u0133!\u0001\u0000\u0000\u0000"+ + "\u0134\u0135\u0005\u0006\u0000\u0000\u0135\u013a\u0003$\u0012\u0000\u0136"+ + "\u0137\u0005!\u0000\u0000\u0137\u0139\u0003$\u0012\u0000\u0138\u0136\u0001"+ + "\u0000\u0000\u0000\u0139\u013c\u0001\u0000\u0000\u0000\u013a\u0138\u0001"+ + "\u0000\u0000\u0000\u013a\u013b\u0001\u0000\u0000\u0000\u013b\u013e\u0001"+ + "\u0000\u0000\u0000\u013c\u013a\u0001\u0000\u0000\u0000\u013d\u013f\u0003"+ + "*\u0015\u0000\u013e\u013d\u0001\u0000\u0000\u0000\u013e\u013f\u0001\u0000"+ + "\u0000\u0000\u013f#\u0001\u0000\u0000\u0000\u0140\u0141\u0003&\u0013\u0000"+ + "\u0141\u0142\u0005h\u0000\u0000\u0142\u0143\u0003(\u0014\u0000\u0143\u0146"+ + "\u0001\u0000\u0000\u0000\u0144\u0146\u0003(\u0014\u0000\u0145\u0140\u0001"+ + "\u0000\u0000\u0000\u0145\u0144\u0001\u0000\u0000\u0000\u0146%\u0001\u0000"+ + "\u0000\u0000\u0147\u0148\u0005L\u0000\u0000\u0148\'\u0001\u0000\u0000"+ + "\u0000\u0149\u014a\u0007\u0002\u0000\u0000\u014a)\u0001\u0000\u0000\u0000"+ + "\u014b\u014e\u0003,\u0016\u0000\u014c\u014e\u0003.\u0017\u0000\u014d\u014b"+ + "\u0001\u0000\u0000\u0000\u014d\u014c\u0001\u0000\u0000\u0000\u014e+\u0001"+ + "\u0000\u0000\u0000\u014f\u0150\u0005K\u0000\u0000\u0150\u0155\u0005L\u0000"+ + "\u0000\u0151\u0152\u0005!\u0000\u0000\u0152\u0154\u0005L\u0000\u0000\u0153"+ + "\u0151\u0001\u0000\u0000\u0000\u0154\u0157\u0001\u0000\u0000\u0000\u0155"+ + "\u0153\u0001\u0000\u0000\u0000\u0155\u0156\u0001\u0000\u0000\u0000\u0156"+ + "-\u0001\u0000\u0000\u0000\u0157\u0155\u0001\u0000\u0000\u0000\u0158\u0159"+ + "\u0005A\u0000\u0000\u0159\u015a\u0003,\u0016\u0000\u015a\u015b\u0005B"+ + "\u0000\u0000\u015b/\u0001\u0000\u0000\u0000\u015c\u015d\u0005\u0013\u0000"+ + "\u0000\u015d\u0162\u0003$\u0012\u0000\u015e\u015f\u0005!\u0000\u0000\u015f"+ + "\u0161\u0003$\u0012\u0000\u0160\u015e\u0001\u0000\u0000\u0000\u0161\u0164"+ + "\u0001\u0000\u0000\u0000\u0162\u0160\u0001\u0000\u0000\u0000\u0162\u0163"+ + "\u0001\u0000\u0000\u0000\u0163\u0166\u0001\u0000\u0000\u0000\u0164\u0162"+ + "\u0001\u0000\u0000\u0000\u0165\u0167\u0003\u001e\u000f\u0000\u0166\u0165"+ + "\u0001\u0000\u0000\u0000\u0166\u0167\u0001\u0000\u0000\u0000\u0167\u016a"+ + "\u0001\u0000\u0000\u0000\u0168\u0169\u0005\u001c\u0000\u0000\u0169\u016b"+ + "\u0003\u001e\u000f\u0000\u016a\u0168\u0001\u0000\u0000\u0000\u016a\u016b"+ + "\u0001\u0000\u0000\u0000\u016b1\u0001\u0000\u0000\u0000\u016c\u016d\u0005"+ + "\u0004\u0000\u0000\u016d\u016e\u0003\u001e\u000f\u0000\u016e3\u0001\u0000"+ + "\u0000\u0000\u016f\u0171\u0005\u000f\u0000\u0000\u0170\u0172\u0003\u001e"+ + "\u000f\u0000\u0171\u0170\u0001\u0000\u0000\u0000\u0171\u0172\u0001\u0000"+ + "\u0000\u0000\u0172\u0175\u0001\u0000\u0000\u0000\u0173\u0174\u0005\u001c"+ + "\u0000\u0000\u0174\u0176\u0003\u001e\u000f\u0000\u0175\u0173\u0001\u0000"+ + "\u0000\u0000\u0175\u0176\u0001\u0000\u0000\u0000\u01765\u0001\u0000\u0000"+ + "\u0000\u0177\u017c\u0003D\"\u0000\u0178\u0179\u0005#\u0000\u0000\u0179"+ + "\u017b\u0003D\"\u0000\u017a\u0178\u0001\u0000\u0000\u0000\u017b\u017e"+ + "\u0001\u0000\u0000\u0000\u017c\u017a\u0001\u0000\u0000\u0000\u017c\u017d"+ + "\u0001\u0000\u0000\u0000\u017d7\u0001\u0000\u0000\u0000\u017e\u017c\u0001"+ + "\u0000\u0000\u0000\u017f\u0184\u0003>\u001f\u0000\u0180\u0181\u0005#\u0000"+ + "\u0000\u0181\u0183\u0003>\u001f\u0000\u0182\u0180\u0001\u0000\u0000\u0000"+ + "\u0183\u0186\u0001\u0000\u0000\u0000\u0184\u0182\u0001\u0000\u0000\u0000"+ + "\u0184\u0185\u0001\u0000\u0000\u0000\u01859\u0001\u0000\u0000\u0000\u0186"+ + "\u0184\u0001\u0000\u0000\u0000\u0187\u018c\u00038\u001c\u0000\u0188\u0189"+ + "\u0005!\u0000\u0000\u0189\u018b\u00038\u001c\u0000\u018a\u0188\u0001\u0000"+ + "\u0000\u0000\u018b\u018e\u0001\u0000\u0000\u0000\u018c\u018a\u0001\u0000"+ + "\u0000\u0000\u018c\u018d\u0001\u0000\u0000\u0000\u018d;\u0001\u0000\u0000"+ + "\u0000\u018e\u018c\u0001\u0000\u0000\u0000\u018f\u0190\u0007\u0003\u0000"+ + "\u0000\u0190=\u0001\u0000\u0000\u0000\u0191\u0194\u0005P\u0000\u0000\u0192"+ + "\u0194\u0003B!\u0000\u0193\u0191\u0001\u0000\u0000\u0000\u0193\u0192\u0001"+ + "\u0000\u0000\u0000\u0194?\u0001\u0000\u0000\u0000\u0195\u01c0\u0005,\u0000"+ + "\u0000\u0196\u0197\u0003d2\u0000\u0197\u0198\u0005C\u0000\u0000\u0198"+ + "\u01c0\u0001\u0000\u0000\u0000\u0199\u01c0\u0003b1\u0000\u019a\u01c0\u0003"+ + "d2\u0000\u019b\u01c0\u0003^/\u0000\u019c\u01c0\u0003B!\u0000\u019d\u01c0"+ + "\u0003f3\u0000\u019e\u019f\u0005A\u0000\u0000\u019f\u01a4\u0003`0\u0000"+ + "\u01a0\u01a1\u0005!\u0000\u0000\u01a1\u01a3\u0003`0\u0000\u01a2\u01a0"+ + "\u0001\u0000\u0000\u0000\u01a3\u01a6\u0001\u0000\u0000\u0000\u01a4\u01a2"+ + "\u0001\u0000\u0000\u0000\u01a4\u01a5\u0001\u0000\u0000\u0000\u01a5\u01a7"+ + "\u0001\u0000\u0000\u0000\u01a6\u01a4\u0001\u0000\u0000\u0000\u01a7\u01a8"+ + "\u0005B\u0000\u0000\u01a8\u01c0\u0001\u0000\u0000\u0000\u01a9\u01aa\u0005"+ + "A\u0000\u0000\u01aa\u01af\u0003^/\u0000\u01ab\u01ac\u0005!\u0000\u0000"+ + "\u01ac\u01ae\u0003^/\u0000\u01ad\u01ab\u0001\u0000\u0000\u0000\u01ae\u01b1"+ + "\u0001\u0000\u0000\u0000\u01af\u01ad\u0001\u0000\u0000\u0000\u01af\u01b0"+ + "\u0001\u0000\u0000\u0000\u01b0\u01b2\u0001\u0000\u0000\u0000\u01b1\u01af"+ + "\u0001\u0000\u0000\u0000\u01b2\u01b3\u0005B\u0000\u0000\u01b3\u01c0\u0001"+ + "\u0000\u0000\u0000\u01b4\u01b5\u0005A\u0000\u0000\u01b5\u01ba\u0003f3"+ + "\u0000\u01b6\u01b7\u0005!\u0000\u0000\u01b7\u01b9\u0003f3\u0000\u01b8"+ + "\u01b6\u0001\u0000\u0000\u0000\u01b9\u01bc\u0001\u0000\u0000\u0000\u01ba"+ + "\u01b8\u0001\u0000\u0000\u0000\u01ba\u01bb\u0001\u0000\u0000\u0000\u01bb"+ + "\u01bd\u0001\u0000\u0000\u0000\u01bc\u01ba\u0001\u0000\u0000\u0000\u01bd"+ + "\u01be\u0005B\u0000\u0000\u01be\u01c0\u0001\u0000\u0000\u0000\u01bf\u0195"+ + "\u0001\u0000\u0000\u0000\u01bf\u0196\u0001\u0000\u0000\u0000\u01bf\u0199"+ + "\u0001\u0000\u0000\u0000\u01bf\u019a\u0001\u0000\u0000\u0000\u01bf\u019b"+ + "\u0001\u0000\u0000\u0000\u01bf\u019c\u0001\u0000\u0000\u0000\u01bf\u019d"+ + "\u0001\u0000\u0000\u0000\u01bf\u019e\u0001\u0000\u0000\u0000\u01bf\u01a9"+ + "\u0001\u0000\u0000\u0000\u01bf\u01b4\u0001\u0000\u0000\u0000\u01c0A\u0001"+ + "\u0000\u0000\u0000\u01c1\u01c4\u0005/\u0000\u0000\u01c2\u01c4\u0005@\u0000"+ + "\u0000\u01c3\u01c1\u0001\u0000\u0000\u0000\u01c3\u01c2\u0001\u0000\u0000"+ + "\u0000\u01c4C\u0001\u0000\u0000\u0000\u01c5\u01c8\u0003<\u001e\u0000\u01c6"+ + "\u01c8\u0003B!\u0000\u01c7\u01c5\u0001\u0000\u0000\u0000\u01c7\u01c6\u0001"+ + "\u0000\u0000\u0000\u01c8E\u0001\u0000\u0000\u0000\u01c9\u01ca\u0005\t"+ + "\u0000\u0000\u01ca\u01cb\u0005\u001a\u0000\u0000\u01cbG\u0001\u0000\u0000"+ + "\u0000\u01cc\u01cd\u0005\u000e\u0000\u0000\u01cd\u01d2\u0003J%\u0000\u01ce"+ + "\u01cf\u0005!\u0000\u0000\u01cf\u01d1\u0003J%\u0000\u01d0\u01ce\u0001"+ + "\u0000\u0000\u0000\u01d1\u01d4\u0001\u0000\u0000\u0000\u01d2\u01d0\u0001"+ + "\u0000\u0000\u0000\u01d2\u01d3\u0001\u0000\u0000\u0000\u01d3I\u0001\u0000"+ + "\u0000\u0000\u01d4\u01d2\u0001\u0000\u0000\u0000\u01d5\u01d7\u0003\n\u0005"+ + "\u0000\u01d6\u01d8\u0007\u0004\u0000\u0000\u01d7\u01d6\u0001\u0000\u0000"+ + "\u0000\u01d7\u01d8\u0001\u0000\u0000\u0000\u01d8\u01db\u0001\u0000\u0000"+ + "\u0000\u01d9\u01da\u0005-\u0000\u0000\u01da\u01dc\u0007\u0005\u0000\u0000"+ + "\u01db\u01d9\u0001\u0000\u0000\u0000\u01db\u01dc\u0001\u0000\u0000\u0000"+ + "\u01dcK\u0001\u0000\u0000\u0000\u01dd\u01de\u0005\b\u0000\u0000\u01de"+ + "\u01df\u0003:\u001d\u0000\u01dfM\u0001\u0000\u0000\u0000\u01e0\u01e1\u0005"+ + "\u0002\u0000\u0000\u01e1\u01e2\u0003:\u001d\u0000\u01e2O\u0001\u0000\u0000"+ + "\u0000\u01e3\u01e4\u0005\u000b\u0000\u0000\u01e4\u01e9\u0003R)\u0000\u01e5"+ + "\u01e6\u0005!\u0000\u0000\u01e6\u01e8\u0003R)\u0000\u01e7\u01e5\u0001"+ + "\u0000\u0000\u0000\u01e8\u01eb\u0001\u0000\u0000\u0000\u01e9\u01e7\u0001"+ + "\u0000\u0000\u0000\u01e9\u01ea\u0001\u0000\u0000\u0000\u01eaQ\u0001\u0000"+ + "\u0000\u0000\u01eb\u01e9\u0001\u0000\u0000\u0000\u01ec\u01ed\u00038\u001c"+ + "\u0000\u01ed\u01ee\u0005T\u0000\u0000\u01ee\u01ef\u00038\u001c\u0000\u01ef"+ + "S\u0001\u0000\u0000\u0000\u01f0\u01f1\u0005\u0001\u0000\u0000\u01f1\u01f2"+ + "\u0003\u0014\n\u0000\u01f2\u01f4\u0003f3\u0000\u01f3\u01f5\u0003Z-\u0000"+ + "\u01f4\u01f3\u0001\u0000\u0000\u0000\u01f4\u01f5\u0001\u0000\u0000\u0000"+ + "\u01f5U\u0001\u0000\u0000\u0000\u01f6\u01f7\u0005\u0007\u0000\u0000\u01f7"+ + "\u01f8\u0003\u0014\n\u0000\u01f8\u01f9\u0003f3\u0000\u01f9W\u0001\u0000"+ + "\u0000\u0000\u01fa\u01fb\u0005\n\u0000\u0000\u01fb\u01fc\u00036\u001b"+ + "\u0000\u01fcY\u0001\u0000\u0000\u0000\u01fd\u0202\u0003\\.\u0000\u01fe"+ + "\u01ff\u0005!\u0000\u0000\u01ff\u0201\u0003\\.\u0000\u0200\u01fe\u0001"+ + "\u0000\u0000\u0000\u0201\u0204\u0001\u0000\u0000\u0000\u0202\u0200\u0001"+ + "\u0000\u0000\u0000\u0202\u0203\u0001\u0000\u0000\u0000\u0203[\u0001\u0000"+ + "\u0000\u0000\u0204\u0202\u0001\u0000\u0000\u0000\u0205\u0206\u0003<\u001e"+ + "\u0000\u0206\u0207\u0005\u001f\u0000\u0000\u0207\u0208\u0003@ \u0000\u0208"+ + "]\u0001\u0000\u0000\u0000\u0209\u020a\u0007\u0006\u0000\u0000\u020a_\u0001"+ + "\u0000\u0000\u0000\u020b\u020e\u0003b1\u0000\u020c\u020e\u0003d2\u0000"+ + "\u020d\u020b\u0001\u0000\u0000\u0000\u020d\u020c\u0001\u0000\u0000\u0000"+ + "\u020ea\u0001\u0000\u0000\u0000\u020f\u0211\u0007\u0000\u0000\u0000\u0210"+ + "\u020f\u0001\u0000\u0000\u0000\u0210\u0211\u0001\u0000\u0000\u0000\u0211"+ + "\u0212\u0001\u0000\u0000\u0000\u0212\u0213\u0005\u001b\u0000\u0000\u0213"+ + "c\u0001\u0000\u0000\u0000\u0214\u0216\u0007\u0000\u0000\u0000\u0215\u0214"+ + "\u0001\u0000\u0000\u0000\u0215\u0216\u0001\u0000\u0000\u0000\u0216\u0217"+ + "\u0001\u0000\u0000\u0000\u0217\u0218\u0005\u001a\u0000\u0000\u0218e\u0001"+ + "\u0000\u0000\u0000\u0219\u021a\u0005\u0019\u0000\u0000\u021ag\u0001\u0000"+ + "\u0000\u0000\u021b\u021c\u0007\u0007\u0000\u0000\u021ci\u0001\u0000\u0000"+ + "\u0000\u021d\u021e\u0005\u0005\u0000\u0000\u021e\u021f\u0003l6\u0000\u021f"+ + "k\u0001\u0000\u0000\u0000\u0220\u0221\u0005A\u0000\u0000\u0221\u0222\u0003"+ + "\u0002\u0001\u0000\u0222\u0223\u0005B\u0000\u0000\u0223m\u0001\u0000\u0000"+ + "\u0000\u0224\u0225\u0005\r\u0000\u0000\u0225\u0226\u0005d\u0000\u0000"+ + "\u0226o\u0001\u0000\u0000\u0000\u0227\u0228\u0005\u0003\u0000\u0000\u0228"+ + "\u022b\u0005Z\u0000\u0000\u0229\u022a\u0005X\u0000\u0000\u022a\u022c\u0003"+ + "8\u001c\u0000\u022b\u0229\u0001\u0000\u0000\u0000\u022b\u022c\u0001\u0000"+ + "\u0000\u0000\u022c\u0236\u0001\u0000\u0000\u0000\u022d\u022e\u0005Y\u0000"+ + "\u0000\u022e\u0233\u0003r9\u0000\u022f\u0230\u0005!\u0000\u0000\u0230"+ + "\u0232\u0003r9\u0000\u0231\u022f\u0001\u0000\u0000\u0000\u0232\u0235\u0001"+ + "\u0000\u0000\u0000\u0233\u0231\u0001\u0000\u0000\u0000\u0233\u0234\u0001"+ + "\u0000\u0000\u0000\u0234\u0237\u0001\u0000\u0000\u0000\u0235\u0233\u0001"+ + "\u0000\u0000\u0000\u0236\u022d\u0001\u0000\u0000\u0000\u0236\u0237\u0001"+ + "\u0000\u0000\u0000\u0237q\u0001\u0000\u0000\u0000\u0238\u0239\u00038\u001c"+ + "\u0000\u0239\u023a\u0005\u001f\u0000\u0000\u023a\u023c\u0001\u0000\u0000"+ + "\u0000\u023b\u0238\u0001\u0000\u0000\u0000\u023b\u023c\u0001\u0000\u0000"+ + "\u0000\u023c\u023d\u0001\u0000\u0000\u0000\u023d\u023e\u00038\u001c\u0000"+ + "\u023es\u0001\u0000\u0000\u0000\u023f\u0240\u0005\u0012\u0000\u0000\u0240"+ + "\u0241\u0003$\u0012\u0000\u0241\u0242\u0005X\u0000\u0000\u0242\u0243\u0003"+ + ":\u001d\u0000\u0243u\u0001\u0000\u0000\u0000\u0244\u0245\u0005\u0011\u0000"+ + "\u0000\u0245\u0248\u0003\u001e\u000f\u0000\u0246\u0247\u0005\u001c\u0000"+ + "\u0000\u0247\u0249\u0003\u001e\u000f\u0000\u0248\u0246\u0001\u0000\u0000"+ + "\u0000\u0248\u0249\u0001\u0000\u0000\u0000\u0249w\u0001\u0000\u0000\u0000"+ + "9\u0083\u008c\u009e\u00aa\u00b3\u00bb\u00c1\u00c9\u00cb\u00d0\u00d7\u00dc"+ + "\u00e7\u00ed\u00f5\u00f7\u0102\u0109\u0114\u0117\u011e\u012a\u0132\u013a"+ + "\u013e\u0145\u014d\u0155\u0162\u0166\u016a\u0171\u0175\u017c\u0184\u018c"+ + "\u0193\u01a4\u01af\u01ba\u01bf\u01c3\u01c7\u01d2\u01d7\u01db\u01e9\u01f4"+ + "\u0202\u020d\u0210\u0215\u022b\u0233\u0236\u023b\u0248"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java index 027281d44b2dc..e2340df954674 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java @@ -332,6 +332,18 @@ public class EsqlBaseParserBaseListener implements EsqlBaseParserListener { *

    The default implementation does nothing.

    */ @Override public void exitFunctionExpression(EsqlBaseParser.FunctionExpressionContext ctx) { } + /** + * {@inheritDoc} + * + *

    The default implementation does nothing.

    + */ + @Override public void enterFunctionName(EsqlBaseParser.FunctionNameContext ctx) { } + /** + * {@inheritDoc} + * + *

    The default implementation does nothing.

    + */ + @Override public void exitFunctionName(EsqlBaseParser.FunctionNameContext ctx) { } /** * {@inheritDoc} * diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java index 463414ab67ea2..99f038b14b5e0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java @@ -202,6 +202,13 @@ public class EsqlBaseParserBaseVisitor extends AbstractParseTreeVisitor im * {@link #visitChildren} on {@code ctx}.

    */ @Override public T visitFunctionExpression(EsqlBaseParser.FunctionExpressionContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitFunctionName(EsqlBaseParser.FunctionNameContext ctx) { return visitChildren(ctx); } /** * {@inheritDoc} * diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java index 1747ff001e162..c6dcaca736e1f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java @@ -313,6 +313,16 @@ public interface EsqlBaseParserListener extends ParseTreeListener { * @param ctx the parse tree */ void exitFunctionExpression(EsqlBaseParser.FunctionExpressionContext ctx); + /** + * Enter a parse tree produced by {@link EsqlBaseParser#functionName}. + * @param ctx the parse tree + */ + void enterFunctionName(EsqlBaseParser.FunctionNameContext ctx); + /** + * Exit a parse tree produced by {@link EsqlBaseParser#functionName}. + * @param ctx the parse tree + */ + void exitFunctionName(EsqlBaseParser.FunctionNameContext ctx); /** * Enter a parse tree produced by the {@code toDataType} * labeled alternative in {@link EsqlBaseParser#dataType}. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java index b7411d0f99c09..310d3dc76dd6d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java @@ -193,6 +193,12 @@ public interface EsqlBaseParserVisitor extends ParseTreeVisitor { * @return the visitor result */ T visitFunctionExpression(EsqlBaseParser.FunctionExpressionContext ctx); + /** + * Visit a parse tree produced by {@link EsqlBaseParser#functionName}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitFunctionName(EsqlBaseParser.FunctionNameContext ctx); /** * Visit a parse tree produced by the {@code toDataType} * labeled alternative in {@link EsqlBaseParser#dataType}. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java index 5696ccea188b1..620a25e0170ea 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java @@ -98,10 +98,8 @@ private class PostProcessor extends EsqlBaseParserBaseListener { @Override public void exitFunctionExpression(EsqlBaseParser.FunctionExpressionContext ctx) { // TODO remove this at some point - EsqlBaseParser.IdentifierOrParameterContext identifierOrParameter = ctx.identifierOrParameter(); - EsqlBaseParser.IdentifierContext idCtx = identifierOrParameter.identifier(); - String functionName = idCtx != null ? idCtx.getText() : identifierOrParameter.parameter().getText(); - if ("is_null".equalsIgnoreCase(functionName)) { + EsqlBaseParser.FunctionNameContext identifier = ctx.functionName(); + if (identifier.getText().equalsIgnoreCase("is_null")) { throw new ParsingException( source(ctx), "is_null function is not supported anymore, please use 'is null'/'is not null' predicates instead" diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java index 50cf4bc998679..39f1758b78733 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java @@ -593,13 +593,7 @@ public UnresolvedAttribute visitDereference(EsqlBaseParser.DereferenceContext ct @Override public Expression visitFunctionExpression(EsqlBaseParser.FunctionExpressionContext ctx) { - EsqlBaseParser.IdentifierOrParameterContext identifierOrParameter = ctx.identifierOrParameter(); - String name; - if (identifierOrParameter.identifier() != null) { - name = visitIdentifier(identifierOrParameter.identifier()); - } else { - name = unresolvedAttributeNameInParam(identifierOrParameter.parameter(), expression(identifierOrParameter.parameter())); - } + String name = visitFunctionName(ctx.functionName()); List args = expressions(ctx.booleanExpression()); if ("is_null".equals(EsqlFunctionRegistry.normalizeName(name))) { throw new ParsingException( @@ -616,6 +610,23 @@ public Expression visitFunctionExpression(EsqlBaseParser.FunctionExpressionConte return new UnresolvedFunction(source(ctx), name, FunctionResolutionStrategy.DEFAULT, args); } + @Override + public String visitFunctionName(EsqlBaseParser.FunctionNameContext ctx) { + if (ctx.DEV_MATCH() != null) { + return ctx.DEV_MATCH().getText(); + } + return visitIdentifierOrParameter(ctx.identifierOrParameter()); + } + + @Override + public String visitIdentifierOrParameter(EsqlBaseParser.IdentifierOrParameterContext ctx) { + if (ctx.identifier() != null) { + return visitIdentifier(ctx.identifier()); + } + + return unresolvedAttributeNameInParam(ctx.parameter(), expression(ctx.parameter())); + } + @Override public Expression visitInlineCast(EsqlBaseParser.InlineCastContext ctx) { Source source = source(ctx); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java index 21aa2cb7d1860..82848fb2f1062 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java @@ -11,10 +11,11 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.GeoDistanceSortBuilder; +import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -38,16 +39,17 @@ public class EsQueryExec extends LeafExec implements EstimatesRowSize { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( PhysicalPlan.class, "EsQueryExec", - EsQueryExec::new + EsQueryExec::deserialize ); public static final EsField DOC_ID_FIELD = new EsField("_doc", DataType.DOC_DATA_TYPE, Map.of(), false); + public static final List NO_SORTS = List.of(); // only exists to mimic older serialization, but we no longer serialize sorts private final EsIndex index; private final IndexMode indexMode; private final QueryBuilder query; private final Expression limit; - private final List sorts; + private final List sorts; private final List attrs; /** @@ -56,8 +58,17 @@ public class EsQueryExec extends LeafExec implements EstimatesRowSize { */ private final Integer estimatedRowSize; - public record FieldSort(FieldAttribute field, Order.OrderDirection direction, Order.NullsPosition nulls) implements Writeable { - public FieldSortBuilder fieldSortBuilder() { + public interface Sort { + SortBuilder sortBuilder(); + + Order.OrderDirection direction(); + + FieldAttribute field(); + } + + public record FieldSort(FieldAttribute field, Order.OrderDirection direction, Order.NullsPosition nulls) implements Sort { + @Override + public SortBuilder sortBuilder() { FieldSortBuilder builder = new FieldSortBuilder(field.name()); builder.order(Direction.from(direction).asOrder()); builder.missing(Missing.from(nulls).searchOrder()); @@ -72,12 +83,14 @@ private static FieldSort readFrom(StreamInput in) throws IOException { in.readEnum(Order.NullsPosition.class) ); } + } + public record GeoDistanceSort(FieldAttribute field, Order.OrderDirection direction, double lat, double lon) implements Sort { @Override - public void writeTo(StreamOutput out) throws IOException { - field().writeTo(out); - out.writeEnum(direction()); - out.writeEnum(nulls()); + public SortBuilder sortBuilder() { + GeoDistanceSortBuilder builder = new GeoDistanceSortBuilder(field.name(), lat, lon); + builder.order(Direction.from(direction).asOrder()); + return builder; } } @@ -92,7 +105,7 @@ public EsQueryExec( List attrs, QueryBuilder query, Expression limit, - List sorts, + List sorts, Integer estimatedRowSize ) { super(source); @@ -105,17 +118,29 @@ public EsQueryExec( this.estimatedRowSize = estimatedRowSize; } - private EsQueryExec(StreamInput in) throws IOException { - this( - Source.readFrom((PlanStreamInput) in), - new EsIndex(in), - EsRelation.readIndexMode(in), - in.readNamedWriteableCollectionAsList(Attribute.class), - in.readOptionalNamedWriteable(QueryBuilder.class), - in.readOptionalNamedWriteable(Expression.class), - in.readOptionalCollectionAsList(FieldSort::readFrom), - in.readOptionalVInt() - ); + /** + * The matching constructor is used during physical plan optimization and needs valid sorts. But we no longer serialize sorts. + * If this cluster node is talking to an older instance it might receive a plan with sorts, but it will ignore them. + */ + public static EsQueryExec deserialize(StreamInput in) throws IOException { + var source = Source.readFrom((PlanStreamInput) in); + var index = new EsIndex(in); + var indexMode = EsRelation.readIndexMode(in); + var attrs = in.readNamedWriteableCollectionAsList(Attribute.class); + var query = in.readOptionalNamedWriteable(QueryBuilder.class); + var limit = in.readOptionalNamedWriteable(Expression.class); + in.readOptionalCollectionAsList(EsQueryExec::readSort); + var rowSize = in.readOptionalVInt(); + // Ignore sorts from the old serialization format + return new EsQueryExec(source, index, indexMode, attrs, query, limit, NO_SORTS, rowSize); + } + + private static Sort readSort(StreamInput in) throws IOException { + return FieldSort.readFrom(in); + } + + private static void writeSort(StreamOutput out, Sort sort) { + throw new IllegalStateException("sorts are no longer serialized"); } @Override @@ -126,7 +151,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeNamedWriteableCollection(output()); out.writeOptionalNamedWriteable(query()); out.writeOptionalNamedWriteable(limit()); - out.writeOptionalCollection(sorts()); + out.writeOptionalCollection(NO_SORTS, EsQueryExec::writeSort); out.writeOptionalVInt(estimatedRowSize()); } @@ -165,7 +190,7 @@ public Expression limit() { return limit; } - public List sorts() { + public List sorts() { return sorts; } @@ -208,7 +233,7 @@ public boolean canPushSorts() { return indexMode != IndexMode.TIME_SERIES; } - public EsQueryExec withSorts(List sorts) { + public EsQueryExec withSorts(List sorts) { if (indexMode == IndexMode.TIME_SERIES) { assert false : "time-series index mode doesn't support sorts"; throw new UnsupportedOperationException("time-series index mode doesn't support sorts"); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index 04be731484267..ab0d68b152262 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -56,7 +56,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; import org.elasticsearch.xpack.esql.plan.physical.AggregateExec; import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec; -import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec.FieldSort; +import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec.Sort; import org.elasticsearch.xpack.esql.plan.physical.FieldExtractExec; import org.elasticsearch.xpack.esql.planner.LocalExecutionPlanner.DriverParallelism; import org.elasticsearch.xpack.esql.planner.LocalExecutionPlanner.LocalExecutionPlannerContext; @@ -161,15 +161,14 @@ public Function querySuppl public final PhysicalOperation sourcePhysicalOperation(EsQueryExec esQueryExec, LocalExecutionPlannerContext context) { final LuceneOperator.Factory luceneFactory; - List sorts = esQueryExec.sorts(); - List> fieldSorts = null; + List sorts = esQueryExec.sorts(); assert esQueryExec.estimatedRowSize() != null : "estimated row size not initialized"; int rowEstimatedSize = esQueryExec.estimatedRowSize(); int limit = esQueryExec.limit() != null ? (Integer) esQueryExec.limit().fold() : NO_LIMIT; if (sorts != null && sorts.isEmpty() == false) { - fieldSorts = new ArrayList<>(sorts.size()); - for (FieldSort sort : sorts) { - fieldSorts.add(sort.fieldSortBuilder()); + List> sortBuilders = new ArrayList<>(sorts.size()); + for (Sort sort : sorts) { + sortBuilders.add(sort.sortBuilder()); } luceneFactory = new LuceneTopNSourceOperator.Factory( shardContexts, @@ -178,7 +177,7 @@ public final PhysicalOperation sourcePhysicalOperation(EsQueryExec esQueryExec, context.queryPragmas().taskConcurrency(), context.pageSize(rowEstimatedSize), limit, - fieldSorts + sortBuilders ); } else { if (esQueryExec.indexMode() == IndexMode.TIME_SERIES) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java index 18aa2628fdc7c..2c8604a7c4a80 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java @@ -24,15 +24,18 @@ import org.elasticsearch.xpack.esql.core.planner.ExpressionTranslators; import org.elasticsearch.xpack.esql.core.planner.TranslatorHandler; import org.elasticsearch.xpack.esql.core.querydsl.query.MatchAll; +import org.elasticsearch.xpack.esql.core.querydsl.query.MatchQuery; import org.elasticsearch.xpack.esql.core.querydsl.query.NotQuery; import org.elasticsearch.xpack.esql.core.querydsl.query.Query; +import org.elasticsearch.xpack.esql.core.querydsl.query.QueryStringQuery; import org.elasticsearch.xpack.esql.core.querydsl.query.RangeQuery; import org.elasticsearch.xpack.esql.core.querydsl.query.TermQuery; import org.elasticsearch.xpack.esql.core.querydsl.query.TermsQuery; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.Check; -import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; +import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; +import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryString; import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils; @@ -55,6 +58,7 @@ import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Set; import static org.elasticsearch.xpack.esql.core.expression.Foldables.valueOf; @@ -85,17 +89,11 @@ public final class EsqlExpressionTranslators { new ExpressionTranslators.StringQueries(), new ExpressionTranslators.Matches(), new ExpressionTranslators.MultiMatches(), - new FullTextFunctions(), + new MatchFunctionTranslator(), + new QueryStringFunctionTranslator(), new Scalars() ); - public static class FullTextFunctions extends ExpressionTranslator { - @Override - protected Query asQuery(FullTextFunction fullTextFunction, TranslatorHandler handler) { - return fullTextFunction.asQuery(); - } - } - public static Query toQuery(Expression e, TranslatorHandler handler) { Query translation = null; for (ExpressionTranslator translator : QUERY_TRANSLATORS) { @@ -528,4 +526,18 @@ private static RangeQuery translate(Range r, TranslatorHandler handler) { ); } } + + public static class MatchFunctionTranslator extends ExpressionTranslator { + @Override + protected Query asQuery(Match match, TranslatorHandler handler) { + return new MatchQuery(match.source(), ((FieldAttribute) match.field()).name(), match.queryAsText()); + } + } + + public static class QueryStringFunctionTranslator extends ExpressionTranslator { + @Override + protected Query asQuery(QueryString queryString, TranslatorHandler handler) { + return new QueryStringQuery(queryString.source(), queryString.queryAsText(), Map.of(), null); + } + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java index ce2a1d7a5f660..f714695504a1d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java @@ -834,7 +834,7 @@ public void messageReceived(ClusterComputeRequest request, TransportChannel chan * execution metadata for ES|QL processing local to this cluster. The execution info will be copied into the * ComputeResponse that is sent back to the primary coordinating cluster. */ - EsqlExecutionInfo execInfo = new EsqlExecutionInfo(); + EsqlExecutionInfo execInfo = new EsqlExecutionInfo(true); execInfo.swapCluster(clusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(clusterAlias, Arrays.toString(request.indices()))); CancellableTask cancellable = (CancellableTask) task; long start = request.configuration().getQueryStartTimeNanos(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlMediaTypeParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlMediaTypeParser.java index 915efe9302a92..17329ca2e0054 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlMediaTypeParser.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlMediaTypeParser.java @@ -37,10 +37,15 @@ public class EsqlMediaTypeParser { * format. If there is a {@code format} parameter we use that. If there * isn't but there is a {@code Accept} header then we use that. If there * isn't then we use the {@code Content-Type} header which is required. + * + * Also validates certain parameter combinations and throws IllegalArgumentException if invalid + * combinations are detected. */ public static MediaType getResponseMediaType(RestRequest request, EsqlQueryRequest esqlRequest) { var mediaType = request.hasParam(URL_PARAM_FORMAT) ? mediaTypeFromParams(request) : mediaTypeFromHeaders(request); - return validateColumnarRequest(esqlRequest.columnar(), mediaType, request); + validateColumnarRequest(esqlRequest.columnar(), mediaType); + validateIncludeCCSMetadata(esqlRequest.includeCCSMetadata(), mediaType); + return checkNonNullMediaType(mediaType, request); } private static MediaType mediaTypeFromHeaders(RestRequest request) { @@ -53,7 +58,7 @@ private static MediaType mediaTypeFromParams(RestRequest request) { return MEDIA_TYPE_REGISTRY.queryParamToMediaType(request.param(URL_PARAM_FORMAT)); } - private static MediaType validateColumnarRequest(boolean requestIsColumnar, MediaType fromMediaType, RestRequest request) { + private static void validateColumnarRequest(boolean requestIsColumnar, MediaType fromMediaType) { if (requestIsColumnar && fromMediaType instanceof TextFormat) { throw new IllegalArgumentException( "Invalid use of [columnar] argument: cannot be used in combination with " @@ -61,7 +66,16 @@ private static MediaType validateColumnarRequest(boolean requestIsColumnar, Medi + " formats" ); } - return checkNonNullMediaType(fromMediaType, request); + } + + private static void validateIncludeCCSMetadata(boolean includeCCSMetadata, MediaType fromMediaType) { + if (includeCCSMetadata && fromMediaType instanceof TextFormat) { + throw new IllegalArgumentException( + "Invalid use of [include_ccs_metadata] argument: cannot be used in combination with " + + Arrays.stream(TextFormat.values()).map(MediaType::queryParameter).toList() + + " formats" + ); + } } private static MediaType checkNonNullMediaType(MediaType mediaType, RestRequest request) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java index 17c795f2de28c..193930cdf711d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java @@ -167,7 +167,10 @@ private void innerExecute(Task task, EsqlQueryRequest request, ActionListener remoteClusterService.isSkipUnavailable(clusterAlias)); + EsqlExecutionInfo executionInfo = new EsqlExecutionInfo( + clusterAlias -> remoteClusterService.isSkipUnavailable(clusterAlias), + request.includeCCSMetadata() + ); BiConsumer> runPhase = (physicalPlan, resultListener) -> computeService.execute( sessionId, (CancellableTask) task, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java index 4f0bcbb43e260..532825290af0d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java @@ -9,6 +9,8 @@ import org.apache.lucene.search.ConstantScoreQuery; import org.elasticsearch.TransportVersion; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.geo.GeoJson; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.geometry.Geometry; @@ -92,9 +94,16 @@ public ShapeRelation shapeRelation() { */ public abstract class ShapeQueryBuilder implements QueryBuilder { - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - throw new UnsupportedOperationException("Unimplemented: toXContent()"); + protected void doToXContent(String queryName, XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.startObject(queryName); + builder.startObject(field); + builder.field("relation", queryRelation); + builder.field("shape"); + GeoJson.toXContent(shape, builder, params); + builder.endObject(); + builder.endObject(); + builder.endObject(); } @Override @@ -157,6 +166,11 @@ public ShapeRelation relation() { public Geometry shape() { return shape; } + + @Override + public String toString() { + return Strings.toString(this, true, true); + } } private class GeoShapeQueryBuilder extends ShapeQueryBuilder { @@ -178,6 +192,13 @@ org.apache.lucene.search.Query buildShapeQuery(SearchExecutionContext context, M final GeoShapeQueryable ft = (GeoShapeQueryable) fieldType; return new ConstantScoreQuery(ft.geoShapeQuery(context, fieldType.name(), shapeRelation(), shape)); } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + // Currently used only in testing and debugging + doToXContent(NAME, builder, params); + return builder; + } } private class CartesianShapeQueryBuilder extends ShapeQueryBuilder { @@ -225,5 +246,13 @@ private static org.apache.lucene.search.Query shapeShapeQuery( throw new QueryShardException(context, "Exception creating query on Field [" + fieldName + "] " + e.getMessage(), e); } } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + // Currently used only in testing and debugging + doToXContent("cartesian_shape", builder, params); + return builder; + } + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index 3eef31e1cc406..f881c0e1a9bba 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -252,6 +252,10 @@ public final void test() throws Throwable { "can't use QSTR function in csv tests", testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.QSTR_FUNCTION.capabilityName()) ); + assumeFalse( + "can't use MATCH function in csv tests", + testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.MATCH_FUNCTION.capabilityName()) + ); if (Build.current().isSnapshot()) { assertThat( @@ -433,7 +437,7 @@ private ActualResults executePlan(BigArrays bigArrays) throws Exception { session.executeOptimizedPlan( new EsqlQueryRequest(), - new EsqlExecutionInfo(), + new EsqlExecutionInfo(randomBoolean()), runPhase(bigArrays, physicalOperationProviders), session.optimizedPlan(analyzed), listener.delegateFailureAndWrap( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java index abf03d4fe06dd..b147cfde21721 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java @@ -134,7 +134,7 @@ EsqlQueryResponse randomResponseAsync(boolean columnar, EsqlQueryResponse.Profil } EsqlExecutionInfo createExecutionInfo() { - EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(); + EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); executionInfo.overallTook(new TimeValue(5000)); executionInfo.swapCluster( "", @@ -426,9 +426,9 @@ static EsqlExecutionInfo parseClusters(XContentParser parser) throws IOException } } if (clusterInfoMap.isEmpty()) { - return new EsqlExecutionInfo(); + return new EsqlExecutionInfo(true); } else { - return new EsqlExecutionInfo(clusterInfoMap); + return new EsqlExecutionInfo(clusterInfoMap, true); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 5d75549893512..6644f9b17055e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -1638,7 +1638,7 @@ public void testCounterTypes() { var attributes = limit.output().stream().collect(Collectors.toMap(NamedExpression::name, a -> a)); assertThat( attributes.keySet(), - equalTo(Set.of("network.connections", "network.bytes_in", "network.bytes_out", "network.message_in")) + equalTo(Set.of("network.connections", "network.bytes_in", "network.bytes_out", "network.message_in", "network.message_out")) ); assertThat(attributes.get("network.connections").dataType(), equalTo(DataType.LONG)); assertThat(attributes.get("network.bytes_in").dataType(), equalTo(DataType.COUNTER_LONG)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 83b3e88c57964..01c020b16ecad 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.analysis; import org.elasticsearch.Build; +import org.elasticsearch.common.logging.LoggerMessageFormat; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; @@ -25,12 +26,16 @@ import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import static org.elasticsearch.xpack.esql.EsqlTestUtils.paramAsConstant; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.loadMapping; +import static org.elasticsearch.xpack.esql.core.type.DataType.COUNTER_DOUBLE; +import static org.elasticsearch.xpack.esql.core.type.DataType.COUNTER_INTEGER; +import static org.elasticsearch.xpack.esql.core.type.DataType.COUNTER_LONG; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -908,6 +913,25 @@ public void testSpatialSort() { assertEquals("1:42: cannot sort on cartesian_shape", error("FROM countries_bbox_web | LIMIT 5 | sort shape", countriesBboxWeb)); } + public void testSourceSorting() { + assertEquals("1:35: cannot sort on _source", error("from test metadata _source | sort _source")); + } + + public void testCountersSorting() { + Map counterDataTypes = Map.of( + COUNTER_DOUBLE, + "network.message_in", + COUNTER_INTEGER, + "network.message_out", + COUNTER_LONG, + "network.bytes_out" + ); + for (DataType counterDT : counterDataTypes.keySet()) { + var fieldName = counterDataTypes.get(counterDT); + assertEquals("1:18: cannot sort on " + counterDT.name().toLowerCase(Locale.ROOT), error("from test | sort " + fieldName, tsdb)); + } + } + public void testInlineImpossibleConvert() { assertEquals("1:5: argument of [false::ip] must be [ip or string], found value [false] type [boolean]", error("ROW false::ip")); } @@ -1086,6 +1110,15 @@ public void testMatchFilter() throws Exception { ); } + public void testMatchFunctionNotAllowedAfterCommands() throws Exception { + assumeTrue("skipping because MATCH is not enabled", EsqlCapabilities.Cap.MATCH_FUNCTION.isEnabled()); + + assertEquals( + "1:24: [MATCH] function cannot be used after LIMIT", + error("from test | limit 10 | where match(first_name, \"Anna\")") + ); + } + public void testQueryStringFunctionsNotAllowedAfterCommands() throws Exception { assumeTrue("skipping because QSTR is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); @@ -1146,26 +1179,167 @@ public void testQueryStringFunctionsNotAllowedAfterCommands() throws Exception { ); } - public void testQueryStringFunctionsOnlyAllowedInWhere() throws Exception { + public void testQueryStringFunctionOnlyAllowedInWhere() throws Exception { assumeTrue("skipping because QSTR is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); - assertEquals("1:22: [QSTR] function is only supported in WHERE commands", error("from test | eval y = qstr(\"Anna\")")); - assertEquals("1:18: [QSTR] function is only supported in WHERE commands", error("from test | sort qstr(\"Connection\") asc")); - assertEquals("1:5: [QSTR] function is only supported in WHERE commands", error("row qstr(\"Connection\")")); + assertEquals("1:9: [QSTR] function is only supported in WHERE commands", error("row a = qstr(\"Anna\")")); + checkFullTextFunctionsOnlyAllowedInWhere("QSTR", "qstr(\"Anna\")"); + } + + public void testMatchFunctionOnlyAllowedInWhere() throws Exception { + assumeTrue("skipping because MATCH is not enabled", EsqlCapabilities.Cap.MATCH_FUNCTION.isEnabled()); + + checkFullTextFunctionsOnlyAllowedInWhere("MATCH", "match(first_name, \"Anna\")"); + } + + private void checkFullTextFunctionsOnlyAllowedInWhere(String functionName, String functionInvocation) throws Exception { + assertEquals( + "1:22: [" + functionName + "] function is only supported in WHERE commands", + error("from test | eval y = " + functionInvocation) + ); + assertEquals( + "1:18: [" + functionName + "] function is only supported in WHERE commands", + error("from test | sort " + functionInvocation + " asc") + ); assertEquals( - "1:23: [QSTR] function is only supported in WHERE commands", - error("from test | STATS c = qstr(\"foo\") BY languages") + "1:23: [" + functionName + "] function is only supported in WHERE commands", + error("from test | STATS c = " + functionInvocation + " BY first_name") ); } public void testQueryStringFunctionArgNotNullOrConstant() throws Exception { assumeTrue("skipping because QSTR is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); - assertEquals("1:19: argument of [QSTR] must be a constant, received [first_name]", error("from test | where qstr(first_name)")); - assertEquals("1:19: argument of [QSTR] cannot be null, received [null]", error("from test | where qstr(null)")); + assertEquals( + "1:19: argument of [qstr(first_name)] must be a constant, received [first_name]", + error("from test | where qstr(first_name)") + ); + assertEquals("1:19: argument of [qstr(null)] cannot be null, received [null]", error("from test | where qstr(null)")); + // Other value types are tested in QueryStringFunctionTests + } + + public void testQueryStringWithDisjunctions() { + assumeTrue("skipping because QSTR is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + + checkWithDisjunctions("QSTR", "qstr(\"first_name: Anna\")"); + } + + public void testMatchWithDisjunctions() { + assumeTrue("skipping because MATCH is not enabled", EsqlCapabilities.Cap.MATCH_FUNCTION.isEnabled()); + + checkWithDisjunctions("MATCH", "match(first_name, \"Anna\")"); + } + + private void checkWithDisjunctions(String functionName, String functionInvocation) { + assertEquals( + LoggerMessageFormat.format( + null, + "1:19: Invalid condition [{} or length(first_name) > 12]. " + "Function {} can't be used as part of an or condition", + functionInvocation, + functionName + ), + error("from test | where " + functionInvocation + " or length(first_name) > 12") + ); + assertEquals( + LoggerMessageFormat.format( + null, + "1:19: Invalid condition [({} and first_name is not null) or (length(first_name) > 12 and first_name is null)]. " + + "Function {} can't be used as part of an or condition", + functionInvocation, + functionName + ), + error( + "from test | where (" + + functionInvocation + + " and first_name is not null) or (length(first_name) > 12 and first_name is null)" + ) + ); + assertEquals( + LoggerMessageFormat.format( + null, + "1:19: Invalid condition [({} and first_name is not null) or first_name is null]. " + + "Function {} can't be used as part of an or condition", + functionInvocation, + functionName + ), + error("from test | where (" + functionInvocation + " and first_name is not null) or first_name is null") + ); + } + + public void testQueryStringFunctionWithNonBooleanFunctions() { + assumeTrue("skipping because QSTR is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + + checkFullTextFunctionsWithNonBooleanFunctions("QSTR", "qstr(\"first_name: Anna\")"); + } + + public void testMatchFunctionWithNonBooleanFunctions() { + assumeTrue("skipping because MATCH is not enabled", EsqlCapabilities.Cap.MATCH_FUNCTION.isEnabled()); + + checkFullTextFunctionsWithNonBooleanFunctions("MATCH", "match(first_name, \"Anna\")"); + } + + private void checkFullTextFunctionsWithNonBooleanFunctions(String functionName, String functionInvocation) { + assertEquals( + "1:19: Invalid condition [" + functionInvocation + " is not null]. Function " + functionName + " can't be used with ISNOTNULL", + error("from test | where " + functionInvocation + " is not null") + ); + assertEquals( + "1:19: Invalid condition [" + functionInvocation + " is null]. Function " + functionName + " can't be used with ISNULL", + error("from test | where " + functionInvocation + " is null") + ); + assertEquals( + "1:19: Invalid condition [" + + functionInvocation + + " in (\"hello\", \"world\")]. Function " + + functionName + + " can't be used with IN", + error("from test | where " + functionInvocation + " in (\"hello\", \"world\")") + ); + } + + public void testMatchFunctionArgNotConstant() throws Exception { + assumeTrue("skipping because MATCH is not enabled", EsqlCapabilities.Cap.MATCH_FUNCTION.isEnabled()); + + assertEquals( + "1:19: second argument of [match(first_name, first_name)] must be a constant, received [first_name]", + error("from test | where match(first_name, first_name)") + ); + assertEquals( + "1:59: second argument of [match(first_name, query)] must be a constant, received [query]", + error("from test | eval query = concat(\"first\", \" name\") | where match(first_name, query)") + ); // Other value types are tested in QueryStringFunctionTests } + // These should pass eventually once we lift some restrictions on match function + public void testMatchFunctionCurrentlyUnsupportedBehaviour() throws Exception { + assumeTrue("skipping because MATCH is not enabled", EsqlCapabilities.Cap.MATCH_FUNCTION.isEnabled()); + + assertEquals( + "1:68: Unknown column [first_name]", + error("from test | stats max_salary = max(salary) by emp_no | where match(first_name, \"Anna\")") + ); + } + + public void testMatchFunctionNullArgs() throws Exception { + assumeTrue("skipping because MATCH is not enabled", EsqlCapabilities.Cap.MATCH_FUNCTION.isEnabled()); + + assertEquals( + "1:19: first argument of [match(null, \"query\")] cannot be null, received [null]", + error("from test | where match(null, \"query\")") + ); + assertEquals( + "1:19: second argument of [match(first_name, null)] cannot be null, received [null]", + error("from test | where match(first_name, null)") + ); + } + + public void testMatchFunctionTargetsExistingField() throws Exception { + assumeTrue("skipping because MATCH is not enabled", EsqlCapabilities.Cap.MATCH_FUNCTION.isEnabled()); + + assertEquals("1:39: Unknown column [first_name]", error("from test | keep emp_no | where match(first_name, \"Anna\")")); + } + public void testCoalesceWithMixedNumericTypes() { assertEquals( "1:22: second argument of [coalesce(languages, height)] must be [integer], found value [height] type [double]", diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchTests.java new file mode 100644 index 0000000000000..f04e9bd495a49 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchTests.java @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.fulltext; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.FieldExpression; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.FunctionName; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; +import org.junit.BeforeClass; + +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.function.Supplier; + +import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.MATCH_FUNCTION; +import static org.hamcrest.Matchers.equalTo; + +@FunctionName("match") +public class MatchTests extends AbstractFunctionTestCase { + + @BeforeClass + public static void checkFunctionEnabled() { + assumeTrue("MATCH function should be enabled ", MATCH_FUNCTION.isEnabled()); + } + + public MatchTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + Set supported = Set.of(DataType.KEYWORD, DataType.TEXT); + List> supportedPerPosition = List.of(supported, supported); + List suppliers = new LinkedList<>(); + for (DataType fieldType : validStringDataTypes()) { + for (DataType queryType : validStringDataTypes()) { + suppliers.add( + new TestCaseSupplier( + "<" + fieldType + "-ES field, " + queryType + ">", + List.of(fieldType, queryType), + () -> testCase(fieldType, randomIdentifier(), queryType, randomAlphaOfLengthBetween(1, 10), equalTo(true)) + ) + ); + suppliers.add( + new TestCaseSupplier( + "<" + fieldType + "-non ES field, " + queryType + ">", + List.of(fieldType, queryType), + typeErrorSupplier(true, supportedPerPosition, List.of(fieldType, queryType), MatchTests::matchTypeErrorSupplier) + ) + ); + } + } + List errorsSuppliers = errorsForCasesWithoutExamples(suppliers, (v, p) -> "string"); + // Don't test null, as it is not allowed but the expected message is not a type error - so we check it separately in VerifierTests + return parameterSuppliersFromTypedData(errorsSuppliers.stream().filter(s -> s.types().contains(DataType.NULL) == false).toList()); + } + + private static String matchTypeErrorSupplier(boolean includeOrdinal, List> validPerPosition, List types) { + return "[] cannot operate on [" + types.getFirst().typeName() + "], which is not a field from an index mapping"; + } + + private static List validStringDataTypes() { + return Arrays.stream(DataType.values()).filter(DataType::isString).toList(); + } + + private static TestCaseSupplier.TestCase testCase( + DataType fieldType, + String field, + DataType queryType, + String query, + Matcher matcher + ) { + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData( + new FieldExpression(field, List.of(new FieldExpression.FieldValue(field))), + fieldType, + "field" + ), + new TestCaseSupplier.TypedData(new BytesRef(query), queryType, "query") + ), + "EndsWithEvaluator[str=Attribute[channel=0], suffix=Attribute[channel=1]]", + DataType.BOOLEAN, + matcher + ); + } + + @Override + protected Expression build(Source source, List args) { + return new Match(source, args.get(0), args.get(1)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunctionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringTests.java similarity index 92% rename from x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunctionTests.java rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringTests.java index 37e16a2499cd9..8b0e4f10b8d54 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunctionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringTests.java @@ -29,14 +29,14 @@ import static org.hamcrest.Matchers.equalTo; @FunctionName("qstr") -public class QueryStringFunctionTests extends AbstractFunctionTestCase { +public class QueryStringTests extends AbstractFunctionTestCase { @BeforeClass public static void checkFunctionEnabled() { assumeTrue("QSTR capability should be enabled ", QSTR_FUNCTION.isEnabled()); } - public QueryStringFunctionTests(@Name("TestCase") Supplier testCaseSupplier) { + public QueryStringTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); } @@ -77,6 +77,6 @@ private static TestCaseSupplier.TestCase testCase(DataType strType, String str, @Override protected Expression build(Source source, List args) { - return new QueryStringFunction(source, args.get(0)); + return new QueryString(source, args.get(0)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/HypotSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/HypotSerializationTests.java new file mode 100644 index 0000000000000..5c2e84fcba8e0 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/HypotSerializationTests.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.math; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; +import org.elasticsearch.xpack.esql.expression.AbstractUnaryScalarSerializationTests; + +import java.io.IOException; + +public class HypotSerializationTests extends AbstractExpressionSerializationTests { + + @Override + protected Hypot createTestInstance() { + Source source = randomSource(); + Expression n1 = randomChild(); + Expression n2 = randomChild(); + return new Hypot(source, n1, n2); + } + + @Override + protected Hypot mutateInstance(Hypot instance) throws IOException { + Source source = instance.source(); + Expression n1 = instance.n1(); + Expression n2 = instance.n2(); + if (randomBoolean()) { + n1 = randomValueOtherThan(n1, AbstractUnaryScalarSerializationTests::randomChild); + } else { + n2 = randomValueOtherThan(n2, AbstractUnaryScalarSerializationTests::randomChild); + } + return new Hypot(source, n1, n2); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/HypotTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/HypotTests.java new file mode 100644 index 0000000000000..0161abc2b9560 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/HypotTests.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.math; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.List; +import java.util.function.Supplier; + +public class HypotTests extends AbstractScalarFunctionTestCase { + public HypotTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + List suppliers = TestCaseSupplier.forBinaryCastingToDouble( + "HypotEvaluator", + "n1", + "n2", + Math::hypot, + Double.NEGATIVE_INFINITY, + Double.POSITIVE_INFINITY, + Double.NEGATIVE_INFINITY, + Double.POSITIVE_INFINITY, + List.of() + ); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); + } + + @Override + protected Expression build(Source source, List args) { + return new Hypot(source, args.get(0), args.get(1)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatterTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatterTests.java index c145d770409da..e735ba83168bb 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatterTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatterTests.java @@ -82,7 +82,7 @@ public class TextFormatterTests extends ESTestCase { null, randomBoolean(), randomBoolean(), - new EsqlExecutionInfo() + new EsqlExecutionInfo(randomBoolean()) ); TextFormatter formatter = new TextFormatter(esqlResponse); @@ -157,7 +157,7 @@ public void testFormatWithoutHeader() { null, randomBoolean(), randomBoolean(), - new EsqlExecutionInfo() + new EsqlExecutionInfo(randomBoolean()) ); String[] result = getTextBodyContent(new TextFormatter(response).format(false)).split("\n"); @@ -198,7 +198,7 @@ public void testVeryLongPadding() { null, randomBoolean(), randomBoolean(), - new EsqlExecutionInfo() + new EsqlExecutionInfo(randomBoolean()) ) ).format(false) ) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index c2779b7dbc46d..3dd0828b82eed 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -50,7 +50,6 @@ import org.elasticsearch.xpack.esql.plan.physical.EvalExec; import org.elasticsearch.xpack.esql.plan.physical.ExchangeExec; import org.elasticsearch.xpack.esql.plan.physical.FieldExtractExec; -import org.elasticsearch.xpack.esql.plan.physical.FilterExec; import org.elasticsearch.xpack.esql.plan.physical.LimitExec; import org.elasticsearch.xpack.esql.plan.physical.LocalSourceExec; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; @@ -438,22 +437,24 @@ public void testQueryStringFunctionConjunctionWhereOperands() { /** * Expecting * LimitExec[1000[INTEGER]] - * \_ExchangeExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, job{f}#10, job.raw{f}#11, languages{f}#6, last_n - * ame{f}#7, long_noidx{f}#12, salary{f}#8],false] - * \_ProjectExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, job{f}#10, job.raw{f}#11, languages{f}#6, last_n - * ame{f}#7, long_noidx{f}#12, salary{f}#8]] - * \_FieldExtractExec[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gen] - * \_EsQueryExec[test], indexMode[standard], query[{"bool":{"should":[{"query_string":{"query":"last_name: Smith","fields":[]}}, - * {"esql_single_value":{"field":"emp_no","next":{"range":{"emp_no":{"gt":10010,"boost":1.0}}},"source":"emp_no > 10010@2:37"}}], - * "boost":1.0}}][_doc{f}#13], limit[1000], sort[] estimatedRowSize[324] + * \_ExchangeExec[[!alias_integer, boolean{f}#4, byte{f}#5, constant_keyword-foo{f}#6, date{f}#7, double{f}#8, float{f}#9, half_ + * float{f}#10, integer{f}#12, ip{f}#13, keyword{f}#14, long{f}#15, scaled_float{f}#11, short{f}#17, text{f}#18, unsigned_long{f}#16], + * false] + * \_ProjectExec[[!alias_integer, boolean{f}#4, byte{f}#5, constant_keyword-foo{f}#6, date{f}#7, double{f}#8, float{f}#9, half_ + * float{f}#10, integer{f}#12, ip{f}#13, keyword{f}#14, long{f}#15, scaled_float{f}#11, short{f}#17, text{f}#18, unsigned_long{f}#16] + * \_FieldExtractExec[!alias_integer, boolean{f}#4, byte{f}#5, constant_k..] + * \_EsQueryExec[test], indexMode[standard], query[{"bool":{"must":[{"query_string":{"query":"last_name: Smith","fields":[]}},{ + * "esql_single_value":{"field":"ip","next":{"terms":{"ip":["127.0.0.1/32"],"boost":1.0}}, + * "source":"cidr_match(ip, \"127.0.0.1/32\")@2:38"}}],"boost":1.0}}][_doc{f}#21], limit[1000], sort[] estimatedRowSize[354] */ - public void testQueryStringFunctionDisjunctionWhereClauses() { + public void testQueryStringFunctionWithFunctionsPushedToLucene() { assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); String queryText = """ from test - | where qstr("last_name: Smith") or emp_no > 10010 + | where qstr("last_name: Smith") and cidr_match(ip, "127.0.0.1/32") """; - var plan = plannerOptimizer.plan(queryText, IS_SV_STATS); + var analyzer = makeAnalyzer("mapping-all-types.json", new EnrichResolution()); + var plan = plannerOptimizer.plan(queryText, IS_SV_STATS, analyzer); var limit = as(plan, LimitExec.class); var exchange = as(limit.child(), ExchangeExec.class); @@ -462,34 +463,34 @@ public void testQueryStringFunctionDisjunctionWhereClauses() { var query = as(field.child(), EsQueryExec.class); assertThat(query.limit().fold(), is(1000)); - Source filterSource = new Source(2, 36, "emp_no > 10000"); - var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); + Source filterSource = new Source(2, 37, "cidr_match(ip, \"127.0.0.1/32\")"); + var terms = wrapWithSingleQuery(queryText, QueryBuilders.termsQuery("ip", "127.0.0.1/32"), "ip", filterSource); var queryString = QueryBuilders.queryStringQuery("last_name: Smith"); - var expected = QueryBuilders.boolQuery().should(queryString).should(range); + var expected = QueryBuilders.boolQuery().must(queryString).must(terms); assertThat(query.query().toString(), is(expected.toString())); } /** * Expecting * LimitExec[1000[INTEGER]] - * \_ExchangeExec[[!alias_integer, boolean{f}#4, byte{f}#5, constant_keyword-foo{f}#6, date{f}#7, double{f}#8, float{f}#9, half_ - * float{f}#10, integer{f}#12, ip{f}#13, keyword{f}#14, long{f}#15, scaled_float{f}#11, short{f}#17, text{f}#18, unsigned_long{f}#16], - * false] - * \_ProjectExec[[!alias_integer, boolean{f}#4, byte{f}#5, constant_keyword-foo{f}#6, date{f}#7, double{f}#8, float{f}#9, half_ - * float{f}#10, integer{f}#12, ip{f}#13, keyword{f}#14, long{f}#15, scaled_float{f}#11, short{f}#17, text{f}#18, unsigned_long{f}#16] - * \_FieldExtractExec[!alias_integer, boolean{f}#4, byte{f}#5, constant_k..] - * \_EsQueryExec[test], indexMode[standard], query[{"bool":{"must":[{"query_string":{"query":"last_name: Smith","fields":[]}},{ - * "esql_single_value":{"field":"ip","next":{"terms":{"ip":["127.0.0.1/32"],"boost":1.0}}, - * "source":"cidr_match(ip, \"127.0.0.1/32\")@2:38"}}],"boost":1.0}}][_doc{f}#21], limit[1000], sort[] estimatedRowSize[354] + * \_ExchangeExec[[_meta_field{f}#1163, emp_no{f}#1157, first_name{f}#1158, gender{f}#1159, job{f}#1164, job.raw{f}#1165, langua + * ges{f}#1160, last_name{f}#1161, long_noidx{f}#1166, salary{f}#1162],false] + * \_ProjectExec[[_meta_field{f}#1163, emp_no{f}#1157, first_name{f}#1158, gender{f}#1159, job{f}#1164, job.raw{f}#1165, langua + * ges{f}#1160, last_name{f}#1161, long_noidx{f}#1166, salary{f}#1162]] + * \_FieldExtractExec[_meta_field{f}#1163, emp_no{f}#1157, first_name{f}#] + * \_EsQueryExec[test], indexMode[standard], + * query[{"bool":{"must":[{"query_string":{"query":"last_name: Smith","fields":[]}}, + * {"esql_single_value":{"field":"emp_no","next":{"range":{"emp_no":{"gt":10010,"boost":1.0}}},"source":"emp_no > 10010@3:9"}}], + * "boost":1.0}}][_doc{f}#1167], limit[1000], sort[] estimatedRowSize[324] */ - public void testQueryStringFunctionWithFunctionsPushedToLucene() { + public void testQueryStringFunctionMultipleWhereClauses() { assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); String queryText = """ from test - | where qstr("last_name: Smith") and cidr_match(ip, "127.0.0.1/32") + | where qstr("last_name: Smith") + | where emp_no > 10010 """; - var analyzer = makeAnalyzer("mapping-all-types.json", new EnrichResolution()); - var plan = plannerOptimizer.plan(queryText, IS_SV_STATS, analyzer); + var plan = plannerOptimizer.plan(queryText, IS_SV_STATS); var limit = as(plan, LimitExec.class); var exchange = as(limit.child(), ExchangeExec.class); @@ -498,45 +499,140 @@ public void testQueryStringFunctionWithFunctionsPushedToLucene() { var query = as(field.child(), EsQueryExec.class); assertThat(query.limit().fold(), is(1000)); - Source filterSource = new Source(2, 37, "cidr_match(ip, \"127.0.0.1/32\")"); - var terms = wrapWithSingleQuery(queryText, QueryBuilders.termsQuery("ip", "127.0.0.1/32"), "ip", filterSource); + Source filterSource = new Source(3, 8, "emp_no > 10000"); + var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); var queryString = QueryBuilders.queryStringQuery("last_name: Smith"); - var expected = QueryBuilders.boolQuery().must(queryString).must(terms); + var expected = QueryBuilders.boolQuery().must(queryString).must(range); assertThat(query.query().toString(), is(expected.toString())); } /** * Expecting - *LimitExec[1000[INTEGER]] - * \_ExchangeExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, job{f}#10, job.raw{f}#11, languages{f}#6, last_n - * ame{f}#7, long_noidx{f}#12, salary{f}#8],false] - * \_ProjectExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, job{f}#10, job.raw{f}#11, languages{f}#6, last_n - * ame{f}#7, long_noidx{f}#12, salary{f}#8]] - * \_FieldExtractExec[_meta_field{f}#9, emp_no{f}#3, gender{f}#5, job{f}#] - * \_LimitExec[1000[INTEGER]] - * \_FilterExec[LENGTH(first_name{f}#4) > 10[INTEGER]] - * \_FieldExtractExec[first_name{f}#4] - * \_EsQueryExec[test], indexMode[standard], - * query[{"query_string":{"query":"last_name: Smith","fields":[]}}][_doc{f}#13], limit[], sort[] estimatedRowSize[324] + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na + * me{f}#6, long_noidx{f}#11, salary{f}#7],false] + * \_ProjectExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na + * me{f}#6, long_noidx{f}#11, salary{f}#7]] + * \_FieldExtractExec[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gen] + * \_EsQueryExec[test], indexMode[standard], query[{"bool": + * {"must":[{"query_string":{"query":"last_name: Smith","fields":[]}}, + * {"query_string":{"query":"emp_no: [10010 TO *]","fields":[]}}],"boost":1.0}}] */ - public void testQueryStringFunctionWithFunctionNotPushedDown() { + public void testQueryStringFunctionMultipleQstrClauses() { assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); String queryText = """ from test - | where qstr("last_name: Smith") and length(first_name) > 10 + | where qstr("last_name: Smith") and qstr("emp_no: [10010 TO *]") """; var plan = plannerOptimizer.plan(queryText, IS_SV_STATS); - var firstLimit = as(plan, LimitExec.class); - var exchange = as(firstLimit.child(), ExchangeExec.class); + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); - var secondLimit = as(field.child(), LimitExec.class); - var filter = as(secondLimit.child(), FilterExec.class); - var fieldExtract = as(filter.child(), FieldExtractExec.class); - var query = as(fieldExtract.child(), EsQueryExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); - var expected = QueryBuilders.queryStringQuery("last_name: Smith"); + var queryStringLeft = QueryBuilders.queryStringQuery("last_name: Smith"); + var queryStringRight = QueryBuilders.queryStringQuery("emp_no: [10010 TO *]"); + var expected = QueryBuilders.boolQuery().must(queryStringLeft).must(queryStringRight); + assertThat(query.query().toString(), is(expected.toString())); + } + + /** + * Expecting + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na + * me{f}#6, long_noidx{f}#11, salary{f}#7],false] + * \_ProjectExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na + * me{f}#6, long_noidx{f}#11, salary{f}#7]] + * \_FieldExtractExec[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gen] + * \_EsQueryExec[test], indexMode[standard], query[{"match":{"last_name":{"query":"Smith"}}}] + */ + public void testMatchFunction() { + assumeTrue("skipping because MATCH function is not enabled", EsqlCapabilities.Cap.MATCH_FUNCTION.isEnabled()); + var plan = plannerOptimizer.plan(""" + from test + | where match(last_name, "Smith") + """, IS_SV_STATS); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); + var expected = QueryBuilders.matchQuery("last_name", "Smith"); + assertThat(query.query().toString(), is(expected.toString())); + } + + /** + * Expecting + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#1419, emp_no{f}#1413, first_name{f}#1414, gender{f}#1415, job{f}#1420, job.raw{f}#1421, langua + * ges{f}#1416, last_name{f}#1417, long_noidx{f}#1422, salary{f}#1418],false] + * \_ProjectExec[[_meta_field{f}#1419, emp_no{f}#1413, first_name{f}#1414, gender{f}#1415, job{f}#1420, job.raw{f}#1421, langua + * ges{f}#1416, last_name{f}#1417, long_noidx{f}#1422, salary{f}#1418]] + * \_FieldExtractExec[_meta_field{f}#1419, emp_no{f}#1413, first_name{f}#] + * \EsQueryExec[test], indexMode[standard], query[{"bool":{"must":[{"match":{"last_name":{"query":"Smith"}}}, + * {"esql_single_value":{"field":"emp_no","next":{"range":{"emp_no":{"gt":10010,"boost":1.0}}}, + * "source":"emp_no > 10010@2:39"}}],"boost":1.0}}][_doc{f}#14], limit[1000], sort[] estimatedRowSize[324] + */ + public void testMatchFunctionConjunctionWhereOperands() { + assumeTrue("skipping because MATCH function is not enabled", EsqlCapabilities.Cap.MATCH_FUNCTION.isEnabled()); + String queryText = """ + from test + | where match(last_name, "Smith") and emp_no > 10010 + """; + var plan = plannerOptimizer.plan(queryText, IS_SV_STATS); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); + + Source filterSource = new Source(2, 38, "emp_no > 10000"); + var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); + var queryString = QueryBuilders.matchQuery("last_name", "Smith"); + var expected = QueryBuilders.boolQuery().must(queryString).must(range); + assertThat(query.query().toString(), is(expected.toString())); + } + + /** + * Expecting + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[!alias_integer, boolean{f}#4, byte{f}#5, constant_keyword-foo{f}#6, date{f}#7, double{f}#8, float{f}#9, half_ + * float{f}#10, integer{f}#12, ip{f}#13, keyword{f}#14, long{f}#15, scaled_float{f}#11, short{f}#17, text{f}#18, unsigned_long{f}#16], + * false] + * \_ProjectExec[[!alias_integer, boolean{f}#4, byte{f}#5, constant_keyword-foo{f}#6, date{f}#7, double{f}#8, float{f}#9, half_ + * float{f}#10, integer{f}#12, ip{f}#13, keyword{f}#14, long{f}#15, scaled_float{f}#11, short{f}#17, text{f}#18, unsigned_long{f}#16] + * \_FieldExtractExec[!alias_integer, boolean{f}#4, byte{f}#5, constant_k..] + * \_EsQueryExec[test], indexMode[standard], query[{"bool":{"must":[{"match":{"text":{"query":"beta"}}}, + * {"esql_single_value":{"field":"ip","next":{"terms":{"ip":["127.0.0.1/32"],"boost":1.0}}, + * "source":"cidr_match(ip, \"127.0.0.1/32\")@2:33"}}],"boost":1.0}}][_doc{f}#22], limit[1000], sort[] estimatedRowSize[354] + */ + public void testMatchFunctionWithFunctionsPushedToLucene() { + assumeTrue("skipping because MATCH function is not enabled", EsqlCapabilities.Cap.MATCH_FUNCTION.isEnabled()); + String queryText = """ + from test + | where match(text, "beta") and cidr_match(ip, "127.0.0.1/32") + """; + var analyzer = makeAnalyzer("mapping-all-types.json", new EnrichResolution()); + var plan = plannerOptimizer.plan(queryText, IS_SV_STATS, analyzer); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); + + Source filterSource = new Source(2, 32, "cidr_match(ip, \"127.0.0.1/32\")"); + var terms = wrapWithSingleQuery(queryText, QueryBuilders.termsQuery("ip", "127.0.0.1/32"), "ip", filterSource); + var queryString = QueryBuilders.matchQuery("text", "beta"); + var expected = QueryBuilders.boolQuery().must(queryString).must(terms); assertThat(query.query().toString(), is(expected.toString())); } @@ -548,16 +644,15 @@ public void testQueryStringFunctionWithFunctionNotPushedDown() { * \_ProjectExec[[_meta_field{f}#1163, emp_no{f}#1157, first_name{f}#1158, gender{f}#1159, job{f}#1164, job.raw{f}#1165, langua * ges{f}#1160, last_name{f}#1161, long_noidx{f}#1166, salary{f}#1162]] * \_FieldExtractExec[_meta_field{f}#1163, emp_no{f}#1157, first_name{f}#] - * \_EsQueryExec[test], indexMode[standard], - * query[{"bool":{"must":[{"query_string":{"query":"last_name: Smith","fields":[]}}, - * {"esql_single_value":{"field":"emp_no","next":{"range":{"emp_no":{"gt":10010,"boost":1.0}}},"source":"emp_no > 10010@3:9"}}], - * "boost":1.0}}][_doc{f}#1167], limit[1000], sort[] estimatedRowSize[324] + * \_EsQueryExec[test], indexMode[standard], query[{"bool":{"must":[{"match":{"last_name":{"query":"Smith"}}}, + * {"esql_single_value":{"field":"emp_no","next":{"range":{"emp_no":{"gt":10010,"boost":1.0}}}, + * "source":"emp_no > 10010@3:9"}}],"boost":1.0}}][_doc{f}#14], limit[1000], sort[] estimatedRowSize[324] */ - public void testQueryStringFunctionMultipleWhereClauses() { - assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + public void testMatchFunctionMultipleWhereClauses() { + assumeTrue("skipping because MATCH function is not enabled", EsqlCapabilities.Cap.MATCH_FUNCTION.isEnabled()); String queryText = """ from test - | where qstr("last_name: Smith") + | where match(last_name, "Smith") | where emp_no > 10010 """; var plan = plannerOptimizer.plan(queryText, IS_SV_STATS); @@ -571,7 +666,7 @@ public void testQueryStringFunctionMultipleWhereClauses() { Source filterSource = new Source(3, 8, "emp_no > 10000"); var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); - var queryString = QueryBuilders.queryStringQuery("last_name: Smith"); + var queryString = QueryBuilders.matchQuery("last_name", "Smith"); var expected = QueryBuilders.boolQuery().must(queryString).must(range); assertThat(query.query().toString(), is(expected.toString())); } @@ -584,15 +679,14 @@ public void testQueryStringFunctionMultipleWhereClauses() { * \_ProjectExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na * me{f}#6, long_noidx{f}#11, salary{f}#7]] * \_FieldExtractExec[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gen] - * \_EsQueryExec[test], indexMode[standard], query[{"bool": - * {"must":[{"query_string":{"query":"last_name: Smith","fields":[]}}, - * {"query_string":{"query":"emp_no: [10010 TO *]","fields":[]}}],"boost":1.0}}] + * \_EsQueryExec[test], indexMode[standard], query[{"bool":{"must":[{"match":{"last_name":{"query":"Smith"}}}, + * {"match":{"first_name":{"query":"John"}}}],"boost":1.0}}][_doc{f}#14], limit[1000], sort[] estimatedRowSize[324] */ - public void testQueryStringFunctionMultipleQstrClauses() { - assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + public void testMatchFunctionMultipleQstrClauses() { + assumeTrue("skipping because MATCH function is not enabled", EsqlCapabilities.Cap.MATCH_FUNCTION.isEnabled()); String queryText = """ from test - | where qstr("last_name: Smith") and qstr("emp_no: [10010 TO *]") + | where match(last_name, "Smith") and match(first_name, "John") """; var plan = plannerOptimizer.plan(queryText, IS_SV_STATS); @@ -603,8 +697,8 @@ public void testQueryStringFunctionMultipleQstrClauses() { var query = as(field.child(), EsQueryExec.class); assertThat(query.limit().fold(), is(1000)); - var queryStringLeft = QueryBuilders.queryStringQuery("last_name: Smith"); - var queryStringRight = QueryBuilders.queryStringQuery("emp_no: [10010 TO *]"); + var queryStringLeft = QueryBuilders.matchQuery("last_name", "Smith"); + var queryStringRight = QueryBuilders.matchQuery("first_name", "John"); var expected = QueryBuilders.boolQuery().must(queryStringLeft).must(queryStringRight); assertThat(query.query().toString(), is(expected.toString())); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index 6ff541de1854a..e8980c99a61f9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.TestBlockFactory; import org.elasticsearch.xpack.esql.VerificationException; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.analysis.Analyzer; import org.elasticsearch.xpack.esql.analysis.AnalyzerContext; import org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils; @@ -5565,6 +5566,42 @@ public void testToDatePeriodToTimeDurationWithField() { assertEquals("1:60: argument of [to_timeduration(x)] must be a constant, received [x]", e.getMessage().substring(header.length())); } + // These should pass eventually once we lift some restrictions on match function + public void testMatchWithNonIndexedColumnCurrentlyUnsupported() { + assumeTrue("skipping because MATCH function is not enabled", EsqlCapabilities.Cap.MATCH_FUNCTION.isEnabled()); + + final String header = "Found 1 problem\nline "; + VerificationException e = expectThrows(VerificationException.class, () -> plan(""" + from test | eval initial = substring(first_name, 1) | where match(initial, "A")""")); + assertTrue(e.getMessage().startsWith("Found ")); + assertEquals( + "1:67: [MATCH] cannot operate on [initial], which is not a field from an index mapping", + e.getMessage().substring(header.length()) + ); + + e = expectThrows(VerificationException.class, () -> plan(""" + from test | eval text=concat(first_name, last_name) | where match(text, "cat")""")); + assertTrue(e.getMessage().startsWith("Found ")); + assertEquals( + "1:67: [MATCH] cannot operate on [text], which is not a field from an index mapping", + e.getMessage().substring(header.length()) + ); + } + + public void testMatchFunctionIsNotNullable() { + assumeTrue("skipping because MATCH function is not enabled", EsqlCapabilities.Cap.MATCH_FUNCTION.isEnabled()); + + String queryText = """ + row n = null | eval text = n + 5 | where match(text::keyword, "Anna") + """; + + VerificationException ve = expectThrows(VerificationException.class, () -> plan(queryText)); + assertThat( + ve.getMessage(), + containsString("[MATCH] cannot operate on [text::keyword], which is not a field from an index mapping") + ); + } + private Literal nullOf(DataType dataType) { return new Literal(Source.EMPTY, null, dataType); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index f66349351bb59..114aed68761fe 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -9,6 +9,7 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.Build; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.lucene.BytesRefs; @@ -20,12 +21,15 @@ import org.elasticsearch.geometry.ShapeType; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.ExistsQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.index.query.RegexpQueryBuilder; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.index.query.WildcardQueryBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.GeoDistanceSortBuilder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.enrich.EnrichPolicy; import org.elasticsearch.xpack.esql.EsqlTestUtils; @@ -41,6 +45,7 @@ import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; +import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; @@ -151,14 +156,17 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; import static org.elasticsearch.xpack.esql.parser.ExpressionBuilder.MAX_EXPRESSION_DEPTH; import static org.elasticsearch.xpack.esql.parser.LogicalPlanBuilder.MAX_QUERY_DEPTH; +import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.isA; import static org.hamcrest.Matchers.matchesRegex; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; @@ -529,11 +537,11 @@ public void testExtractorForField() { assertThat(names(extract.attributesToExtract()), contains("salary", "emp_no", "last_name")); var source = source(extract.child()); assertThat(source.limit(), is(topN.limit())); - assertThat(source.sorts(), is(sorts(topN.order()))); + assertThat(source.sorts(), is(fieldSorts(topN.order()))); assertThat(source.limit(), is(l(10))); assertThat(source.sorts().size(), is(1)); - FieldSort order = source.sorts().get(0); + EsQueryExec.Sort order = source.sorts().get(0); assertThat(order.direction(), is(Order.OrderDirection.ASC)); assertThat(name(order.field()), is("last_name")); // last name is keyword, salary, emp_no, doc id, segment, forwards and backwards doc id maps are all ints @@ -802,6 +810,19 @@ public void testQueryForStatWithMultiAgg() { assertThat(query.query(), is(boolQuery().should(existsQuery("emp_no")).should(existsQuery("salary")))); } + /** + * This used to not allow pushing the sort down to the source, but now it does, since the eval is not used for the sort + * + * TopNExec[[Order[emp_no{f}#6,ASC,LAST]],1[INTEGER],0] + * \_ExchangeExec[[_meta_field{f}#12, emp_no{f}#6, first_name{f}#7, gender{f}#8, job{f}#13, job.raw{f}#14, ..],false] + * \_ProjectExec[[_meta_field{f}#12, emp_no{f}#6, first_name{f}#7, gender{f}#8, job{f}#13, job.raw{f}#14, ..]] + * \_FieldExtractExec[_meta_field{f}#12, emp_no{f}#6, first_name{f}#7, ge..][] + * \_EvalExec[[null[INTEGER] AS nullsum]] + * \_EsQueryExec[test], indexMode[standard], query[][_doc{f}#27], limit[1], sort[[ + * FieldSort[field=emp_no{f}#6, direction=ASC, nulls=LAST] + * ]] estimatedRowSize[340] + * + */ public void testQueryWithNull() { var plan = physicalPlan(""" from test @@ -818,15 +839,10 @@ public void testQueryWithNull() { var exchange = asRemoteExchange(topN.child()); var project = as(exchange.child(), ProjectExec.class); var extract = as(project.child(), FieldExtractExec.class); - var topNLocal = as(extract.child(), TopNExec.class); - // All fields except emp_no are loaded after this topn. We load an extra int for the doc and segment mapping. - assertThat(topNLocal.estimatedRowSize(), equalTo(allFieldRowSize + Integer.BYTES)); - - var extractForEval = as(topNLocal.child(), FieldExtractExec.class); - var eval = as(extractForEval.child(), EvalExec.class); + var eval = as(extract.child(), EvalExec.class); var source = source(eval.child()); - // emp_no and nullsum are longs, doc id is an int - assertThat(source.estimatedRowSize(), equalTo(Integer.BYTES * 2 + Integer.BYTES)); + // All fields loaded + assertThat(source.estimatedRowSize(), equalTo(allFieldRowSize + 3 * Integer.BYTES + Long.BYTES)); } public void testPushAndInequalitiesFilter() { @@ -1065,7 +1081,7 @@ public void testProjectAfterTopN() throws Exception { var extract = as(project.child(), FieldExtractExec.class); var source = source(extract.child()); assertThat(source.limit(), is(topN.limit())); - assertThat(source.sorts(), is(sorts(topN.order()))); + assertThat(source.sorts(), is(fieldSorts(topN.order()))); // an int for doc id, an int for segment id, two ints for doc id map, and int for emp_no. assertThat(source.estimatedRowSize(), equalTo(Integer.BYTES * 5 + KEYWORD_EST)); } @@ -1221,11 +1237,11 @@ public void testDoNotAliasesDefinedAfterTheExchange() throws Exception { assertThat(names(extract.attributesToExtract()), contains("languages", "salary")); var source = source(extract.child()); assertThat(source.limit(), is(topN.limit())); - assertThat(source.sorts(), is(sorts(topN.order()))); + assertThat(source.sorts(), is(fieldSorts(topN.order()))); assertThat(source.limit(), is(l(1))); assertThat(source.sorts().size(), is(1)); - FieldSort order = source.sorts().get(0); + EsQueryExec.Sort order = source.sorts().get(0); assertThat(order.direction(), is(Order.OrderDirection.ASC)); assertThat(name(order.field()), is("salary")); // ints for doc id, segment id, forwards and backwards mapping, languages, and salary @@ -1754,6 +1770,124 @@ public void testPushDownNotRLike() { assertThat(regexpQuery.value(), is(".*foo.*")); } + /** + * + * TopNExec[[Order[name{r}#4,ASC,LAST]],1000[INTEGER],0] + * \_ExchangeExec[[_meta_field{f}#20, emp_no{f}#14, gender{f}#16, job{f}#21, job.raw{f}#22, languages{f}#17, + * long_noidx{f}#23, salary{f}#19, name{r}#4, first_name{r}#7, last_name{r}#10 + * ],false] + * \_ProjectExec[[_meta_field{f}#20, emp_no{f}#14, gender{f}#16, job{f}#21, job.raw{f}#22, languages{f}#17, + * long_noidx{f}#23, salary{f}#19, name{r}#4, first_name{r}#7, last_name{r}#10 + * ]] + * \_FieldExtractExec[_meta_field{f}#20, emp_no{f}#14, gender{f}#16, job{..][] + * \_EvalExec[[first_name{f}#15 AS name, last_name{f}#18 AS first_name, name{r}#4 AS last_name]] + * \_FieldExtractExec[first_name{f}#15, last_name{f}#18][] + * \_EsQueryExec[test], indexMode[standard], query[{ + * "bool":{"must":[ + * {"esql_single_value":{"field":"last_name","next":{"term":{"last_name":{"value":"foo"}}},"source":...}}, + * {"esql_single_value":{"field":"first_name","next":{"term":{"first_name":{"value":"bar"}}},"source":...}} + * ],"boost":1.0}}][_doc{f}#37], limit[1000], sort[[ + * FieldSort[field=first_name{f}#15, direction=ASC, nulls=LAST] + * ]] estimatedRowSize[486] + * + */ + public void testPushDownEvalFilter() { + var plan = physicalPlan(""" + FROM test + | EVAL name = first_name, first_name = last_name, last_name = name + | WHERE first_name == "foo" AND last_name == "bar" + | SORT name + """); + var optimized = optimizedPlan(plan); + + var topN = as(optimized, TopNExec.class); + var exchange = asRemoteExchange(topN.child()); + var project = as(exchange.child(), ProjectExec.class); + var extract = as(project.child(), FieldExtractExec.class); + assertThat(extract.attributesToExtract().size(), greaterThan(5)); + var eval = as(extract.child(), EvalExec.class); + extract = as(eval.child(), FieldExtractExec.class); + assertThat( + extract.attributesToExtract().stream().map(Attribute::name).collect(Collectors.toList()), + contains("first_name", "last_name") + ); + + // Now verify the correct Lucene push-down of both the filter and the sort + var source = source(extract.child()); + QueryBuilder query = source.query(); + assertNotNull(query); + assertThat(query, instanceOf(BoolQueryBuilder.class)); + var boolQuery = (BoolQueryBuilder) query; + var must = boolQuery.must(); + assertThat(must.size(), is(2)); + var range1 = (TermQueryBuilder) ((SingleValueQuery.Builder) must.get(0)).next(); + assertThat(range1.fieldName(), is("last_name")); + var range2 = (TermQueryBuilder) ((SingleValueQuery.Builder) must.get(1)).next(); + assertThat(range2.fieldName(), is("first_name")); + var sort = source.sorts(); + assertThat(sort.size(), is(1)); + assertThat(sort.get(0).field().fieldName(), is("first_name")); + } + + /** + * + * ProjectExec[[last_name{f}#21 AS name, first_name{f}#18 AS last_name, last_name{f}#21 AS first_name]] + * \_TopNExec[[Order[last_name{f}#21,ASC,LAST]],10[INTEGER],0] + * \_ExchangeExec[[last_name{f}#21, first_name{f}#18],false] + * \_ProjectExec[[last_name{f}#21, first_name{f}#18]] + * \_FieldExtractExec[last_name{f}#21, first_name{f}#18][] + * \_EsQueryExec[test], indexMode[standard], query[{ + * "bool":{"must":[ + * {"esql_single_value":{ + * "field":"last_name", + * "next":{"range":{"last_name":{"gt":"B","boost":1.0}}}, + * "source":"first_name > \"B\"@3:9" + * }}, + * {"exists":{"field":"first_name","boost":1.0}} + * ],"boost":1.0}}][_doc{f}#40], limit[10], sort[[ + * FieldSort[field=last_name{f}#21, direction=ASC, nulls=LAST] + * ]] estimatedRowSize[116] + * + */ + public void testPushDownEvalSwapFilter() { + var plan = physicalPlan(""" + FROM test + | EVAL name = last_name, last_name = first_name, first_name = name + | WHERE first_name > "B" AND last_name IS NOT NULL + | SORT name + | LIMIT 10 + | KEEP name, last_name, first_name + """); + var optimized = optimizedPlan(plan); + + var topProject = as(optimized, ProjectExec.class); + var topN = as(topProject.child(), TopNExec.class); + var exchange = asRemoteExchange(topN.child()); + var project = as(exchange.child(), ProjectExec.class); + var extract = as(project.child(), FieldExtractExec.class); + assertThat( + extract.attributesToExtract().stream().map(Attribute::name).collect(Collectors.toList()), + contains("last_name", "first_name") + ); + + // Now verify the correct Lucene push-down of both the filter and the sort + var source = source(extract.child()); + QueryBuilder query = source.query(); + assertNotNull(query); + assertThat(query, instanceOf(BoolQueryBuilder.class)); + var boolQuery = (BoolQueryBuilder) query; + var must = boolQuery.must(); + assertThat(must.size(), is(2)); + var svq = (SingleValueQuery.Builder) must.get(0); + var range = (RangeQueryBuilder) svq.next(); + assertThat(range.fieldName(), is("last_name")); + var exists = (ExistsQueryBuilder) must.get(1); + assertThat(exists.fieldName(), is("first_name")); + var sort = source.sorts(); + assertThat(sort.size(), is(1)); + assertThat(sort.get(0).field().fieldName(), is("last_name")); + } + /** * EnrichExec[first_name{f}#3,foo,fld,idx,[a{r}#11, b{r}#12]] * \_LimitExec[10000[INTEGER]] @@ -3074,6 +3208,105 @@ public void testPushSpatialIntersectsStringToSource() { } } + /** + * Plan: + * + * EvalExec[[scalerank{f}#8 AS rank]] + * \_LimitExec[1000[INTEGER]] + * \_ExchangeExec[[],false] + * \_FragmentExec[filter=null, estimatedRowSize=0, reducer=[], fragment=[ + * Limit[1000[INTEGER]] + * \_Filter[scalerank{f}#8 < 4[INTEGER]] + * \_EsRelation[airports][abbrev{f}#6, city{f}#12, city_location{f}#13, count..]]] + * + * Optimized: + * + * EvalExec[[scalerank{f}#8 AS rank]] + * \_LimitExec[1000[INTEGER]] + * \_ExchangeExec[[abbrev{f}#6, city{f}#12, city_location{f}#13, country{f}#11, location{f}#10, name{f}#7, scalerank{f}#8, + * type{f}#9],false + * ] + * \_ProjectExec[[abbrev{f}#6, city{f}#12, city_location{f}#13, country{f}#11, location{f}#10, name{f}#7, scalerank{f}#8, + * type{f}#9] + * ] + * \_FieldExtractExec[abbrev{f}#6, city{f}#12, city_location{f}#13, count..][] + * \_EsQueryExec[airports], indexMode[standard], query[{ + * "esql_single_value":{"field":"scalerank","next":{"range":{"scalerank":{"lt":4,"boost":1.0}}},"source":"rank < 4@3:9"} + * ][_doc{f}#23], limit[1000], sort[] estimatedRowSize[304] + * + */ + public void testPushWhereEvalToSource() { + String query = """ + FROM airports + | EVAL rank = scalerank + | WHERE rank < 4 + """; + + var plan = this.physicalPlan(query, airports); + var eval = as(plan, EvalExec.class); + var limit = as(eval.child(), LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var fragment = as(exchange.child(), FragmentExec.class); + var limit2 = as(fragment.fragment(), Limit.class); + var filter = as(limit2.child(), Filter.class); + assertThat("filter contains LessThan", filter.condition(), instanceOf(LessThan.class)); + + var optimized = optimizedPlan(plan); + eval = as(optimized, EvalExec.class); + var topLimit = as(eval.child(), LimitExec.class); + exchange = as(topLimit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var fieldExtract = as(project.child(), FieldExtractExec.class); + assertThat(fieldExtract.attributesToExtract().size(), greaterThan(5)); + var source = source(fieldExtract.child()); + assertThat(source.limit(), is(topLimit.limit())); + var condition = as(source.query(), SingleValueQuery.Builder.class); + assertThat("Expected predicate to be passed to Lucene query", condition.source().text(), equalTo("rank < 4")); + assertThat("Expected field to be passed to Lucene query", condition.field(), equalTo("scalerank")); + var range = as(condition.next(), RangeQueryBuilder.class); + assertThat("Expected range have no lower bound", range.from(), nullValue()); + assertThat("Expected range to be less than 4", range.to(), equalTo(4)); + } + + public void testPushSpatialIntersectsEvalToSource() { + for (String query : new String[] { """ + FROM airports + | EVAL point = location + | WHERE ST_INTERSECTS(point, TO_GEOSHAPE("POLYGON((42 14, 43 14, 43 15, 42 15, 42 14))")) + """, """ + FROM airports + | EVAL point = location + | WHERE ST_INTERSECTS(TO_GEOSHAPE("POLYGON((42 14, 43 14, 43 15, 42 15, 42 14))"), point) + """ }) { + + var plan = this.physicalPlan(query, airports); + var eval = as(plan, EvalExec.class); + var limit = as(eval.child(), LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var fragment = as(exchange.child(), FragmentExec.class); + var limit2 = as(fragment.fragment(), Limit.class); + var filter = as(limit2.child(), Filter.class); + assertThat("filter contains ST_INTERSECTS", filter.condition(), instanceOf(SpatialIntersects.class)); + + var optimized = optimizedPlan(plan); + eval = as(optimized, EvalExec.class); + var topLimit = as(eval.child(), LimitExec.class); + exchange = as(topLimit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var fieldExtract = as(project.child(), FieldExtractExec.class); + assertThat(fieldExtract.attributesToExtract().size(), greaterThan(5)); + var source = source(fieldExtract.child()); + assertThat(source.limit(), is(topLimit.limit())); + var condition = as(source.query(), SpatialRelatesQuery.ShapeQueryBuilder.class); + assertThat("Geometry field name", condition.fieldName(), equalTo("location")); + assertThat("Spatial relationship", condition.relation(), equalTo(ShapeRelation.INTERSECTS)); + assertThat("Geometry is Polygon", condition.shape().type(), equalTo(ShapeType.POLYGON)); + var polygon = as(condition.shape(), Polygon.class); + assertThat("Polygon shell length", polygon.getPolygon().length(), equalTo(5)); + assertThat("Polygon holes", polygon.getNumberOfHoles(), equalTo(0)); + } + } + private record TestSpatialRelation(ShapeRelation relation, TestDataSource index, boolean literalRight, boolean canPushToSource) { String function() { return switch (relation) { @@ -3257,30 +3490,22 @@ public void testPushDownSpatialRelatesStringToSourceAndUseDocValuesForCentroid() * * Optimized: * - * LimitExec[500[INTEGER]] - * \_AggregateExec[[],[SPATIALCENTROID(location{f}#12) AS centroid, COUNT([2a][KEYWORD]) AS count],FINAL,58] + * LimitExec[1000[INTEGER]] + * \_AggregateExec[[],[SPATIALCENTROID(location{f}#12) AS centroid, COUNT([2a][KEYWORD]) AS count],FINAL,[...],29] * \_ExchangeExec[[xVal{r}#16, xDel{r}#17, yVal{r}#18, yDel{r}#19, count{r}#20, count{r}#21, seen{r}#22],true] - * \_AggregateExec[[],[SPATIALCENTROID(location{f}#12) AS centroid, COUNT([2a][KEYWORD]) AS count],PARTIAL,58] + * \_AggregateExec[[],[SPATIALCENTROID(location{f}#12) AS centroid, COUNT([2a][KEYWORD]) AS count],INITIAL,[...],29] * \_FieldExtractExec[location{f}#12][location{f}#12] - * \_EsQueryExec[airports], query[{ - * "esql_single_value":{ - * "field":"location", - * "next":{ - * "geo_shape":{ - * "location":{ - * "shape":{ - * "type":"Polygon", - * "coordinates":[[[42.0,14.0],[43.0,14.0],[43.0,15.0],[42.0,15.0],[42.0,14.0]]] - * }, - * "relation":"intersects" - * }, - * "ignore_unmapped":false, - * "boost":1.0 + * \_EsQueryExec[airports], indexMode[standard], query[{ + * "geo_shape":{ + * "location":{ + * "relation":"INTERSECTS", + * "shape":{ + * "type":"Polygon", + * "coordinates":[[[42.0,14.0],[43.0,14.0],[43.0,15.0],[42.0,15.0],[42.0,14.0]]] * } - * }, - * "source":"ST_INTERSECTS(location, \"POLYGON((42 14, 43 14, 43 15, 42 15, 42 14))\")@2:9" + * } * } - * }][_doc{f}#140, limit[], sort[] estimatedRowSize[54] + * }][_doc{f}#47], limit[], sort[] estimatedRowSize[25] * */ public void testPushSpatialIntersectsStringToSourceAndUseDocValuesForCentroid() { @@ -3732,6 +3957,167 @@ AND ST_DISTANCE(location, TO_GEOPOINT("POINT(12.565 55.673)")) >= 400000 assertShapeQueryRange(shapeQueryBuilders, 400000.0, 600000.0); } + /** + * Plan: + * + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[],false] + * \_FragmentExec[filter=null, estimatedRowSize=0, reducer=[], fragment=[ + * Limit[1000[INTEGER]] + * \_Filter[distance{r}#4 le 600000[INTEGER] AND distance{r}#4 ge 400000[INTEGER]] + * \_Eval[[STDISTANCE(location{f}#11,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) + * AS distance]] + * \_EsRelation[airports][abbrev{f}#7, city{f}#13, city_location{f}#14, count..]]] + * + * Optimized: + * + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[abbrev{f}#7, city{f}#13, city_location{f}#14, country{f}#12, location{f}#11, name{f}#8, scalerank{f}#9, type{ + * f}#10, distance{r}#4],false] + * \_ProjectExec[[abbrev{f}#7, city{f}#13, city_location{f}#14, country{f}#12, location{f}#11, name{f}#8, scalerank{f}#9, type{ + * f}#10, distance{r}#4]] + * \_FieldExtractExec[abbrev{f}#7, city{f}#13, city_location{f}#14, count..][] + * \_LimitExec[1000[INTEGER]] + * \_EvalExec[[STDISTANCE(location{f}#11,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) + * AS distance]] + * \_FieldExtractExec[location{f}#11][] + * \_EsQueryExec[airports], indexMode[standard], query[{ + * "bool":{ + * "must":[ + * { + * "geo_shape":{ + * "location":{ + * "relation":"INTERSECTS", + * "shape":{ + * "type":"Circle", + * "radius":"600000.0m", + * "coordinates":[12.565,55.673] + * } + * } + * } + * }, + * { + * "geo_shape":{ + * "location":{ + * "relation":"DISJOINT", + * "shape":{ + * "type":"Circle", + * "radius":"400000.0m", + * "coordinates":[12.565,55.673] + * } + * } + * } + * } + * ], + * "boost":1.0 + * }}][_doc{f}#24], limit[], sort[] estimatedRowSize[308] + * + */ + public void testPushSpatialDistanceEvalToSource() { + var query = """ + FROM airports + | EVAL distance = ST_DISTANCE(location, TO_GEOPOINT("POINT(12.565 55.673)")) + | WHERE distance <= 600000 + AND distance >= 400000 + """; + var plan = this.physicalPlan(query, airports); + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var fragment = as(exchange.child(), FragmentExec.class); + var limit2 = as(fragment.fragment(), Limit.class); + var filter = as(limit2.child(), Filter.class); + + // Validate the EVAL expression + var eval = as(filter.child(), Eval.class); + var alias = as(eval.fields().get(0), Alias.class); + assertThat(alias.name(), is("distance")); + var stDistance = as(alias.child(), StDistance.class); + var location = as(stDistance.left(), FieldAttribute.class); + assertThat(location.fieldName(), is("location")); + + // Validate the filter condition + var and = as(filter.condition(), And.class); + for (Expression expression : and.arguments()) { + var comp = as(expression, EsqlBinaryComparison.class); + var expectedComp = comp.equals(and.left()) ? LessThanOrEqual.class : GreaterThanOrEqual.class; + assertThat("filter contains expected binary comparison", comp, instanceOf(expectedComp)); + var distance = as(comp.left(), ReferenceAttribute.class); + assertThat(distance.name(), is("distance")); + } + + var optimized = optimizedPlan(plan); + var topLimit = as(optimized, LimitExec.class); + exchange = as(topLimit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var fieldExtract = as(project.child(), FieldExtractExec.class); + var limit3 = as(fieldExtract.child(), LimitExec.class); + var evalExec = as(limit3.child(), EvalExec.class); + var fieldExtract2 = as(evalExec.child(), FieldExtractExec.class); + var source = source(fieldExtract2.child()); + var bool = as(source.query(), BoolQueryBuilder.class); + var rangeQueryBuilders = bool.filter().stream().filter(p -> p instanceof SingleValueQuery.Builder).toList(); + assertThat("Expected zero range query builder", rangeQueryBuilders.size(), equalTo(0)); + var shapeQueryBuilders = bool.must().stream().filter(p -> p instanceof SpatialRelatesQuery.ShapeQueryBuilder).toList(); + assertShapeQueryRange(shapeQueryBuilders, 400000.0, 600000.0); + } + + public void testPushSpatialDistanceMultiEvalToSource() { + var query = """ + FROM airports + | EVAL poi = TO_GEOPOINT("POINT(12.565 55.673)") + | EVAL distance = ST_DISTANCE(location, poi) + | WHERE distance <= 600000 + AND distance >= 400000 + """; + var plan = this.physicalPlan(query, airports); + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var fragment = as(exchange.child(), FragmentExec.class); + var limit2 = as(fragment.fragment(), Limit.class); + var filter = as(limit2.child(), Filter.class); + + // Validate the EVAL expression + var eval = as(filter.child(), Eval.class); + assertThat(eval.fields().size(), is(2)); + var alias1 = as(eval.fields().get(0), Alias.class); + assertThat(alias1.name(), is("poi")); + var poi = as(alias1.child(), Literal.class); + assertThat(poi.fold(), instanceOf(BytesRef.class)); + var alias2 = as(eval.fields().get(1), Alias.class); + assertThat(alias2.name(), is("distance")); + var stDistance = as(alias2.child(), StDistance.class); + var location = as(stDistance.left(), FieldAttribute.class); + assertThat(location.fieldName(), is("location")); + var poiRef = as(stDistance.right(), Literal.class); + assertThat(poiRef.fold(), instanceOf(BytesRef.class)); + assertThat(poiRef.fold().toString(), is(poi.fold().toString())); + + // Validate the filter condition + var and = as(filter.condition(), And.class); + for (Expression expression : and.arguments()) { + var comp = as(expression, EsqlBinaryComparison.class); + var expectedComp = comp.equals(and.left()) ? LessThanOrEqual.class : GreaterThanOrEqual.class; + assertThat("filter contains expected binary comparison", comp, instanceOf(expectedComp)); + var distance = as(comp.left(), ReferenceAttribute.class); + assertThat(distance.name(), is("distance")); + } + + var optimized = optimizedPlan(plan); + var topLimit = as(optimized, LimitExec.class); + exchange = as(topLimit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var fieldExtract = as(project.child(), FieldExtractExec.class); + var limit3 = as(fieldExtract.child(), LimitExec.class); + var evalExec = as(limit3.child(), EvalExec.class); + var fieldExtract2 = as(evalExec.child(), FieldExtractExec.class); + var source = source(fieldExtract2.child()); + var bool = as(source.query(), BoolQueryBuilder.class); + var rangeQueryBuilders = bool.filter().stream().filter(p -> p instanceof SingleValueQuery.Builder).toList(); + assertThat("Expected zero range query builder", rangeQueryBuilders.size(), equalTo(0)); + var shapeQueryBuilders = bool.must().stream().filter(p -> p instanceof SpatialRelatesQuery.ShapeQueryBuilder).toList(); + assertShapeQueryRange(shapeQueryBuilders, 400000.0, 600000.0); + } + public void testPushSpatialDistanceDisjointBandsToSource() { var query = """ FROM airports @@ -3826,6 +4212,1361 @@ AND ST_DISTANCE(location, TO_GEOPOINT("POINT(12.565 55.673)")) >= 200000)) } } + /** + * + * \_ExchangeExec[[abbrev{f}#22, city{f}#28, city_location{f}#29, country{f}#27, location{f}#26, name{f}#23, scalerank{f}#24, + * type{f}#25, poi_x{r}#3, distance_x{r}#7, poi{r}#10, distance{r}#13],false] + * \_ProjectExec[[abbrev{f}#22, city{f}#28, city_location{f}#29, country{f}#27, location{f}#26, name{f}#23, scalerank{f}#24, + * type{f}#25, poi_x{r}#3, distance_x{r}#7, poi{r}#10, distance{r}#13]] + * \_FieldExtractExec[abbrev{f}#22, city{f}#28, city_location{f}#29, coun..][] + * \_LimitExec[1000[INTEGER]] + * \_EvalExec[[ + * [1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT] AS poi_x, + * DISTANCE(location{f}#26,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) AS distance_x, + * [1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT] AS poi, + * distance_x{r}#7 AS distance + * ]] + * \_FieldExtractExec[location{f}#26][] + * \_EsQueryExec[airports], indexMode[standard], query[{ + * "bool":{ + * "must":[ + * {"esql_single_value":{ + * "field":"abbrev", + * "next":{"bool":{"must_not":[{"term":{"abbrev":{"value":"PLQ"}}}],"boost":1.0}}, + * "source":"NOT abbrev == \"PLQ\"@10:9" + * }}, + * {"esql_single_value":{ + * "field":"scalerank", + * "next":{"range":{"scalerank":{"lt":6,"boost":1.0}}}, + * "source":"scalerank lt 6@11:9" + * }} + * ], + * "filter":[ + * {"bool":{ + * "should":[ + * {"bool":{"must":[ + * {"geo_shape":{"location":{"relation":"INTERSECTS","shape":{...}}}}, + * {"geo_shape":{"location":{"relation":"DISJOINT","shape":{...}}}}, + * {"bool":{"must_not":[ + * {"bool":{"must":[ + * {"geo_shape":{"location":{"relation":"INTERSECTS","shape":{...}}}}, + * {"geo_shape":{"location":{"relation":"DISJOINT","shape":{...}}}} + * ],"boost":1.0}} + * ],"boost":1.0}} + * ],"boost":1.0}}, + * {"bool":{"must":[ + * {"geo_shape":{"location":{"relation":"INTERSECTS","shape":{...}}}}, + * {"geo_shape":{"location":{"relation":"DISJOINT","shape":{...}}}} + * ],"boost":1.0}} + * ],"boost":1.0 + * }} + * ],"boost":1.0}}][_doc{f}#34], limit[], sort[] estimatedRowSize[329] + * + */ + public void testPushSpatialDistanceComplexPredicateWithEvalToSource() { + var query = """ + FROM airports + | EVAL poi_x = TO_GEOPOINT("POINT(12.565 55.673)") + | EVAL distance_x = ST_DISTANCE(location, poi_x) + | EVAL poi = poi_x + | EVAL distance = distance_x + | WHERE ((distance <= 600000 + AND distance >= 400000 + AND NOT (distance <= 500000 + AND distance >= 430000)) + OR (distance <= 300000 + AND distance >= 200000)) + AND NOT abbrev == "PLQ" + AND scalerank < 6 + """; + var plan = this.physicalPlan(query, airports); + var optimized = optimizedPlan(plan); + var topLimit = as(optimized, LimitExec.class); + var exchange = as(topLimit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var fieldExtract = as(project.child(), FieldExtractExec.class); + var limit2 = as(fieldExtract.child(), LimitExec.class); + var evalExec = as(limit2.child(), EvalExec.class); + var fieldExtract2 = as(evalExec.child(), FieldExtractExec.class); + var source = source(fieldExtract2.child()); + var bool = as(source.query(), BoolQueryBuilder.class); + assertThat("Expected boolean query of three MUST clauses", bool.must().size(), equalTo(2)); + assertThat("Expected boolean query of one FILTER clause", bool.filter().size(), equalTo(1)); + var boolDisjuntive = as(bool.filter().get(0), BoolQueryBuilder.class); + var disjuntiveQueryBuilders = boolDisjuntive.should().stream().filter(p -> p instanceof BoolQueryBuilder).toList(); + assertThat("Expected two disjunctive query builders", disjuntiveQueryBuilders.size(), equalTo(2)); + for (int i = 0; i < disjuntiveQueryBuilders.size(); i++) { + var subRangeBool = as(disjuntiveQueryBuilders.get(i), BoolQueryBuilder.class); + var shapeQueryBuilders = subRangeBool.must().stream().filter(p -> p instanceof SpatialRelatesQuery.ShapeQueryBuilder).toList(); + assertShapeQueryRange(shapeQueryBuilders, i == 0 ? 400000.0 : 200000.0, i == 0 ? 600000.0 : 300000.0); + } + } + + /** + * Plan: + * + * LimitExec[1000[INTEGER]] + * \_AggregateExec[[],[COUNT([2a][KEYWORD]) AS count],FINAL,[count{r}#17, seen{r}#18],null] + * \_ExchangeExec[[count{r}#17, seen{r}#18],true] + * \_FragmentExec[filter=null, estimatedRowSize=0, reducer=[], fragment=[ + * Aggregate[STANDARD,[],[COUNT([2a][KEYWORD]) AS count]] + * \_Filter[distance{r}#4 lt 1000000[INTEGER] AND distance{r}#4 gt 10000[INTEGER]] + * \_Eval[[ + * STDISTANCE(location{f}#13,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) AS distance + * ]] + * \_EsRelation[airports][abbrev{f}#9, city{f}#15, city_location{f}#16, count..]]] + * + * Optimized: + * + * LimitExec[1000[INTEGER]] + * \_AggregateExec[[],[COUNT([2a][KEYWORD]) AS count],FINAL,[count{r}#17, seen{r}#18],8] + * \_ExchangeExec[[count{r}#17, seen{r}#18],true] + * \_AggregateExec[[],[COUNT([2a][KEYWORD]) AS count],INITIAL,[count{r}#31, seen{r}#32],8] + * \_EvalExec[[ + * STDISTANCE(location{f}#13,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) AS distance + * ]] + * \_FieldExtractExec[location{f}#13][] + * \_EsQueryExec[airports], indexMode[standard], query[{ + * "bool":{ + * "must":[ + * {"geo_shape":{"location":{"relation":"INTERSECTS","shape":{...}}}}, + * {"geo_shape":{"location":{"relation":"DISJOINT","shape":{...}}}} + * ],"boost":1.0}}][_doc{f}#33], limit[], sort[] estimatedRowSize[33] + * + */ + public void testPushSpatialDistanceEvalWithSimpleStatsToSource() { + var query = """ + FROM airports + | EVAL distance = ST_DISTANCE(location, TO_GEOPOINT("POINT(12.565 55.673)")) + | WHERE distance < 1000000 AND distance > 10000 + | STATS count=COUNT(*) + """; + var plan = this.physicalPlan(query, airports); + var limit = as(plan, LimitExec.class); + var agg = as(limit.child(), AggregateExec.class); + var exchange = as(agg.child(), ExchangeExec.class); + var fragment = as(exchange.child(), FragmentExec.class); + var agg2 = as(fragment.fragment(), Aggregate.class); + var filter = as(agg2.child(), Filter.class); + + // Validate the filter condition (two distance filters) + var and = as(filter.condition(), And.class); + for (Expression expression : and.arguments()) { + var comp = as(expression, EsqlBinaryComparison.class); + var expectedComp = comp.equals(and.left()) ? LessThan.class : GreaterThan.class; + assertThat("filter contains expected binary comparison", comp, instanceOf(expectedComp)); + var distance = as(comp.left(), ReferenceAttribute.class); + assertThat(distance.name(), is("distance")); + } + + // Validate the eval (calculating distance) + var eval = as(filter.child(), Eval.class); + var alias = as(eval.fields().get(0), Alias.class); + assertThat(alias.name(), is("distance")); + as(eval.child(), EsRelation.class); + + // Now optimize the plan + var optimized = optimizedPlan(plan); + var topLimit = as(optimized, LimitExec.class); + var aggExec = as(topLimit.child(), AggregateExec.class); + var exchangeExec = as(aggExec.child(), ExchangeExec.class); + var aggExec2 = as(exchangeExec.child(), AggregateExec.class); + // TODO: Remove the eval entirely, since the distance is no longer required after filter pushdown + // Right now we don't mark the distance field as doc-values, introducing a performance hit + // However, fixing this to doc-values is not as good as removing the EVAL entirely, which is a more sensible optimization + var evalExec = as(aggExec2.child(), EvalExec.class); + var stDistance = as(evalExec.fields().get(0).child(), StDistance.class); + assertThat("Expect distance function to expect doc-values", stDistance.leftDocValues(), is(false)); + var source = assertChildIsGeoPointExtract(evalExec, false); + + // No sort is pushed down + assertThat(source.limit(), nullValue()); + assertThat(source.sorts(), nullValue()); + + // Fine-grained checks on the pushed down query + var bool = as(source.query(), BoolQueryBuilder.class); + var shapeQueryBuilders = bool.must().stream().filter(p -> p instanceof SpatialRelatesQuery.ShapeQueryBuilder).toList(); + assertShapeQueryRange(shapeQueryBuilders, 10000.0, 1000000.0); + } + + /** + * Plan: + * + * TopNExec[[Order[count{r}#10,DESC,FIRST], Order[country{f}#21,ASC,LAST]],1000[INTEGER],null] + * \_AggregateExec[[country{f}#21],[COUNT([2a][KEYWORD]) AS count, SPATIALCENTROID(location{f}#20) AS centroid, country{f}#21],FINA + * L,[country{f}#21, count{r}#24, seen{r}#25, xVal{r}#26, xDel{r}#27, yVal{r}#28, yDel{r}#29, count{r}#30],null] + * \_ExchangeExec[[country{f}#21, count{r}#24, seen{r}#25, xVal{r}#26, xDel{r}#27, yVal{r}#28, yDel{r}#29, count{r}#30],true] + * \_FragmentExec[filter=null, estimatedRowSize=0, reducer=[], fragment=[ + * Aggregate[STANDARD,[country{f}#21],[COUNT([2a][KEYWORD]) AS count, SPATIALCENTROID(location{f}#20) AS centroid, country{f} + * #21]] + * \_Filter[distance{r}#4 lt 1000000[INTEGER] AND distance{r}#4 gt 10000[INTEGER]] + * \_Eval[[STDISTANCE(location{f}#20,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) + * AS distance]] + * \_Filter[scalerank{f}#18 lt 6[INTEGER]] + * \_EsRelation[airports][abbrev{f}#16, city{f}#22, city_location{f}#23, coun..]]] + * + * Optimized: + * + * TopNExec[[Order[count{r}#10,DESC,FIRST], Order[country{f}#21,ASC,LAST]],1000[INTEGER],0] + * \_AggregateExec[[country{f}#21],[COUNT([2a][KEYWORD]) AS count, SPATIALCENTROID(location{f}#20) AS centroid, country{f}#21],FINA + * L,[country{f}#21, count{r}#24, seen{r}#25, xVal{r}#26, xDel{r}#27, yVal{r}#28, yDel{r}#29, count{r}#30],79] + * \_ExchangeExec[[country{f}#21, count{r}#24, seen{r}#25, xVal{r}#26, xDel{r}#27, yVal{r}#28, yDel{r}#29, count{r}#30],true] + * \_AggregateExec[[country{f}#21],[COUNT([2a][KEYWORD]) AS count, SPATIALCENTROID(location{f}#20) AS centroid, country{f}#21],INIT + * IAL,[country{f}#21, count{r}#49, seen{r}#50, xVal{r}#51, xDel{r}#52, yVal{r}#53, yDel{r}#54, count{r}#55],79] + * \_EvalExec[[STDISTANCE(location{f}#20,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) + * AS distance]] + * \_FieldExtractExec[location{f}#20][location{f}#20] + * \_EsQueryExec[airports], indexMode[standard], query[{ + * "bool":{ + * "filter":[ + * { + * "esql_single_value":{ + * "field":"scalerank", + * "next":{"range":{"scalerank":{"lt":6,"boost":1.0}}}, + * "source":"scalerank lt 6@3:31" + * } + * }, + * { + * "bool":{ + * "must":[ + * {"geo_shape":{ + * "location":{ + * "relation":"INTERSECTS", + * "shape":{"type":"Circle","radius":"1000000m","coordinates":[12.565,55.673]} + * } + * }}, + * {"geo_shape":{ + * "location":{ + * "relation":"DISJOINT", + * "shape":{"type":"Circle","radius":"10000m","coordinates":[12.565,55.673]} + * } + * }} + * ], + * "boost":1.0 + * } + * } + * ], + * "boost":1.0 + * }}][_doc{f}#56], limit[], sort[] estimatedRowSize[33] + * + */ + public void testPushSpatialDistanceEvalWithStatsToSource() { + var query = """ + FROM airports + | EVAL distance = ST_DISTANCE(location, TO_GEOPOINT("POINT(12.565 55.673)")) + | WHERE distance < 1000000 AND scalerank < 6 AND distance > 10000 + | STATS count=COUNT(*), centroid=ST_CENTROID_AGG(location) BY country + | SORT count DESC, country ASC + """; + var plan = this.physicalPlan(query, airports); + var topN = as(plan, TopNExec.class); + var agg = as(topN.child(), AggregateExec.class); + var exchange = as(agg.child(), ExchangeExec.class); + var fragment = as(exchange.child(), FragmentExec.class); + var agg2 = as(fragment.fragment(), Aggregate.class); + var filter = as(agg2.child(), Filter.class); + + // Validate the filter condition (two distance filters) + var and = as(filter.condition(), And.class); + for (Expression expression : and.arguments()) { + var comp = as(expression, EsqlBinaryComparison.class); + var expectedComp = comp.equals(and.left()) ? LessThan.class : GreaterThan.class; + assertThat("filter contains expected binary comparison", comp, instanceOf(expectedComp)); + var distance = as(comp.left(), ReferenceAttribute.class); + assertThat(distance.name(), is("distance")); + } + + // Validate the eval (calculating distance) + var eval = as(filter.child(), Eval.class); + var alias = as(eval.fields().get(0), Alias.class); + assertThat(alias.name(), is("distance")); + var filter2 = as(eval.child(), Filter.class); + + // Now optimize the plan + var optimized = optimizedPlan(plan); + var topLimit = as(optimized, TopNExec.class); + var aggExec = as(topLimit.child(), AggregateExec.class); + var exchangeExec = as(aggExec.child(), ExchangeExec.class); + var aggExec2 = as(exchangeExec.child(), AggregateExec.class); + // TODO: Remove the eval entirely, since the distance is no longer required after filter pushdown + var evalExec = as(aggExec2.child(), EvalExec.class); + var stDistance = as(evalExec.fields().get(0).child(), StDistance.class); + assertThat("Expect distance function to expect doc-values", stDistance.leftDocValues(), is(true)); + var source = assertChildIsGeoPointExtract(evalExec, true); + + // No sort is pushed down + assertThat(source.limit(), nullValue()); + assertThat(source.sorts(), nullValue()); + + // Fine-grained checks on the pushed down query + var bool = as(source.query(), BoolQueryBuilder.class); + var rangeQueryBuilders = bool.filter().stream().filter(p -> p instanceof SingleValueQuery.Builder).toList(); + assertThat("Expected one range query builder", rangeQueryBuilders.size(), equalTo(1)); + assertThat(((SingleValueQuery.Builder) rangeQueryBuilders.get(0)).field(), equalTo("scalerank")); + var filterBool = bool.filter().stream().filter(p -> p instanceof BoolQueryBuilder).toList(); + var fb = as(filterBool.get(0), BoolQueryBuilder.class); + var shapeQueryBuilders = fb.must().stream().filter(p -> p instanceof SpatialRelatesQuery.ShapeQueryBuilder).toList(); + assertShapeQueryRange(shapeQueryBuilders, 10000.0, 1000000.0); + } + + /** + * ProjectExec[[languages{f}#8, salary{f}#10]] + * \_TopNExec[[Order[salary{f}#10,DESC,FIRST]],10[INTEGER],0] + * \_ExchangeExec[[languages{f}#8, salary{f}#10],false] + * \_ProjectExec[[languages{f}#8, salary{f}#10]] + * \_FieldExtractExec[languages{f}#8, salary{f}#10][] + * \_EsQueryExec[test], + * indexMode[standard], + * query[][_doc{f}#25], + * limit[10], + * sort[[FieldSort[field=salary{f}#10, direction=DESC, nulls=FIRST]]] estimatedRowSize[24] + */ + public void testPushTopNToSource() { + var optimized = optimizedPlan(physicalPlan(""" + FROM test + | SORT salary DESC + | LIMIT 10 + | KEEP languages, salary + """)); + + var project = as(optimized, ProjectExec.class); + var topN = as(project.child(), TopNExec.class); + var exchange = asRemoteExchange(topN.child()); + + project = as(exchange.child(), ProjectExec.class); + assertThat(names(project.projections()), contains("languages", "salary")); + var extract = as(project.child(), FieldExtractExec.class); + assertThat(names(extract.attributesToExtract()), contains("languages", "salary")); + var source = source(extract.child()); + assertThat(source.limit(), is(topN.limit())); + assertThat(source.sorts(), is(fieldSorts(topN.order()))); + + assertThat(source.limit(), is(l(10))); + assertThat(source.sorts().size(), is(1)); + EsQueryExec.Sort sort = source.sorts().get(0); + assertThat(sort.direction(), is(Order.OrderDirection.DESC)); + assertThat(name(sort.field()), is("salary")); + assertThat(sort.sortBuilder(), isA(FieldSortBuilder.class)); + assertNull(source.query()); + } + + /** + * ProjectExec[[languages{f}#9, salary{f}#11]] + * \_TopNExec[[Order[salary{f}#11,DESC,FIRST]],10[INTEGER],0] + * \_ExchangeExec[[languages{f}#9, salary{f}#11],false] + * \_ProjectExec[[languages{f}#9, salary{f}#11]] + * \_FieldExtractExec[languages{f}#9, salary{f}#11][] + * \_EsQueryExec[test], + * indexMode[standard], + * query[{"esql_single_value":{ + * "field":"salary", + * "next":{"range":{"salary":{"gt":50000,"boost":1.0}}}, + * "source":"salary > 50000@2:9" + * }}][_doc{f}#26], + * limit[10], + * sort[[FieldSort[field=salary{f}#11, direction=DESC, nulls=FIRST]]] estimatedRowSize[24] + */ + public void testPushTopNWithFilterToSource() { + var optimized = optimizedPlan(physicalPlan(""" + FROM test + | WHERE salary > 50000 + | SORT salary DESC + | LIMIT 10 + | KEEP languages, salary + """)); + + var project = as(optimized, ProjectExec.class); + var topN = as(project.child(), TopNExec.class); + var exchange = asRemoteExchange(topN.child()); + + project = as(exchange.child(), ProjectExec.class); + assertThat(names(project.projections()), contains("languages", "salary")); + var extract = as(project.child(), FieldExtractExec.class); + assertThat(names(extract.attributesToExtract()), contains("languages", "salary")); + var source = source(extract.child()); + assertThat(source.limit(), is(topN.limit())); + assertThat(source.sorts(), is(fieldSorts(topN.order()))); + + assertThat(source.limit(), is(l(10))); + assertThat(source.sorts().size(), is(1)); + EsQueryExec.Sort sort = source.sorts().get(0); + assertThat(sort.direction(), is(Order.OrderDirection.DESC)); + assertThat(name(sort.field()), is("salary")); + assertThat(sort.sortBuilder(), isA(FieldSortBuilder.class)); + var rq = as(sv(source.query(), "salary"), RangeQueryBuilder.class); + assertThat(rq.fieldName(), equalTo("salary")); + assertThat(rq.from(), equalTo(50000)); + assertThat(rq.includeLower(), equalTo(false)); + assertThat(rq.to(), nullValue()); + } + + /** + * ProjectExec[[abbrev{f}#12321, name{f}#12322, location{f}#12325, country{f}#12326, city{f}#12327]] + * \_TopNExec[[Order[abbrev{f}#12321,ASC,LAST]],5[INTEGER],0] + * \_ExchangeExec[[abbrev{f}#12321, name{f}#12322, location{f}#12325, country{f}#12326, city{f}#12327],false] + * \_ProjectExec[[abbrev{f}#12321, name{f}#12322, location{f}#12325, country{f}#12326, city{f}#12327]] + * \_FieldExtractExec[abbrev{f}#12321, name{f}#12322, location{f}#12325, ..][] + * \_EsQueryExec[airports], + * indexMode[standard], + * query[][_doc{f}#12337], + * limit[5], + * sort[[FieldSort[field=abbrev{f}#12321, direction=ASC, nulls=LAST]]] estimatedRowSize[237] + */ + public void testPushTopNKeywordToSource() { + var optimized = optimizedPlan(physicalPlan(""" + FROM airports + | SORT abbrev + | LIMIT 5 + | KEEP abbrev, name, location, country, city + """, airports)); + + var project = as(optimized, ProjectExec.class); + var topN = as(project.child(), TopNExec.class); + var exchange = asRemoteExchange(topN.child()); + + project = as(exchange.child(), ProjectExec.class); + assertThat(names(project.projections()), contains("abbrev", "name", "location", "country", "city")); + var extract = as(project.child(), FieldExtractExec.class); + assertThat(names(extract.attributesToExtract()), contains("abbrev", "name", "location", "country", "city")); + var source = source(extract.child()); + assertThat(source.limit(), is(topN.limit())); + assertThat(source.sorts(), is(fieldSorts(topN.order()))); + + assertThat(source.limit(), is(l(5))); + assertThat(source.sorts().size(), is(1)); + EsQueryExec.Sort sort = source.sorts().get(0); + assertThat(sort.direction(), is(Order.OrderDirection.ASC)); + assertThat(name(sort.field()), is("abbrev")); + assertThat(sort.sortBuilder(), isA(FieldSortBuilder.class)); + assertNull(source.query()); + } + + /** + * + * ProjectExec[[abbrev{f}#12, name{f}#13, location{f}#16, country{f}#17, city{f}#18, abbrev{f}#12 AS code]] + * \_TopNExec[[Order[abbrev{f}#12,ASC,LAST]],5[INTEGER],0] + * \_ExchangeExec[[abbrev{f}#12, name{f}#13, location{f}#16, country{f}#17, city{f}#18],false] + * \_ProjectExec[[abbrev{f}#12, name{f}#13, location{f}#16, country{f}#17, city{f}#18]] + * \_FieldExtractExec[abbrev{f}#12, name{f}#13, location{f}#16, country{f..][] + * \_EsQueryExec[airports], indexMode[standard], query[][_doc{f}#29], limit[5], + * sort[[FieldSort[field=abbrev{f}#12, direction=ASC, nulls=LAST]]] estimatedRowSize[237] + * + */ + public void testPushTopNAliasedKeywordToSource() { + var optimized = optimizedPlan(physicalPlan(""" + FROM airports + | EVAL code = abbrev + | SORT code + | LIMIT 5 + | KEEP abbrev, name, location, country, city, code + """, airports)); + + var project = as(optimized, ProjectExec.class); + assertThat(names(project.projections()), contains("abbrev", "name", "location", "country", "city", "code")); + var topN = as(project.child(), TopNExec.class); + var exchange = asRemoteExchange(topN.child()); + + project = as(exchange.child(), ProjectExec.class); + assertThat(names(project.projections()), contains("abbrev", "name", "location", "country", "city")); + var extract = as(project.child(), FieldExtractExec.class); + assertThat(names(extract.attributesToExtract()), contains("abbrev", "name", "location", "country", "city")); + var source = source(extract.child()); + assertThat(source.limit(), is(topN.limit())); + assertThat(source.sorts(), is(fieldSorts(topN.order()))); + + assertThat(source.limit(), is(l(5))); + assertThat(source.sorts().size(), is(1)); + EsQueryExec.Sort sort = source.sorts().get(0); + assertThat(sort.direction(), is(Order.OrderDirection.ASC)); + assertThat(name(sort.field()), is("abbrev")); + assertThat(sort.sortBuilder(), isA(FieldSortBuilder.class)); + assertNull(source.query()); + } + + /** + * ProjectExec[[abbrev{f}#11, name{f}#12, location{f}#15, country{f}#16, city{f}#17]] + * \_TopNExec[[Order[distance{r}#4,ASC,LAST]],5[INTEGER],0] + * \_ExchangeExec[[abbrev{f}#11, name{f}#12, location{f}#15, country{f}#16, city{f}#17, distance{r}#4],false] + * \_ProjectExec[[abbrev{f}#11, name{f}#12, location{f}#15, country{f}#16, city{f}#17, distance{r}#4]] + * \_FieldExtractExec[abbrev{f}#11, name{f}#12, country{f}#16, city{f}#17][] + * \_EvalExec[[STDISTANCE(location{f}#15,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) + * AS distance]] + * \_FieldExtractExec[location{f}#15][] + * \_EsQueryExec[airports], + * indexMode[standard], + * query[][_doc{f}#28], + * limit[5], + * sort[[GeoDistanceSort[field=location{f}#15, direction=ASC, lat=55.673, lon=12.565]]] estimatedRowSize[245] + */ + public void testPushTopNDistanceToSource() { + var optimized = optimizedPlan(physicalPlan(""" + FROM airports + | EVAL distance = ST_DISTANCE(location, TO_GEOPOINT("POINT(12.565 55.673)")) + | SORT distance ASC + | LIMIT 5 + | KEEP abbrev, name, location, country, city + """, airports)); + + var project = as(optimized, ProjectExec.class); + var topN = as(project.child(), TopNExec.class); + var exchange = asRemoteExchange(topN.child()); + + project = as(exchange.child(), ProjectExec.class); + assertThat(names(project.projections()), contains("abbrev", "name", "location", "country", "city", "distance")); + var extract = as(project.child(), FieldExtractExec.class); + assertThat(names(extract.attributesToExtract()), contains("abbrev", "name", "country", "city")); + var evalExec = as(extract.child(), EvalExec.class); + var alias = as(evalExec.fields().get(0), Alias.class); + assertThat(alias.name(), is("distance")); + var stDistance = as(alias.child(), StDistance.class); + assertThat(stDistance.left().toString(), startsWith("location")); + extract = as(evalExec.child(), FieldExtractExec.class); + assertThat(names(extract.attributesToExtract()), contains("location")); + var source = source(extract.child()); + + // Assert that the TopN(distance) is pushed down as geo-sort(location) + assertThat(source.limit(), is(topN.limit())); + Set orderSet = orderAsSet(topN.order()); + Set sortsSet = sortsAsSet(source.sorts(), Map.of("location", "distance")); + assertThat(orderSet, is(sortsSet)); + + // Fine-grained checks on the pushed down sort + assertThat(source.limit(), is(l(5))); + assertThat(source.sorts().size(), is(1)); + EsQueryExec.Sort sort = source.sorts().get(0); + assertThat(sort.direction(), is(Order.OrderDirection.ASC)); + assertThat(name(sort.field()), is("location")); + assertThat(sort.sortBuilder(), isA(GeoDistanceSortBuilder.class)); + assertNull(source.query()); + } + + /** + * ProjectExec[[abbrev{f}#8, name{f}#9, location{f}#12, country{f}#13, city{f}#14]] + * \_TopNExec[[Order[$$order_by$0$0{r}#16,ASC,LAST]],5[INTEGER],0] + * \_ExchangeExec[[abbrev{f}#8, name{f}#9, location{f}#12, country{f}#13, city{f}#14, $$order_by$0$0{r}#16],false] + * \_ProjectExec[[abbrev{f}#8, name{f}#9, location{f}#12, country{f}#13, city{f}#14, $$order_by$0$0{r}#16]] + * \_FieldExtractExec[abbrev{f}#8, name{f}#9, country{f}#13, city{f}#14][] + * \_EvalExec[[ + * STDISTANCE(location{f}#12,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) AS $$order_by$0$0 + * ]] + * \_FieldExtractExec[location{f}#12][] + * \_EsQueryExec[airports], + * indexMode[standard], + * query[][_doc{f}#26], + * limit[5], + * sort[[GeoDistanceSort[field=location{f}#12, direction=ASC, lat=55.673, lon=12.565]]] estimatedRowSize[245] + */ + public void testPushTopNInlineDistanceToSource() { + var optimized = optimizedPlan(physicalPlan(""" + FROM airports + | SORT ST_DISTANCE(location, TO_GEOPOINT("POINT(12.565 55.673)")) ASC + | LIMIT 5 + | KEEP abbrev, name, location, country, city + """, airports)); + + var project = as(optimized, ProjectExec.class); + var topN = as(project.child(), TopNExec.class); + var exchange = asRemoteExchange(topN.child()); + + project = as(exchange.child(), ProjectExec.class); + assertThat(names(project.projections()), contains("abbrev", "name", "location", "country", "city", "$$order_by$0$0")); + var extract = as(project.child(), FieldExtractExec.class); + assertThat(names(extract.attributesToExtract()), contains("abbrev", "name", "country", "city")); + var evalExec = as(extract.child(), EvalExec.class); + var alias = as(evalExec.fields().get(0), Alias.class); + assertThat(alias.name(), is("$$order_by$0$0")); + var stDistance = as(alias.child(), StDistance.class); + assertThat(stDistance.left().toString(), startsWith("location")); + extract = as(evalExec.child(), FieldExtractExec.class); + assertThat(names(extract.attributesToExtract()), contains("location")); + var source = source(extract.child()); + + // Assert that the TopN(distance) is pushed down as geo-sort(location) + assertThat(source.limit(), is(topN.limit())); + Set orderSet = orderAsSet(topN.order()); + Set sortsSet = sortsAsSet(source.sorts(), Map.of("location", "$$order_by$0$0")); + assertThat(orderSet, is(sortsSet)); + + // Fine-grained checks on the pushed down sort + assertThat(source.limit(), is(l(5))); + assertThat(source.sorts().size(), is(1)); + EsQueryExec.Sort sort = source.sorts().get(0); + assertThat(sort.direction(), is(Order.OrderDirection.ASC)); + assertThat(name(sort.field()), is("location")); + assertThat(sort.sortBuilder(), isA(GeoDistanceSortBuilder.class)); + assertNull(source.query()); + } + + /** + * + * ProjectExec[[abbrev{f}#12, name{f}#13, location{f}#16, country{f}#17, city{f}#18]] + * \_TopNExec[[Order[distance{r}#4,ASC,LAST]],5[INTEGER],0] + * \_ExchangeExec[[abbrev{f}#12, name{f}#13, location{f}#16, country{f}#17, city{f}#18, distance{r}#4],false] + * \_ProjectExec[[abbrev{f}#12, name{f}#13, location{f}#16, country{f}#17, city{f}#18, distance{r}#4]] + * \_FieldExtractExec[abbrev{f}#12, name{f}#13, country{f}#17, city{f}#18][] + * \_EvalExec[[STDISTANCE(location{f}#16,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) AS distance + * ]] + * \_FieldExtractExec[location{f}#16][] + * \_EsQueryExec[airports], indexMode[standard], query[ + * { + * "geo_shape":{ + * "location":{ + * "relation":"DISJOINT", + * "shape":{ + * "type":"Circle", + * "radius":"50000.00000000001m", + * "coordinates":[12.565,55.673] + * } + * } + * } + * }][_doc{f}#29], limit[5], sort[[GeoDistanceSort[field=location{f}#16, direction=ASC, lat=55.673, lon=12.565]]] estimatedRowSize[245] + * + */ + public void testPushTopNDistanceWithFilterToSource() { + var optimized = optimizedPlan(physicalPlan(""" + FROM airports + | EVAL distance = ST_DISTANCE(location, TO_GEOPOINT("POINT(12.565 55.673)")) + | WHERE distance > 50000 + | SORT distance ASC + | LIMIT 5 + | KEEP abbrev, name, location, country, city + """, airports)); + + var project = as(optimized, ProjectExec.class); + var topN = as(project.child(), TopNExec.class); + var exchange = asRemoteExchange(topN.child()); + + project = as(exchange.child(), ProjectExec.class); + assertThat(names(project.projections()), contains("abbrev", "name", "location", "country", "city", "distance")); + var extract = as(project.child(), FieldExtractExec.class); + assertThat(names(extract.attributesToExtract()), contains("abbrev", "name", "country", "city")); + var evalExec = as(extract.child(), EvalExec.class); + var alias = as(evalExec.fields().get(0), Alias.class); + assertThat(alias.name(), is("distance")); + var stDistance = as(alias.child(), StDistance.class); + assertThat(stDistance.left().toString(), startsWith("location")); + extract = as(evalExec.child(), FieldExtractExec.class); + assertThat(names(extract.attributesToExtract()), contains("location")); + var source = source(extract.child()); + + // Assert that the TopN(distance) is pushed down as geo-sort(location) + assertThat(source.limit(), is(topN.limit())); + Set orderSet = orderAsSet(topN.order()); + Set sortsSet = sortsAsSet(source.sorts(), Map.of("location", "distance")); + assertThat(orderSet, is(sortsSet)); + + // Fine-grained checks on the pushed down sort + assertThat(source.limit(), is(l(5))); + assertThat(source.sorts().size(), is(1)); + EsQueryExec.Sort sort = source.sorts().get(0); + assertThat(sort.direction(), is(Order.OrderDirection.ASC)); + assertThat(name(sort.field()), is("location")); + assertThat(sort.sortBuilder(), isA(GeoDistanceSortBuilder.class)); + + var condition = as(source.query(), SpatialRelatesQuery.ShapeQueryBuilder.class); + assertThat("Geometry field name", condition.fieldName(), equalTo("location")); + assertThat("Spatial relationship", condition.relation(), equalTo(ShapeRelation.DISJOINT)); + assertThat("Geometry is Circle", condition.shape().type(), equalTo(ShapeType.CIRCLE)); + var circle = as(condition.shape(), Circle.class); + assertThat("Circle center-x", circle.getX(), equalTo(12.565)); + assertThat("Circle center-y", circle.getY(), equalTo(55.673)); + assertThat("Circle radius for predicate", circle.getRadiusMeters(), closeTo(50000.0, 1e-9)); + } + + /** + * + * ProjectExec[[abbrev{f}#14, name{f}#15, location{f}#18, country{f}#19, city{f}#20]] + * \_TopNExec[[Order[distance{r}#4,ASC,LAST]],5[INTEGER],0] + * \_ExchangeExec[[abbrev{f}#14, name{f}#15, location{f}#18, country{f}#19, city{f}#20, distance{r}#4],false] + * \_ProjectExec[[abbrev{f}#14, name{f}#15, location{f}#18, country{f}#19, city{f}#20, distance{r}#4]] + * \_FieldExtractExec[abbrev{f}#14, name{f}#15, country{f}#19, city{f}#20][] + * \_EvalExec[[STDISTANCE(location{f}#18,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) + * AS distance]] + * \_FieldExtractExec[location{f}#18][] + * \_EsQueryExec[airports], indexMode[standard], query[{ + * "bool":{ + * "filter":[ + * { + * "esql_single_value":{ + * "field":"scalerank", + * "next":{"range":{"scalerank":{"lt":6,"boost":1.0}}}, + * "source":"scalerank lt 6@3:31" + * } + * }, + * { + * "bool":{ + * "must":[ + * {"geo_shape":{ + * "location":{ + * "relation":"INTERSECTS", + * "shape":{"type":"Circle","radius":"499999.99999999994m","coordinates":[12.565,55.673]} + * } + * }}, + * {"geo_shape":{ + * "location":{ + * "relation":"DISJOINT", + * "shape":{"type":"Circle","radius":"10000.000000000002m","coordinates":[12.565,55.673]} + * } + * }} + * ], + * "boost":1.0 + * } + * } + * ], + * "boost":1.0 + * }}][_doc{f}#31], limit[5], sort[[ + * GeoDistanceSort[field=location{f}#18, direction=ASC, lat=55.673, lon=12.565] + * ]] estimatedRowSize[245] + * + */ + public void testPushTopNDistanceWithCompoundFilterToSource() { + var optimized = optimizedPlan(physicalPlan(""" + FROM airports + | EVAL distance = ST_DISTANCE(location, TO_GEOPOINT("POINT(12.565 55.673)")) + | WHERE distance < 500000 AND scalerank < 6 AND distance > 10000 + | SORT distance ASC + | LIMIT 5 + | KEEP abbrev, name, location, country, city + """, airports)); + + var project = as(optimized, ProjectExec.class); + var topN = as(project.child(), TopNExec.class); + var exchange = asRemoteExchange(topN.child()); + + project = as(exchange.child(), ProjectExec.class); + assertThat(names(project.projections()), contains("abbrev", "name", "location", "country", "city", "distance")); + var extract = as(project.child(), FieldExtractExec.class); + assertThat(names(extract.attributesToExtract()), contains("abbrev", "name", "country", "city")); + var evalExec = as(extract.child(), EvalExec.class); + var alias = as(evalExec.fields().get(0), Alias.class); + assertThat(alias.name(), is("distance")); + var stDistance = as(alias.child(), StDistance.class); + assertThat(stDistance.left().toString(), startsWith("location")); + extract = as(evalExec.child(), FieldExtractExec.class); + assertThat(names(extract.attributesToExtract()), contains("location")); + var source = source(extract.child()); + + // Assert that the TopN(distance) is pushed down as geo-sort(location) + assertThat(source.limit(), is(topN.limit())); + Set orderSet = orderAsSet(topN.order()); + Set sortsSet = sortsAsSet(source.sorts(), Map.of("location", "distance")); + assertThat(orderSet, is(sortsSet)); + + // Fine-grained checks on the pushed down sort + assertThat(source.limit(), is(l(5))); + assertThat(source.sorts().size(), is(1)); + EsQueryExec.Sort sort = source.sorts().get(0); + assertThat(sort.direction(), is(Order.OrderDirection.ASC)); + assertThat(name(sort.field()), is("location")); + assertThat(sort.sortBuilder(), isA(GeoDistanceSortBuilder.class)); + + // Fine-grained checks on the pushed down query + var bool = as(source.query(), BoolQueryBuilder.class); + var rangeQueryBuilders = bool.filter().stream().filter(p -> p instanceof SingleValueQuery.Builder).toList(); + assertThat("Expected one range query builder", rangeQueryBuilders.size(), equalTo(1)); + assertThat(((SingleValueQuery.Builder) rangeQueryBuilders.get(0)).field(), equalTo("scalerank")); + var filterBool = bool.filter().stream().filter(p -> p instanceof BoolQueryBuilder).toList(); + var fb = as(filterBool.get(0), BoolQueryBuilder.class); + var shapeQueryBuilders = fb.must().stream().filter(p -> p instanceof SpatialRelatesQuery.ShapeQueryBuilder).toList(); + assertShapeQueryRange(shapeQueryBuilders, 10000.0, 500000.0); + } + + /** + * This test shows that with an additional EVAL used in the filter, we can no longer push down the SORT distance. + * TODO: This could be optimized in future work. Consider moving much of EnableSpatialDistancePushdown into logical planning. + * + * ProjectExec[[abbrev{f}#23, name{f}#24, location{f}#27, country{f}#28, city{f}#29, scalerank{f}#25 AS scale]] + * \_TopNExec[[Order[distance{r}#4,ASC,LAST], Order[scalerank{f}#25,ASC,LAST]],5[INTEGER],0] + * \_ExchangeExec[[abbrev{f}#23, name{f}#24, location{f}#27, country{f}#28, city{f}#29, scalerank{f}#25, distance{r}#4],false] + * \_ProjectExec[[abbrev{f}#23, name{f}#24, location{f}#27, country{f}#28, city{f}#29, scalerank{f}#25, distance{r}#4]] + * \_FieldExtractExec[abbrev{f}#23, name{f}#24, country{f}#28, city{f}#29][] + * \_TopNExec[[Order[distance{r}#4,ASC,LAST], Order[scalerank{f}#25,ASC,LAST]],5[INTEGER],208] + * \_FieldExtractExec[scalerank{f}#25][] + * \_FilterExec[SUBSTRING(position{r}#7,1[INTEGER],5[INTEGER]) == [50 4f 49 4e 54][KEYWORD]] + * \_EvalExec[[ + * STDISTANCE(location{f}#27,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) AS distance, + * TOSTRING(location{f}#27) AS position + * ]] + * \_FieldExtractExec[location{f}#27][] + * \_EsQueryExec[airports], indexMode[standard], query[{ + * "bool":{"filter":[ + * {"esql_single_value":{"field":"scalerank","next":{"range":{"scalerank":{"lt":6,"boost":1.0}}},"source":...}}, + * {"bool":{"must":[ + * {"geo_shape":{"location":{"relation":"INTERSECTS","shape":{...}}}}, + * {"geo_shape":{"location":{"relation":"DISJOINT","shape":{...}}}} + * ],"boost":1.0}}],"boost":1.0}}][_doc{f}#42], limit[], sort[] estimatedRowSize[87] + * + */ + public void testPushTopNDistanceAndNonPushableEvalWithCompoundFilterToSource() { + var optimized = optimizedPlan(physicalPlan(""" + FROM airports + | EVAL distance = ST_DISTANCE(location, TO_GEOPOINT("POINT(12.565 55.673)")), position = location::keyword, scale = scalerank + | WHERE distance < 500000 AND SUBSTRING(position, 1, 5) == "POINT" AND distance > 10000 AND scale < 6 + | SORT distance ASC, scale ASC + | LIMIT 5 + | KEEP abbrev, name, location, country, city, scale + """, airports)); + + var project = as(optimized, ProjectExec.class); + var topN = as(project.child(), TopNExec.class); + var exchange = asRemoteExchange(topN.child()); + + project = as(exchange.child(), ProjectExec.class); + assertThat(names(project.projections()), contains("abbrev", "name", "location", "country", "city", "scalerank", "distance")); + var extract = as(project.child(), FieldExtractExec.class); + assertThat(names(extract.attributesToExtract()), contains("abbrev", "name", "country", "city")); + var topNChild = as(extract.child(), TopNExec.class); + extract = as(topNChild.child(), FieldExtractExec.class); + assertThat(names(extract.attributesToExtract()), contains("scalerank")); + var filter = as(extract.child(), FilterExec.class); + var evalExec = as(filter.child(), EvalExec.class); + assertThat(evalExec.fields().size(), is(2)); + var aliasDistance = as(evalExec.fields().get(0), Alias.class); + assertThat(aliasDistance.name(), is("distance")); + var stDistance = as(aliasDistance.child(), StDistance.class); + assertThat(stDistance.left().toString(), startsWith("location")); + var aliasPosition = as(evalExec.fields().get(1), Alias.class); + assertThat(aliasPosition.name(), is("position")); + extract = as(evalExec.child(), FieldExtractExec.class); + assertThat(names(extract.attributesToExtract()), contains("location")); + var source = source(extract.child()); + + // In this example TopN is not pushed down (we can optimize that in later work) + assertThat(source.limit(), nullValue()); + assertThat(source.sorts(), nullValue()); + + // Fine-grained checks on the pushed down query + var bool = as(source.query(), BoolQueryBuilder.class); + var rangeQueryBuilders = bool.filter().stream().filter(p -> p instanceof SingleValueQuery.Builder).toList(); + assertThat("Expected one range query builder", rangeQueryBuilders.size(), equalTo(1)); + assertThat(((SingleValueQuery.Builder) rangeQueryBuilders.get(0)).field(), equalTo("scalerank")); + var filterBool = bool.filter().stream().filter(p -> p instanceof BoolQueryBuilder).toList(); + var fb = as(filterBool.get(0), BoolQueryBuilder.class); + var shapeQueryBuilders = fb.must().stream().filter(p -> p instanceof SpatialRelatesQuery.ShapeQueryBuilder).toList(); + assertShapeQueryRange(shapeQueryBuilders, 10000.0, 500000.0); + } + + /** + * This test further shows that with a non-aliasing function, with the same name, less gets pushed down. + * + * ProjectExec[[abbrev{f}#23, name{f}#24, location{f}#27, country{f}#28, city{f}#29, scale{r}#10]] + * \_TopNExec[[Order[distance{r}#4,ASC,LAST], Order[scale{r}#10,ASC,LAST]],5[INTEGER],0] + * \_ExchangeExec[[abbrev{f}#23, name{f}#24, location{f}#27, country{f}#28, city{f}#29, scale{r}#10, distance{r}#4],false] + * \_ProjectExec[[abbrev{f}#23, name{f}#24, location{f}#27, country{f}#28, city{f}#29, scale{r}#10, distance{r}#4]] + * \_FieldExtractExec[abbrev{f}#23, name{f}#24, country{f}#28, city{f}#29][] + * \_TopNExec[[Order[distance{r}#4,ASC,LAST], Order[scale{r}#10,ASC,LAST]],5[INTEGER],208] + * \_FilterExec[ + * SUBSTRING(position{r}#7,1[INTEGER],5[INTEGER]) == [50 4f 49 4e 54][KEYWORD] + * AND scale{r}#10 > 3[INTEGER] + * ] + * \_EvalExec[[ + * STDISTANCE(location{f}#27,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) AS distance, + * TOSTRING(location{f}#27) AS position, + * 10[INTEGER] - scalerank{f}#25 AS scale + * ]] + * \_FieldExtractExec[location{f}#27, scalerank{f}#25][] + * \_EsQueryExec[airports], indexMode[standard], query[{ + * "bool":{"must":[ + * {"geo_shape":{"location":{"relation":"INTERSECTS","shape":{...}}}}, + * {"geo_shape":{"location":{"relation":"DISJOINT","shape":{...}}}} + * ],"boost":1.0}}][_doc{f}#42], limit[], sort[] estimatedRowSize[91] + * + */ + public void testPushTopNDistanceAndNonPushableEvalsWithCompoundFilterToSource() { + var optimized = optimizedPlan(physicalPlan(""" + FROM airports + | EVAL distance = ST_DISTANCE(location, TO_GEOPOINT("POINT(12.565 55.673)")), + position = location::keyword, scalerank = 10 - scalerank + | WHERE distance < 500000 AND SUBSTRING(position, 1, 5) == "POINT" AND distance > 10000 AND scalerank > 3 + | SORT distance ASC, scalerank ASC + | LIMIT 5 + | KEEP abbrev, name, location, country, city, scalerank + """, airports)); + var project = as(optimized, ProjectExec.class); + var topN = as(project.child(), TopNExec.class); + var exchange = asRemoteExchange(topN.child()); + + project = as(exchange.child(), ProjectExec.class); + assertThat(names(project.projections()), contains("abbrev", "name", "location", "country", "city", "scalerank", "distance")); + var extract = as(project.child(), FieldExtractExec.class); + assertThat(names(extract.attributesToExtract()), contains("abbrev", "name", "country", "city")); + var topNChild = as(extract.child(), TopNExec.class); + var filter = as(topNChild.child(), FilterExec.class); + assertThat(filter.condition(), isA(And.class)); + var and = (And) filter.condition(); + assertThat(and.left(), isA(Equals.class)); + assertThat(and.right(), isA(GreaterThan.class)); + var evalExec = as(filter.child(), EvalExec.class); + assertThat(evalExec.fields().size(), is(3)); + var aliasDistance = as(evalExec.fields().get(0), Alias.class); + assertThat(aliasDistance.name(), is("distance")); + var stDistance = as(aliasDistance.child(), StDistance.class); + assertThat(stDistance.left().toString(), startsWith("location")); + var aliasPosition = as(evalExec.fields().get(1), Alias.class); + assertThat(aliasPosition.name(), is("position")); + var aliasScale = as(evalExec.fields().get(2), Alias.class); + assertThat(aliasScale.name(), is("scalerank")); + extract = as(evalExec.child(), FieldExtractExec.class); + assertThat(names(extract.attributesToExtract()), contains("location", "scalerank")); + var source = source(extract.child()); + + // In this example TopN is not pushed down (we can optimize that in later work) + assertThat(source.limit(), nullValue()); + assertThat(source.sorts(), nullValue()); + + // Fine-grained checks on the pushed down query, only the spatial distance gets pushed down, not the scale filter + var bool = as(source.query(), BoolQueryBuilder.class); + var shapeQueryBuilders = bool.must().stream().filter(p -> p instanceof SpatialRelatesQuery.ShapeQueryBuilder).toList(); + assertShapeQueryRange(shapeQueryBuilders, 10000.0, 500000.0); + } + + /** + * This test shows that with if the top level AND'd predicate contains a non-pushable component, we should not push anything. + * + * ProjectExec[[abbrev{f}#8612, name{f}#8613, location{f}#8616, country{f}#8617, city{f}#8618, scalerank{f}#8614 AS scale]] + * \_TopNExec[[Order[distance{r}#8596,ASC,LAST], Order[scalerank{f}#8614,ASC,LAST]],5[INTEGER],0] + * \_ExchangeExec[[abbrev{f}#8612, name{f}#8613, location{f}#8616, country{f}#8617, city{f}#8618, + * scalerank{f}#8614, distance{r}#8596 + * ],false] + * \_ProjectExec[[abbrev{f}#8612, name{f}#8613, location{f}#8616, country{f}#8617, city{f}#8618, + * scalerank{f}#8614, distance{r}#8596 + * ]] + * \_FieldExtractExec[abbrev{f}#8612, name{f}#8613, country{f}#8617, city..][] + * \_TopNExec[[Order[distance{r}#8596,ASC,LAST], Order[scalerank{f}#8614,ASC,LAST]],5[INTEGER],208] + * \_FilterExec[ + * distance{r}#8596 < 500000[INTEGER] + * AND distance{r}#8596 > 10000[INTEGER] + * AND scalerank{f}#8614 < 6[INTEGER] + * OR SUBSTRING(TOSTRING(location{f}#8616),1[INTEGER],5[INTEGER]) == [50 4f 49 4e 54][KEYWORD] + * ] + * \_FieldExtractExec[scalerank{f}#8614][] + * \_EvalExec[[ + * STDISTANCE(location{f}#8616,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) AS distance + * ]] + * \_FieldExtractExec[location{f}#8616][] + * \_EsQueryExec[airports], indexMode[standard], query[][_doc{f}#8630], limit[], sort[] estimatedRowSize[37] + * + */ + public void testPushTopNDistanceWithCompoundFilterToSourceAndDisjunctiveNonPushableEval() { + var optimized = optimizedPlan(physicalPlan(""" + FROM airports + | EVAL distance = ST_DISTANCE(location, TO_GEOPOINT("POINT(12.565 55.673)")), scale = scalerank + | WHERE distance < 500000 AND distance > 10000 AND scale < 6 OR SUBSTRING(location::keyword, 1, 5) == "POINT" + | SORT distance ASC, scale ASC + | LIMIT 5 + | KEEP abbrev, name, location, country, city, scale + """, airports)); + + var project = as(optimized, ProjectExec.class); + var topN = as(project.child(), TopNExec.class); + var exchange = asRemoteExchange(topN.child()); + + project = as(exchange.child(), ProjectExec.class); + assertThat(names(project.projections()), contains("abbrev", "name", "location", "country", "city", "scalerank", "distance")); + var extract = as(project.child(), FieldExtractExec.class); + assertThat(names(extract.attributesToExtract()), contains("abbrev", "name", "country", "city")); + var topNChild = as(extract.child(), TopNExec.class); + var filter = as(topNChild.child(), FilterExec.class); + assertThat(filter.condition(), isA(Or.class)); + var filterOr = (Or) filter.condition(); + assertThat(filterOr.left(), isA(And.class)); + assertThat(filterOr.right(), isA(Equals.class)); + extract = as(filter.child(), FieldExtractExec.class); + assertThat(names(extract.attributesToExtract()), contains("scalerank")); + var evalExec = as(extract.child(), EvalExec.class); + assertThat(evalExec.fields().size(), is(1)); + var aliasDistance = as(evalExec.fields().get(0), Alias.class); + assertThat(aliasDistance.name(), is("distance")); + var stDistance = as(aliasDistance.child(), StDistance.class); + assertThat(stDistance.left().toString(), startsWith("location")); + extract = as(evalExec.child(), FieldExtractExec.class); + assertThat(names(extract.attributesToExtract()), contains("location")); + var source = source(extract.child()); + + // In this example neither TopN not filter is pushed down + assertThat(source.limit(), nullValue()); + assertThat(source.sorts(), nullValue()); + assertThat(source.query(), nullValue()); + } + + /** + * + * ProjectExec[[abbrev{f}#15, name{f}#16, location{f}#19, country{f}#20, city{f}#21]] + * \_TopNExec[[Order[scalerank{f}#17,ASC,LAST], Order[distance{r}#4,ASC,LAST]],15[INTEGER],0] + * \_ExchangeExec[[abbrev{f}#15, name{f}#16, location{f}#19, country{f}#20, city{f}#21, scalerank{f}#17, distance{r}#4],false] + * \_ProjectExec[[abbrev{f}#15, name{f}#16, location{f}#19, country{f}#20, city{f}#21, scalerank{f}#17, distance{r}#4]] + * \_FieldExtractExec[abbrev{f}#15, name{f}#16, country{f}#20, city{f}#21, ..][] + * \_EvalExec[[STDISTANCE(location{f}#19,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) + * AS distance]] + * \_FieldExtractExec[location{f}#19][] + * \_EsQueryExec[airports], indexMode[standard], query[{ + * "bool":{ + * "filter":[ + * {"esql_single_value":{"field":"scalerank",...,"source":"scalerank lt 6@3:31"}}, + * {"bool":{"must":[ + * {"geo_shape":{"location":{"relation":"INTERSECTS","shape":{...}}}}, + * {"geo_shape":{"location":{"relation":"DISJOINT","shape":{...}}}} + * ],"boost":1.0}} + * ],"boost":1.0 + * } + * }][_doc{f}#32], limit[], sort[[ + * FieldSort[field=scalerank{f}#17, direction=ASC, nulls=LAST], + * GeoDistanceSort[field=location{f}#19, direction=ASC, lat=55.673, lon=12.565] + * ]] estimatedRowSize[37] + * + */ + public void testPushCompoundTopNDistanceWithCompoundFilterToSource() { + var optimized = optimizedPlan(physicalPlan(""" + FROM airports + | EVAL distance = ST_DISTANCE(location, TO_GEOPOINT("POINT(12.565 55.673)")) + | WHERE distance < 500000 AND scalerank < 6 AND distance > 10000 + | SORT scalerank, distance + | LIMIT 15 + | KEEP abbrev, name, location, country, city + """, airports)); + + var project = as(optimized, ProjectExec.class); + assertThat(names(project.projections()), contains("abbrev", "name", "location", "country", "city")); + var topN = as(project.child(), TopNExec.class); + var exchange = asRemoteExchange(topN.child()); + + project = as(exchange.child(), ProjectExec.class); + assertThat(names(project.projections()), contains("abbrev", "name", "location", "country", "city", "scalerank", "distance")); + var extract = as(project.child(), FieldExtractExec.class); + assertThat(names(extract.attributesToExtract()), contains("abbrev", "name", "country", "city", "scalerank")); + var evalExec = as(extract.child(), EvalExec.class); + var alias = as(evalExec.fields().get(0), Alias.class); + assertThat(alias.name(), is("distance")); + var stDistance = as(alias.child(), StDistance.class); + assertThat(stDistance.left().toString(), startsWith("location")); + extract = as(evalExec.child(), FieldExtractExec.class); + assertThat(names(extract.attributesToExtract()), contains("location")); + var source = source(extract.child()); + + // Assert that the TopN(distance) is pushed down as geo-sort(location) + assertThat(source.limit(), is(topN.limit())); + Set orderSet = orderAsSet(topN.order()); + Set sortsSet = sortsAsSet(source.sorts(), Map.of("location", "distance")); + assertThat(orderSet, is(sortsSet)); + + // Fine-grained checks on the pushed down sort + assertThat(source.limit(), is(l(15))); + assertThat(source.sorts().size(), is(2)); + EsQueryExec.Sort fieldSort = source.sorts().get(0); + assertThat(fieldSort.direction(), is(Order.OrderDirection.ASC)); + assertThat(name(fieldSort.field()), is("scalerank")); + assertThat(fieldSort.sortBuilder(), isA(FieldSortBuilder.class)); + EsQueryExec.Sort distSort = source.sorts().get(1); + assertThat(distSort.direction(), is(Order.OrderDirection.ASC)); + assertThat(name(distSort.field()), is("location")); + assertThat(distSort.sortBuilder(), isA(GeoDistanceSortBuilder.class)); + + // Fine-grained checks on the pushed down query + var bool = as(source.query(), BoolQueryBuilder.class); + var rangeQueryBuilders = bool.filter().stream().filter(p -> p instanceof SingleValueQuery.Builder).toList(); + assertThat("Expected one range query builder", rangeQueryBuilders.size(), equalTo(1)); + assertThat(((SingleValueQuery.Builder) rangeQueryBuilders.get(0)).field(), equalTo("scalerank")); + var filterBool = bool.filter().stream().filter(p -> p instanceof BoolQueryBuilder).toList(); + var fb = as(filterBool.get(0), BoolQueryBuilder.class); + var shapeQueryBuilders = fb.must().stream().filter(p -> p instanceof SpatialRelatesQuery.ShapeQueryBuilder).toList(); + assertShapeQueryRange(shapeQueryBuilders, 10000.0, 500000.0); + } + + /** + * + * TopNExec[[Order[scalerank{f}#15,ASC,LAST], Order[distance{r}#7,ASC,LAST]],15[INTEGER],0] + * \_ExchangeExec[[abbrev{f}#13, city{f}#19, city_location{f}#20, country{f}#18, location{f}#17, name{f}#14, scalerank{f}#15, + * type{f}#16, poi{r}#3, distance{r}#7],false] + * \_ProjectExec[[abbrev{f}#13, city{f}#19, city_location{f}#20, country{f}#18, location{f}#17, name{f}#14, scalerank{f}#15, + * type{f}#16, poi{r}#3, distance{r}#7]] + * \_FieldExtractExec[abbrev{f}#13, city{f}#19, city_location{f}#20, coun..][] + * \_EvalExec[[ + * [1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT] AS poi, + * STDISTANCE(location{f}#17,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) AS distance + * ]] + * \_FieldExtractExec[location{f}#17][] + * \_EsQueryExec[airports], indexMode[standard], query[{ + * "bool":{ + * "filter":[ + * {"esql_single_value":{"field":"scalerank",...,"source":"scalerank lt 6@4:31"}}, + * {"bool":{"must":[ + * {"geo_shape":{"location":{"relation":"INTERSECTS","shape":{...}}}}, + * {"geo_shape":{"location":{"relation":"DISJOINT","shape":{...}}}} + * ],"boost":1.0}} + * ],"boost":1.0 + * } + * }][_doc{f}#31], limit[15], sort[[ + * FieldSort[field=scalerank{f}#15, direction=ASC, nulls=LAST], + * GeoDistanceSort[field=location{f}#17, direction=ASC, lat=55.673, lon=12.565] + * ]] estimatedRowSize[341] + * + */ + public void testPushCompoundTopNDistanceWithCompoundFilterAndCompoundEvalToSource() { + var optimized = optimizedPlan(physicalPlan(""" + FROM airports + | EVAL poi = TO_GEOPOINT("POINT(12.565 55.673)") + | EVAL distance = ST_DISTANCE(location, poi) + | WHERE distance < 500000 AND scalerank < 6 AND distance > 10000 + | SORT scalerank, distance + | LIMIT 15 + """, airports)); + + var topN = as(optimized, TopNExec.class); + var exchange = asRemoteExchange(topN.child()); + + var project = as(exchange.child(), ProjectExec.class); + assertThat( + names(project.projections()), + containsInAnyOrder("abbrev", "name", "type", "location", "country", "city", "city_location", "scalerank", "poi", "distance") + ); + var extract = as(project.child(), FieldExtractExec.class); + assertThat( + names(extract.attributesToExtract()), + containsInAnyOrder("abbrev", "name", "type", "country", "city", "city_location", "scalerank") + ); + var evalExec = as(extract.child(), EvalExec.class); + assertThat(evalExec.fields().size(), is(2)); + var alias1 = as(evalExec.fields().get(0), Alias.class); + assertThat(alias1.name(), is("poi")); + var poi = as(alias1.child(), Literal.class); + assertThat(poi.fold(), instanceOf(BytesRef.class)); + var alias2 = as(evalExec.fields().get(1), Alias.class); + assertThat(alias2.name(), is("distance")); + var stDistance = as(alias2.child(), StDistance.class); + var location = as(stDistance.left(), FieldAttribute.class); + assertThat(location.fieldName(), is("location")); + var poiRef = as(stDistance.right(), Literal.class); + assertThat(poiRef.fold(), instanceOf(BytesRef.class)); + assertThat(poiRef.fold().toString(), is(poi.fold().toString())); + extract = as(evalExec.child(), FieldExtractExec.class); + assertThat(names(extract.attributesToExtract()), contains("location")); + var source = source(extract.child()); + + // Assert that the TopN(distance) is pushed down as geo-sort(location) + assertThat(source.limit(), is(topN.limit())); + Set orderSet = orderAsSet(topN.order()); + Set sortsSet = sortsAsSet(source.sorts(), Map.of("location", "distance")); + assertThat(orderSet, is(sortsSet)); + + // Fine-grained checks on the pushed down sort + assertThat(source.limit(), is(l(15))); + assertThat(source.sorts().size(), is(2)); + EsQueryExec.Sort fieldSort = source.sorts().get(0); + assertThat(fieldSort.direction(), is(Order.OrderDirection.ASC)); + assertThat(name(fieldSort.field()), is("scalerank")); + assertThat(fieldSort.sortBuilder(), isA(FieldSortBuilder.class)); + EsQueryExec.Sort distSort = source.sorts().get(1); + assertThat(distSort.direction(), is(Order.OrderDirection.ASC)); + assertThat(name(distSort.field()), is("location")); + assertThat(distSort.sortBuilder(), isA(GeoDistanceSortBuilder.class)); + + // Fine-grained checks on the pushed down query + var bool = as(source.query(), BoolQueryBuilder.class); + var rangeQueryBuilders = bool.filter().stream().filter(p -> p instanceof SingleValueQuery.Builder).toList(); + assertThat("Expected one range query builder", rangeQueryBuilders.size(), equalTo(1)); + assertThat(((SingleValueQuery.Builder) rangeQueryBuilders.get(0)).field(), equalTo("scalerank")); + var filterBool = bool.filter().stream().filter(p -> p instanceof BoolQueryBuilder).toList(); + var fb = as(filterBool.get(0), BoolQueryBuilder.class); + var shapeQueryBuilders = fb.must().stream().filter(p -> p instanceof SpatialRelatesQuery.ShapeQueryBuilder).toList(); + assertShapeQueryRange(shapeQueryBuilders, 10000.0, 500000.0); + } + + public void testPushCompoundTopNDistanceWithDeeplyNestedCompoundEvalToSource() { + var optimized = optimizedPlan(physicalPlan(""" + FROM airports + | EVAL poi = TO_GEOPOINT("POINT(12.565 55.673)") + | EVAL poi2 = poi, poi3 = poi2 + | EVAL loc2 = location + | EVAL loc3 = loc2 + | EVAL dist = ST_DISTANCE(loc3, poi3) + | EVAL distance = dist + | SORT scalerank, distance + | LIMIT 15 + """, airports)); + + var topN = as(optimized, TopNExec.class); + var exchange = asRemoteExchange(topN.child()); + + var project = as(exchange.child(), ProjectExec.class); + assertThat( + names(project.projections()), + containsInAnyOrder( + "abbrev", + "name", + "type", + "location", + "country", + "city", + "city_location", + "scalerank", + "poi", + "poi2", + "poi3", + "loc2", + "loc3", + "dist", + "distance" + ) + ); + var extract = as(project.child(), FieldExtractExec.class); + assertThat( + names(extract.attributesToExtract()), + containsInAnyOrder("abbrev", "name", "type", "country", "city", "city_location", "scalerank") + ); + var evalExec = as(extract.child(), EvalExec.class); + assertThat(evalExec.fields().size(), is(7)); + var alias1 = as(evalExec.fields().get(0), Alias.class); + assertThat(alias1.name(), is("poi")); + var poi = as(alias1.child(), Literal.class); + assertThat(poi.fold(), instanceOf(BytesRef.class)); + var alias4 = as(evalExec.fields().get(3), Alias.class); + assertThat(alias4.name(), is("loc2")); + as(alias4.child(), FieldAttribute.class); + var alias5 = as(evalExec.fields().get(4), Alias.class); + assertThat(alias5.name(), is("loc3")); + as(alias5.child(), ReferenceAttribute.class); + var alias6 = as(evalExec.fields().get(5), Alias.class); + assertThat(alias6.name(), is("dist")); + var stDistance = as(alias6.child(), StDistance.class); + var refLocation = as(stDistance.left(), ReferenceAttribute.class); + assertThat(refLocation.name(), is("loc3")); + var poiRef = as(stDistance.right(), Literal.class); + assertThat(poiRef.fold(), instanceOf(BytesRef.class)); + assertThat(poiRef.fold().toString(), is(poi.fold().toString())); + var alias7 = as(evalExec.fields().get(6), Alias.class); + assertThat(alias7.name(), is("distance")); + as(alias7.child(), ReferenceAttribute.class); + extract = as(evalExec.child(), FieldExtractExec.class); + assertThat(names(extract.attributesToExtract()), contains("location")); + var source = source(extract.child()); + + // Assert that the TopN(distance) is pushed down as geo-sort(location) + assertThat(source.limit(), is(topN.limit())); + Set orderSet = orderAsSet(topN.order()); + Set sortsSet = sortsAsSet(source.sorts(), Map.of("location", "distance")); + assertThat(orderSet, is(sortsSet)); + + // Fine-grained checks on the pushed down sort + assertThat(source.limit(), is(l(15))); + assertThat(source.sorts().size(), is(2)); + EsQueryExec.Sort fieldSort = source.sorts().get(0); + assertThat(fieldSort.direction(), is(Order.OrderDirection.ASC)); + assertThat(name(fieldSort.field()), is("scalerank")); + assertThat(fieldSort.sortBuilder(), isA(FieldSortBuilder.class)); + EsQueryExec.Sort distSort = source.sorts().get(1); + assertThat(distSort.direction(), is(Order.OrderDirection.ASC)); + assertThat(name(distSort.field()), is("location")); + assertThat(distSort.sortBuilder(), isA(GeoDistanceSortBuilder.class)); + + // No filter is pushed down + assertThat(source.query(), nullValue()); + } + + /** + * TopNExec[[Order[scalerank{f}#15,ASC,LAST], Order[distance{r}#7,ASC,LAST]],15[INTEGER],0] + * \_ExchangeExec[[abbrev{f}#13, city{f}#19, city_location{f}#20, country{f}#18, location{f}#17, name{f}#14, scalerank{f}#15, + * type{f}#16, poi{r}#3, distance{r}#7],false] + * \_ProjectExec[[abbrev{f}#13, city{f}#19, city_location{f}#20, country{f}#18, location{f}#17, name{f}#14, scalerank{f}#15, + * type{f}#16, poi{r}#3, distance{r}#7]] + * \_FieldExtractExec[abbrev{f}#13, city{f}#19, city_location{f}#20, coun..][] + * \_EvalExec[[ + * [1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT] AS poi, + * STDISTANCE(location{f}#17,[1 1 0 0 0 e1 7a 14 ae 47 21 29 40 a0 1a 2f dd 24 d6 4b 40][GEO_POINT]) AS distance] + * ] + * \_FieldExtractExec[location{f}#17][] + * \_EsQueryExec[airports], indexMode[standard], query[{"bool":{ + * "filter":[ + * {"esql_single_value":{"field":"scalerank","next":{"range":{...}},"source":"scalerank lt 6@4:31"}}, + * {"bool":{"must":[ + * {"geo_shape":{"location":{"relation":"INTERSECTS","shape":{...}}}}, + * {"geo_shape":{"location":{"relation":"DISJOINT","shape":{...}}}} + * ],"boost":1.0}} + * ],"boost":1.0 + * }}][_doc{f}#31], limit[15], sort[[ + * FieldSort[field=scalerank{f}#15, direction=ASC, nulls=LAST], + * GeoDistanceSort[field=location{f}#17, direction=ASC, lat=55.673, lon=12.565] + * ]] estimatedRowSize[341] + */ + public void testPushCompoundTopNDistanceWithCompoundFilterAndNestedCompoundEvalToSource() { + var optimized = optimizedPlan(physicalPlan(""" + FROM airports + | EVAL poi = TO_GEOPOINT("POINT(12.565 55.673)") + | EVAL distance = ST_DISTANCE(location, poi) + | WHERE distance < 500000 AND scalerank < 6 AND distance > 10000 + | SORT scalerank, distance + | LIMIT 15 + """, airports)); + + var topN = as(optimized, TopNExec.class); + var exchange = asRemoteExchange(topN.child()); + + var project = as(exchange.child(), ProjectExec.class); + assertThat( + names(project.projections()), + containsInAnyOrder("abbrev", "name", "type", "location", "country", "city", "city_location", "scalerank", "poi", "distance") + ); + var extract = as(project.child(), FieldExtractExec.class); + assertThat( + names(extract.attributesToExtract()), + containsInAnyOrder("abbrev", "name", "type", "country", "city", "city_location", "scalerank") + ); + var evalExec = as(extract.child(), EvalExec.class); + assertThat(evalExec.fields().size(), is(2)); + var alias1 = as(evalExec.fields().get(0), Alias.class); + assertThat(alias1.name(), is("poi")); + var poi = as(alias1.child(), Literal.class); + assertThat(poi.fold(), instanceOf(BytesRef.class)); + var alias2 = as(evalExec.fields().get(1), Alias.class); + assertThat(alias2.name(), is("distance")); + var stDistance = as(alias2.child(), StDistance.class); + var location = as(stDistance.left(), FieldAttribute.class); + assertThat(location.fieldName(), is("location")); + var poiRef = as(stDistance.right(), Literal.class); + assertThat(poiRef.fold(), instanceOf(BytesRef.class)); + assertThat(poiRef.fold().toString(), is(poi.fold().toString())); + extract = as(evalExec.child(), FieldExtractExec.class); + assertThat(names(extract.attributesToExtract()), contains("location")); + var source = source(extract.child()); + + // Assert that the TopN(distance) is pushed down as geo-sort(location) + assertThat(source.limit(), is(topN.limit())); + Set orderSet = orderAsSet(topN.order()); + Set sortsSet = sortsAsSet(source.sorts(), Map.of("location", "distance")); + assertThat(orderSet, is(sortsSet)); + + // Fine-grained checks on the pushed down sort + assertThat(source.limit(), is(l(15))); + assertThat(source.sorts().size(), is(2)); + EsQueryExec.Sort fieldSort = source.sorts().get(0); + assertThat(fieldSort.direction(), is(Order.OrderDirection.ASC)); + assertThat(name(fieldSort.field()), is("scalerank")); + assertThat(fieldSort.sortBuilder(), isA(FieldSortBuilder.class)); + EsQueryExec.Sort distSort = source.sorts().get(1); + assertThat(distSort.direction(), is(Order.OrderDirection.ASC)); + assertThat(name(distSort.field()), is("location")); + assertThat(distSort.sortBuilder(), isA(GeoDistanceSortBuilder.class)); + + // Fine-grained checks on the pushed down query + var bool = as(source.query(), BoolQueryBuilder.class); + var rangeQueryBuilders = bool.filter().stream().filter(p -> p instanceof SingleValueQuery.Builder).toList(); + assertThat("Expected one range query builder", rangeQueryBuilders.size(), equalTo(1)); + assertThat(((SingleValueQuery.Builder) rangeQueryBuilders.get(0)).field(), equalTo("scalerank")); + var filterBool = bool.filter().stream().filter(p -> p instanceof BoolQueryBuilder).toList(); + var fb = as(filterBool.get(0), BoolQueryBuilder.class); + var shapeQueryBuilders = fb.must().stream().filter(p -> p instanceof SpatialRelatesQuery.ShapeQueryBuilder).toList(); + assertShapeQueryRange(shapeQueryBuilders, 10000.0, 500000.0); + } + + private Set orderAsSet(List sorts) { + return sorts.stream().map(o -> ((Attribute) o.child()).name() + "->" + o.direction()).collect(Collectors.toSet()); + } + + private Set sortsAsSet(List sorts, Map fieldMap) { + return sorts.stream() + .map(s -> fieldMap.getOrDefault(s.field().name(), s.field().name()) + "->" + s.direction()) + .collect(Collectors.toSet()); + } + private void assertShapeQueryRange(List shapeQueryBuilders, double min, double max) { assertThat("Expected two shape query builders", shapeQueryBuilders.size(), equalTo(2)); var relationStats = new HashMap(); @@ -3838,7 +5579,7 @@ private void assertShapeQueryRange(List shapeQueryBuilders, double var circle = as(condition.shape(), Circle.class); assertThat("Circle center-x", circle.getX(), equalTo(12.565)); assertThat("Circle center-y", circle.getY(), equalTo(55.673)); - assertThat("Circle radius for shape relation " + condition.relation(), circle.getRadiusMeters(), equalTo(expected)); + assertThat("Circle radius for shape relation " + condition.relation(), circle.getRadiusMeters(), closeTo(expected, 1e-9)); } assertThat("Expected one INTERSECTS and one DISJOINT", relationStats.size(), equalTo(2)); assertThat("Expected one INTERSECTS", relationStats.get(ShapeRelation.INTERSECTS), equalTo(1)); @@ -4796,7 +6537,7 @@ private PhysicalPlan physicalPlan(String query, TestDataSource dataSource) { return physical; } - private List sorts(List orders) { + private List fieldSorts(List orders) { return orders.stream().map(o -> new FieldSort((FieldAttribute) o.child(), o.direction(), o.nullsPosition())).toList(); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFiltersTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFiltersTests.java index 121c0ef337817..49a738f4f4fa3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFiltersTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFiltersTests.java @@ -9,9 +9,14 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.predicate.Predicates; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.expression.function.aggregate.Count; +import org.elasticsearch.xpack.esql.expression.function.scalar.math.Pow; import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike; import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; @@ -20,17 +25,23 @@ import org.elasticsearch.xpack.esql.index.EsIndex; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; +import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.Filter; +import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.plan.logical.Project; import org.elasticsearch.xpack.esql.plan.logical.local.EsqlProject; +import java.util.ArrayList; import java.util.List; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.FOUR; import static org.elasticsearch.xpack.esql.EsqlTestUtils.ONE; import static org.elasticsearch.xpack.esql.EsqlTestUtils.THREE; import static org.elasticsearch.xpack.esql.EsqlTestUtils.TWO; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.as; import static org.elasticsearch.xpack.esql.EsqlTestUtils.getFieldAttribute; import static org.elasticsearch.xpack.esql.EsqlTestUtils.greaterThanOf; import static org.elasticsearch.xpack.esql.EsqlTestUtils.greaterThanOrEqualOf; @@ -77,6 +88,115 @@ public void testPushDownFilter() { assertEquals(new EsqlProject(EMPTY, combinedFilter, projections), new PushDownAndCombineFilters().apply(fb)); } + public void testPushDownFilterPastRenamingProject() { + FieldAttribute a = getFieldAttribute("a"); + FieldAttribute b = getFieldAttribute("b"); + EsRelation relation = relation(List.of(a, b)); + + Alias aRenamed = new Alias(EMPTY, "a_renamed", a); + Alias aRenamedTwice = new Alias(EMPTY, "a_renamed_twice", aRenamed.toAttribute()); + Alias bRenamed = new Alias(EMPTY, "b_renamed", b); + + Project project = new Project(EMPTY, relation, List.of(aRenamed, aRenamedTwice, bRenamed)); + + GreaterThan aRenamedTwiceGreaterThanOne = greaterThanOf(aRenamedTwice.toAttribute(), ONE); + LessThan bRenamedLessThanTwo = lessThanOf(bRenamed.toAttribute(), TWO); + Filter filter = new Filter(EMPTY, project, Predicates.combineAnd(List.of(aRenamedTwiceGreaterThanOne, bRenamedLessThanTwo))); + + LogicalPlan optimized = new PushDownAndCombineFilters().apply(filter); + + Project optimizedProject = as(optimized, Project.class); + assertEquals(optimizedProject.projections(), project.projections()); + Filter optimizedFilter = as(optimizedProject.child(), Filter.class); + assertEquals(optimizedFilter.condition(), Predicates.combineAnd(List.of(greaterThanOf(a, ONE), lessThanOf(b, TWO)))); + EsRelation optimizedRelation = as(optimizedFilter.child(), EsRelation.class); + assertEquals(optimizedRelation, relation); + } + + // ... | eval a_renamed = a, a_renamed_twice = a_renamed, a_squared = pow(a, 2) + // | where a_renamed > 1 and a_renamed_twice < 2 and a_squared < 4 + // -> + // ... | where a > 1 and a < 2 | eval a_renamed = a, a_renamed_twice = a_renamed, non_pushable = pow(a, 2) | where a_squared < 4 + public void testPushDownFilterOnAliasInEval() { + FieldAttribute a = getFieldAttribute("a"); + FieldAttribute b = getFieldAttribute("b"); + EsRelation relation = relation(List.of(a, b)); + + Alias aRenamed = new Alias(EMPTY, "a_renamed", a); + Alias aRenamedTwice = new Alias(EMPTY, "a_renamed_twice", aRenamed.toAttribute()); + Alias bRenamed = new Alias(EMPTY, "b_renamed", b); + Alias aSquared = new Alias(EMPTY, "a_squared", new Pow(EMPTY, a, TWO)); + Eval eval = new Eval(EMPTY, relation, List.of(aRenamed, aRenamedTwice, aSquared, bRenamed)); + + // We'll construct a Filter after the Eval that has conditions that can or cannot be pushed before the Eval. + List pushableConditionsBefore = List.of( + greaterThanOf(a.toAttribute(), TWO), + greaterThanOf(aRenamed.toAttribute(), ONE), + lessThanOf(aRenamedTwice.toAttribute(), TWO), + lessThanOf(aRenamedTwice.toAttribute(), bRenamed.toAttribute()) + ); + List pushableConditionsAfter = List.of( + greaterThanOf(a.toAttribute(), TWO), + greaterThanOf(a.toAttribute(), ONE), + lessThanOf(a.toAttribute(), TWO), + lessThanOf(a.toAttribute(), b.toAttribute()) + ); + List nonPushableConditions = List.of( + lessThanOf(aSquared.toAttribute(), FOUR), + greaterThanOf(aRenamedTwice.toAttribute(), aSquared.toAttribute()) + ); + + // Try different combinations of pushable and non-pushable conditions in the filter while also randomizing their order a bit. + for (int numPushable = 0; numPushable <= pushableConditionsBefore.size(); numPushable++) { + for (int numNonPushable = 0; numNonPushable <= nonPushableConditions.size(); numNonPushable++) { + if (numPushable == 0 && numNonPushable == 0) { + continue; + } + + List conditions = new ArrayList<>(); + + int pushableIndex = 0, nonPushableIndex = 0; + // Loop and add either a pushable or non-pushable condition to the filter. + boolean addPushable; + while (pushableIndex < numPushable || nonPushableIndex < numNonPushable) { + if (pushableIndex == numPushable) { + addPushable = false; + } else if (nonPushableIndex == numNonPushable) { + addPushable = true; + } else { + addPushable = randomBoolean(); + } + + if (addPushable) { + conditions.add(pushableConditionsBefore.get(pushableIndex++)); + } else { + conditions.add(nonPushableConditions.get(nonPushableIndex++)); + } + } + + Filter filter = new Filter(EMPTY, eval, Predicates.combineAnd(conditions)); + + LogicalPlan plan = new PushDownAndCombineFilters().apply(filter); + + if (numNonPushable > 0) { + Filter optimizedFilter = as(plan, Filter.class); + assertEquals(optimizedFilter.condition(), Predicates.combineAnd(nonPushableConditions.subList(0, numNonPushable))); + plan = optimizedFilter.child(); + } + Eval optimizedEval = as(plan, Eval.class); + assertEquals(optimizedEval.fields(), eval.fields()); + plan = optimizedEval.child(); + if (numPushable > 0) { + Filter pushedFilter = as(plan, Filter.class); + assertEquals(pushedFilter.condition(), Predicates.combineAnd(pushableConditionsAfter.subList(0, numPushable))); + plan = pushedFilter.child(); + } + EsRelation optimizedRelation = as(plan, EsRelation.class); + assertEquals(optimizedRelation, relation); + } + } + } + public void testPushDownLikeRlikeFilter() { EsRelation relation = relation(); org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLike conditionA = rlike(getFieldAttribute("a"), "foo"); @@ -125,7 +245,17 @@ public void testSelectivelyPushDownFilterPastFunctionAgg() { assertEquals(expected, new PushDownAndCombineFilters().apply(fb)); } - private EsRelation relation() { - return new EsRelation(EMPTY, new EsIndex(randomAlphaOfLength(8), emptyMap()), randomFrom(IndexMode.values()), randomBoolean()); + private static EsRelation relation() { + return relation(List.of()); + } + + private static EsRelation relation(List fieldAttributes) { + return new EsRelation( + EMPTY, + new EsIndex(randomAlphaOfLength(8), emptyMap()), + fieldAttributes, + randomFrom(IndexMode.values()), + randomBoolean() + ); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSourceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSourceTests.java new file mode 100644 index 0000000000000..0fe7eb6b3d43b --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSourceTests.java @@ -0,0 +1,466 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules.physical.local; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.utils.GeometryValidator; +import org.elasticsearch.geometry.utils.WellKnownBinary; +import org.elasticsearch.geometry.utils.WellKnownText; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.EsqlTestUtils; +import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.Nullability; +import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.EsField; +import org.elasticsearch.xpack.esql.expression.Order; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StDistance; +import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add; +import org.elasticsearch.xpack.esql.index.EsIndex; +import org.elasticsearch.xpack.esql.optimizer.LocalPhysicalOptimizerContext; +import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec; +import org.elasticsearch.xpack.esql.plan.physical.EvalExec; +import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; +import org.elasticsearch.xpack.esql.plan.physical.TopNExec; +import org.elasticsearch.xpack.esql.stats.DisabledSearchStats; + +import java.io.IOException; +import java.nio.ByteOrder; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; +import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; +import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; +import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; +import static org.elasticsearch.xpack.esql.optimizer.rules.physical.local.PushTopNToSourceTests.TestPhysicalPlanBuilder.from; +import static org.elasticsearch.xpack.esql.plan.physical.AbstractPhysicalPlanSerializationTests.randomEstimatedRowSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; + +public class PushTopNToSourceTests extends ESTestCase { + + public void testSimpleSortField() { + // FROM index | SORT field | LIMIT 10 + var query = from("index").sort("field").limit(10); + assertPushdownSort(query); + assertNoPushdownSort(query.asTimeSeries(), "for time series index mode"); + } + + public void testSimpleSortMultipleFields() { + // FROM index | SORT field, integer, double | LIMIT 10 + var query = from("index").sort("field").sort("integer").sort("double").limit(10); + assertPushdownSort(query); + assertNoPushdownSort(query.asTimeSeries(), "for time series index mode"); + } + + public void testSimpleSortFieldAndEvalLiteral() { + // FROM index | EVAL x = 1 | SORT field | LIMIT 10 + var query = from("index").eval("x", e -> e.i(1)).sort("field").limit(10); + assertPushdownSort(query, List.of(EvalExec.class, EsQueryExec.class)); + assertNoPushdownSort(query.asTimeSeries(), "for time series index mode"); + } + + public void testSimpleSortFieldWithAlias() { + // FROM index | EVAL x = field | SORT field | LIMIT 10 + var query = from("index").eval("x", b -> b.field("field")).sort("field").limit(10); + assertPushdownSort(query, List.of(EvalExec.class, EsQueryExec.class)); + assertNoPushdownSort(query.asTimeSeries(), "for time series index mode"); + } + + public void testSimpleSortMultipleFieldsWithAliases() { + // FROM index | EVAL x = field, y = integer, z = double | SORT field, integer, double | LIMIT 10 + var query = from("index").eval("x", b -> b.field("field")) + .eval("y", b -> b.field("integer")) + .eval("z", b -> b.field("double")) + .sort("field") + .sort("integer") + .sort("double") + .limit(10); + assertPushdownSort(query, List.of(EvalExec.class, EsQueryExec.class)); + assertNoPushdownSort(query.asTimeSeries(), "for time series index mode"); + } + + public void testSimpleSortFieldAsAlias() { + // FROM index | EVAL x = field | SORT x | LIMIT 10 + var query = from("index").eval("x", b -> b.field("field")).sort("x").limit(10); + assertPushdownSort(query, Map.of("x", "field"), List.of(EvalExec.class, EsQueryExec.class)); + assertNoPushdownSort(query.asTimeSeries(), "for time series index mode"); + } + + public void testSimpleSortFieldAndEvalSumLiterals() { + // FROM index | EVAL sum = 1 + 2 | SORT field | LIMIT 10 + var query = from("index").eval("sum", b -> b.add(b.i(1), b.i(2))).sort("field").limit(10); + assertPushdownSort(query, List.of(EvalExec.class, EsQueryExec.class)); + assertNoPushdownSort(query.asTimeSeries(), "for time series index mode"); + } + + public void testSimpleSortFieldAndEvalSumLiteralAndField() { + // FROM index | EVAL sum = 1 + integer | SORT integer | LIMIT 10 + var query = from("index").eval("sum", b -> b.add(b.i(1), b.field("integer"))).sort("integer").limit(10); + assertPushdownSort(query, List.of(EvalExec.class, EsQueryExec.class)); + assertNoPushdownSort(query.asTimeSeries(), "for time series index mode"); + } + + public void testSimpleSortEvalSumLiteralAndField() { + // FROM index | EVAL sum = 1 + integer | SORT sum | LIMIT 10 + var query = from("index").eval("sum", b -> b.add(b.i(1), b.field("integer"))).sort("sum").limit(10); + // TODO: Consider supporting this if we can determine that the eval function maintains the same order + assertNoPushdownSort(query, "when sorting on a derived field"); + assertNoPushdownSort(query.asTimeSeries(), "for time series index mode"); + } + + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/114515") + public void testPartiallyPushableSort() { + // FROM index | EVAL sum = 1 + integer | SORT integer, sum, field | LIMIT 10 + var query = from("index").eval("sum", b -> b.add(b.i(1), b.field("integer"))).sort("integer").sort("sum").sort("field").limit(10); + // Both integer and field can be pushed down, but we can only push down the leading sortable fields, so the 'sum' blocks 'field' + assertPushdownSort(query, List.of(query.orders.get(0)), null, List.of(EvalExec.class, EsQueryExec.class)); + assertNoPushdownSort(query.asTimeSeries(), "for time series index mode"); + } + + public void testSortGeoPointField() { + // FROM index | SORT location | LIMIT 10 + var query = from("index").sort("location", Order.OrderDirection.ASC).limit(10); + // NOTE: while geo_point is not sortable, this is checked during logical planning and the physical planner does not know or care + assertPushdownSort(query); + assertNoPushdownSort(query.asTimeSeries(), "for time series index mode"); + } + + public void testSortGeoDistanceFunction() { + // FROM index | EVAL distance = ST_DISTANCE(location, POINT(1 2)) | SORT distance | LIMIT 10 + var query = from("index").eval("distance", b -> b.distance("location", "POINT(1 2)")) + .sort("distance", Order.OrderDirection.ASC) + .limit(10); + // The pushed-down sort will use the underlying field 'location', not the sorted reference field 'distance' + assertPushdownSort(query, Map.of("distance", "location"), List.of(EvalExec.class, EsQueryExec.class)); + assertNoPushdownSort(query.asTimeSeries(), "for time series index mode"); + } + + public void testSortGeoDistanceFunctionInverted() { + // FROM index | EVAL distance = ST_DISTANCE(POINT(1 2), location) | SORT distance | LIMIT 10 + var query = from("index").eval("distance", b -> b.distance("POINT(1 2)", "location")) + .sort("distance", Order.OrderDirection.ASC) + .limit(10); + // The pushed-down sort will use the underlying field 'location', not the sorted reference field 'distance' + assertPushdownSort(query, Map.of("distance", "location"), List.of(EvalExec.class, EsQueryExec.class)); + assertNoPushdownSort(query.asTimeSeries(), "for time series index mode"); + } + + public void testSortGeoDistanceFunctionLiterals() { + // FROM index | EVAL distance = ST_DISTANCE(POINT(2 1), POINT(1 2)) | SORT distance | LIMIT 10 + var query = from("index").eval("distance", b -> b.distance("POINT(2 1)", "POINT(1 2)")) + .sort("distance", Order.OrderDirection.ASC) + .limit(10); + // The pushed-down sort will use the underlying field 'location', not the sorted reference field 'distance' + assertNoPushdownSort(query, "sort on foldable distance function"); + assertNoPushdownSort(query.asTimeSeries(), "for time series index mode"); + } + + public void testSortGeoDistanceFunctionAndFieldsWithAliases() { + // FROM index | EVAL distance = ST_DISTANCE(location, POINT(1 2)), x = field | SORT distance, field, integer | LIMIT 10 + var query = from("index").eval("distance", b -> b.distance("location", "POINT(1 2)")) + .eval("x", b -> b.field("field")) + .sort("distance", Order.OrderDirection.ASC) + .sort("field", Order.OrderDirection.DESC) + .sort("integer", Order.OrderDirection.DESC) + .limit(10); + // The pushed-down sort will use the underlying field 'location', not the sorted reference field 'distance' + assertPushdownSort(query, query.orders, Map.of("distance", "location"), List.of(EvalExec.class, EsQueryExec.class)); + assertNoPushdownSort(query.asTimeSeries(), "for time series index mode"); + } + + public void testSortGeoDistanceFunctionAndFieldsAndAliases() { + // FROM index | EVAL distance = ST_DISTANCE(location, POINT(1 2)), x = field | SORT distance, x, integer | LIMIT 10 + var query = from("index").eval("distance", b -> b.distance("location", "POINT(1 2)")) + .eval("x", b -> b.field("field")) + .sort("distance", Order.OrderDirection.ASC) + .sort("x", Order.OrderDirection.DESC) + .sort("integer", Order.OrderDirection.DESC) + .limit(10); + // The pushed-down sort will use the underlying field 'location', not the sorted reference field 'distance' + assertPushdownSort(query, query.orders, Map.of("distance", "location", "x", "field"), List.of(EvalExec.class, EsQueryExec.class)); + assertNoPushdownSort(query.asTimeSeries(), "for time series index mode"); + } + + public void testSortGeoDistanceFunctionAndFieldsAndManyAliases() { + // FROM index + // | EVAL loc = location, loc2 = loc, loc3 = loc2, distance = ST_DISTANCE(loc3, POINT(1 2)), x = field + // | SORT distance, x, integer + // | LIMIT 10 + var query = from("index").eval("loc", b -> b.field("location")) + .eval("loc2", b -> b.ref("loc")) + .eval("loc3", b -> b.ref("loc2")) + .eval("distance", b -> b.distance("loc3", "POINT(1 2)")) + .eval("x", b -> b.field("field")) + .sort("distance", Order.OrderDirection.ASC) + .sort("x", Order.OrderDirection.DESC) + .sort("integer", Order.OrderDirection.DESC) + .limit(10); + // The pushed-down sort will use the underlying field 'location', not the sorted reference field 'distance' + assertPushdownSort(query, Map.of("distance", "location", "x", "field"), List.of(EvalExec.class, EsQueryExec.class)); + assertNoPushdownSort(query.asTimeSeries(), "for time series index mode"); + } + + private static void assertPushdownSort(TestPhysicalPlanBuilder builder) { + assertPushdownSort(builder, null, List.of(EsQueryExec.class)); + } + + private static void assertPushdownSort(TestPhysicalPlanBuilder builder, List> topClass) { + assertPushdownSort(builder, null, topClass); + } + + private static void assertPushdownSort( + TestPhysicalPlanBuilder builder, + Map fieldMap, + List> topClass + ) { + var topNExec = builder.build(); + var result = pushTopNToSource(topNExec); + assertPushdownSort(result, builder.orders, fieldMap, topClass); + } + + private static void assertPushdownSort( + TestPhysicalPlanBuilder builder, + List expectedSorts, + Map fieldMap, + List> topClass + ) { + var topNExec = builder.build(); + var result = pushTopNToSource(topNExec); + assertPushdownSort(result, expectedSorts, fieldMap, topClass); + } + + private static void assertNoPushdownSort(TestPhysicalPlanBuilder builder, String message) { + var topNExec = builder.build(); + var result = pushTopNToSource(topNExec); + assertNoPushdownSort(result, message); + } + + private static PhysicalPlan pushTopNToSource(TopNExec topNExec) { + var configuration = EsqlTestUtils.configuration("from test"); + var searchStats = new DisabledSearchStats(); + var ctx = new LocalPhysicalOptimizerContext(configuration, searchStats); + var pushTopNToSource = new PushTopNToSource(); + return pushTopNToSource.rule(topNExec, ctx); + } + + private static void assertNoPushdownSort(PhysicalPlan plan, String message) { + var esQueryExec = findEsQueryExec(plan); + var sorts = esQueryExec.sorts(); + assertThat("Expect no sorts " + message, sorts.size(), is(0)); + } + + private static void assertPushdownSort( + PhysicalPlan plan, + List expectedSorts, + Map fieldMap, + List> topClass + ) { + if (topClass != null && topClass.size() > 0) { + PhysicalPlan current = plan; + for (var clazz : topClass) { + assertThat("Expect non-null physical plan class to match " + clazz.getSimpleName(), current, notNullValue()); + assertThat("Expect top physical plan class to match", current.getClass(), is(clazz)); + current = current.children().size() > 0 ? current.children().get(0) : null; + } + if (current != null) { + fail("No more child classes expected in plan, but found: " + current.getClass().getSimpleName()); + } + } + var esQueryExec = findEsQueryExec(plan); + var sorts = esQueryExec.sorts(); + assertThat("Expect sorts count to match", sorts.size(), is(expectedSorts.size())); + for (int i = 0; i < expectedSorts.size(); i++) { + String name = ((Attribute) expectedSorts.get(i).child()).name(); + String fieldName = sorts.get(i).field().fieldName(); + assertThat("Expect sort[" + i + "] name to match", fieldName, is(sortName(name, fieldMap))); + assertThat("Expect sort[" + i + "] direction to match", sorts.get(i).direction(), is(expectedSorts.get(i).direction())); + } + } + + private static String sortName(String name, Map fieldMap) { + return fieldMap != null ? fieldMap.getOrDefault(name, name) : name; + } + + private static EsQueryExec findEsQueryExec(PhysicalPlan plan) { + if (plan instanceof EsQueryExec esQueryExec) { + return esQueryExec; + } + // We assume no physical plans with multiple children would be generated + return findEsQueryExec(plan.children().get(0)); + } + + /** + * This builder allows for easy creation of physical plans using a syntax like `from("index").sort("field").limit(10)`. + * The idea is to create tests that are clearly related to real queries, but also easy to make assertions on. + * It only supports a very small subset of possible plans, with FROM, EVAL and SORT+LIMIT, in that order, matching + * the physical plan rules that are being tested: TopNExec, EvalExec and EsQueryExec. + */ + static class TestPhysicalPlanBuilder { + private final String index; + private final LinkedHashMap fields; + private final LinkedHashMap refs; + private IndexMode indexMode; + private final List aliases = new ArrayList<>(); + private final List orders = new ArrayList<>(); + private int limit = Integer.MAX_VALUE; + + private TestPhysicalPlanBuilder(String index, IndexMode indexMode) { + this.index = index; + this.indexMode = indexMode; + this.fields = new LinkedHashMap<>(); + this.refs = new LinkedHashMap<>(); + addSortableFieldAttributes(this.fields); + } + + private static void addSortableFieldAttributes(Map fields) { + addFieldAttribute(fields, "field", KEYWORD); + addFieldAttribute(fields, "integer", INTEGER); + addFieldAttribute(fields, "double", DOUBLE); + addFieldAttribute(fields, "keyword", KEYWORD); + addFieldAttribute(fields, "location", GEO_POINT); + } + + private static void addFieldAttribute(Map fields, String name, DataType type) { + fields.put(name, new FieldAttribute(Source.EMPTY, name, new EsField(name, type, new HashMap<>(), true))); + } + + static TestPhysicalPlanBuilder from(String index) { + return new TestPhysicalPlanBuilder(index, IndexMode.STANDARD); + } + + public TestPhysicalPlanBuilder eval(Alias... aliases) { + if (orders.isEmpty() == false) { + throw new IllegalArgumentException("Eval must be before sort"); + } + if (aliases.length == 0) { + throw new IllegalArgumentException("At least one alias must be provided"); + } + for (Alias alias : aliases) { + if (refs.containsKey(alias.name())) { + throw new IllegalArgumentException("Reference already exists: " + alias.name()); + } + refs.put( + alias.name(), + new ReferenceAttribute(Source.EMPTY, alias.name(), alias.dataType(), Nullability.FALSE, alias.id(), alias.synthetic()) + ); + this.aliases.add(alias); + } + return this; + } + + public TestPhysicalPlanBuilder eval(String name, Function builder) { + var testExpressionBuilder = new TestExpressionBuilder(); + Expression expression = builder.apply(testExpressionBuilder); + return eval(new Alias(Source.EMPTY, name, expression)); + } + + public TestPhysicalPlanBuilder sort(String field) { + return sort(field, Order.OrderDirection.ASC); + } + + public TestPhysicalPlanBuilder sort(String field, Order.OrderDirection direction) { + Attribute attr = refs.get(field); + if (attr == null) { + attr = fields.get(field); + } + if (attr == null) { + throw new IllegalArgumentException("Field not found: " + field); + } + orders.add(new Order(Source.EMPTY, attr, direction, Order.NullsPosition.LAST)); + return this; + } + + public TestPhysicalPlanBuilder limit(int limit) { + this.limit = limit; + return this; + } + + public TopNExec build() { + EsIndex esIndex = new EsIndex(this.index, Map.of()); + List attributes = new ArrayList<>(fields.values()); + PhysicalPlan child = new EsQueryExec(Source.EMPTY, esIndex, indexMode, attributes, null, null, List.of(), 0); + if (aliases.isEmpty() == false) { + child = new EvalExec(Source.EMPTY, child, aliases); + } + return new TopNExec(Source.EMPTY, child, orders, new Literal(Source.EMPTY, limit, INTEGER), randomEstimatedRowSize()); + } + + public TestPhysicalPlanBuilder asTimeSeries() { + this.indexMode = IndexMode.TIME_SERIES; + return this; + } + + class TestExpressionBuilder { + Expression field(String name) { + return fields.get(name); + } + + Expression ref(String name) { + return refs.get(name); + } + + Expression literal(Object value, DataType dataType) { + return new Literal(Source.EMPTY, value, dataType); + } + + Expression i(int value) { + return new Literal(Source.EMPTY, value, DataType.INTEGER); + } + + Expression d(double value) { + return new Literal(Source.EMPTY, value, DOUBLE); + } + + Expression k(String value) { + return new Literal(Source.EMPTY, value, KEYWORD); + } + + public Expression add(Expression left, Expression right) { + return new Add(Source.EMPTY, left, right); + } + + public Expression distance(String left, String right) { + return new StDistance(Source.EMPTY, geoExpr(left), geoExpr(right)); + } + + private Expression geoExpr(String text) { + if (text.startsWith("POINT")) { + try { + Geometry geometry = WellKnownText.fromWKT(GeometryValidator.NOOP, false, text); + BytesRef bytes = new BytesRef(WellKnownBinary.toWKB(geometry, ByteOrder.LITTLE_ENDIAN)); + return new Literal(Source.EMPTY, bytes, GEO_POINT); + } catch (IOException | ParseException e) { + throw new IllegalArgumentException("Failed to parse WKT: " + text, e); + } + } + if (fields.containsKey(text)) { + return fields.get(text); + } + if (refs.containsKey(text)) { + return refs.get(text); + } + throw new IllegalArgumentException("Unknown field: " + text); + } + } + + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExecSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExecSerializationTests.java index 6bb5111b154e6..6104069769085 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExecSerializationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExecSerializationTests.java @@ -13,20 +13,15 @@ import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.expression.Order; -import org.elasticsearch.xpack.esql.expression.function.FieldAttributeTests; import org.elasticsearch.xpack.esql.index.EsIndex; import org.elasticsearch.xpack.esql.index.EsIndexSerializationTests; import java.io.IOException; import java.util.List; -import static org.elasticsearch.xpack.esql.plan.logical.AbstractLogicalPlanSerializationTests.randomFieldAttributes; - public class EsQueryExecSerializationTests extends AbstractPhysicalPlanSerializationTests { public static EsQueryExec randomEsQueryExec() { Source source = randomSource(); @@ -35,26 +30,14 @@ public static EsQueryExec randomEsQueryExec() { List attrs = randomFieldAttributes(1, 10, false); QueryBuilder query = randomQuery(); Expression limit = new Literal(randomSource(), between(0, Integer.MAX_VALUE), DataType.INTEGER); - List sorts = randomFieldSorts(); Integer estimatedRowSize = randomEstimatedRowSize(); - return new EsQueryExec(source, index, indexMode, attrs, query, limit, sorts, estimatedRowSize); + return new EsQueryExec(source, index, indexMode, attrs, query, limit, EsQueryExec.NO_SORTS, estimatedRowSize); } public static QueryBuilder randomQuery() { return randomBoolean() ? new MatchAllQueryBuilder() : new TermQueryBuilder(randomAlphaOfLength(4), randomAlphaOfLength(4)); } - public static List randomFieldSorts() { - return randomList(0, 4, EsQueryExecSerializationTests::randomFieldSort); - } - - public static EsQueryExec.FieldSort randomFieldSort() { - FieldAttribute field = FieldAttributeTests.createFieldAttribute(0, false); - Order.OrderDirection direction = randomFrom(Order.OrderDirection.values()); - Order.NullsPosition nulls = randomFrom(Order.NullsPosition.values()); - return new EsQueryExec.FieldSort(field, direction, nulls); - } - @Override protected EsQueryExec createTestInstance() { return randomEsQueryExec(); @@ -67,9 +50,8 @@ protected EsQueryExec mutateInstance(EsQueryExec instance) throws IOException { List attrs = instance.attrs(); QueryBuilder query = instance.query(); Expression limit = instance.limit(); - List sorts = instance.sorts(); Integer estimatedRowSize = instance.estimatedRowSize(); - switch (between(0, 6)) { + switch (between(0, 5)) { case 0 -> index = randomValueOtherThan(index, EsIndexSerializationTests::randomEsIndex); case 1 -> indexMode = randomValueOtherThan(indexMode, () -> randomFrom(IndexMode.values())); case 2 -> attrs = randomValueOtherThan(attrs, () -> randomFieldAttributes(1, 10, false)); @@ -78,13 +60,12 @@ protected EsQueryExec mutateInstance(EsQueryExec instance) throws IOException { limit, () -> new Literal(randomSource(), between(0, Integer.MAX_VALUE), DataType.INTEGER) ); - case 5 -> sorts = randomValueOtherThan(sorts, EsQueryExecSerializationTests::randomFieldSorts); - case 6 -> estimatedRowSize = randomValueOtherThan( + case 5 -> estimatedRowSize = randomValueOtherThan( estimatedRowSize, AbstractPhysicalPlanSerializationTests::randomEstimatedRowSize ); } - return new EsQueryExec(instance.source(), index, indexMode, attrs, query, limit, sorts, estimatedRowSize); + return new EsQueryExec(instance.source(), index, indexMode, attrs, query, limit, EsQueryExec.NO_SORTS, estimatedRowSize); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java index 272321b0f350b..f60e5384e1a6f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java @@ -107,6 +107,21 @@ public void testLuceneTopNSourceOperator() throws IOException { assertThat(factory.limit(), equalTo(10)); } + public void testLuceneTopNSourceOperatorDistanceSort() throws IOException { + int estimatedRowSize = randomEstimatedRowSize(estimatedRowSizeIsHuge); + FieldAttribute sortField = new FieldAttribute(Source.EMPTY, "point", new EsField("point", DataType.GEO_POINT, Map.of(), true)); + EsQueryExec.GeoDistanceSort sort = new EsQueryExec.GeoDistanceSort(sortField, Order.OrderDirection.ASC, 1, -1); + Literal limit = new Literal(Source.EMPTY, 10, DataType.INTEGER); + LocalExecutionPlanner.LocalExecutionPlan plan = planner().plan( + new EsQueryExec(Source.EMPTY, index(), IndexMode.STANDARD, List.of(), null, limit, List.of(sort), estimatedRowSize) + ); + assertThat(plan.driverFactories.size(), lessThanOrEqualTo(pragmas.taskConcurrency())); + LocalExecutionPlanner.DriverSupplier supplier = plan.driverFactories.get(0).driverSupplier(); + var factory = (LuceneTopNSourceOperator.Factory) supplier.physicalOperation().sourceOperatorFactory; + assertThat(factory.maxPageSize(), maxPageSizeMatcher(estimatedRowSizeIsHuge, estimatedRowSize)); + assertThat(factory.limit(), equalTo(10)); + } + private int randomEstimatedRowSize(boolean huge) { int hugeBoundary = SourceOperator.MIN_TARGET_PAGE_SIZE * 10; return huge ? between(hugeBoundary, Integer.MAX_VALUE) : between(1, hugeBoundary); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ComputeListenerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ComputeListenerTests.java index da11a790e6f2f..8cfcb605a19d5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ComputeListenerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ComputeListenerTests.java @@ -125,7 +125,7 @@ private ComputeResponse randomResponse(boolean includeExecutionInfo) { public void testEmpty() { PlainActionFuture results = new PlainActionFuture<>(); - EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(); + EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(randomBoolean()); try ( ComputeListener ignored = ComputeListener.create( RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY, @@ -145,7 +145,7 @@ public void testEmpty() { public void testCollectComputeResults() { PlainActionFuture future = new PlainActionFuture<>(); List allProfiles = new ArrayList<>(); - EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(); + EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(randomBoolean()); try ( ComputeListener computeListener = ComputeListener.create( RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY, @@ -194,7 +194,7 @@ public void testAcquireComputeCCSListener() { PlainActionFuture future = new PlainActionFuture<>(); List allProfiles = new ArrayList<>(); String remoteAlias = "rc1"; - EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(); + EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); executionInfo.swapCluster(remoteAlias, (k, v) -> new EsqlExecutionInfo.Cluster(remoteAlias, "logs*", false)); try ( ComputeListener computeListener = ComputeListener.create( @@ -248,7 +248,7 @@ public void testAcquireComputeCCSListener() { public void testAcquireComputeRunningOnRemoteClusterFillsInTookTime() { PlainActionFuture future = new PlainActionFuture<>(); List allProfiles = new ArrayList<>(); - EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(); + EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); String remoteAlias = "rc1"; executionInfo.swapCluster( remoteAlias, @@ -318,7 +318,7 @@ public void testAcquireComputeRunningOnRemoteClusterFillsInTookTime() { public void testAcquireComputeRunningOnQueryingClusterFillsInTookTime() { PlainActionFuture future = new PlainActionFuture<>(); List allProfiles = new ArrayList<>(); - EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(); + EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); String localCluster = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; // we need a remote cluster in the ExecutionInfo in order to simulate a CCS, since ExecutionInfo is only // fully filled in for cross-cluster searches @@ -372,7 +372,7 @@ public void testCancelOnFailure() throws Exception { int failedTasks = between(1, 100); PlainActionFuture rootListener = new PlainActionFuture<>(); CancellableTask rootTask = newTask(); - EsqlExecutionInfo execInfo = new EsqlExecutionInfo(); + EsqlExecutionInfo execInfo = new EsqlExecutionInfo(randomBoolean()); try ( ComputeListener computeListener = ComputeListener.create( RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY, @@ -436,7 +436,7 @@ public void onFailure(Exception e) { } }; CountDownLatch latch = new CountDownLatch(1); - EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(); + EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(randomBoolean()); try ( ComputeListener computeListener = ComputeListener.create( RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/EsqlMediaTypeParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/EsqlMediaTypeParserTests.java index 789d6e5adbfc7..4b9166c621940 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/EsqlMediaTypeParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/EsqlMediaTypeParserTests.java @@ -80,6 +80,18 @@ public void testColumnarWithAcceptText() { assertEquals(e.getMessage(), "Invalid use of [columnar] argument: cannot be used in combination with [txt, csv, tsv] formats"); } + public void testIncludeCCSMetadataWithAcceptText() { + var accept = randomFrom("text/plain", "text/csv", "text/tab-separated-values"); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> getResponseMediaType(reqWithAccept(accept), createTestInstance(false, true)) + ); + assertEquals( + "Invalid use of [include_ccs_metadata] argument: cannot be used in combination with [txt, csv, tsv] formats", + e.getMessage() + ); + } + public void testColumnarWithParamText() { IllegalArgumentException e = expectThrows( IllegalArgumentException.class, @@ -88,6 +100,26 @@ public void testColumnarWithParamText() { assertEquals(e.getMessage(), "Invalid use of [columnar] argument: cannot be used in combination with [txt, csv, tsv] formats"); } + public void testIncludeCCSMetadataWithNonJSONMediaTypesInParams() { + { + RestRequest restRequest = reqWithParams(Map.of("format", randomFrom("txt", "csv", "tsv"))); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> getResponseMediaType(restRequest, createTestInstance(false, true)) + ); + assertEquals( + "Invalid use of [include_ccs_metadata] argument: cannot be used in combination with [txt, csv, tsv] formats", + e.getMessage() + ); + } + { + // check that no exception is thrown for the XContent types + RestRequest restRequest = reqWithParams(Map.of("format", randomFrom("SMILE", "YAML", "CBOR", "JSON"))); + MediaType responseMediaType = getResponseMediaType(restRequest, createTestInstance(true, true)); + assertNotNull(responseMediaType); + } + } + public void testNoFormat() { IllegalArgumentException e = expectThrows( IllegalArgumentException.class, @@ -113,4 +145,10 @@ protected EsqlQueryRequest createTestInstance(boolean columnar) { request.columnar(columnar); return request; } + + protected EsqlQueryRequest createTestInstance(boolean columnar, boolean includeCCSMetadata) { + var request = createTestInstance(columnar); + request.includeCCSMetadata(includeCCSMetadata); + return request; + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java index 326756ad0b5f4..7e93213fcee21 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java @@ -30,7 +30,7 @@ public void testUpdateExecutionInfoWithUnavailableClusters() { final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; final String remote1Alias = "remote1"; final String remote2Alias = "remote2"; - EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(); + EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false)); executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true)); executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", true)); @@ -59,7 +59,7 @@ public void testUpdateExecutionInfoWithUnavailableClusters() { final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; final String remote1Alias = "remote1"; final String remote2Alias = "remote2"; - EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(); + EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false)); executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true)); executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", false)); @@ -87,7 +87,7 @@ public void testUpdateExecutionInfoWithUnavailableClusters() { final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; final String remote1Alias = "remote1"; final String remote2Alias = "remote2"; - EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(); + EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false)); executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true)); executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", false)); @@ -117,7 +117,7 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() { final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; final String remote1Alias = "remote1"; final String remote2Alias = "remote2"; - EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(); + EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false)); executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true)); executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", false)); @@ -160,7 +160,7 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() { final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; final String remote1Alias = "remote1"; final String remote2Alias = "remote2"; - EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(); + EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false)); executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true)); executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", false)); @@ -206,7 +206,7 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() { final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; final String remote1Alias = "remote1"; final String remote2Alias = "remote2"; - EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(); + EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false)); executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true)); executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", false)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java index adc449bfc092e..9edc85223e7b3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java @@ -120,7 +120,7 @@ public void testFailedMetric() { randomAlphaOfLength(10), EsqlTestUtils.TEST_CFG, enrichResolver, - new EsqlExecutionInfo(), + new EsqlExecutionInfo(randomBoolean()), groupIndicesByCluster, runPhase, new ActionListener<>() { @@ -149,7 +149,7 @@ public void onFailure(Exception e) { randomAlphaOfLength(10), EsqlTestUtils.TEST_CFG, enrichResolver, - new EsqlExecutionInfo(), + new EsqlExecutionInfo(randomBoolean()), groupIndicesByCluster, runPhase, new ActionListener<>() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java index 0ab3980f112ef..d186b4c199d77 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java @@ -87,6 +87,7 @@ import static java.util.Collections.emptyList; import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomConfiguration; +import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; import static org.mockito.Mockito.mock; /** @@ -421,7 +422,11 @@ public void accept(Page page) { // Grok.Parser is a record / final, cannot be mocked return Grok.pattern(Source.EMPTY, randomGrokPattern()); } else if (argClass == EsQueryExec.FieldSort.class) { + // TODO: It appears neither FieldSort nor GeoDistanceSort are ever actually tested return randomFieldSort(); + } else if (argClass == EsQueryExec.GeoDistanceSort.class) { + // TODO: It appears neither FieldSort nor GeoDistanceSort are ever actually tested + return randomGeoDistanceSort(); } else if (toBuildClass == Pow.class && Expression.class.isAssignableFrom(argClass)) { return randomResolvedExpression(randomBoolean() ? FieldAttribute.class : Literal.class); } else if (isPlanNodeClass(toBuildClass) && Expression.class.isAssignableFrom(argClass)) { @@ -679,6 +684,15 @@ static EsQueryExec.FieldSort randomFieldSort() { ); } + static EsQueryExec.GeoDistanceSort randomGeoDistanceSort() { + return new EsQueryExec.GeoDistanceSort( + field(randomAlphaOfLength(16), GEO_POINT), + randomFrom(EnumSet.allOf(Order.OrderDirection.class)), + randomDoubleBetween(-90, 90, false), + randomDoubleBetween(-180, 180, false) + ); + } + static FieldAttribute field(String name, DataType type) { return new FieldAttribute(Source.EMPTY, name, new EsField(name, type, Collections.emptyMap(), false)); } diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java index f82b6f155c0a0..3ca6b45c2948e 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java @@ -81,6 +81,21 @@ static String mockSparseServiceModelConfig(@Nullable TaskType taskTypeInBody) { """, taskType); } + static String updateConfig(@Nullable TaskType taskTypeInBody, String apiKey, int temperature) { + var taskType = taskTypeInBody == null ? "" : "\"task_type\": \"" + taskTypeInBody + "\","; + return Strings.format(""" + { + %s + "service_settings": { + "api_key": "%s" + }, + "task_settings": { + "temperature": %d + } + } + """, taskType, apiKey, temperature); + } + static String mockCompletionServiceModelConfig(@Nullable TaskType taskTypeInBody) { var taskType = taskTypeInBody == null ? "" : "\"task_type\": \"" + taskTypeInBody + "\","; return Strings.format(""" @@ -196,6 +211,11 @@ protected Map putModel(String modelId, String modelConfig, TaskT return putRequest(endpoint, modelConfig); } + protected Map updateEndpoint(String inferenceID, String modelConfig, TaskType taskType) throws IOException { + String endpoint = Strings.format("_inference/%s/%s/_update", taskType, inferenceID); + return putRequest(endpoint, modelConfig); + } + protected Map putPipeline(String pipelineId, String modelId) throws IOException { String endpoint = Strings.format("_ingest/pipeline/%s", pipelineId); String body = """ diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java index 5a84fd8985504..98c8d43707219 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java @@ -16,6 +16,8 @@ import java.io.IOException; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.function.Function; import java.util.stream.IntStream; @@ -29,7 +31,7 @@ public class InferenceCrudIT extends InferenceBaseRestTest { @SuppressWarnings("unchecked") - public void testGet() throws IOException { + public void testCRUD() throws IOException { for (int i = 0; i < 5; i++) { putModel("se_model_" + i, mockSparseServiceModelConfig(), TaskType.SPARSE_EMBEDDING); } @@ -53,11 +55,29 @@ public void testGet() throws IOException { for (var denseModel : getDenseModels) { assertEquals("text_embedding", denseModel.get("task_type")); } - - var singleModel = getModels("se_model_1", TaskType.SPARSE_EMBEDDING); - assertThat(singleModel, hasSize(1)); - assertEquals("se_model_1", singleModel.get(0).get("inference_id")); - + String oldApiKey; + { + var singleModel = getModels("se_model_1", TaskType.SPARSE_EMBEDDING); + assertThat(singleModel, hasSize(1)); + assertEquals("se_model_1", singleModel.get(0).get("inference_id")); + oldApiKey = (String) singleModel.get(0).get("api_key"); + } + var newApiKey = randomAlphaOfLength(10); + int temperature = randomIntBetween(1, 10); + Map updatedEndpoint = updateEndpoint( + "se_model_1", + updateConfig(TaskType.SPARSE_EMBEDDING, newApiKey, temperature), + TaskType.SPARSE_EMBEDDING + ); + Map updatedTaskSettings = (Map) updatedEndpoint.get("task_settings"); + assertEquals(temperature, updatedTaskSettings.get("temperature")); + { + var singleModel = getModels("se_model_1", TaskType.SPARSE_EMBEDDING); + assertThat(singleModel, hasSize(1)); + assertEquals("se_model_1", singleModel.get(0).get("inference_id")); + assertNotEquals(oldApiKey, newApiKey); + assertEquals(updatedEndpoint, singleModel.get(0)); + } for (int i = 0; i < 5; i++) { deleteModel("se_model_" + i, TaskType.SPARSE_EMBEDDING); } diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/AbstractTestInferenceService.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/AbstractTestInferenceService.java index 02dfff1b5c2e6..6496bcdd89f21 100644 --- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/AbstractTestInferenceService.java +++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/AbstractTestInferenceService.java @@ -163,6 +163,11 @@ public String getWriteableName() { public TransportVersion getMinimalSupportedVersion() { return TransportVersion.current(); // fine for these tests but will not work for cluster upgrade tests } + + @Override + public TaskSettings updatedTaskSettings(Map newSettings) { + return fromMap(new HashMap<>(newSettings)); + } } public record TestSecretSettings(String apiKey) implements SecretSettings { @@ -211,5 +216,10 @@ public String getWriteableName() { public TransportVersion getMinimalSupportedVersion() { return TransportVersion.current(); // fine for these tests but will not work for cluster upgrade tests } + + @Override + public SecretSettings newSecretSettings(Map newSecrets) { + return TestSecretSettings.fromMap(new HashMap<>(newSecrets)); + } } } diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java index 1a2f0fb6a1137..a76c4303268e4 100644 --- a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java +++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java @@ -600,6 +600,10 @@ public void writeTo(StreamOutput out) throws IOException { public boolean isEmpty() { return true; } + + public TaskSettings updatedTaskSettings(Map newSettings) { + return this; + } } record TestSecretSettings(String key) implements SecretSettings { @@ -625,6 +629,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); return builder; } + + @Override + public SecretSettings newSecretSettings(Map newSecrets) { + return new TestSecretSettings(newSecrets.get("secret").toString()); + } } TestModelOfAnyKind(String inferenceEntityId, TaskType taskType, String service) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java index 927fd94809886..d361ce0837b93 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java @@ -47,12 +47,14 @@ import org.elasticsearch.xpack.core.inference.action.GetInferenceModelAction; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.core.inference.action.PutInferenceModelAction; +import org.elasticsearch.xpack.core.inference.action.UpdateInferenceModelAction; import org.elasticsearch.xpack.inference.action.TransportDeleteInferenceEndpointAction; import org.elasticsearch.xpack.inference.action.TransportGetInferenceDiagnosticsAction; import org.elasticsearch.xpack.inference.action.TransportGetInferenceModelAction; import org.elasticsearch.xpack.inference.action.TransportInferenceAction; import org.elasticsearch.xpack.inference.action.TransportInferenceUsageAction; import org.elasticsearch.xpack.inference.action.TransportPutInferenceModelAction; +import org.elasticsearch.xpack.inference.action.TransportUpdateInferenceModelAction; import org.elasticsearch.xpack.inference.action.filter.ShardBulkInferenceActionFilter; import org.elasticsearch.xpack.inference.common.Truncator; import org.elasticsearch.xpack.inference.external.amazonbedrock.AmazonBedrockRequestSender; @@ -76,6 +78,7 @@ import org.elasticsearch.xpack.inference.rest.RestInferenceAction; import org.elasticsearch.xpack.inference.rest.RestPutInferenceModelAction; import org.elasticsearch.xpack.inference.rest.RestStreamInferenceAction; +import org.elasticsearch.xpack.inference.rest.RestUpdateInferenceModelAction; import org.elasticsearch.xpack.inference.services.ServiceComponents; import org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchService; import org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockService; @@ -149,6 +152,7 @@ public InferencePlugin(Settings settings) { new ActionHandler<>(InferenceAction.INSTANCE, TransportInferenceAction.class), new ActionHandler<>(GetInferenceModelAction.INSTANCE, TransportGetInferenceModelAction.class), new ActionHandler<>(PutInferenceModelAction.INSTANCE, TransportPutInferenceModelAction.class), + new ActionHandler<>(UpdateInferenceModelAction.INSTANCE, TransportUpdateInferenceModelAction.class), new ActionHandler<>(DeleteInferenceEndpointAction.INSTANCE, TransportDeleteInferenceEndpointAction.class), new ActionHandler<>(XPackUsageFeatureAction.INFERENCE, TransportInferenceUsageAction.class), new ActionHandler<>(GetInferenceDiagnosticsAction.INSTANCE, TransportGetInferenceDiagnosticsAction.class) @@ -172,6 +176,7 @@ public List getRestHandlers( new RestStreamInferenceAction(), new RestGetInferenceModelAction(), new RestPutInferenceModelAction(), + new RestUpdateInferenceModelAction(), new RestDeleteInferenceEndpointAction(), new RestGetInferenceDiagnosticsAction() ); @@ -212,8 +217,10 @@ public Collection createComponents(PluginServices services) { // reference correctly var registry = new InferenceServiceRegistry(inferenceServices, factoryContext); registry.init(services.client()); - for (var service : registry.getServices().values()) { - service.defaultConfigs().forEach(modelRegistry::addDefaultConfiguration); + if (DefaultElserFeatureFlag.isEnabled()) { + for (var service : registry.getServices().values()) { + service.defaultConfigs().forEach(modelRegistry::addDefaultConfiguration); + } } inferenceServiceRegistry.set(registry); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java index 49d65b6e0dc59..64eeed82ee1b9 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java @@ -41,6 +41,7 @@ import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.inference.InferencePlugin; import org.elasticsearch.xpack.inference.registry.ModelRegistry; +import org.elasticsearch.xpack.inference.services.ServiceUtils; import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService; import java.io.IOException; @@ -100,7 +101,7 @@ protected void masterOperation( ActionListener listener ) throws Exception { var requestAsMap = requestToMap(request); - var resolvedTaskType = resolveTaskType(request.getTaskType(), (String) requestAsMap.remove(TaskType.NAME)); + var resolvedTaskType = ServiceUtils.resolveTaskType(request.getTaskType(), (String) requestAsMap.remove(TaskType.NAME)); String serviceName = (String) requestAsMap.remove(ModelConfigurations.SERVICE); if (serviceName == null) { @@ -227,37 +228,4 @@ protected ClusterBlockException checkBlock(PutInferenceModelAction.Request reque return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); } - /** - * task_type can be specified as either a URL parameter or in the - * request body. Resolve which to use or throw if the settings are - * inconsistent - * @param urlTaskType Taken from the URL parameter. ANY means not specified. - * @param bodyTaskType Taken from the request body. Maybe null - * @return The resolved task type - */ - static TaskType resolveTaskType(TaskType urlTaskType, String bodyTaskType) { - if (bodyTaskType == null) { - if (urlTaskType == TaskType.ANY) { - throw new ElasticsearchStatusException("model is missing required setting [task_type]", RestStatus.BAD_REQUEST); - } else { - return urlTaskType; - } - } - - TaskType parsedBodyTask = TaskType.fromStringOrStatusException(bodyTaskType); - if (parsedBodyTask == TaskType.ANY) { - throw new ElasticsearchStatusException("task_type [any] is not valid type for inference", RestStatus.BAD_REQUEST); - } - - if (parsedBodyTask.isAnyOrSame(urlTaskType) == false) { - throw new ElasticsearchStatusException( - "Cannot resolve conflicting task_type parameter in the request URL [{}] and the request body [{}]", - RestStatus.BAD_REQUEST, - urlTaskType.toString(), - bodyTaskType - ); - } - - return parsedBodyTask; - } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUpdateInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUpdateInferenceModelAction.java new file mode 100644 index 0000000000000..03a88e5228fa8 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUpdateInferenceModelAction.java @@ -0,0 +1,328 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.action; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.SubscribableListener; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.inference.InferenceService; +import org.elasticsearch.inference.InferenceServiceRegistry; +import org.elasticsearch.inference.Model; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.ModelSecrets; +import org.elasticsearch.inference.SecretSettings; +import org.elasticsearch.inference.ServiceSettings; +import org.elasticsearch.inference.TaskSettings; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.UnparsedModel; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xpack.core.inference.action.UpdateInferenceModelAction; +import org.elasticsearch.xpack.core.ml.action.CreateTrainedModelAssignmentAction; +import org.elasticsearch.xpack.core.ml.action.UpdateTrainedModelDeploymentAction; +import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignmentUtils; +import org.elasticsearch.xpack.core.ml.job.messages.Messages; +import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.inference.registry.ModelRegistry; +import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService; +import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalServiceSettings; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; + +import static org.elasticsearch.xpack.inference.services.ServiceUtils.resolveTaskType; +import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalServiceSettings.NUM_ALLOCATIONS; + +public class TransportUpdateInferenceModelAction extends TransportMasterNodeAction< + UpdateInferenceModelAction.Request, + UpdateInferenceModelAction.Response> { + + private static final Logger logger = LogManager.getLogger(TransportUpdateInferenceModelAction.class); + + private final ModelRegistry modelRegistry; + private final InferenceServiceRegistry serviceRegistry; + private final Client client; + + @Inject + public TransportUpdateInferenceModelAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + ModelRegistry modelRegistry, + InferenceServiceRegistry serviceRegistry, + Client client, + Settings settings + ) { + super( + UpdateInferenceModelAction.NAME, + transportService, + clusterService, + threadPool, + actionFilters, + UpdateInferenceModelAction.Request::new, + indexNameExpressionResolver, + UpdateInferenceModelAction.Response::new, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.modelRegistry = modelRegistry; + this.serviceRegistry = serviceRegistry; + this.client = client; + } + + @Override + protected void masterOperation( + Task task, + UpdateInferenceModelAction.Request request, + ClusterState state, + ActionListener masterListener + ) { + var bodyTaskType = request.getContentAsSettings().taskType(); + var resolvedTaskType = resolveTaskType(request.getTaskType(), bodyTaskType != null ? bodyTaskType.toString() : null); + + AtomicReference service = new AtomicReference<>(); + + var inferenceEntityId = request.getInferenceEntityId(); + + SubscribableListener.newForked(listener -> { checkEndpointExists(inferenceEntityId, listener); }) + .andThen((listener, unparsedModel) -> { + + Optional optionalService = serviceRegistry.getService(unparsedModel.service()); + if (optionalService.isEmpty()) { + listener.onFailure( + new ElasticsearchStatusException( + "Service [{}] not found", + RestStatus.INTERNAL_SERVER_ERROR, + unparsedModel.service() + ) + ); + } else { + service.set(optionalService.get()); + listener.onResponse(unparsedModel); + } + }) + .andThen((listener, existingUnparsedModel) -> { + + Model existingParsedModel = service.get() + .parsePersistedConfigWithSecrets( + request.getInferenceEntityId(), + existingUnparsedModel.taskType(), + new HashMap<>(existingUnparsedModel.settings()), + new HashMap<>(existingUnparsedModel.secrets()) + ); + + Model newModel = combineExistingModelWithNewSettings( + existingParsedModel, + request.getContentAsSettings(), + service.get().name(), + resolvedTaskType + ); + + if (isInClusterService(service.get().name())) { + updateInClusterEndpoint(request, newModel, existingParsedModel, listener); + } else { + modelRegistry.updateModelTransaction(newModel, existingParsedModel, listener); + } + }) + .andThen((listener, didUpdate) -> { + if (didUpdate) { + modelRegistry.getModel(inferenceEntityId, ActionListener.wrap((unparsedModel) -> { + if (unparsedModel == null) { + listener.onFailure( + new ElasticsearchStatusException( + "Failed to update model, updated model not found", + RestStatus.INTERNAL_SERVER_ERROR + ) + ); + } else { + listener.onResponse( + service.get() + .parsePersistedConfig( + request.getInferenceEntityId(), + resolvedTaskType, + new HashMap<>(unparsedModel.settings()) + ) + .getConfigurations() + ); + } + }, listener::onFailure)); + } else { + listener.onFailure(new ElasticsearchStatusException("Failed to update model", RestStatus.INTERNAL_SERVER_ERROR)); + } + + }).andThen((listener, modelConfig) -> { + listener.onResponse(new UpdateInferenceModelAction.Response(modelConfig)); + }) + .addListener(masterListener); + } + + /** + * Combines the existing model with the new settings to create a new model using the + * SecretSettings and TaskSettings implementations for each service, as well as specifically handling NUM_ALLOCATIONS. + * + * @param existingParsedModel the Model representing a third-party service endpoint + * @param settingsToUpdate new settings + * @param serviceName + * @return a new object representing the updated model + */ + private Model combineExistingModelWithNewSettings( + Model existingParsedModel, + UpdateInferenceModelAction.Settings settingsToUpdate, + String serviceName, + TaskType resolvedTaskType + ) { + ModelConfigurations existingConfigs = existingParsedModel.getConfigurations(); + TaskSettings existingTaskSettings = existingConfigs.getTaskSettings(); + SecretSettings existingSecretSettings = existingParsedModel.getSecretSettings(); + + SecretSettings newSecretSettings = existingSecretSettings; + TaskSettings newTaskSettings = existingTaskSettings; + ServiceSettings newServiceSettings = existingConfigs.getServiceSettings(); + + if (settingsToUpdate.serviceSettings() != null && existingSecretSettings != null) { + newSecretSettings = existingSecretSettings.newSecretSettings(settingsToUpdate.serviceSettings()); + } + if (settingsToUpdate.serviceSettings() != null && settingsToUpdate.serviceSettings().containsKey(NUM_ALLOCATIONS)) { + // In cluster services can only have their num_allocations updated, so this is a special case + if (newServiceSettings instanceof ElasticsearchInternalServiceSettings elasticServiceSettings) { + newServiceSettings = new ElasticsearchInternalServiceSettings( + elasticServiceSettings, + (Integer) settingsToUpdate.serviceSettings().get(NUM_ALLOCATIONS) + ); + } + } + if (settingsToUpdate.taskSettings() != null && existingTaskSettings != null) { + newTaskSettings = existingTaskSettings.updatedTaskSettings(settingsToUpdate.taskSettings()); + } + + if (existingParsedModel.getTaskType().equals(resolvedTaskType) == false) { + throw new ElasticsearchStatusException("Task type must match the task type of the existing endpoint", RestStatus.BAD_REQUEST); + } + + ModelConfigurations newModelConfigs = new ModelConfigurations( + existingParsedModel.getInferenceEntityId(), + existingParsedModel.getTaskType(), + serviceName, + newServiceSettings, + newTaskSettings + ); + + return new Model(newModelConfigs, new ModelSecrets(newSecretSettings)); + } + + private void updateInClusterEndpoint( + UpdateInferenceModelAction.Request request, + Model newModel, + Model existingParsedModel, + ActionListener listener + ) throws IOException { + // The model we are trying to update must have a trained model associated with it if it is an in-cluster deployment + throwIfTrainedModelDoesntExist(request); + + Map serviceSettings = request.getContentAsSettings().serviceSettings(); + if (serviceSettings != null && serviceSettings.get(NUM_ALLOCATIONS) instanceof Integer numAllocations) { + + UpdateTrainedModelDeploymentAction.Request updateRequest = new UpdateTrainedModelDeploymentAction.Request( + request.getInferenceEntityId() + ); + updateRequest.setNumberOfAllocations(numAllocations); + + var delegate = listener.delegateFailure((l2, response) -> { + modelRegistry.updateModelTransaction(newModel, existingParsedModel, l2); + }); + + logger.info( + "Updating trained model deployment for inference entity [{}] with [{}] num_allocations", + request.getInferenceEntityId(), + numAllocations + ); + client.execute(UpdateTrainedModelDeploymentAction.INSTANCE, updateRequest, delegate); + + } else { + listener.onFailure( + new ElasticsearchStatusException( + "Failed to parse [{}] of update request [{}]", + RestStatus.BAD_REQUEST, + NUM_ALLOCATIONS, + request.getContent().utf8ToString() + ) + ); + } + + } + + private boolean isInClusterService(String name) { + return List.of(ElasticsearchInternalService.NAME, ElasticsearchInternalService.OLD_ELSER_SERVICE_NAME).contains(name); + } + + private void throwIfTrainedModelDoesntExist(UpdateInferenceModelAction.Request request) throws ElasticsearchStatusException { + var assignments = TrainedModelAssignmentUtils.modelAssignments(request.getInferenceEntityId(), clusterService.state()); + if ((assignments == null || assignments.isEmpty())) { + throw ExceptionsHelper.entityNotFoundException( + Messages.MODEL_ID_DOES_NOT_MATCH_EXISTING_MODEL_IDS_BUT_MUST_FOR_IN_CLUSTER_SERVICE, + request.getInferenceEntityId() + + ); + } + } + + private void checkEndpointExists(String inferenceEntityId, ActionListener listener) { + modelRegistry.getModelWithSecrets(inferenceEntityId, ActionListener.wrap((model) -> { + if (model == null) { + listener.onFailure( + ExceptionsHelper.entityNotFoundException(Messages.INFERENCE_ENTITY_NON_EXISTANT_NO_UPDATE, inferenceEntityId) + ); + } else { + listener.onResponse(model); + } + }, e -> { + if (e instanceof ResourceNotFoundException) { + listener.onFailure( + // provide a more specific error message if the inference entity does not exist + ExceptionsHelper.entityNotFoundException(Messages.INFERENCE_ENTITY_NON_EXISTANT_NO_UPDATE, inferenceEntityId) + ); + } else { + listener.onFailure(e); + } + })); + } + + private static XContentParser getParser(UpdateInferenceModelAction.Request request) throws IOException { + return XContentHelper.createParser(XContentParserConfiguration.EMPTY, request.getContent(), request.getContentType()); + } + + @Override + protected ClusterBlockException checkBlock(UpdateInferenceModelAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsOptions.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsOptions.java index a85b92dd1a055..93d435eb0b69f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsOptions.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsOptions.java @@ -10,7 +10,8 @@ public enum ChunkingSettingsOptions { STRATEGY("strategy"), MAX_CHUNK_SIZE("max_chunk_size"), - OVERLAP("overlap"); + OVERLAP("overlap"), + SENTENCE_OVERLAP("sentence_overlap"); private final String chunkingSettingsOption; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunker.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunker.java index 3a53ecc7ae958..5df940d6a3fba 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunker.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunker.java @@ -34,6 +34,7 @@ public class SentenceBoundaryChunker implements Chunker { public SentenceBoundaryChunker() { sentenceIterator = BreakIterator.getSentenceInstance(Locale.ROOT); wordIterator = BreakIterator.getWordInstance(Locale.ROOT); + } /** @@ -46,7 +47,7 @@ public SentenceBoundaryChunker() { @Override public List chunk(String input, ChunkingSettings chunkingSettings) { if (chunkingSettings instanceof SentenceBoundaryChunkingSettings sentenceBoundaryChunkingSettings) { - return chunk(input, sentenceBoundaryChunkingSettings.maxChunkSize); + return chunk(input, sentenceBoundaryChunkingSettings.maxChunkSize, sentenceBoundaryChunkingSettings.sentenceOverlap > 0); } else { throw new IllegalArgumentException( Strings.format( @@ -64,7 +65,7 @@ public List chunk(String input, ChunkingSettings chunkingSettings) { * @param maxNumberWordsPerChunk Maximum size of the chunk * @return The input text chunked */ - public List chunk(String input, int maxNumberWordsPerChunk) { + public List chunk(String input, int maxNumberWordsPerChunk, boolean includePrecedingSentence) { var chunks = new ArrayList(); sentenceIterator.setText(input); @@ -75,24 +76,46 @@ public List chunk(String input, int maxNumberWordsPerChunk) { int sentenceStart = 0; int chunkWordCount = 0; + int wordsInPrecedingSentenceCount = 0; + int previousSentenceStart = 0; + int boundary = sentenceIterator.next(); while (boundary != BreakIterator.DONE) { int sentenceEnd = sentenceIterator.current(); - int countWordsInSentence = countWords(sentenceStart, sentenceEnd); + int wordsInSentenceCount = countWords(sentenceStart, sentenceEnd); - if (chunkWordCount + countWordsInSentence > maxNumberWordsPerChunk) { + if (chunkWordCount + wordsInSentenceCount > maxNumberWordsPerChunk) { // over the max chunk size, roll back to the last sentence + int nextChunkWordCount = wordsInSentenceCount; if (chunkWordCount > 0) { // add a new chunk containing all the input up to this sentence chunks.add(input.substring(chunkStart, chunkEnd)); - chunkStart = chunkEnd; - chunkWordCount = countWordsInSentence; // the next chunk will contain this sentence + + if (includePrecedingSentence) { + if (wordsInPrecedingSentenceCount + wordsInSentenceCount > maxNumberWordsPerChunk) { + // cut the last sentence + int numWordsToSkip = numWordsToSkipInPreviousSentence(wordsInPrecedingSentenceCount, maxNumberWordsPerChunk); + + chunkStart = skipWords(input, previousSentenceStart, numWordsToSkip); + chunkWordCount = (wordsInPrecedingSentenceCount - numWordsToSkip) + wordsInSentenceCount; + } else { + chunkWordCount = wordsInPrecedingSentenceCount + wordsInSentenceCount; + chunkStart = previousSentenceStart; + } + + nextChunkWordCount = chunkWordCount; + } else { + chunkStart = chunkEnd; + chunkWordCount = wordsInSentenceCount; // the next chunk will contain this sentence + } } - if (countWordsInSentence > maxNumberWordsPerChunk) { - // This sentence is bigger than the max chunk size. + // Is the next chunk larger than max chunk size? + // If so split it + if (nextChunkWordCount > maxNumberWordsPerChunk) { + // This sentence (and optional overlap) is bigger than the max chunk size. // Split the sentence on the word boundary var sentenceSplits = splitLongSentence( input.substring(chunkStart, sentenceEnd), @@ -113,7 +136,12 @@ public List chunk(String input, int maxNumberWordsPerChunk) { chunkWordCount = sentenceSplits.get(i).wordCount(); } } else { - chunkWordCount += countWordsInSentence; + chunkWordCount += wordsInSentenceCount; + } + + if (includePrecedingSentence) { + previousSentenceStart = sentenceStart; + wordsInPrecedingSentenceCount = wordsInSentenceCount; } sentenceStart = sentenceEnd; @@ -133,6 +161,45 @@ static List splitLongSentence(String text, in return new WordBoundaryChunker().chunkPositions(text, maxNumberOfWords, overlap); } + static int numWordsToSkipInPreviousSentence(int wordsInPrecedingSentenceCount, int maxNumberWordsPerChunk) { + var maxWordsInOverlap = maxWordsInOverlap(maxNumberWordsPerChunk); + if (wordsInPrecedingSentenceCount > maxWordsInOverlap) { + return wordsInPrecedingSentenceCount - maxWordsInOverlap; + } else { + return 0; + } + } + + static int maxWordsInOverlap(int maxNumberWordsPerChunk) { + return Math.min(maxNumberWordsPerChunk / 2, 20); + } + + private int skipWords(String input, int start, int numWords) { + var itr = BreakIterator.getWordInstance(Locale.ROOT); + itr.setText(input); + return skipWords(start, numWords, itr); + } + + static int skipWords(int start, int numWords, BreakIterator wordIterator) { + wordIterator.preceding(start); // start of the current word + + int boundary = wordIterator.current(); + int wordCount = 0; + while (boundary != BreakIterator.DONE && wordCount < numWords) { + int wordStatus = wordIterator.getRuleStatus(); + if (wordStatus != BreakIterator.WORD_NONE) { + wordCount++; + } + boundary = wordIterator.next(); + } + + if (boundary == BreakIterator.DONE) { + return wordIterator.last(); + } else { + return boundary; + } + } + private int countWords(int start, int end) { return countWords(start, end, this.wordIterator); } @@ -157,6 +224,6 @@ static int countWords(int start, int end, BreakIterator wordIterator) { } private static int overlapForChunkSize(int chunkSize) { - return (chunkSize - 1) / 2; + return Math.min(20, (chunkSize - 1) / 2); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkingSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkingSettings.java index 0d1903895f615..758dd5d04e268 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkingSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkingSettings.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.ChunkingStrategy; import org.elasticsearch.inference.ModelConfigurations; @@ -30,16 +31,25 @@ public class SentenceBoundaryChunkingSettings implements ChunkingSettings { private static final ChunkingStrategy STRATEGY = ChunkingStrategy.SENTENCE; private static final Set VALID_KEYS = Set.of( ChunkingSettingsOptions.STRATEGY.toString(), - ChunkingSettingsOptions.MAX_CHUNK_SIZE.toString() + ChunkingSettingsOptions.MAX_CHUNK_SIZE.toString(), + ChunkingSettingsOptions.SENTENCE_OVERLAP.toString() ); + + private static int DEFAULT_OVERLAP = 0; + protected final int maxChunkSize; + protected int sentenceOverlap = DEFAULT_OVERLAP; - public SentenceBoundaryChunkingSettings(Integer maxChunkSize) { + public SentenceBoundaryChunkingSettings(Integer maxChunkSize, @Nullable Integer sentenceOverlap) { this.maxChunkSize = maxChunkSize; + this.sentenceOverlap = sentenceOverlap == null ? DEFAULT_OVERLAP : sentenceOverlap; } public SentenceBoundaryChunkingSettings(StreamInput in) throws IOException { maxChunkSize = in.readInt(); + if (in.getTransportVersion().onOrAfter(TransportVersions.CHUNK_SENTENCE_OVERLAP_SETTING_ADDED)) { + sentenceOverlap = in.readVInt(); + } } public static SentenceBoundaryChunkingSettings fromMap(Map map) { @@ -59,11 +69,24 @@ public static SentenceBoundaryChunkingSettings fromMap(Map map) validationException ); + Integer sentenceOverlap = ServiceUtils.extractOptionalPositiveInteger( + map, + ChunkingSettingsOptions.SENTENCE_OVERLAP.toString(), + ModelConfigurations.CHUNKING_SETTINGS, + validationException + ); + + if (sentenceOverlap != null && sentenceOverlap > 1) { + validationException.addValidationError( + ChunkingSettingsOptions.SENTENCE_OVERLAP.toString() + "[" + sentenceOverlap + "] must be either 0 or 1" + ); // todo better + } + if (validationException.validationErrors().isEmpty() == false) { throw validationException; } - return new SentenceBoundaryChunkingSettings(maxChunkSize); + return new SentenceBoundaryChunkingSettings(maxChunkSize, sentenceOverlap); } @Override @@ -72,6 +95,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws { builder.field(ChunkingSettingsOptions.STRATEGY.toString(), STRATEGY); builder.field(ChunkingSettingsOptions.MAX_CHUNK_SIZE.toString(), maxChunkSize); + builder.field(ChunkingSettingsOptions.SENTENCE_OVERLAP.toString(), sentenceOverlap); } builder.endObject(); return builder; @@ -90,6 +114,9 @@ public TransportVersion getMinimalSupportedVersion() { @Override public void writeTo(StreamOutput out) throws IOException { out.writeInt(maxChunkSize); + if (out.getTransportVersion().onOrAfter(TransportVersions.CHUNK_SENTENCE_OVERLAP_SETTING_ADDED)) { + out.writeVInt(sentenceOverlap); + } } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkingSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkingSettings.java index 6517e0eea14d9..5b91e122b9c80 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkingSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkingSettings.java @@ -52,7 +52,7 @@ public static WordBoundaryChunkingSettings fromMap(Map map) { var invalidSettings = map.keySet().stream().filter(key -> VALID_KEYS.contains(key) == false).toArray(); if (invalidSettings.length > 0) { validationException.addValidationError( - Strings.format("Sentence based chunking settings can not have the following settings: %s", Arrays.toString(invalidSettings)) + Strings.format("Word based chunking settings can not have the following settings: %s", Arrays.toString(invalidSettings)) ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankBuilder.java index 6bc43a4309b0c..fd0ad220faa3b 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankBuilder.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankBuilder.java @@ -23,8 +23,6 @@ import org.elasticsearch.search.rank.context.RankFeaturePhaseRankCoordinatorContext; import org.elasticsearch.search.rank.context.RankFeaturePhaseRankShardContext; import org.elasticsearch.search.rank.feature.RankFeatureDoc; -import org.elasticsearch.search.rank.rerank.RerankingQueryPhaseRankCoordinatorContext; -import org.elasticsearch.search.rank.rerank.RerankingQueryPhaseRankShardContext; import org.elasticsearch.search.rank.rerank.RerankingRankFeaturePhaseRankShardContext; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.XContentBuilder; @@ -157,12 +155,12 @@ public Explanation explainHit(Explanation baseExplanation, RankDoc scoreDoc, Lis @Override public QueryPhaseRankShardContext buildQueryPhaseShardContext(List queries, int from) { - return new RerankingQueryPhaseRankShardContext(queries, rankWindowSize()); + return null; } @Override public QueryPhaseRankCoordinatorContext buildQueryPhaseCoordinatorContext(int size, int from) { - return new RerankingQueryPhaseRankCoordinatorContext(rankWindowSize()); + return null; } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java index d756c0ef26f14..62571c13aebf4 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java @@ -3,6 +3,8 @@ * or more contributor license agreements. Licensed under the Elastic License * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. + * + * this file contains code contributed by a generative AI */ package org.elasticsearch.xpack.inference.registry; @@ -21,6 +23,7 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.OriginSettingClient; @@ -49,10 +52,13 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; import java.util.stream.Collectors; @@ -83,6 +89,8 @@ public static UnparsedModel unparsedModelFromMap(ModelConfigMap modelConfigMap) private final OriginSettingClient client; private Map defaultConfigs; + private final Set preventDeletionLock = Collections.newSetFromMap(new ConcurrentHashMap<>()); + public ModelRegistry(Client client) { this.client = new OriginSettingClient(client, ClientHelper.INFERENCE_ORIGIN); this.defaultConfigs = new HashMap<>(); @@ -306,7 +314,139 @@ private ModelConfigMap createModelConfigMap(SearchHits hits, String inferenceEnt ); } + public void updateModelTransaction(Model newModel, Model existingModel, ActionListener finalListener) { + + String inferenceEntityId = newModel.getConfigurations().getInferenceEntityId(); + logger.info("Attempting to store update to inference endpoint [{}]", inferenceEntityId); + + if (preventDeletionLock.contains(inferenceEntityId)) { + logger.warn(format("Attempted to update endpoint [{}] that is already being updated", inferenceEntityId)); + finalListener.onFailure( + new ElasticsearchStatusException( + "Endpoint [{}] is currently being updated. Try again once the update completes", + RestStatus.CONFLICT, + inferenceEntityId + ) + ); + return; + } else { + preventDeletionLock.add(inferenceEntityId); + } + + SubscribableListener.newForked((subListener) -> { + // in this block, we try to update the stored model configurations + IndexRequest configRequest = createIndexRequest( + Model.documentId(inferenceEntityId), + InferenceIndex.INDEX_NAME, + newModel.getConfigurations(), + true + ); + + ActionListener storeConfigListener = subListener.delegateResponse((l, e) -> { + // this block will only be called if the bulk unexpectedly throws an exception + preventDeletionLock.remove(inferenceEntityId); + l.onFailure(e); + }); + + client.prepareBulk().add(configRequest).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).execute(storeConfigListener); + + }).andThen((subListener, configResponse) -> { + // in this block, we respond to the success or failure of updating the model configurations, then try to store the new secrets + if (configResponse.hasFailures()) { + // if storing the model configurations failed, it won't throw an exception, we need to check the BulkResponse and handle the + // exceptions ourselves. + logger.error( + format("Failed to update inference endpoint [%s] due to [%s]", inferenceEntityId, configResponse.buildFailureMessage()) + ); + // Since none of our updates succeeded at this point, we can simply return. + finalListener.onFailure( + new ElasticsearchStatusException( + format("Failed to update inference endpoint [%s] due to [%s]", inferenceEntityId), + RestStatus.INTERNAL_SERVER_ERROR, + configResponse.buildFailureMessage() + ) + ); + } else { + // Since the model configurations were successfully updated, we can now try to store the new secrets + IndexRequest secretsRequest = createIndexRequest( + Model.documentId(newModel.getConfigurations().getInferenceEntityId()), + InferenceSecretsIndex.INDEX_NAME, + newModel.getSecrets(), + true + ); + + ActionListener storeSecretsListener = subListener.delegateResponse((l, e) -> { + // this block will only be called if the bulk unexpectedly throws an exception + preventDeletionLock.remove(inferenceEntityId); + l.onFailure(e); + }); + + client.prepareBulk() + .add(secretsRequest) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .execute(storeSecretsListener); + } + }).andThen((subListener, secretsResponse) -> { + // in this block, we respond to the success or failure of updating the model secrets + if (secretsResponse.hasFailures()) { + // since storing the secrets failed, we will try to restore / roll-back-to the previous model configurations + IndexRequest configRequest = createIndexRequest( + Model.documentId(inferenceEntityId), + InferenceIndex.INDEX_NAME, + existingModel.getConfigurations(), + true + ); + logger.error( + "Failed to update inference endpoint secrets [{}], attempting rolling back to previous state", + inferenceEntityId + ); + + ActionListener rollbackConfigListener = subListener.delegateResponse((l, e) -> { + // this block will only be called if the bulk unexpectedly throws an exception + preventDeletionLock.remove(inferenceEntityId); + l.onFailure(e); + }); + client.prepareBulk() + .add(configRequest) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .execute(rollbackConfigListener); + } else { + // since updating the secrets was successful, we can remove the lock and respond to the final listener + preventDeletionLock.remove(inferenceEntityId); + finalListener.onResponse(true); + } + }).andThen((subListener, configResponse) -> { + // this block will be called if the secrets response failed, and the rollback didn't throw an exception. + // The rollback still could have failed though, so we need to check for that. + preventDeletionLock.remove(inferenceEntityId); + if (configResponse.hasFailures()) { + logger.error( + format("Failed to update inference endpoint [%s] due to [%s]", inferenceEntityId, configResponse.buildFailureMessage()) + ); + finalListener.onFailure( + new ElasticsearchStatusException( + format( + "Failed to rollback while handling failure to update inference endpoint [%s]. " + + "Endpoint may be in an inconsistent state due to [%s]", + inferenceEntityId + ), + RestStatus.INTERNAL_SERVER_ERROR, + configResponse.buildFailureMessage() + ) + ); + } else { + logger.warn("Failed to update inference endpoint [{}], successfully rolled back to previous state", inferenceEntityId); + finalListener.onResponse(false); + } + }); + + } + + /** + * Note: storeModel does not overwrite existing models and thus does not need to check the lock + */ public void storeModel(Model model, ActionListener listener) { + ActionListener bulkResponseActionListener = getStoreModelListener(model, listener); IndexRequest configRequest = createIndexRequest( @@ -405,6 +545,16 @@ private static BulkItemResponse.Failure getFirstBulkFailure(BulkResponse bulkRes } public void deleteModel(String inferenceEntityId, ActionListener listener) { + if (preventDeletionLock.contains(inferenceEntityId)) { + listener.onFailure( + new ElasticsearchStatusException( + "Model is currently being updated, you may delete the model once the update completes", + RestStatus.CONFLICT + ) + ); + return; + } + DeleteByQueryRequest request = new DeleteByQueryRequest().setAbortOnVersionConflict(false); request.indices(InferenceIndex.INDEX_PATTERN, InferenceSecretsIndex.INDEX_PATTERN); request.setQuery(documentIdQuery(inferenceEntityId)); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/Paths.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/Paths.java index 9f64b58e48b55..2dec72e6692a6 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/Paths.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/Paths.java @@ -14,6 +14,12 @@ public final class Paths { static final String INFERENCE_ID_PATH = "_inference/{" + TASK_TYPE_OR_INFERENCE_ID + "}"; static final String TASK_TYPE_INFERENCE_ID_PATH = "_inference/{" + TASK_TYPE_OR_INFERENCE_ID + "}/{" + INFERENCE_ID + "}"; static final String INFERENCE_DIAGNOSTICS_PATH = "_inference/.diagnostics"; + static final String TASK_TYPE_INFERENCE_ID_UPDATE_PATH = "_inference/{" + + TASK_TYPE_OR_INFERENCE_ID + + "}/{" + + INFERENCE_ID + + "}/_update"; + static final String INFERENCE_ID_UPDATE_PATH = "_inference/{" + TASK_TYPE_OR_INFERENCE_ID + "}/_update"; static final String STREAM_INFERENCE_ID_PATH = "_inference/{" + TASK_TYPE_OR_INFERENCE_ID + "}/_stream"; static final String STREAM_TASK_TYPE_INFERENCE_ID_PATH = "_inference/{" diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestUpdateInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestUpdateInferenceModelAction.java new file mode 100644 index 0000000000000..9405a6752538c --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestUpdateInferenceModelAction.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.rest; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.rest.RestUtils; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.core.inference.action.UpdateInferenceModelAction; + +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.PUT; +import static org.elasticsearch.xpack.inference.rest.Paths.INFERENCE_ID; +import static org.elasticsearch.xpack.inference.rest.Paths.INFERENCE_ID_UPDATE_PATH; +import static org.elasticsearch.xpack.inference.rest.Paths.TASK_TYPE_INFERENCE_ID_UPDATE_PATH; +import static org.elasticsearch.xpack.inference.rest.Paths.TASK_TYPE_OR_INFERENCE_ID; + +@ServerlessScope(Scope.PUBLIC) +public class RestUpdateInferenceModelAction extends BaseRestHandler { + @Override + public String getName() { + return "update_inference_model_action"; + } + + @Override + public List routes() { + return List.of(new Route(PUT, INFERENCE_ID_UPDATE_PATH), new Route(PUT, TASK_TYPE_INFERENCE_ID_UPDATE_PATH)); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) { + String inferenceEntityId; + TaskType taskType; + if (restRequest.hasParam(INFERENCE_ID)) { + inferenceEntityId = restRequest.param(INFERENCE_ID); + taskType = TaskType.fromStringOrStatusException(restRequest.param(TASK_TYPE_OR_INFERENCE_ID)); + } else { + throw new ElasticsearchStatusException("Inference ID must be provided in the path", RestStatus.BAD_REQUEST); + } + + var request = new UpdateInferenceModelAction.Request( + inferenceEntityId, + restRequest.requiredContent(), + restRequest.getXContentType(), + taskType, + RestUtils.getMasterNodeTimeout(restRequest) + ); + return channel -> client.execute(UpdateInferenceModelAction.INSTANCE, request, new RestToXContentListener<>(channel)); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java index 32c1d17373e53..c0e3c78b12f13 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java @@ -625,6 +625,40 @@ public static String mustBeAPositiveLongErrorMessage(String settingName, String return format("[%s] Invalid value [%s]. [%s] must be a positive long", scope, value, settingName); } + /** + * task_type can be specified as either a URL parameter or in the + * request body. Resolve which to use or throw if the settings are + * inconsistent + * @param urlTaskType Taken from the URL parameter. ANY means not specified. + * @param bodyTaskType Taken from the request body. Maybe null + * @return The resolved task type + */ + public static TaskType resolveTaskType(TaskType urlTaskType, String bodyTaskType) { + if (bodyTaskType == null) { + if (urlTaskType == TaskType.ANY) { + throw new ElasticsearchStatusException("model is missing required setting [task_type]", RestStatus.BAD_REQUEST); + } else { + return urlTaskType; + } + } + + TaskType parsedBodyTask = TaskType.fromStringOrStatusException(bodyTaskType); + if (parsedBodyTask == TaskType.ANY) { + throw new ElasticsearchStatusException("task_type [any] is not valid type for inference", RestStatus.BAD_REQUEST); + } + + if (parsedBodyTask.isAnyOrSame(urlTaskType) == false) { + throw new ElasticsearchStatusException( + "Cannot resolve conflicting task_type parameter in the request URL [{}] and the request body [{}]", + RestStatus.BAD_REQUEST, + urlTaskType.toString(), + bodyTaskType + ); + } + + return parsedBodyTask; + } + /** * Functional interface for creating an enum from a string. * @param diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionTaskSettings.java index 63f82a8eceb98..05b5873a81d8d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionTaskSettings.java @@ -139,4 +139,12 @@ public int hashCode() { public Map getParameters() { return parameters; } + + @Override + public TaskSettings updatedTaskSettings(Map newSettings) { + AlibabaCloudSearchCompletionTaskSettings updatedSettings = AlibabaCloudSearchCompletionTaskSettings.fromMap( + new HashMap<>(newSettings) + ); + return of(this, updatedSettings); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsTaskSettings.java index c908c219e4053..9a431717d9fb9 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsTaskSettings.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.EnumSet; +import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -174,4 +175,10 @@ public int hashCode() { public static String invalidInputTypeMessage(InputType inputType) { return Strings.format("received invalid input type value [%s]", inputType.toString()); } + + @Override + public TaskSettings updatedTaskSettings(Map newSettings) { + AlibabaCloudSearchEmbeddingsTaskSettings newSettingsOnly = fromMap(new HashMap<>(newSettings)); + return of(this, newSettingsOnly, newSettingsOnly.inputType != null ? newSettingsOnly.inputType : this.getInputType()); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/rerank/AlibabaCloudSearchRerankTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/rerank/AlibabaCloudSearchRerankTaskSettings.java index 97e7ecd41223d..40c3dee00d6c7 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/rerank/AlibabaCloudSearchRerankTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/rerank/AlibabaCloudSearchRerankTaskSettings.java @@ -102,4 +102,10 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(); } + + @Override + public TaskSettings updatedTaskSettings(Map newSettings) { + AlibabaCloudSearchRerankTaskSettings updatedSettings = new AlibabaCloudSearchRerankTaskSettings(); + return of(this, updatedSettings); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseTaskSettings.java index 873cdf31fbe9d..0f4ebce920167 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseTaskSettings.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.EnumSet; +import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -188,4 +189,10 @@ public int hashCode() { public static String invalidInputTypeMessage(InputType inputType) { return Strings.format("received invalid input type value [%s]", inputType.toString()); } + + @Override + public TaskSettings updatedTaskSettings(Map newSettings) { + AlibabaCloudSearchSparseTaskSettings updatedSettings = fromMap(new HashMap<>(newSettings)); + return of(this, updatedSettings, updatedSettings.getInputType() != null ? updatedSettings.getInputType() : this.inputType); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockSecretSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockSecretSettings.java index 9e6328ce1c358..30a7dc9ad5a2e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockSecretSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockSecretSettings.java @@ -18,6 +18,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -107,4 +108,9 @@ public boolean equals(Object object) { public int hashCode() { return Objects.hash(accessKey, secretKey); } + + @Override + public SecretSettings newSecretSettings(Map newSecrets) { + return fromMap(new HashMap<>(newSecrets)); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/completion/AmazonBedrockChatCompletionTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/completion/AmazonBedrockChatCompletionTaskSettings.java index 13787ed8cb6a4..c3db1465863e4 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/completion/AmazonBedrockChatCompletionTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/completion/AmazonBedrockChatCompletionTaskSettings.java @@ -17,6 +17,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -192,4 +193,12 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(temperature, topP, topK, maxNewTokens); } + + @Override + public TaskSettings updatedTaskSettings(Map newSettings) { + AmazonBedrockChatCompletionRequestTaskSettings requestSettings = AmazonBedrockChatCompletionRequestTaskSettings.fromMap( + new HashMap<>(newSettings) + ); + return of(this, requestSettings); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionTaskSettings.java index bb2c027127371..e8a6ca638c916 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionTaskSettings.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -59,6 +60,11 @@ private static AnthropicChatCompletionTaskSettings fromPersistedMap(Map newSettings) { + return fromRequestMap(new HashMap<>(newSettings)); + } + private record CommonFields(int maxTokens, Double temperature, Double topP, Integer topK) {} private static CommonFields fromMap(Map map, ValidationException validationException) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionTaskSettings.java index b8e33bac410fe..544c52f59a3c4 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionTaskSettings.java @@ -20,6 +20,7 @@ import org.elasticsearch.xpack.inference.services.azureopenai.embeddings.AzureOpenAiEmbeddingsTaskSettings; import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -178,6 +179,20 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + @Override + public String toString() { + return "AzureAiStudioChatCompletionTaskSettings{" + + "temperature=" + + temperature + + ", topP=" + + topP + + ", doSample=" + + doSample + + ", maxNewTokens=" + + maxNewTokens + + '}'; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -194,4 +209,11 @@ public int hashCode() { return Objects.hash(temperature, topP, doSample, maxNewTokens); } + @Override + public TaskSettings updatedTaskSettings(Map newSettings) { + AzureAiStudioChatCompletionRequestTaskSettings requestSettings = AzureAiStudioChatCompletionRequestTaskSettings.fromMap( + new HashMap<>(newSettings) + ); + return of(this, requestSettings); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsTaskSettings.java index bdb6ae74e5ab3..340ee95cd7b0c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsTaskSettings.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.inference.services.azureopenai.embeddings.AzureOpenAiEmbeddingsTaskSettings; import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -111,4 +112,12 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hashCode(user); } + + @Override + public TaskSettings updatedTaskSettings(Map newSettings) { + AzureAiStudioEmbeddingsRequestTaskSettings requestSettings = AzureAiStudioEmbeddingsRequestTaskSettings.fromMap( + new HashMap<>(newSettings) + ); + return AzureAiStudioEmbeddingsTaskSettings.of(this, requestSettings); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettings.java index 06217e8079b06..a2bd4f6175989 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettings.java @@ -19,6 +19,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -125,4 +126,9 @@ public boolean equals(Object object) { public int hashCode() { return Objects.hash(entraId, apiKey); } + + @Override + public SecretSettings newSecretSettings(Map newSecrets) { + return AzureOpenAiSecretSettings.fromMap(new HashMap<>(newSecrets)); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionTaskSettings.java index de0a0897a93c5..3008a543b8fea 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionTaskSettings.java @@ -18,6 +18,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -107,4 +108,12 @@ public boolean equals(Object object) { public int hashCode() { return Objects.hash(user); } + + @Override + public TaskSettings updatedTaskSettings(Map newSettings) { + AzureOpenAiCompletionRequestTaskSettings updatedSettings = AzureOpenAiCompletionRequestTaskSettings.fromMap( + new HashMap<>(newSettings) + ); + return of(this, updatedSettings); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsTaskSettings.java index 28ccade0a06b0..4157d7748d789 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsTaskSettings.java @@ -18,6 +18,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -116,4 +117,12 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(user); } + + @Override + public TaskSettings updatedTaskSettings(Map newSettings) { + AzureOpenAiEmbeddingsRequestTaskSettings requestSettings = AzureOpenAiEmbeddingsRequestTaskSettings.fromMap( + new HashMap<>(newSettings) + ); + return of(this, requestSettings); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsTaskSettings.java index 34d37d0003adf..b789d1578290a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsTaskSettings.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.EnumSet; +import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -204,4 +205,10 @@ public int hashCode() { public static String invalidInputTypeMessage(InputType inputType) { return Strings.format("received invalid input type value [%s]", inputType.toString()); } + + @Override + public TaskSettings updatedTaskSettings(Map newSettings) { + CohereEmbeddingsTaskSettings updatedSettings = CohereEmbeddingsTaskSettings.fromMap(new HashMap<>(newSettings)); + return of(this, updatedSettings, updatedSettings.inputType != null ? updatedSettings.inputType : this.inputType); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankTaskSettings.java index f5893c825efcf..479000f840502 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankTaskSettings.java @@ -20,6 +20,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -186,4 +187,9 @@ public Integer getMaxChunksPerDoc() { return maxChunksPerDoc; } + @Override + public TaskSettings updatedTaskSettings(Map newSettings) { + CohereRerankTaskSettings updatedSettings = CohereRerankTaskSettings.fromMap(new HashMap<>(newSettings)); + return CohereRerankTaskSettings.of(this, updatedSettings); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettings.java index 70d787152121f..a0be1661b860d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettings.java @@ -17,6 +17,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -87,7 +88,11 @@ public CustomElandRerankTaskSettings(StreamInput in) throws IOException { } public CustomElandRerankTaskSettings(@Nullable Boolean doReturnDocuments) { - this.returnDocuments = doReturnDocuments; + if (doReturnDocuments == null) { + this.returnDocuments = true; + } else { + this.returnDocuments = doReturnDocuments; + } } @Override @@ -136,4 +141,10 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(returnDocuments); } + + @Override + public TaskSettings updatedTaskSettings(Map newSettings) { + CustomElandRerankTaskSettings updatedSettings = CustomElandRerankTaskSettings.fromMap(new HashMap<>(newSettings)); + return of(this, updatedSettings); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java index f8b5837ef387e..37e0f28dfb3fe 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java @@ -122,6 +122,18 @@ protected ElasticsearchInternalServiceSettings(ElasticsearchInternalServiceSetti this.adaptiveAllocationsSettings = other.adaptiveAllocationsSettings; } + /** + * Copy constructor with the ability to set the number of allocations. Used for Update API. + * @param other the existing settings + * @param numAllocations the new number of allocations + */ + public ElasticsearchInternalServiceSettings(ElasticsearchInternalServiceSettings other, int numAllocations) { + this.numAllocations = numAllocations; + this.numThreads = other.numThreads; + this.modelId = other.modelId; + this.adaptiveAllocationsSettings = other.adaptiveAllocationsSettings; + } + public ElasticsearchInternalServiceSettings(StreamInput in) throws IOException { if (in.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_ADAPTIVE_ALLOCATIONS)) { this.numAllocations = in.readOptionalVInt(); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserMlNodeTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserMlNodeTaskSettings.java index 33696231668a5..3bcaa57827fdb 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserMlNodeTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserMlNodeTaskSettings.java @@ -15,6 +15,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Map; import java.util.Objects; public class ElserMlNodeTaskSettings implements TaskSettings { @@ -65,4 +66,9 @@ public int hashCode() { // Return the hash of NAME to make the serialization tests pass return Objects.hash(NAME); } + + @Override + public TaskSettings updatedTaskSettings(Map newSettings) { + return DEFAULT; + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiSecretSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiSecretSettings.java index 57c8d61f9f9a5..20dbadb9b3eae 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiSecretSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiSecretSettings.java @@ -19,6 +19,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -30,7 +31,7 @@ public class GoogleVertexAiSecretSettings implements SecretSettings { public static final String SERVICE_ACCOUNT_JSON = "service_account_json"; - private final SecureString serviceAccountJson; + final SecureString serviceAccountJson; public static GoogleVertexAiSecretSettings fromMap(@Nullable Map map) { if (map == null) { @@ -101,4 +102,9 @@ public boolean equals(Object object) { public int hashCode() { return Objects.hash(serviceAccountJson); } + + @Override + public SecretSettings newSecretSettings(Map newSecrets) { + return GoogleVertexAiSecretSettings.fromMap(new HashMap<>(newSecrets)); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsTaskSettings.java index 5e0185a7abb36..b7242100178a3 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsTaskSettings.java @@ -17,6 +17,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -107,4 +108,12 @@ public boolean equals(Object object) { public int hashCode() { return Objects.hash(autoTruncate); } + + @Override + public TaskSettings updatedTaskSettings(Map newSettings) { + GoogleVertexAiEmbeddingsRequestTaskSettings requestSettings = GoogleVertexAiEmbeddingsRequestTaskSettings.fromMap( + new HashMap<>(newSettings) + ); + return of(this, requestSettings); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankTaskSettings.java index 8256eed7a5cba..64bec7e6cfeef 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankTaskSettings.java @@ -18,6 +18,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -107,4 +108,12 @@ public boolean equals(Object object) { public int hashCode() { return Objects.hash(topN); } + + @Override + public TaskSettings updatedTaskSettings(Map newSettings) { + GoogleVertexAiRerankRequestTaskSettings requestSettings = GoogleVertexAiRerankRequestTaskSettings.fromMap( + new HashMap<>(newSettings) + ); + return of(this, requestSettings); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionTaskSettings.java index 3c2586fb5a264..44064f61f5180 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionTaskSettings.java @@ -18,6 +18,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -107,4 +108,12 @@ public boolean equals(Object object) { public int hashCode() { return Objects.hash(user); } + + @Override + public TaskSettings updatedTaskSettings(Map newSettings) { + OpenAiChatCompletionRequestTaskSettings updatedSettings = OpenAiChatCompletionRequestTaskSettings.fromMap( + new HashMap<>(newSettings) + ); + return of(this, updatedSettings); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsTaskSettings.java index c7cc60043ef47..64f852822703c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsTaskSettings.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -127,4 +128,10 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(user); } + + @Override + public TaskSettings updatedTaskSettings(Map newSettings) { + OpenAiEmbeddingsRequestTaskSettings requestSettings = OpenAiEmbeddingsRequestTaskSettings.fromMap(new HashMap<>(newSettings)); + return of(this, requestSettings); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/DefaultSecretSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/DefaultSecretSettings.java index 6affa998c089d..c68d4bc801724 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/DefaultSecretSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/DefaultSecretSettings.java @@ -19,6 +19,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -78,4 +79,9 @@ public TransportVersion getMinimalSupportedVersion() { public void writeTo(StreamOutput out) throws IOException { out.writeSecureString(apiKey); } + + @Override + public SecretSettings newSecretSettings(Map newSecrets) { + return fromMap(new HashMap<>(newSecrets)); + } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/EmptySecretSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/EmptySecretSettingsTests.java index b50ea9e5ee224..d27a326d5fa1e 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/EmptySecretSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/EmptySecretSettingsTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.inference; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.core.Tuple; import org.elasticsearch.inference.EmptySecretSettings; import org.elasticsearch.test.AbstractWireSerializingTestCase; @@ -32,4 +33,13 @@ protected EmptySecretSettings mutateInstance(EmptySecretSettings instance) { // All instances are the same and have no fields, nothing to mutate return null; } + + public void testNewSecretSettings() { + + EmptySecretSettings newSecretSettings = (EmptySecretSettings) EmptySecretSettings.INSTANCE.newSecretSettings( + randomMap(0, 3, () -> new Tuple<>(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10))) + ); + + assertSame(EmptySecretSettings.INSTANCE, newSecretSettings); + } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/EmptyTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/EmptyTaskSettingsTests.java index 060dc23b935cc..7bc0cc57e31ab 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/EmptyTaskSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/EmptyTaskSettingsTests.java @@ -11,12 +11,20 @@ import org.elasticsearch.inference.EmptyTaskSettings; import org.elasticsearch.test.AbstractWireSerializingTestCase; +import java.util.Map; + public class EmptyTaskSettingsTests extends AbstractWireSerializingTestCase { public static EmptyTaskSettings createRandom() { return EmptyTaskSettings.INSTANCE; // no options to randomise } + public void testUpdatedTaskSettings() { + EmptyTaskSettings initialSettings = createRandom(); + EmptyTaskSettings updatedSettings = (EmptyTaskSettings) initialSettings.updatedTaskSettings(Map.of()); + assertEquals(EmptyTaskSettings.INSTANCE, updatedSettings); + } + @Override protected Writeable.Reader instanceReader() { return EmptyTaskSettings::new; diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/ModelSecretsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/ModelSecretsTests.java index d6d139190c12c..ea2f41bf5c6cf 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/ModelSecretsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/ModelSecretsTests.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.List; +import java.util.Map; public class ModelSecretsTests extends AbstractWireSerializingTestCase { @@ -83,5 +84,10 @@ public String getWriteableName() { public TransportVersion getMinimalSupportedVersion() { return TransportVersions.V_8_11_X; } + + @Override + public SecretSettings newSecretSettings(Map newSecrets) { + return new FakeSecretSettings(newSecrets.get(API_KEY).toString()); + } } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelActionTests.java index 27e56c1bd973d..991c5a581eb35 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelActionTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.inference.TaskType; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.inference.services.ServiceUtils; import static org.hamcrest.Matchers.containsString; @@ -17,27 +18,18 @@ public class TransportPutInferenceModelActionTests extends ESTestCase { public void testResolveTaskType() { - assertEquals(TaskType.SPARSE_EMBEDDING, TransportPutInferenceModelAction.resolveTaskType(TaskType.SPARSE_EMBEDDING, null)); - assertEquals( - TaskType.SPARSE_EMBEDDING, - TransportPutInferenceModelAction.resolveTaskType(TaskType.ANY, TaskType.SPARSE_EMBEDDING.toString()) - ); + assertEquals(TaskType.SPARSE_EMBEDDING, ServiceUtils.resolveTaskType(TaskType.SPARSE_EMBEDDING, null)); + assertEquals(TaskType.SPARSE_EMBEDDING, ServiceUtils.resolveTaskType(TaskType.ANY, TaskType.SPARSE_EMBEDDING.toString())); - var e = expectThrows( - ElasticsearchStatusException.class, - () -> TransportPutInferenceModelAction.resolveTaskType(TaskType.ANY, null) - ); + var e = expectThrows(ElasticsearchStatusException.class, () -> ServiceUtils.resolveTaskType(TaskType.ANY, null)); assertThat(e.getMessage(), containsString("model is missing required setting [task_type]")); - e = expectThrows( - ElasticsearchStatusException.class, - () -> TransportPutInferenceModelAction.resolveTaskType(TaskType.ANY, TaskType.ANY.toString()) - ); + e = expectThrows(ElasticsearchStatusException.class, () -> ServiceUtils.resolveTaskType(TaskType.ANY, TaskType.ANY.toString())); assertThat(e.getMessage(), containsString("task_type [any] is not valid type for inference")); e = expectThrows( ElasticsearchStatusException.class, - () -> TransportPutInferenceModelAction.resolveTaskType(TaskType.SPARSE_EMBEDDING, TaskType.TEXT_EMBEDDING.toString()) + () -> ServiceUtils.resolveTaskType(TaskType.SPARSE_EMBEDDING, TaskType.TEXT_EMBEDDING.toString()) ); assertThat( e.getMessage(), diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsBuilderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsBuilderTests.java index 061ea677e6fe1..3c09984ac0162 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsBuilderTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsBuilderTests.java @@ -56,7 +56,7 @@ private Map, ChunkingSettings> chunkingSettingsMapToChunking ChunkingSettingsOptions.MAX_CHUNK_SIZE.toString(), maxChunkSize ), - new SentenceBoundaryChunkingSettings(maxChunkSize) + new SentenceBoundaryChunkingSettings(maxChunkSize, 1) ); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsTests.java index 2482586c75595..8373ae93354b1 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsTests.java @@ -25,7 +25,7 @@ public static ChunkingSettings createRandomChunkingSettings() { return new WordBoundaryChunkingSettings(maxChunkSize, randomIntBetween(1, maxChunkSize / 2)); } case SENTENCE -> { - return new SentenceBoundaryChunkingSettings(randomNonNegativeInt()); + return new SentenceBoundaryChunkingSettings(randomNonNegativeInt(), randomBoolean() ? 0 : 1); } default -> throw new IllegalArgumentException("Unsupported random strategy [" + randomStrategy + "]"); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkerTests.java index 335752faa6b22..5687ebc4dbae7 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkerTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkerTests.java @@ -13,19 +13,24 @@ import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; +import java.util.ArrayList; import java.util.Arrays; import java.util.Locale; import static org.elasticsearch.xpack.inference.chunking.WordBoundaryChunkerTests.TEST_TEXT; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.startsWith; public class SentenceBoundaryChunkerTests extends ESTestCase { public void testChunkSplitLargeChunkSizes() { for (int maxWordsPerChunk : new int[] { 100, 200 }) { var chunker = new SentenceBoundaryChunker(); - var chunks = chunker.chunk(TEST_TEXT, maxWordsPerChunk); + var chunks = chunker.chunk(TEST_TEXT, maxWordsPerChunk, false); int numChunks = expectedNumberOfChunks(sentenceSizes(TEST_TEXT), maxWordsPerChunk); assertThat("words per chunk " + maxWordsPerChunk, chunks, hasSize(numChunks)); @@ -39,11 +44,94 @@ public void testChunkSplitLargeChunkSizes() { } } + public void testChunkSplitLargeChunkSizes_withOverlap() { + boolean overlap = true; + for (int maxWordsPerChunk : new int[] { 70, 80, 100, 120, 150, 200 }) { + var chunker = new SentenceBoundaryChunker(); + var chunks = chunker.chunk(TEST_TEXT, maxWordsPerChunk, overlap); + + int[] overlaps = chunkOverlaps(sentenceSizes(TEST_TEXT), maxWordsPerChunk, overlap); + assertThat("words per chunk " + maxWordsPerChunk, chunks, hasSize(overlaps.length)); + + assertTrue(Character.isUpperCase(chunks.get(0).charAt(0))); + + for (int i = 0; i < overlaps.length; i++) { + if (overlaps[i] == 0) { + // start of a sentence + assertTrue(Character.isUpperCase(chunks.get(i).charAt(0))); + } else { + // The start of this chunk should contain some text from the end of the previous + var previousChunk = chunks.get(i - 1); + assertThat(chunks.get(i), containsString(previousChunk.substring(previousChunk.length() - 20))); + } + } + + var trailingWhiteSpaceRemoved = chunks.get(0).strip(); + var lastChar = trailingWhiteSpaceRemoved.charAt(trailingWhiteSpaceRemoved.length() - 1); + assertThat(lastChar, Matchers.is('.')); + trailingWhiteSpaceRemoved = chunks.get(chunks.size() - 1).strip(); + lastChar = trailingWhiteSpaceRemoved.charAt(trailingWhiteSpaceRemoved.length() - 1); + assertThat(lastChar, Matchers.is('.')); + } + } + + public void testWithOverlap_SentencesFitInChunks() { + int numChunks = 4; + int chunkSize = 100; + + var sb = new StringBuilder(); + + int[] sentenceStartIndexes = new int[numChunks]; + sentenceStartIndexes[0] = 0; + + int numSentences = randomIntBetween(2, 5); + int sentenceIndex = 0; + int lastSentenceSize = 0; + int roughSentenceSize = (chunkSize / numSentences) - 1; + for (int j = 0; j < numSentences; j++) { + sb.append(makeSentence(roughSentenceSize, sentenceIndex++)); + lastSentenceSize = roughSentenceSize; + } + + for (int i = 1; i < numChunks; i++) { + sentenceStartIndexes[i] = sentenceIndex - 1; + + roughSentenceSize = (chunkSize / numSentences) - 1; + int wordCount = lastSentenceSize; + + while (wordCount + roughSentenceSize < chunkSize) { + sb.append(makeSentence(roughSentenceSize, sentenceIndex++)); + lastSentenceSize = roughSentenceSize; + wordCount += roughSentenceSize; + } + } + + var chunker = new SentenceBoundaryChunker(); + var chunks = chunker.chunk(sb.toString(), chunkSize, true); + assertThat(chunks, hasSize(numChunks)); + for (int i = 0; i < numChunks; i++) { + assertThat("num sentences " + numSentences, chunks.get(i), startsWith("SStart" + sentenceStartIndexes[i])); + assertThat("num sentences " + numSentences, chunks.get(i).trim(), endsWith(".")); + } + } + + private String makeSentence(int numWords, int sentenceIndex) { + StringBuilder sb = new StringBuilder(); + sb.append("SStart").append(sentenceIndex).append(' '); + for (int i = 1; i < numWords - 1; i++) { + sb.append(i).append(' '); + } + sb.append(numWords - 1).append(". "); + return sb.toString(); + } + public void testChunk_ChunkSizeLargerThanText() { int maxWordsPerChunk = 500; var chunker = new SentenceBoundaryChunker(); - var chunks = chunker.chunk(TEST_TEXT, maxWordsPerChunk); + var chunks = chunker.chunk(TEST_TEXT, maxWordsPerChunk, false); + assertEquals(chunks.get(0), TEST_TEXT); + chunks = chunker.chunk(TEST_TEXT, maxWordsPerChunk, true); assertEquals(chunks.get(0), TEST_TEXT); } @@ -54,7 +142,7 @@ public void testChunkSplit_SentencesLongerThanChunkSize() { for (int i = 0; i < chunkSizes.length; i++) { int maxWordsPerChunk = chunkSizes[i]; var chunker = new SentenceBoundaryChunker(); - var chunks = chunker.chunk(TEST_TEXT, maxWordsPerChunk); + var chunks = chunker.chunk(TEST_TEXT, maxWordsPerChunk, false); assertThat("words per chunk " + maxWordsPerChunk, chunks, hasSize(expectedNumberOFChunks[i])); for (var chunk : chunks) { @@ -76,6 +164,48 @@ public void testChunkSplit_SentencesLongerThanChunkSize() { } } + public void testChunkSplit_SentencesLongerThanChunkSize_WithOverlap() { + var chunkSizes = new int[] { 10, 30, 50 }; + + // Chunk sizes are shorter the sentences most of the sentences will be split. + for (int i = 0; i < chunkSizes.length; i++) { + int maxWordsPerChunk = chunkSizes[i]; + var chunker = new SentenceBoundaryChunker(); + var chunks = chunker.chunk(TEST_TEXT, maxWordsPerChunk, true); + assertThat(chunks.get(0), containsString("Word segmentation is the problem of dividing")); + assertThat(chunks.get(chunks.size() - 1), containsString(", with solidification being a stronger norm.")); + } + } + + public void testShortLongShortSentences_WithOverlap() { + int maxWordsPerChunk = 40; + var sb = new StringBuilder(); + int[] sentenceLengths = new int[] { 15, 30, 20, 5 }; + for (int l = 0; l < sentenceLengths.length; l++) { + sb.append("SStart").append(l).append(" "); + for (int i = 1; i < sentenceLengths[l] - 1; i++) { + sb.append(i).append(' '); + } + sb.append(sentenceLengths[l] - 1).append(". "); + } + + var chunker = new SentenceBoundaryChunker(); + var chunks = chunker.chunk(sb.toString(), maxWordsPerChunk, true); + assertThat(chunks, hasSize(5)); + assertTrue(chunks.get(0).trim().startsWith("SStart0")); // Entire sentence + assertTrue(chunks.get(0).trim().endsWith(".")); // Entire sentence + + assertTrue(chunks.get(1).trim().startsWith("SStart0")); // contains previous sentence + assertFalse(chunks.get(1).trim().endsWith(".")); // not a full sentence(s) + + assertTrue(chunks.get(2).trim().endsWith(".")); + assertTrue(chunks.get(3).trim().endsWith(".")); + + assertTrue(chunks.get(4).trim().startsWith("SStart2")); // contains previous sentence + assertThat(chunks.get(4), containsString("SStart3")); // last chunk contains 2 sentences + assertTrue(chunks.get(4).trim().endsWith(".")); // full sentence(s) + } + public void testCountWords() { // Test word count matches the whitespace separated word count. var splitByWhiteSpaceSentenceSizes = sentenceSizes(TEST_TEXT); @@ -102,6 +232,30 @@ public void testCountWords() { assertEquals(BreakIterator.DONE, sentenceIterator.next()); } + public void testSkipWords() { + int numWords = 50; + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < numWords; i++) { + sb.append("word").append(i).append(" "); + } + var text = sb.toString(); + + var wordIterator = BreakIterator.getWordInstance(Locale.ROOT); + wordIterator.setText(text); + + int start = 0; + int pos = SentenceBoundaryChunker.skipWords(start, 3, wordIterator); + assertThat(text.substring(pos), startsWith("word3 ")); + pos = SentenceBoundaryChunker.skipWords(pos + 1, 1, wordIterator); + assertThat(text.substring(pos), startsWith("word4 ")); + pos = SentenceBoundaryChunker.skipWords(pos + 1, 5, wordIterator); + assertThat(text.substring(pos), startsWith("word9 ")); + + // past the end of the input + pos = SentenceBoundaryChunker.skipWords(0, numWords + 10, wordIterator); + assertThat(pos, greaterThan(0)); + } + public void testCountWords_short() { // Test word count matches the whitespace separated word count. var text = "This is a short sentence. Followed by another."; @@ -148,7 +302,7 @@ public void testCountWords_WithSymbols() { public void testChunkSplitLargeChunkSizesWithChunkingSettings() { for (int maxWordsPerChunk : new int[] { 100, 200 }) { var chunker = new SentenceBoundaryChunker(); - SentenceBoundaryChunkingSettings chunkingSettings = new SentenceBoundaryChunkingSettings(maxWordsPerChunk); + SentenceBoundaryChunkingSettings chunkingSettings = new SentenceBoundaryChunkingSettings(maxWordsPerChunk, 0); var chunks = chunker.chunk(TEST_TEXT, chunkingSettings); int numChunks = expectedNumberOfChunks(sentenceSizes(TEST_TEXT), maxWordsPerChunk); @@ -182,16 +336,30 @@ private int[] sentenceSizes(String text) { } private int expectedNumberOfChunks(int[] sentenceLengths, int maxWordsPerChunk) { - int numChunks = 1; + return chunkOverlaps(sentenceLengths, maxWordsPerChunk, false).length; + } + + private int[] chunkOverlaps(int[] sentenceLengths, int maxWordsPerChunk, boolean includeSingleSentenceOverlap) { + int maxOverlap = SentenceBoundaryChunker.maxWordsInOverlap(maxWordsPerChunk); + + var overlaps = new ArrayList(); + overlaps.add(0); int runningWordCount = 0; for (int i = 0; i < sentenceLengths.length; i++) { if (runningWordCount + sentenceLengths[i] > maxWordsPerChunk) { - numChunks++; runningWordCount = sentenceLengths[i]; + if (includeSingleSentenceOverlap && i > 0) { + // include what is carried over from the previous + int overlap = Math.min(maxOverlap, sentenceLengths[i - 1]); + overlaps.add(overlap); + runningWordCount += overlap; + } else { + overlaps.add(0); + } } else { runningWordCount += sentenceLengths[i]; } } - return numChunks; + return overlaps.stream().mapToInt(Integer::intValue).toArray(); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkingSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkingSettingsTests.java index 3f304a593144b..fe97d7eb3af54 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkingSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/SentenceBoundaryChunkingSettingsTests.java @@ -59,13 +59,12 @@ protected Writeable.Reader instanceReader() { @Override protected SentenceBoundaryChunkingSettings createTestInstance() { - return new SentenceBoundaryChunkingSettings(randomNonNegativeInt()); + return new SentenceBoundaryChunkingSettings(randomNonNegativeInt(), randomBoolean() ? 0 : 1); } @Override protected SentenceBoundaryChunkingSettings mutateInstance(SentenceBoundaryChunkingSettings instance) throws IOException { var chunkSize = randomValueOtherThan(instance.maxChunkSize, ESTestCase::randomNonNegativeInt); - - return new SentenceBoundaryChunkingSettings(chunkSize); + return new SentenceBoundaryChunkingSettings(chunkSize, instance.sentenceOverlap); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkerTests.java index 21d8c65ad7dcd..08c0724f36270 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkerTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkerTests.java @@ -54,9 +54,6 @@ public class WordBoundaryChunkerTests extends ESTestCase { + " خليفہ المومنين يا خليفہ المسلمين يا صحابی يا رضي الله عنه چئي۔ (ب) آنحضور ﷺ جي گھروارين کان علاوه ڪنھن کي ام المومنين " + "چئي۔ (ج) آنحضور ﷺ جي خاندان جي اھل بيت کان علاوہڍه ڪنھن کي اھل بيت چئي۔ (د) پنھنجي عبادت گاھ کي مسجد چئي۔" }; - private static final int DEFAULT_MAX_CHUNK_SIZE = 250; - private static final int DEFAULT_OVERLAP = 100; - public static int NUM_WORDS_IN_TEST_TEXT; static { var wordIterator = BreakIterator.getWordInstance(Locale.ROOT); @@ -139,7 +136,7 @@ public void testNumberOfChunksWithWordBoundaryChunkingSettings() { } public void testInvalidChunkingSettingsProvided() { - ChunkingSettings chunkingSettings = new SentenceBoundaryChunkingSettings(randomNonNegativeInt()); + ChunkingSettings chunkingSettings = new SentenceBoundaryChunkingSettings(randomNonNegativeInt(), 0); assertThrows(IllegalArgumentException.class, () -> { new WordBoundaryChunker().chunk(TEST_TEXT, chunkingSettings); }); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/model/TestModel.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/model/TestModel.java index d8c25fb5a6d88..779a98e023455 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/model/TestModel.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/model/TestModel.java @@ -25,6 +25,7 @@ import org.elasticsearch.xpack.inference.services.ServiceUtils; import java.io.IOException; +import java.util.HashMap; import java.util.Map; import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength; @@ -217,6 +218,11 @@ public String getWriteableName() { public TransportVersion getMinimalSupportedVersion() { return TransportVersion.current(); // fine for these tests but will not work for cluster upgrade tests } + + @Override + public TaskSettings updatedTaskSettings(Map newSettings) { + return TestTaskSettings.fromMap(new HashMap<>(newSettings)); + } } public record TestSecretSettings(String apiKey) implements SecretSettings { @@ -265,5 +271,10 @@ public String getWriteableName() { public TransportVersion getMinimalSupportedVersion() { return TransportVersion.current(); // fine for these tests but will not work for cluster upgrade tests } + + @Override + public SecretSettings newSecretSettings(Map newSecrets) { + return new TestSecretSettings(newSecrets.get("api_key").toString()); + } } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankMultiNodeTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankMultiNodeTests.java index a3605aade1fa1..6d6403b69ea11 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankMultiNodeTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankMultiNodeTests.java @@ -42,4 +42,12 @@ protected RankBuilder getThrowingRankBuilder(int rankWindowSize, String rankFeat protected Collection> pluginsNeeded() { return List.of(InferencePlugin.class, TextSimilarityTestPlugin.class); } + + public void testQueryPhaseShardThrowingAllShardsFail() throws Exception { + // no-op + } + + public void testQueryPhaseCoordinatorThrowingAllShardsFail() throws Exception { + // no-op + } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityTestPlugin.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityTestPlugin.java index 120527f489549..358aa9804b916 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityTestPlugin.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityTestPlugin.java @@ -7,14 +7,10 @@ package org.elasticsearch.xpack.inference.rank.textsimilarity; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.TopDocs; import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.action.search.SearchPhaseController; import org.elasticsearch.action.support.ActionFilter; import org.elasticsearch.action.support.ActionFilterChain; import org.elasticsearch.client.internal.Client; @@ -28,11 +24,8 @@ import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.rank.RankBuilder; import org.elasticsearch.search.rank.RankShardResult; -import org.elasticsearch.search.rank.context.QueryPhaseRankCoordinatorContext; -import org.elasticsearch.search.rank.context.QueryPhaseRankShardContext; import org.elasticsearch.search.rank.context.RankFeaturePhaseRankCoordinatorContext; import org.elasticsearch.search.rank.context.RankFeaturePhaseRankShardContext; import org.elasticsearch.search.rank.rerank.AbstractRerankerIT; @@ -246,37 +239,6 @@ public void doXContent(XContentBuilder builder, ToXContent.Params params) throws builder.field(THROWING_TYPE_FIELD.getPreferredName(), throwingRankBuilderType); } - @Override - public QueryPhaseRankShardContext buildQueryPhaseShardContext(List queries, int from) { - if (this.throwingRankBuilderType == AbstractRerankerIT.ThrowingRankBuilderType.THROWING_QUERY_PHASE_SHARD_CONTEXT) - return new QueryPhaseRankShardContext(queries, rankWindowSize()) { - @Override - public RankShardResult combineQueryPhaseResults(List rankResults) { - throw new UnsupportedOperationException("qps - simulated failure"); - } - }; - else { - return super.buildQueryPhaseShardContext(queries, from); - } - } - - @Override - public QueryPhaseRankCoordinatorContext buildQueryPhaseCoordinatorContext(int size, int from) { - if (this.throwingRankBuilderType == AbstractRerankerIT.ThrowingRankBuilderType.THROWING_QUERY_PHASE_COORDINATOR_CONTEXT) - return new QueryPhaseRankCoordinatorContext(rankWindowSize()) { - @Override - public ScoreDoc[] rankQueryPhaseResults( - List querySearchResults, - SearchPhaseController.TopDocsStats topDocStats - ) { - throw new UnsupportedOperationException("qpc - simulated failure"); - } - }; - else { - return super.buildQueryPhaseCoordinatorContext(size, from); - } - } - @Override public RankFeaturePhaseRankShardContext buildRankFeaturePhaseShardContext() { if (this.throwingRankBuilderType == AbstractRerankerIT.ThrowingRankBuilderType.THROWING_RANK_FEATURE_PHASE_SHARD_CONTEXT) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionTaskSettingsTests.java index c48d57cf3e03b..7acba78b3066b 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionTaskSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionTaskSettingsTests.java @@ -7,16 +7,17 @@ package org.elasticsearch.xpack.inference.services.alibabacloudsearch.completion; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.Nullable; import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.hamcrest.MatcherAssert; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.Map; +import static org.elasticsearch.xpack.inference.services.alibabacloudsearch.completion.AlibabaCloudSearchCompletionTaskSettings.PARAMETERS; import static org.hamcrest.Matchers.is; public class AlibabaCloudSearchCompletionTaskSettingsTests extends AbstractWireSerializingTestCase< @@ -34,10 +35,20 @@ public void testFromMap() { ); } - public void testIsEmpty() { - var randomSettings = createRandom(); - var stringRep = Strings.toString(randomSettings); - assertEquals(stringRep, randomSettings.isEmpty(), stringRep.equals("{}")); + public void testUpdatedTaskSettings() { + var initialSettings = createRandom(); + var newSettings = createRandom(); + Map newSettingsMap = new HashMap<>(); + if (newSettings.getParameters() != null) { + newSettingsMap.put(PARAMETERS, newSettings.getParameters()); + } + AlibabaCloudSearchCompletionTaskSettings updatedSettings = (AlibabaCloudSearchCompletionTaskSettings) initialSettings + .updatedTaskSettings(Collections.unmodifiableMap(newSettingsMap)); + if (newSettings.getParameters() == null) { + assertEquals(initialSettings.getParameters(), updatedSettings.getParameters()); + } else { + assertEquals(newSettings.getParameters(), updatedSettings.getParameters()); + } } @Override @@ -60,7 +71,7 @@ public static Map getTaskSettingsMap(@Nullable Map(); if (params != null) { - map.put(AlibabaCloudSearchCompletionTaskSettings.PARAMETERS, params); + map.put(PARAMETERS, params); } return map; diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsTaskSettingsTests.java index 9e75a2f475051..4b558949fdc4a 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsTaskSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsTaskSettingsTests.java @@ -15,10 +15,12 @@ import org.hamcrest.MatcherAssert; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import static org.elasticsearch.xpack.inference.InputTypeTests.randomWithIngestAndSearch; +import static org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsTaskSettings.INPUT_TYPE; import static org.hamcrest.Matchers.is; public class AlibabaCloudSearchEmbeddingsTaskSettingsTests extends AbstractWireSerializingTestCase< @@ -31,13 +33,27 @@ public static AlibabaCloudSearchEmbeddingsTaskSettings createRandom() { public void testFromMap() { MatcherAssert.assertThat( - AlibabaCloudSearchEmbeddingsTaskSettings.fromMap( - new HashMap<>(Map.of(AlibabaCloudSearchEmbeddingsTaskSettings.INPUT_TYPE, "ingest")) - ), + AlibabaCloudSearchEmbeddingsTaskSettings.fromMap(new HashMap<>(Map.of(INPUT_TYPE, "ingest"))), is(new AlibabaCloudSearchEmbeddingsTaskSettings(InputType.INGEST)) ); } + public void testUpdatedTaskSettings() { + var initialSettings = createRandom(); + var newSettings = createRandom(); + Map newSettingsMap = new HashMap<>(); + if (newSettings.getInputType() != null) { + newSettingsMap.put(INPUT_TYPE, newSettings.getInputType().toString()); + } + AlibabaCloudSearchEmbeddingsTaskSettings updatedSettings = (AlibabaCloudSearchEmbeddingsTaskSettings) initialSettings + .updatedTaskSettings(Collections.unmodifiableMap(newSettingsMap)); + if (newSettings.getInputType() == null) { + assertEquals(initialSettings.getInputType(), updatedSettings.getInputType()); + } else { + assertEquals(newSettings.getInputType(), updatedSettings.getInputType()); + } + } + public void testFromMap_WhenInputTypeIsNull() { InputType inputType = null; MatcherAssert.assertThat( @@ -72,7 +88,7 @@ public static Map getTaskSettingsMap(@Nullable InputType inputTy var map = new HashMap(); if (inputType != null) { - map.put(AlibabaCloudSearchEmbeddingsTaskSettings.INPUT_TYPE, inputType.toString()); + map.put(INPUT_TYPE, inputType.toString()); } return map; diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseTaskSettingsTests.java index 2c134c6765078..fa78b24d1a4bb 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseTaskSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseTaskSettingsTests.java @@ -19,6 +19,8 @@ import java.util.Map; import static org.elasticsearch.xpack.inference.InputTypeTests.randomWithIngestAndSearch; +import static org.elasticsearch.xpack.inference.services.alibabacloudsearch.sparse.AlibabaCloudSearchSparseTaskSettings.INPUT_TYPE; +import static org.elasticsearch.xpack.inference.services.alibabacloudsearch.sparse.AlibabaCloudSearchSparseTaskSettings.RETURN_TOKEN; import static org.hamcrest.Matchers.is; public class AlibabaCloudSearchSparseTaskSettingsTests extends AbstractWireSerializingTestCase { @@ -31,11 +33,33 @@ public static AlibabaCloudSearchSparseTaskSettings createRandom() { public void testFromMap() { MatcherAssert.assertThat( - AlibabaCloudSearchSparseTaskSettings.fromMap(new HashMap<>(Map.of(AlibabaCloudSearchSparseTaskSettings.INPUT_TYPE, "ingest"))), + AlibabaCloudSearchSparseTaskSettings.fromMap(new HashMap<>(Map.of(INPUT_TYPE, "ingest"))), is(new AlibabaCloudSearchSparseTaskSettings(InputType.INGEST, null)) ); } + public void testUpdatedTaskSettings() { + { + var initialSettings = createRandom(); + var newSettings = createRandom(); + AlibabaCloudSearchSparseTaskSettings updatedSettings = (AlibabaCloudSearchSparseTaskSettings) initialSettings + .updatedTaskSettings(Map.of(RETURN_TOKEN, newSettings.isReturnToken())); + } + { + var initialSettings = createRandom(); + var newSettings = createRandom(); + AlibabaCloudSearchSparseTaskSettings updatedSettings = (AlibabaCloudSearchSparseTaskSettings) initialSettings + .updatedTaskSettings( + Map.of( + INPUT_TYPE, + newSettings.getInputType() == null ? InputType.SEARCH.toString() : newSettings.getInputType().toString(), + RETURN_TOKEN, + newSettings.isReturnToken() + ) + ); + } + } + public void testIsEmpty() { var randomSettings = createRandom(); var stringRep = Strings.toString(randomSettings); @@ -69,11 +93,11 @@ public static Map getTaskSettingsMap(@Nullable InputType inputTy var map = new HashMap(); if (inputType != null) { - map.put(AlibabaCloudSearchSparseTaskSettings.INPUT_TYPE, inputType.toString()); + map.put(INPUT_TYPE, inputType.toString()); } if (returnToken != null) { - map.put(AlibabaCloudSearchSparseTaskSettings.RETURN_TOKEN, returnToken); + map.put(RETURN_TOKEN, returnToken); } return map; diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockSecretSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockSecretSettingsTests.java index 904851842a6c8..88aebd2d9d42b 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockSecretSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockSecretSettingsTests.java @@ -29,6 +29,17 @@ public class AmazonBedrockSecretSettingsTests extends AbstractBWCWireSerializationTestCase { + public void testNewSecretSettings() { + AmazonBedrockSecretSettings initialSettings = createRandom(); + AmazonBedrockSecretSettings newSettings = createRandom(); + + AmazonBedrockSecretSettings finalSettings = (AmazonBedrockSecretSettings) initialSettings.newSecretSettings( + Map.of(ACCESS_KEY_FIELD, newSettings.accessKey.toString(), SECRET_KEY_FIELD, newSettings.secretKey.toString()) + ); + + assertEquals(newSettings, finalSettings); + } + public void testIt_CreatesSettings_ReturnsNullFromMap_null() { var secrets = AmazonBedrockSecretSettings.fromMap(null); assertNull(secrets); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/completion/AmazonBedrockChatCompletionTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/completion/AmazonBedrockChatCompletionTaskSettingsTests.java index 69dd3b1e6257b..adbf2c66ca0e2 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/completion/AmazonBedrockChatCompletionTaskSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/completion/AmazonBedrockChatCompletionTaskSettingsTests.java @@ -38,6 +38,68 @@ public void testIsEmpty() { assertEquals(stringRep, randomSettings.isEmpty(), stringRep.equals("{}")); } + public void updatedTaskSettings_WithEmptyMap_ReturnsSameSettings() { + var initialSettings = createRandom(); + AmazonBedrockChatCompletionTaskSettings updatedSettings = (AmazonBedrockChatCompletionTaskSettings) initialSettings + .updatedTaskSettings(Map.of()); + assertEquals(initialSettings, updatedSettings); + } + + public void updatedTaskSettings_WithNewTemperature_ReturnsUpdatedSettings() { + var initialSettings = createRandom(); + Map newSettings = Map.of(TEMPERATURE_FIELD, 0.7); + AmazonBedrockChatCompletionTaskSettings updatedSettings = (AmazonBedrockChatCompletionTaskSettings) initialSettings + .updatedTaskSettings(newSettings); + assertEquals(0.7, (double) updatedSettings.temperature(), 0.001); + assertEquals(initialSettings.topP(), updatedSettings.topP()); + assertEquals(initialSettings.topK(), updatedSettings.topK()); + assertEquals(initialSettings.maxNewTokens(), updatedSettings.maxNewTokens()); + } + + public void updatedTaskSettings_WithNewTopP_ReturnsUpdatedSettings() { + var initialSettings = createRandom(); + Map newSettings = Map.of(TOP_P_FIELD, 0.8); + AmazonBedrockChatCompletionTaskSettings updatedSettings = (AmazonBedrockChatCompletionTaskSettings) initialSettings + .updatedTaskSettings(newSettings); + assertEquals(0.8, (double) updatedSettings.topP(), 0.001); + assertEquals(initialSettings.temperature(), updatedSettings.temperature()); + assertEquals(initialSettings.topK(), updatedSettings.topK()); + assertEquals(initialSettings.maxNewTokens(), updatedSettings.maxNewTokens()); + } + + public void updatedTaskSettings_WithNewTopK_ReturnsUpdatedSettings() { + var initialSettings = createRandom(); + Map newSettings = Map.of(TOP_K_FIELD, 0.9); + AmazonBedrockChatCompletionTaskSettings updatedSettings = (AmazonBedrockChatCompletionTaskSettings) initialSettings + .updatedTaskSettings(newSettings); + assertEquals(0.9, (double) updatedSettings.topK(), 0.001); + assertEquals(initialSettings.temperature(), updatedSettings.temperature()); + assertEquals(initialSettings.topP(), updatedSettings.topP()); + assertEquals(initialSettings.maxNewTokens(), updatedSettings.maxNewTokens()); + } + + public void updatedTaskSettings_WithNewMaxNewTokens_ReturnsUpdatedSettings() { + var initialSettings = createRandom(); + Map newSettings = Map.of(MAX_NEW_TOKENS_FIELD, 256); + AmazonBedrockChatCompletionTaskSettings updatedSettings = (AmazonBedrockChatCompletionTaskSettings) initialSettings + .updatedTaskSettings(newSettings); + assertEquals(256, (double) updatedSettings.maxNewTokens(), 0.001); + assertEquals(initialSettings.temperature(), updatedSettings.temperature()); + assertEquals(initialSettings.topP(), updatedSettings.topP()); + assertEquals(initialSettings.topK(), updatedSettings.topK()); + } + + public void updatedTaskSettings_WithMultipleNewValues_ReturnsUpdatedSettings() { + var initialSettings = createRandom(); + Map newSettings = Map.of(TEMPERATURE_FIELD, 0.7, TOP_P_FIELD, 0.8, TOP_K_FIELD, 0.9, MAX_NEW_TOKENS_FIELD, 256); + AmazonBedrockChatCompletionTaskSettings updatedSettings = (AmazonBedrockChatCompletionTaskSettings) initialSettings + .updatedTaskSettings(newSettings); + assertEquals(0.7, (double) updatedSettings.temperature(), 0.001); + assertEquals(0.8, (double) updatedSettings.topP(), 0.001); + assertEquals(0.9, (double) updatedSettings.topK(), 0.001); + assertEquals(256, (int) updatedSettings.maxNewTokens(), 0.001); + } + public void testFromMap_AllValues() { var taskMap = getChatCompletionTaskSettingsMap(1.0, 0.5, 0.6, 512); assertEquals( diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionTaskSettingsTests.java index e00de80e8709e..5f6823770345f 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionTaskSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionTaskSettingsTests.java @@ -24,6 +24,25 @@ public class AnthropicChatCompletionTaskSettingsTests extends AbstractBWCWireSerializationTestCase { + public void testUpdatedTaskSettings() { + var initialSettings = createRandom(); + var newSettings = createRandom(); + AnthropicChatCompletionTaskSettings updatedSettings = (AnthropicChatCompletionTaskSettings) initialSettings.updatedTaskSettings( + Map.of( + AnthropicServiceFields.MAX_TOKENS, + newSettings.maxTokens(), + AnthropicServiceFields.TEMPERATURE_FIELD, + newSettings.temperature(), + AnthropicServiceFields.TOP_P_FIELD, + newSettings.topP(), + AnthropicServiceFields.TOP_K_FIELD, + newSettings.topK() + ) + ); + + assertEquals(newSettings, updatedSettings); + } + public static Map getChatCompletionTaskSettingsMap( @Nullable Integer maxTokens, @Nullable Double temperature, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionTaskSettingsTests.java index 8d7dcf1ef5170..21c1a233348fe 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionTaskSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionTaskSettingsTests.java @@ -19,6 +19,7 @@ import org.hamcrest.MatcherAssert; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -38,6 +39,30 @@ public void testIsEmpty() { assertEquals(stringRep, randomSettings.isEmpty(), stringRep.equals("{}")); } + public void testUpdatedTaskSettings() { + var initialSettings = createRandom(); + var newSettings = createRandom(); + var settingsMap = new HashMap(); + if (newSettings.doSample() != null) settingsMap.put(DO_SAMPLE_FIELD, newSettings.doSample()); + if (newSettings.temperature() != null) settingsMap.put(TEMPERATURE_FIELD, newSettings.temperature()); + if (newSettings.topP() != null) settingsMap.put(TOP_P_FIELD, newSettings.topP()); + if (newSettings.maxNewTokens() != null) settingsMap.put(MAX_NEW_TOKENS_FIELD, newSettings.maxNewTokens()); + + AzureAiStudioChatCompletionTaskSettings updatedSettings = (AzureAiStudioChatCompletionTaskSettings) initialSettings + .updatedTaskSettings(Collections.unmodifiableMap(settingsMap)); + + assertEquals( + newSettings.temperature() == null ? initialSettings.temperature() : newSettings.temperature(), + updatedSettings.temperature() + ); + assertEquals(newSettings.topP() == null ? initialSettings.topP() : newSettings.topP(), updatedSettings.topP()); + assertEquals(newSettings.doSample() == null ? initialSettings.doSample() : newSettings.doSample(), updatedSettings.doSample()); + assertEquals( + newSettings.maxNewTokens() == null ? initialSettings.maxNewTokens() : newSettings.maxNewTokens(), + updatedSettings.maxNewTokens() + ); + } + public void testFromMap_AllValues() { var taskMap = getTaskSettingsMap(1.0, 2.0, true, 512); assertEquals( diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsTaskSettingsTests.java index 4b6b38bd15c0d..cdfde5fcb09c9 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsTaskSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsTaskSettingsTests.java @@ -20,6 +20,7 @@ import org.hamcrest.MatcherAssert; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -32,6 +33,23 @@ public void testIsEmpty() { assertEquals(stringRep, randomSettings.isEmpty(), stringRep.equals("{}")); } + public void testUpdatedTaskSettings() { + var initialSettings = createRandom(); + var newSettings = createRandom(); + Map newSettingsMap = new HashMap<>(); + if (newSettings.user() != null) { + newSettingsMap.put(AzureAiStudioConstants.USER_FIELD, newSettings.user()); + } + AzureAiStudioEmbeddingsTaskSettings updatedSettings = (AzureAiStudioEmbeddingsTaskSettings) initialSettings.updatedTaskSettings( + Collections.unmodifiableMap(newSettingsMap) + ); + if (newSettings.user() == null) { + assertEquals(initialSettings.user(), updatedSettings.user()); + } else { + assertEquals(newSettings.user(), updatedSettings.user()); + } + } + public void testFromMap_WithUser() { assertEquals( new AzureAiStudioEmbeddingsTaskSettings("user"), diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettingsTests.java index e08365e7ca3bf..dbbf90054a55b 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettingsTests.java @@ -30,7 +30,31 @@ public class AzureOpenAiSecretSettingsTests extends AbstractBWCWireSerializationTestCase { public static AzureOpenAiSecretSettings createRandom() { - return new AzureOpenAiSecretSettings(randomSecureStringOfLength(15), randomSecureStringOfLength(15)); + boolean isApiKeyNotEntraId = randomBoolean(); + return new AzureOpenAiSecretSettings( + isApiKeyNotEntraId ? randomSecureStringOfLength(15) : null, + isApiKeyNotEntraId == false ? randomSecureStringOfLength(15) : null + ); + } + + public void testNewSecretSettingsApiKey() { + AzureOpenAiSecretSettings initialSettings = createRandom(); + AzureOpenAiSecretSettings newSettings = new AzureOpenAiSecretSettings(randomSecureStringOfLength(15), null); + AzureOpenAiSecretSettings finalSettings = (AzureOpenAiSecretSettings) initialSettings.newSecretSettings( + Map.of(API_KEY, newSettings.apiKey().toString()) + ); + + assertEquals(newSettings, finalSettings); + } + + public void testNewSecretSettingsEntraId() { + AzureOpenAiSecretSettings initialSettings = createRandom(); + AzureOpenAiSecretSettings newSettings = new AzureOpenAiSecretSettings(null, randomSecureStringOfLength(15)); + AzureOpenAiSecretSettings finalSettings = (AzureOpenAiSecretSettings) initialSettings.newSecretSettings( + Map.of(ENTRA_ID, newSettings.entraId().toString()) + ); + + assertEquals(newSettings, finalSettings); } public void testFromMap_ApiKey_Only() { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionTaskSettingsTests.java index 8e8d9c4f92800..9d77abfe6d512 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionTaskSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionTaskSettingsTests.java @@ -38,6 +38,16 @@ public void testIsEmpty() { assertEquals(stringRep, randomSettings.isEmpty(), stringRep.equals("{}")); } + public void testUpdatedTaskSettings() { + var initialSettings = createRandom(); + var newSettings = createRandom(); + AzureOpenAiCompletionTaskSettings updatedSettings = (AzureOpenAiCompletionTaskSettings) initialSettings.updatedTaskSettings( + newSettings.user() == null ? Map.of() : Map.of(AzureOpenAiServiceFields.USER, newSettings.user()) + ); + + assertEquals(newSettings.user() == null ? initialSettings.user() : newSettings.user(), updatedSettings.user()); + } + public void testFromMap_WithUser() { var user = "user"; diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsTaskSettingsTests.java index 72a063af37b90..4df9f2f6bcce0 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsTaskSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsTaskSettingsTests.java @@ -12,13 +12,13 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.Nullable; import org.elasticsearch.test.AbstractWireSerializingTestCase; -import org.elasticsearch.xpack.inference.services.azureopenai.AzureOpenAiServiceFields; import org.hamcrest.MatcherAssert; import java.io.IOException; import java.util.HashMap; import java.util.Map; +import static org.elasticsearch.xpack.inference.services.azureopenai.AzureOpenAiServiceFields.USER; import static org.hamcrest.Matchers.is; public class AzureOpenAiEmbeddingsTaskSettingsTests extends AbstractWireSerializingTestCase { @@ -41,17 +41,31 @@ public static AzureOpenAiEmbeddingsTaskSettings createRandom() { return new AzureOpenAiEmbeddingsTaskSettings(user); } + public void testUpdatedTaskSettings() { + var initialSettings = createRandom(); + var newSettings = createRandom(); + AzureOpenAiEmbeddingsTaskSettings updatedSettings = (AzureOpenAiEmbeddingsTaskSettings) initialSettings.updatedTaskSettings( + newSettings.user() == null ? Map.of() : Map.of(USER, newSettings.user()) + ); + + if (newSettings.user() == null) { + assertEquals(initialSettings.user(), updatedSettings.user()); + } else { + assertEquals(newSettings.user(), updatedSettings.user()); + } + } + public void testFromMap_WithUser() { assertEquals( new AzureOpenAiEmbeddingsTaskSettings("user"), - AzureOpenAiEmbeddingsTaskSettings.fromMap(new HashMap<>(Map.of(AzureOpenAiServiceFields.USER, "user"))) + AzureOpenAiEmbeddingsTaskSettings.fromMap(new HashMap<>(Map.of(USER, "user"))) ); } public void testFromMap_UserIsEmptyString() { var thrownException = expectThrows( ValidationException.class, - () -> AzureOpenAiEmbeddingsTaskSettings.fromMap(new HashMap<>(Map.of(AzureOpenAiServiceFields.USER, ""))) + () -> AzureOpenAiEmbeddingsTaskSettings.fromMap(new HashMap<>(Map.of(USER, ""))) ); MatcherAssert.assertThat( @@ -66,7 +80,7 @@ public void testFromMap_MissingUser_DoesNotThrowException() { } public void testOverrideWith_KeepsOriginalValuesWithOverridesAreNull() { - var taskSettings = AzureOpenAiEmbeddingsTaskSettings.fromMap(new HashMap<>(Map.of(AzureOpenAiServiceFields.USER, "user"))); + var taskSettings = AzureOpenAiEmbeddingsTaskSettings.fromMap(new HashMap<>(Map.of(USER, "user"))); var overriddenTaskSettings = AzureOpenAiEmbeddingsTaskSettings.of( taskSettings, @@ -76,11 +90,9 @@ public void testOverrideWith_KeepsOriginalValuesWithOverridesAreNull() { } public void testOverrideWith_UsesOverriddenSettings() { - var taskSettings = AzureOpenAiEmbeddingsTaskSettings.fromMap(new HashMap<>(Map.of(AzureOpenAiServiceFields.USER, "user"))); + var taskSettings = AzureOpenAiEmbeddingsTaskSettings.fromMap(new HashMap<>(Map.of(USER, "user"))); - var requestTaskSettings = AzureOpenAiEmbeddingsRequestTaskSettings.fromMap( - new HashMap<>(Map.of(AzureOpenAiServiceFields.USER, "user2")) - ); + var requestTaskSettings = AzureOpenAiEmbeddingsRequestTaskSettings.fromMap(new HashMap<>(Map.of(USER, "user2"))); var overriddenTaskSettings = AzureOpenAiEmbeddingsTaskSettings.of(taskSettings, requestTaskSettings); MatcherAssert.assertThat(overriddenTaskSettings, is(new AzureOpenAiEmbeddingsTaskSettings("user2"))); @@ -105,7 +117,7 @@ public static Map getAzureOpenAiRequestTaskSettingsMap(@Nullable var map = new HashMap(); if (user != null) { - map.put(AzureOpenAiServiceFields.USER, user); + map.put(USER, user); } return map; diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsTaskSettingsTests.java index 90c9b032465c6..3df8fcaf5d6b8 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsTaskSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsTaskSettingsTests.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.Arrays; +import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.Locale; @@ -43,6 +44,31 @@ public void testIsEmpty() { assertEquals(stringRep, randomSettings.isEmpty(), stringRep.equals("{}")); } + public void testUpdatedTaskSettings() { + var initialSettings = createRandom(); + var newSettings = createRandom(); + Map newSettingsMap = new HashMap<>(); + if (newSettings.getInputType() != null) { + newSettingsMap.put(CohereEmbeddingsTaskSettings.INPUT_TYPE, newSettings.getInputType().toString()); + } + if (newSettings.getTruncation() != null) { + newSettingsMap.put(CohereServiceFields.TRUNCATE, newSettings.getTruncation().toString()); + } + CohereEmbeddingsTaskSettings updatedSettings = (CohereEmbeddingsTaskSettings) initialSettings.updatedTaskSettings( + Collections.unmodifiableMap(newSettingsMap) + ); + if (newSettings.getInputType() == null) { + assertEquals(initialSettings.getInputType(), updatedSettings.getInputType()); + } else { + assertEquals(newSettings.getInputType(), updatedSettings.getInputType()); + } + if (newSettings.getTruncation() == null) { + assertEquals(initialSettings.getTruncation(), updatedSettings.getTruncation()); + } else { + assertEquals(newSettings.getTruncation(), updatedSettings.getTruncation()); + } + } + public void testFromMap_CreatesEmptySettings_WhenAllFieldsAreNull() { MatcherAssert.assertThat( CohereEmbeddingsTaskSettings.fromMap(new HashMap<>(Map.of())), diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankTaskSettingsTests.java new file mode 100644 index 0000000000000..6924ee05ecbb8 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankTaskSettingsTests.java @@ -0,0 +1,154 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.cohere.rerank; + +import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.Matchers.containsString; + +public class CohereRerankTaskSettingsTests extends AbstractWireSerializingTestCase { + + public static CohereRerankTaskSettings createRandom() { + var returnDocuments = randomBoolean() ? randomBoolean() : null; + var topNDocsOnly = randomBoolean() ? randomIntBetween(1, 10) : null; + var maxChunksPerDoc = randomBoolean() ? randomIntBetween(1, 20) : null; + + return new CohereRerankTaskSettings(topNDocsOnly, returnDocuments, maxChunksPerDoc); + } + + public void testFromMap_WithValidValues_ReturnsSettings() { + Map taskMap = Map.of( + CohereRerankTaskSettings.RETURN_DOCUMENTS, + true, + CohereRerankTaskSettings.TOP_N_DOCS_ONLY, + 5, + CohereRerankTaskSettings.MAX_CHUNKS_PER_DOC, + 10 + ); + var settings = CohereRerankTaskSettings.fromMap(new HashMap<>(taskMap)); + assertTrue(settings.getReturnDocuments()); + assertEquals(5, settings.getTopNDocumentsOnly().intValue()); + assertEquals(10, settings.getMaxChunksPerDoc().intValue()); + } + + public void testFromMap_WithNullValues_ReturnsSettingsWithNulls() { + var settings = CohereRerankTaskSettings.fromMap(Map.of()); + assertNull(settings.getReturnDocuments()); + assertNull(settings.getTopNDocumentsOnly()); + assertNull(settings.getMaxChunksPerDoc()); + } + + public void testFromMap_WithInvalidReturnDocuments_ThrowsValidationException() { + Map taskMap = Map.of( + CohereRerankTaskSettings.RETURN_DOCUMENTS, + "invalid", + CohereRerankTaskSettings.TOP_N_DOCS_ONLY, + 5, + CohereRerankTaskSettings.MAX_CHUNKS_PER_DOC, + 10 + ); + var thrownException = expectThrows(ValidationException.class, () -> CohereRerankTaskSettings.fromMap(new HashMap<>(taskMap))); + assertThat(thrownException.getMessage(), containsString("field [return_documents] is not of the expected type")); + } + + public void testFromMap_WithInvalidTopNDocsOnly_ThrowsValidationException() { + Map taskMap = Map.of( + CohereRerankTaskSettings.RETURN_DOCUMENTS, + true, + CohereRerankTaskSettings.TOP_N_DOCS_ONLY, + "invalid", + CohereRerankTaskSettings.MAX_CHUNKS_PER_DOC, + 10 + ); + var thrownException = expectThrows(ValidationException.class, () -> CohereRerankTaskSettings.fromMap(new HashMap<>(taskMap))); + assertThat(thrownException.getMessage(), containsString("field [top_n] is not of the expected type")); + } + + public void testFromMap_WithInvalidMaxChunksPerDoc_ThrowsValidationException() { + Map taskMap = Map.of( + CohereRerankTaskSettings.RETURN_DOCUMENTS, + true, + CohereRerankTaskSettings.TOP_N_DOCS_ONLY, + 5, + CohereRerankTaskSettings.MAX_CHUNKS_PER_DOC, + "invalid" + ); + var thrownException = expectThrows(ValidationException.class, () -> CohereRerankTaskSettings.fromMap(new HashMap<>(taskMap))); + assertThat(thrownException.getMessage(), containsString("field [max_chunks_per_doc] is not of the expected type")); + } + + public void UpdatedTaskSettings_WithEmptyMap_ReturnsSameSettings() { + var initialSettings = new CohereRerankTaskSettings(5, true, 10); + CohereRerankTaskSettings updatedSettings = (CohereRerankTaskSettings) initialSettings.updatedTaskSettings(Map.of()); + assertEquals(initialSettings, updatedSettings); + } + + public void testUpdatedTaskSettings_WithNewReturnDocuments_ReturnsUpdatedSettings() { + var initialSettings = new CohereRerankTaskSettings(5, true, 10); + Map newSettings = Map.of(CohereRerankTaskSettings.RETURN_DOCUMENTS, false); + CohereRerankTaskSettings updatedSettings = (CohereRerankTaskSettings) initialSettings.updatedTaskSettings(newSettings); + assertFalse(updatedSettings.getReturnDocuments()); + assertEquals(initialSettings.getTopNDocumentsOnly(), updatedSettings.getTopNDocumentsOnly()); + assertEquals(initialSettings.getMaxChunksPerDoc(), updatedSettings.getMaxChunksPerDoc()); + } + + public void testUpdatedTaskSettings_WithNewTopNDocsOnly_ReturnsUpdatedSettings() { + var initialSettings = new CohereRerankTaskSettings(5, true, 10); + Map newSettings = Map.of(CohereRerankTaskSettings.TOP_N_DOCS_ONLY, 7); + CohereRerankTaskSettings updatedSettings = (CohereRerankTaskSettings) initialSettings.updatedTaskSettings(newSettings); + assertEquals(7, updatedSettings.getTopNDocumentsOnly().intValue()); + assertEquals(initialSettings.getReturnDocuments(), updatedSettings.getReturnDocuments()); + assertEquals(initialSettings.getMaxChunksPerDoc(), updatedSettings.getMaxChunksPerDoc()); + } + + public void testUpdatedTaskSettings_WithNewMaxChunksPerDoc_ReturnsUpdatedSettings() { + var initialSettings = new CohereRerankTaskSettings(5, true, 10); + Map newSettings = Map.of(CohereRerankTaskSettings.MAX_CHUNKS_PER_DOC, 15); + CohereRerankTaskSettings updatedSettings = (CohereRerankTaskSettings) initialSettings.updatedTaskSettings(newSettings); + assertEquals(15, updatedSettings.getMaxChunksPerDoc().intValue()); + assertEquals(initialSettings.getReturnDocuments(), updatedSettings.getReturnDocuments()); + assertEquals(initialSettings.getTopNDocumentsOnly(), updatedSettings.getTopNDocumentsOnly()); + } + + public void testUpdatedTaskSettings_WithMultipleNewValues_ReturnsUpdatedSettings() { + var initialSettings = new CohereRerankTaskSettings(5, true, 10); + Map newSettings = Map.of( + CohereRerankTaskSettings.RETURN_DOCUMENTS, + false, + CohereRerankTaskSettings.TOP_N_DOCS_ONLY, + 7, + CohereRerankTaskSettings.MAX_CHUNKS_PER_DOC, + 15 + ); + CohereRerankTaskSettings updatedSettings = (CohereRerankTaskSettings) initialSettings.updatedTaskSettings(newSettings); + assertFalse(updatedSettings.getReturnDocuments()); + assertEquals(7, updatedSettings.getTopNDocumentsOnly().intValue()); + assertEquals(15, updatedSettings.getMaxChunksPerDoc().intValue()); + } + + @Override + protected Writeable.Reader instanceReader() { + return CohereRerankTaskSettings::new; + } + + @Override + protected CohereRerankTaskSettings createTestInstance() { + return createRandom(); + } + + @Override + protected CohereRerankTaskSettings mutateInstance(CohereRerankTaskSettings instance) throws IOException { + return randomValueOtherThan(instance, CohereRerankTaskSettingsTests::createRandom); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettingsTests.java index 72e6daa911c1d..4207896fc54f3 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettingsTests.java @@ -15,7 +15,9 @@ import org.elasticsearch.xcontent.XContentType; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; +import java.util.Map; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.sameInstance; @@ -28,6 +30,23 @@ public void testIsEmpty() { assertEquals(stringRep, randomSettings.isEmpty(), stringRep.equals("{}")); } + public void testUpdatedTaskSettings() { + var initialSettings = createRandom(); + var newSettings = createRandom(); + Map newSettingsMap = new HashMap<>(); + if (newSettings.returnDocuments() != null) { + newSettingsMap.put(CustomElandRerankTaskSettings.RETURN_DOCUMENTS, newSettings.returnDocuments()); + } + CustomElandRerankTaskSettings updatedSettings = (CustomElandRerankTaskSettings) initialSettings.updatedTaskSettings( + Collections.unmodifiableMap(newSettingsMap) + ); + if (newSettings.returnDocuments() == null) { + assertEquals(initialSettings.returnDocuments(), updatedSettings.returnDocuments()); + } else { + assertEquals(newSettings.returnDocuments(), updatedSettings.returnDocuments()); + } + } + public void testDefaultsFromMap_MapIsNull_ReturnsDefaultSettings() { var customElandRerankTaskSettings = CustomElandRerankTaskSettings.defaultsFromMap(null); @@ -69,18 +88,6 @@ public void testToXContent_WritesAllValues() throws IOException { {"return_documents":true}""")); } - public void testToXContent_DoesNotWriteReturnDocuments_IfNull() throws IOException { - Boolean bool = null; - var serviceSettings = new CustomElandRerankTaskSettings(bool); - - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); - serviceSettings.toXContent(builder, null); - String xContentResult = Strings.toString(builder); - - assertThat(xContentResult, is(""" - {}""")); - } - public void testOf_PrefersNonNullRequestTaskSettings() { var originalSettings = new CustomElandRerankTaskSettings(Boolean.FALSE); var requestTaskSettings = new CustomElandRerankTaskSettings(Boolean.TRUE); @@ -90,16 +97,6 @@ public void testOf_PrefersNonNullRequestTaskSettings() { assertThat(taskSettings, sameInstance(requestTaskSettings)); } - public void testOf_UseOriginalSettings_IfRequestSettingsValuesAreNull() { - Boolean bool = null; - var originalSettings = new CustomElandRerankTaskSettings(Boolean.TRUE); - var requestTaskSettings = new CustomElandRerankTaskSettings(bool); - - var taskSettings = CustomElandRerankTaskSettings.of(originalSettings, requestTaskSettings); - - assertThat(taskSettings, sameInstance(originalSettings)); - } - private static CustomElandRerankTaskSettings createRandom() { return new CustomElandRerankTaskSettings(randomOptionalBoolean()); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiSecretSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiSecretSettingsTests.java index 95d3522b863a9..90738d43aacb3 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiSecretSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiSecretSettingsTests.java @@ -29,6 +29,15 @@ public static GoogleVertexAiSecretSettings createRandom() { return new GoogleVertexAiSecretSettings(randomSecureStringOfLength(30)); } + public void testNewSecretSettings() { + GoogleVertexAiSecretSettings initialSettings = createRandom(); + GoogleVertexAiSecretSettings newSettings = createRandom(); + GoogleVertexAiSecretSettings newGoogleVertexAiSecretSettings = (GoogleVertexAiSecretSettings) initialSettings.newSecretSettings( + Map.of(GoogleVertexAiSecretSettings.SERVICE_ACCOUNT_JSON, newSettings.serviceAccountJson.toString()) + ); + assertEquals(newSettings, newGoogleVertexAiSecretSettings); + } + public void testFromMap_ReturnsNull_WhenMapIsNUll() { assertNull(GoogleVertexAiSecretSettings.fromMap(null)); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsTaskSettingsTests.java index ac7e9348b370b..5b87bbc3c42c8 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsTaskSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsTaskSettingsTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -31,6 +32,23 @@ public void testIsEmpty() { assertEquals(stringRep, randomSettings.isEmpty(), stringRep.equals("{}")); } + public void testUpdatedTaskSettings() { + var initialSettings = createRandom(); + var newSettings = createRandom(); + Map newSettingsMap = new HashMap<>(); + if (newSettings.autoTruncate() != null) { + newSettingsMap.put(GoogleVertexAiEmbeddingsTaskSettings.AUTO_TRUNCATE, newSettings.autoTruncate()); + } + GoogleVertexAiEmbeddingsTaskSettings updatedSettings = (GoogleVertexAiEmbeddingsTaskSettings) initialSettings.updatedTaskSettings( + Collections.unmodifiableMap(newSettingsMap) + ); + if (newSettings.autoTruncate() == null) { + assertEquals(initialSettings.autoTruncate(), updatedSettings.autoTruncate()); + } else { + assertEquals(newSettings.autoTruncate(), updatedSettings.autoTruncate()); + } + } + public void testFromMap_AutoTruncateIsSet() { var autoTruncate = true; var taskSettingsMap = getTaskSettingsMap(autoTruncate); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankTaskSettingsTests.java index 03f89b6a2c042..957defb54d846 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankTaskSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankTaskSettingsTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -33,6 +34,23 @@ public void testIsEmpty() { assertEquals(stringRep, randomSettings.isEmpty(), stringRep.equals("{}")); } + public void testUpdatedTaskSettings() { + var initialSettings = createRandom(); + var newSettings = createRandom(); + Map newSettingsMap = new HashMap<>(); + if (newSettings.topN() != null) { + newSettingsMap.put(GoogleVertexAiRerankTaskSettings.TOP_N, newSettings.topN()); + } + GoogleVertexAiRerankTaskSettings updatedSettings = (GoogleVertexAiRerankTaskSettings) initialSettings.updatedTaskSettings( + Collections.unmodifiableMap(newSettingsMap) + ); + if (newSettings.topN() == null) { + assertEquals(initialSettings.topN(), updatedSettings.topN()); + } else { + assertEquals(newSettings.topN(), updatedSettings.topN()); + } + } + public void testFromMap_TopNIsSet() { var topN = 1; var taskSettingsMap = getTaskSettingsMap(topN); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionTaskSettingsTests.java index 16d7e8f1db9be..9d1170bb23dbb 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionTaskSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionTaskSettingsTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.xpack.inference.services.openai.OpenAiServiceFields; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -31,6 +32,23 @@ public void testIsEmpty() { assertEquals(stringRep, randomSettings.isEmpty(), stringRep.equals("{}")); } + public void testUpdatedTaskSettings() { + var initialSettings = createRandomWithUser(); + var newSettings = createRandomWithUser(); + Map newSettingsMap = new HashMap<>(); + if (newSettings.user() != null) { + newSettingsMap.put(OpenAiServiceFields.USER, newSettings.user()); + } + OpenAiChatCompletionTaskSettings updatedSettings = (OpenAiChatCompletionTaskSettings) initialSettings.updatedTaskSettings( + Collections.unmodifiableMap(newSettingsMap) + ); + if (newSettings.user() == null) { + assertEquals(initialSettings.user(), updatedSettings.user()); + } else { + assertEquals(newSettings.user(), updatedSettings.user()); + } + } + public void testFromMap_WithUser() { assertEquals( new OpenAiChatCompletionTaskSettings("user"), diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsTaskSettingsTests.java index a5ae2f0a3a44b..0512c36e64de5 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsTaskSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsTaskSettingsTests.java @@ -14,9 +14,11 @@ import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.openai.OpenAiServiceFields; +import org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionTaskSettings; import org.hamcrest.MatcherAssert; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -37,11 +39,28 @@ public static OpenAiEmbeddingsTaskSettings createRandom() { } public void testIsEmpty() { - var randomSettings = createRandom(); + var randomSettings = new OpenAiChatCompletionTaskSettings(randomBoolean() ? null : "username"); var stringRep = Strings.toString(randomSettings); assertEquals(stringRep, randomSettings.isEmpty(), stringRep.equals("{}")); } + public void testUpdatedTaskSettings() { + var initialSettings = createRandom(); + var newSettings = createRandom(); + Map newSettingsMap = new HashMap<>(); + if (newSettings.user() != null) { + newSettingsMap.put(OpenAiServiceFields.USER, newSettings.user()); + } + OpenAiEmbeddingsTaskSettings updatedSettings = (OpenAiEmbeddingsTaskSettings) initialSettings.updatedTaskSettings( + Collections.unmodifiableMap(newSettingsMap) + ); + if (newSettings.user() == null) { + assertEquals(initialSettings.user(), updatedSettings.user()); + } else { + assertEquals(newSettings.user(), updatedSettings.user()); + } + } + public void testFromMap_WithUser() { assertEquals( new OpenAiEmbeddingsTaskSettings("user"), diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/settings/DefaultSecretSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/settings/DefaultSecretSettingsTests.java index 212a867349e5c..118cf25a452a7 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/settings/DefaultSecretSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/settings/DefaultSecretSettingsTests.java @@ -26,6 +26,15 @@ public static DefaultSecretSettings createRandom() { return new DefaultSecretSettings(new SecureString(randomAlphaOfLength(15).toCharArray())); } + public void testNewSecretSettings() { + DefaultSecretSettings initialSettings = createRandom(); + DefaultSecretSettings newSettings = createRandom(); + DefaultSecretSettings finalSettings = (DefaultSecretSettings) initialSettings.newSecretSettings( + Map.of(DefaultSecretSettings.API_KEY, newSettings.apiKey().toString()) + ); + assertEquals(newSettings, finalSettings); + } + public void testFromMap() { var apiKey = "abc"; var serviceSettings = DefaultSecretSettings.fromMap(new HashMap<>(Map.of(DefaultSecretSettings.API_KEY, apiKey))); diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java index 759fa6af98868..6e139cc3ce9e6 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java @@ -79,15 +79,17 @@ boolean newIndexHasSyntheticSourceUsage( return false; } - var tmpIndexMetadata = buildIndexMetadataForMapperService(indexName, isTimeSeries, indexTemplateAndCreateRequestSettings); - try (var mapperService = mapperServiceFactory.apply(tmpIndexMetadata)) { - // combinedTemplateMappings can be null when creating system indices - // combinedTemplateMappings can be empty when creating a normal index that doesn't match any template and without mapping. - if (combinedTemplateMappings == null || combinedTemplateMappings.isEmpty()) { - combinedTemplateMappings = List.of(new CompressedXContent("{}")); + try { + var tmpIndexMetadata = buildIndexMetadataForMapperService(indexName, isTimeSeries, indexTemplateAndCreateRequestSettings); + try (var mapperService = mapperServiceFactory.apply(tmpIndexMetadata)) { + // combinedTemplateMappings can be null when creating system indices + // combinedTemplateMappings can be empty when creating a normal index that doesn't match any template and without mapping. + if (combinedTemplateMappings == null || combinedTemplateMappings.isEmpty()) { + combinedTemplateMappings = List.of(new CompressedXContent("{}")); + } + mapperService.merge(MapperService.SINGLE_MAPPING_NAME, combinedTemplateMappings, MapperService.MergeReason.INDEX_TEMPLATE); + return mapperService.documentMapper().sourceMapper().isSynthetic(); } - mapperService.merge(MapperService.SINGLE_MAPPING_NAME, combinedTemplateMappings, MapperService.MergeReason.INDEX_TEMPLATE); - return mapperService.documentMapper().sourceMapper().isSynthetic(); } catch (AssertionError | Exception e) { // In case invalid mappings or setting are provided, then mapper service creation can fail. // In that case it is ok to return false here. The index creation will fail anyway later, so need to fallback to stored source. diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/40_source_mode_setting.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/40_source_mode_setting.yml new file mode 100644 index 0000000000000..33fedce3b59c1 --- /dev/null +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/40_source_mode_setting.yml @@ -0,0 +1,847 @@ +--- +create an index with disabled source mode and standard index mode without setting: + - do: + indices.create: + index: test_disabled_standard + body: + settings: + index: + mode: standard + mappings: + _source: + mode: disabled + + - do: + indices.get_mapping: + index: test_disabled_standard + + - match: { test_disabled_standard.mappings._source.mode: disabled } + +--- +create an index with stored source mode and standard index mode without setting: + - do: + indices.create: + index: test_stored_standard + body: + settings: + index: + mode: standard + mappings: + _source: + mode: stored + + - do: + indices.get_mapping: + index: test_stored_standard + + - match: { test_stored_standard.mappings._source.mode: stored } + +--- +create an index with synthetic source mode and standard index mode without setting: + - do: + indices.create: + index: test_synthetic_standard + body: + settings: + index: + mode: standard + mappings: + _source: + mode: synthetic + + - do: + indices.get_mapping: + index: test_synthetic_standard + + - match: { test_synthetic_standard.mappings._source.mode: synthetic } + +--- +create an index with disabled source mode and logsdb index mode without setting: + - do: + catch: bad_request + indices.create: + index: test_disabled_logsdb + body: + settings: + index: + mode: logsdb + mappings: + _source: + mode: disabled + + - match: { error.type: "mapper_parsing_exception" } + - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode" } + +--- +create an index with stored source mode and logsdb index mode without setting: + - do: + indices.create: + index: test_stored_logsdb + body: + settings: + index: + mode: logsdb + mappings: + _source: + mode: stored + + - do: + indices.get_settings: + index: "test_stored_logsdb" + - match: { test_stored_logsdb.settings.index.mode: logsdb } + + - do: + indices.get_mapping: + index: test_stored_logsdb + + - match: { test_stored_logsdb.mappings._source.mode: stored } + +--- +create an index with synthetic source mode and logsdb index mode without setting: + - do: + indices.create: + index: test_synthetic_logsdb + body: + settings: + index: + mode: logsdb + mappings: + _source: + mode: synthetic + + - do: + indices.get_mapping: + index: test_synthetic_logsdb + + - match: { test_synthetic_logsdb.mappings._source.mode: synthetic } + +--- +create an index with disabled source mode and time series index mode without setting: + - do: + catch: bad_request + indices.create: + index: test_disabled_time_series + body: + settings: + index: + mode: time_series + routing_path: [ keyword ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + _source: + mode: disabled + properties: + keyword: + type: keyword + time_series_dimension: true + + - match: { error.type: "mapper_parsing_exception" } + - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [time_series] index mode" } + +--- +create an index with stored source mode and time series index mode without setting: + - do: + indices.create: + index: test_stored_time_series + body: + settings: + index: + mode: time_series + routing_path: [ keyword ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + _source: + mode: stored + properties: + keyword: + type: keyword + time_series_dimension: true + + - do: + indices.get_settings: + index: "test_stored_time_series" + - match: { test_stored_time_series.settings.index.mode: time_series } + + - do: + indices.get_mapping: + index: test_stored_time_series + + - match: { test_stored_time_series.mappings._source.mode: stored } + + +--- +create an index with synthetic source mode and time series index mode without setting: + - do: + indices.create: + index: test_synthetic_time_series + body: + settings: + index: + mode: time_series + routing_path: [ keyword ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + _source: + mode: synthetic + properties: + keyword: + type: keyword + time_series_dimension: true + + - do: + indices.get_settings: + index: "test_synthetic_time_series" + - match: { test_synthetic_time_series.settings.index.mode: time_series } + + - do: + indices.get_mapping: + index: test_synthetic_time_series + + - match: { test_synthetic_time_series.mappings._source.mode: synthetic } + +--- +create an index with stored source mode: + - do: + indices.create: + index: test_stored_default + body: + mappings: + _source: + mode: stored + + - do: + indices.get_mapping: + index: test_stored_default + + - match: { test_stored_default.mappings._source.mode: stored } + +--- +override stored to synthetic source mode: + - do: + indices.create: + index: test_stored_override + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + _source: + mode: stored + + - do: + indices.get_mapping: + index: test_stored_override + + - match: { test_stored_override.mappings._source.mode: synthetic } + +--- +override stored to disabled source mode: + - do: + indices.create: + index: test_stored_disabled + body: + settings: + index: + mapping.source.mode: disabled + mappings: + _source: + mode: stored + + - do: + indices.get_mapping: + index: test_stored_disabled + + - match: { test_stored_disabled.mappings._source.mode: disabled } + +--- +create an index with disabled source mode: + - do: + indices.create: + index: test_disabled_default + body: + mappings: + _source: + mode: disabled + + - do: + indices.get_mapping: + index: test_disabled_default + + - match: { test_disabled_default.mappings._source.mode: disabled } + +--- +override disabled to synthetic source mode: + - do: + indices.create: + index: test_disabled_synthetic + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + _source: + mode: disabled + + - do: + indices.get_mapping: + index: test_disabled_synthetic + + - match: { test_disabled_synthetic.mappings._source.mode: synthetic } + +--- +override disabled to stored source mode: + - do: + indices.create: + index: test_disabled_stored + body: + settings: + index: + mapping.source.mode: stored + mappings: + _source: + mode: disabled + + - do: + indices.get_mapping: + index: test_disabled_stored + + - match: { test_disabled_stored.mappings._source.mode: stored } + +--- +create an index with synthetic source mode: + - do: + indices.create: + index: test_synthetic_default + body: + mappings: + _source: + mode: synthetic + + - do: + indices.get_mapping: + index: test_synthetic_default + + - match: { test_synthetic_default.mappings._source.mode: synthetic } + +--- +override synthetic to stored source mode: + - do: + indices.create: + index: test_synthetic_stored + body: + settings: + index: + mapping.source.mode: stored + mappings: + _source: + mode: synthetic + + - do: + indices.get_mapping: + index: test_synthetic_stored + + - match: { test_synthetic_stored.mappings._source.mode: stored } + +--- +override synthetic to disabled source mode: + - do: + indices.create: + index: test_synthetic_disabled + body: + settings: + index: + mapping.source.mode: disabled + mappings: + _source: + mode: synthetic + + - do: + indices.get_mapping: + index: test_synthetic_disabled + + - match: { test_synthetic_disabled.mappings._source.mode: disabled } + +--- +create an index with unspecified source mode: + - do: + indices.create: + index: test_unset_default + + - do: + indices.get_mapping: + index: test_unset_default + + - match: { test_unset_default.mappings._source.mode: null } + +--- +override unspecified to stored source mode: + - do: + indices.create: + index: test_unset_stored + body: + settings: + index: + mapping.source.mode: stored + + - do: + indices.get_mapping: + index: test_unset_stored + + - match: { test_unset_stored.mappings: { } } + +--- +override unspecified to disabled source mode: + - do: + indices.create: + index: test_unset_disabled + body: + settings: + index: + mapping.source.mode: disabled + + - do: + indices.get_mapping: + index: test_unset_disabled + + - match: { test_unset_disabled.mappings: { } } + +--- +override unspecified to synthetic source mode: + - do: + indices.create: + index: test_unset_synthetic + body: + settings: + index: + mapping.source.mode: synthetic + + - do: + indices.get_mapping: + index: test_unset_synthetic + + - match: { test_unset_synthetic.mappings: { } } + +--- +create an index with standard index mode: + - do: + indices.create: + index: test_standard_index_mode + body: + settings: + index: + mode: standard + mappings: + _source: + mode: stored + + - do: + indices.get_mapping: + index: test_standard_index_mode + + - match: { test_standard_index_mode.mappings._source.mode: stored } + +--- +create an index with time_series index mode and synthetic source: + - do: + indices.create: + index: test_time_series_index_mode_synthetic + body: + settings: + index: + mode: time_series + mapping.source.mode: synthetic + routing_path: [ keyword ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + keyword: + type: keyword + time_series_dimension: true + + - do: + indices.get_settings: + index: "test_time_series_index_mode_synthetic" + - match: { test_time_series_index_mode_synthetic.settings.index.mode: time_series } + + + - do: + indices.get_mapping: + index: test_time_series_index_mode_synthetic + + - match: { test_time_series_index_mode_synthetic.mappings._source.mode: synthetic } + +--- +create an index with logsdb index mode and synthetic source: + - do: + indices.create: + index: test_logsdb_index_mode_synthetic + body: + settings: + index: + mode: logsdb + mapping.source.mode: synthetic + + - do: + indices.get_settings: + index: "test_logsdb_index_mode_synthetic" + - match: { test_logsdb_index_mode_synthetic.settings.index.mode: logsdb } + + - do: + indices.get_mapping: + index: test_logsdb_index_mode_synthetic + + - match: { test_logsdb_index_mode_synthetic.mappings._source.mode: synthetic } + +--- +create an index with time_series index mode and stored source: + - do: + indices.create: + index: test_time_series_index_mode_undefined + body: + settings: + index: + mode: time_series + mapping.source.mode: stored + routing_path: [ keyword ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + keyword: + type: keyword + time_series_dimension: true + + - do: + indices.get_settings: + index: "test_time_series_index_mode_undefined" + - match: { test_time_series_index_mode_undefined.settings.index.mode: time_series } + + - do: + indices.get_mapping: + index: test_time_series_index_mode_undefined + + - match: { test_time_series_index_mode_undefined.mappings._source.mode: stored } + +--- +create an index with logsdb index mode and stored source: + - do: + indices.create: + index: test_logsdb_index_mode_undefined + body: + settings: + index: + mode: logsdb + mapping.source.mode: stored + + - do: + indices.get_settings: + index: "test_logsdb_index_mode_undefined" + - match: { test_logsdb_index_mode_undefined.settings.index.mode: logsdb } + + - do: + indices.get_mapping: + index: test_logsdb_index_mode_undefined + + - match: { test_logsdb_index_mode_undefined.mappings._source.mode: stored } + +--- +create an index with time_series index mode and disabled source: + - do: + catch: bad_request + indices.create: + index: test_time_series_index_mode + body: + settings: + index: + mode: time_series + mapping.source.mode: disabled + routing_path: [ keyword ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + keyword: + type: keyword + time_series_dimension: true + + - match: { error.type: "mapper_parsing_exception" } + - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [time_series] index mode" } + +--- +create an index with logsdb index mode and disabled source: + - do: + catch: bad_request + indices.create: + index: test_logsdb_index_mode + body: + settings: + index: + mode: logsdb + mapping.source.mode: disabled + + - match: { error.type: "mapper_parsing_exception" } + - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode" } + +--- +modify final setting after index creation: + - do: + indices.create: + index: test_modify_setting + body: + settings: + index: + mapping.source.mode: stored + + - do: + catch: /.*Can't update non dynamic setting.*/ + indices.put_settings: + index: test_modify_setting + body: + index: + mapping.source.mode: synthetic + +--- +modify source mapping from stored to disabled after index creation: + - do: + indices.create: + index: test_modify_source_mode_stored_disabled + body: + settings: + index: + mapping.source.mode: stored + + - do: + indices.put_mapping: + index: test_modify_source_mode_stored_disabled + body: + _source: + mode: disabled + - is_true: acknowledged + + - do: + indices.get_mapping: + index: test_modify_source_mode_stored_disabled + - match: { test_modify_source_mode_stored_disabled.mappings._source.mode: stored } + +--- +modify source mapping from stored to synthetic after index creation: + - do: + indices.create: + index: test_modify_source_mode_stored_synthetic + body: + settings: + index: + mapping.source.mode: stored + + - do: + indices.put_mapping: + index: test_modify_source_mode_stored_synthetic + body: + _source: + mode: synthetic + - is_true: acknowledged + + - do: + indices.get_mapping: + index: test_modify_source_mode_stored_synthetic + - match: { test_modify_source_mode_stored_synthetic.mappings._source.mode: stored } + +--- +modify source mapping from disabled to stored after index creation: + - do: + indices.create: + index: test_modify_source_mode_disabled_stored + body: + settings: + index: + mapping.source.mode: disabled + + - do: + indices.put_mapping: + index: test_modify_source_mode_disabled_stored + body: + _source: + mode: stored + - is_true: acknowledged + + - do: + indices.get_mapping: + index: test_modify_source_mode_disabled_stored + - match: { test_modify_source_mode_disabled_stored.mappings._source.mode: disabled } + +--- +modify source mapping from disabled to synthetic after index creation: + - do: + indices.create: + index: test_modify_source_mode_disabled_synthetic + body: + settings: + index: + mapping.source.mode: disabled + + - do: + indices.put_mapping: + index: test_modify_source_mode_disabled_synthetic + body: + _source: + mode: synthetic + - is_true: acknowledged + + - do: + indices.get_mapping: + index: test_modify_source_mode_disabled_synthetic + - match: { test_modify_source_mode_disabled_synthetic.mappings._source.mode: disabled } + +--- +modify source mapping from synthetic to stored after index creation: + - do: + indices.create: + index: test_modify_source_mode_synthetic_stored + body: + settings: + index: + mapping.source.mode: synthetic + + - do: + indices.put_mapping: + index: test_modify_source_mode_synthetic_stored + body: + _source: + mode: stored + - is_true: acknowledged + + - do: + indices.get_mapping: + index: test_modify_source_mode_synthetic_stored + - match: { test_modify_source_mode_synthetic_stored.mappings._source.mode: synthetic } + +--- +modify source mapping from synthetic to disabled after index creation: + - do: + indices.create: + index: test_modify_source_mode_synthetic_disabled + body: + settings: + index: + mapping.source.mode: synthetic + + - do: + indices.put_mapping: + index: test_modify_source_mode_synthetic_disabled + body: + _source: + mode: disabled + - is_true: acknowledged + + - do: + indices.get_mapping: + index: test_modify_source_mode_synthetic_disabled + - match: { test_modify_source_mode_synthetic_disabled.mappings._source.mode: synthetic } + +--- +modify logsdb index source mode to disabled after index creation: + - do: + indices.create: + index: test_modify_logsdb_disabled_after_creation + body: + settings: + index: + mode: logsdb + + - do: + catch: bad_request + indices.put_mapping: + index: test_modify_logsdb_disabled_after_creation + body: + _source: + mode: disabled + - match: { error.type: "mapper_parsing_exception" } + - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode" } + +--- +modify logsdb index source mode to stored after index creation: + - do: + indices.create: + index: test_modify_logsdb_stored_after_creation + body: + settings: + index: + mode: logsdb + + - do: + catch: bad_request + indices.put_mapping: + index: test_modify_logsdb_stored_after_creation + body: + _source: + mode: stored + - match: { error.type: "illegal_argument_exception" } + - match: { error.reason: "Mapper for [_source] conflicts with existing mapper:\n\tCannot update parameter [mode] from [synthetic] to [stored]" } + +--- +modify time_series index source mode to disabled after index creation: + - do: + indices.create: + index: test_modify_time_series_disabled_after_creation + body: + settings: + index: + mode: time_series + routing_path: [ keyword ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + keyword: + type: keyword + time_series_dimension: true + + - do: + catch: bad_request + indices.put_mapping: + index: test_modify_time_series_disabled_after_creation + body: + _source: + mode: disabled + - match: { error.type: "mapper_parsing_exception" } + - match: { error.reason: "Failed to parse mapping: _source can not be disabled in index using [time_series] index mode" } + +--- +modify time_series index source mode to stored after index creation: + - do: + indices.create: + index: test_modify_time_series_stored_after_creation + body: + settings: + index: + mode: time_series + routing_path: [ keyword ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + keyword: + type: keyword + time_series_dimension: true + + - do: + catch: bad_request + indices.put_mapping: + index: test_modify_time_series_stored_after_creation + body: + _source: + mode: stored + - match: { error.type: "illegal_argument_exception" } + - match: { error.reason: "Mapper for [_source] conflicts with existing mapper:\n\tCannot update parameter [mode] from [synthetic] to [stored]" } diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml index 513e1a857787e..4a039886ecc4e 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml @@ -17,6 +17,9 @@ template: type: constant_keyword data_stream.namespace: type: constant_keyword + event.dataset: + type: alias + path: data_stream.dataset attributes: type: passthrough dynamic: true diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml index 0957a79552ad3..be4de6dca6c76 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml @@ -145,3 +145,21 @@ Structured log body: index: $datastream-backing-index - is_true: $datastream-backing-index - match: { .$datastream-backing-index.mappings.properties.body.properties.flattened.type: "flattened" } +--- +"event.dataset alias must point to data_stream.dataset": + - do: + bulk: + index: logs-generic.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:49:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default"}, "body_text":"error1"}' + - is_false: errors + - is_false: errors + - do: + search: + index: logs-generic.otel-default + body: + fields: ["event.dataset"] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields.event\.dataset: ["generic.otel"] } diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index 853d0fd9318ae..b29dc0fa410b6 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -174,6 +174,7 @@ public class Constants { "cluster:admin/xpack/enrich/reindex", "cluster:admin/xpack/inference/delete", "cluster:admin/xpack/inference/put", + "cluster:admin/xpack/inference/update", "cluster:admin/xpack/license/basic_status", // "cluster:admin/xpack/license/delete", "cluster:admin/xpack/license/feature_usage", @@ -501,6 +502,9 @@ public class Constants { "indices:admin/data_stream/lifecycle/get", "indices:admin/data_stream/lifecycle/put", "indices:admin/data_stream/lifecycle/explain", + "indices:admin/data_stream/options/delete", + "indices:admin/data_stream/options/get", + "indices:admin/data_stream/options/put", "indices:admin/delete", "indices:admin/flush", "indices:admin/flush[s]", diff --git a/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownShardsIT.java b/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownShardsIT.java index d12d093dd5b8d..ee7438dfca428 100644 --- a/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownShardsIT.java +++ b/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownShardsIT.java @@ -327,11 +327,8 @@ public void testAutoExpandDuringRestart() throws Exception { ensureGreen("myindex"); putNodeShutdown(primaryNodeId, SingleNodeShutdownMetadata.Type.RESTART, null); - // registering node shutdown entry does not perform reroute, neither should it. - // we provoke it here in the test to ensure that auto-expansion has run. - updateIndexSettings(Settings.builder().put("index.routing.allocation.exclude.name", "non-existent"), "myindex"); - assertBusy(() -> assertIndexSetting("myindex", "index.number_of_replicas", "1")); + assertIndexSetting("myindex", "index.number_of_replicas", "1"); indexRandomData("myindex"); internalCluster().restartNode(primaryNode, new InternalTestCluster.RestartCallback() { @@ -361,9 +358,6 @@ public void testAutoExpandDuringReplace() throws Exception { var replacementNodeName = "node_t2"; putNodeShutdown(nodeIdToReplace, SingleNodeShutdownMetadata.Type.REPLACE, replacementNodeName); - // registering node shutdown entry does not perform reroute, neither should it. - // we provoke it here in the test to ensure that auto-expansion has run. - updateIndexSettings(Settings.builder().put("index.routing.allocation.exclude.name", "non-existent"), "index"); ensureGreen("index"); assertIndexSetting("index", "index.number_of_replicas", "1"); @@ -381,6 +375,32 @@ public void testAutoExpandDuringReplace() throws Exception { assertIndexSetting("index", "index.number_of_replicas", "1"); } + public void testAutoExpandDuringShutdown() throws Exception { + + var node1 = internalCluster().startNode(); + var node2 = internalCluster().startNode(); + + createIndex("index", indexSettings(1, 0).put("index.auto_expand_replicas", randomFrom("0-all", "0-1")).build()); + indexRandomData("index"); + + ensureGreen("index"); + assertIndexSetting("index", "index.number_of_replicas", "1"); + + var nodeNameToShutdown = randomFrom(node1, node2); + var nodeIdToShutdown = getNodeId(nodeNameToShutdown); + + putNodeShutdown(nodeIdToShutdown, SingleNodeShutdownMetadata.Type.REMOVE, null); + + ensureGreen("index"); + assertIndexSetting("index", "index.number_of_replicas", "0"); + + assertBusy(() -> assertNodeShutdownStatus(nodeIdToShutdown, COMPLETE)); + internalCluster().stopNode(nodeIdToShutdown); + + ensureGreen("index"); + assertIndexSetting("index", "index.number_of_replicas", "0"); + } + public void testNodeShutdownWithUnassignedShards() throws Exception { final String nodeA = internalCluster().startNode(); final String nodeAId = getNodeId(nodeA); diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsDataSource.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsDataSource.java index ad8c39b6345ba..b54a425836df2 100644 --- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsDataSource.java +++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsDataSource.java @@ -25,7 +25,7 @@ public class EsDataSource implements DataSource, Wrapper { static { - // invoke Version to perform classpath/jar sanity checks + // invoke version to perform classpath/jar sanity checks ClientVersion.CURRENT.toString(); } diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsDriver.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsDriver.java index 9c5734368bcd1..7a22f0df0f798 100644 --- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsDriver.java +++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsDriver.java @@ -23,7 +23,7 @@ public class EsDriver implements Driver { private static final EsDriver INSTANCE = new EsDriver(); static { - // invoke Version to perform classpath/jar sanity checks + // invoke version to perform classpath/jar sanity checks ClientVersion.CURRENT.toString(); try { diff --git a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/DriverManagerRegistrationTests.java b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/DriverManagerRegistrationTests.java index 273b7f75c5a2d..eff050993ed4e 100644 --- a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/DriverManagerRegistrationTests.java +++ b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/DriverManagerRegistrationTests.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.sql.jdbc; -import org.elasticsearch.Version; import org.elasticsearch.test.ESTestCase; import java.security.AccessController; @@ -27,8 +26,8 @@ public void testVersioning() throws Exception { /* This test will only work properly in gradle because in gradle we run the tests * using the jar. */ - assertNotEquals(String.valueOf(Version.CURRENT.major), d.getMajorVersion()); - assertNotEquals(String.valueOf(Version.CURRENT.minor), d.getMinorVersion()); + assertNotEquals(String.valueOf(VersionTests.current().major), d.getMajorVersion()); + assertNotEquals(String.valueOf(VersionTests.current().minor), d.getMinorVersion()); }); } diff --git a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/VersionParityTests.java b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/VersionParityTests.java index 524a653de7e76..8fc11681c2303 100644 --- a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/VersionParityTests.java +++ b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/VersionParityTests.java @@ -7,17 +7,21 @@ package org.elasticsearch.xpack.sql.jdbc; -import org.elasticsearch.Version; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.rest.root.MainResponse; -import org.elasticsearch.test.VersionUtils; import org.elasticsearch.test.http.MockResponse; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.sql.client.ClientVersion; import org.elasticsearch.xpack.sql.proto.SqlVersion; +import org.elasticsearch.xpack.sql.proto.SqlVersions; import java.io.IOException; import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.INTRODUCING_VERSION_COMPATIBILITY; /** * Test class for JDBC-ES server versions checks. @@ -29,15 +33,11 @@ public class VersionParityTests extends WebServerTestCase { public void testExceptionThrownOnIncompatibleVersions() throws IOException, SQLException { String url = JdbcConfiguration.URL_PREFIX + webServerAddress(); - Version firstVersion = VersionUtils.getFirstVersion(); - Version version = Version.V_7_7_0; - do { - version = VersionUtils.getPreviousVersion(version); + for (var version = SqlVersions.getFirstVersion(); version.onOrAfter(INTRODUCING_VERSION_COMPATIBILITY) == false; version = + SqlVersions.getNextVersion(version)) { logger.info("Checking exception is thrown for version {}", version); prepareResponse(version); - // Client's version is wired up to patch level, excluding the qualifier => generate the test version as the server does it. - String versionString = SqlVersion.fromString(version.toString()).toString(); SQLException ex = expectThrows( SQLException.class, @@ -48,27 +48,30 @@ public void testExceptionThrownOnIncompatibleVersions() throws IOException, SQLE + ClientVersion.CURRENT.majorMinorToString() + " or newer; attempting to connect to a server " + "version " - + versionString, + + version, ex.getMessage() ); - } while (version.compareTo(firstVersion) > 0); + } } public void testNoExceptionThrownForCompatibleVersions() throws IOException { String url = JdbcConfiguration.URL_PREFIX + webServerAddress(); - Version version = Version.CURRENT; - try { - do { + List afterVersionCompatibility = SqlVersions.getAllVersions() + .stream() + .filter(v -> v.onOrAfter(INTRODUCING_VERSION_COMPATIBILITY)) + .collect(Collectors.toCollection(ArrayList::new)); + afterVersionCompatibility.add(VersionTests.current()); + for (var version : afterVersionCompatibility) { + try { prepareResponse(version); new JdbcHttpClient(new JdbcConnection(JdbcConfiguration.create(url, null, 0), false)); - version = VersionUtils.getPreviousVersion(version); - } while (version.compareTo(Version.V_7_7_0) >= 0); - } catch (SQLException sqle) { - fail("JDBC driver version and Elasticsearch server version should be compatible. Error: " + sqle); + } catch (SQLException sqle) { + fail("JDBC driver version and Elasticsearch server version should be compatible. Error: " + sqle); + } } } - void prepareResponse(Version version) throws IOException { + void prepareResponse(SqlVersion version) throws IOException { MainResponse response = version == null ? createCurrentVersionMainResponse() : createMainResponse(version); webServer().enqueue( new MockResponse().setResponseCode(200) diff --git a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/VersionTests.java b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/VersionTests.java index 1553077250c7e..de6a67ab9805a 100644 --- a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/VersionTests.java +++ b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/VersionTests.java @@ -6,15 +6,20 @@ */ package org.elasticsearch.xpack.sql.jdbc; +import org.elasticsearch.Build; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.sql.client.ClientVersion; +import org.elasticsearch.xpack.sql.proto.SqlVersion; public class VersionTests extends ESTestCase { public void testVersionIsCurrent() { /* This test will only work properly in gradle because in gradle we run the tests * using the jar. */ - assertEquals(org.elasticsearch.Version.CURRENT.major, ClientVersion.CURRENT.major); - assertEquals(org.elasticsearch.Version.CURRENT.minor, ClientVersion.CURRENT.minor); - assertEquals(org.elasticsearch.Version.CURRENT.revision, ClientVersion.CURRENT.revision); + assertEquals(current(), ClientVersion.CURRENT); + } + + /** Returns the current stack version. Can be unreleased. */ + public static SqlVersion current() { + return SqlVersion.fromString(Build.current().version()); } } diff --git a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/WebServerTestCase.java b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/WebServerTestCase.java index 85e296935676a..1d940e059af4d 100644 --- a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/WebServerTestCase.java +++ b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/WebServerTestCase.java @@ -8,13 +8,13 @@ package org.elasticsearch.xpack.sql.jdbc; import org.elasticsearch.Build; -import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.rest.root.MainResponse; import org.elasticsearch.test.BuildUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.http.MockWebServer; +import org.elasticsearch.xpack.sql.proto.SqlVersion; import org.junit.After; import org.junit.Before; @@ -42,10 +42,10 @@ public MockWebServer webServer() { } MainResponse createCurrentVersionMainResponse() { - return createMainResponse(Version.CURRENT); + return createMainResponse(VersionTests.current()); } - MainResponse createMainResponse(Version version) { + MainResponse createMainResponse(SqlVersion version) { // the SQL client only cares about node version, // so ignore index & transport versions here (just set them to current) String clusterUuid = randomAlphaOfLength(10); diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ConnectionTestCase.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ConnectionTestCase.java index db384964b3c84..15b9d3613bbb1 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ConnectionTestCase.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ConnectionTestCase.java @@ -6,8 +6,6 @@ */ package org.elasticsearch.xpack.sql.qa.jdbc; -import org.elasticsearch.Version; - import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.SQLException; @@ -22,8 +20,8 @@ public void testConnectionProperties() throws SQLException { assertFalse(c.isClosed()); assertTrue(c.isReadOnly()); DatabaseMetaData md = c.getMetaData(); - assertEquals(Version.CURRENT.major, md.getDatabaseMajorVersion()); - assertEquals(Version.CURRENT.minor, md.getDatabaseMinorVersion()); + assertEquals(JdbcTestUtils.CURRENT.major, md.getDatabaseMajorVersion()); + assertEquals(JdbcTestUtils.CURRENT.minor, md.getDatabaseMinorVersion()); } } diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java index fb5b69a053f13..01d2df1d710d2 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java @@ -6,8 +6,9 @@ */ package org.elasticsearch.xpack.sql.qa.jdbc; -import org.elasticsearch.Version; +import org.elasticsearch.Build; import org.elasticsearch.xpack.sql.jdbc.EsType; +import org.elasticsearch.xpack.sql.proto.SqlVersion; import org.elasticsearch.xpack.sql.proto.StringUtils; import java.math.BigInteger; @@ -26,10 +27,11 @@ import java.util.LinkedHashMap; import java.util.Map; -import static org.elasticsearch.Version.V_8_2_0; -import static org.elasticsearch.Version.V_8_4_0; import static org.elasticsearch.common.time.DateUtils.toMilliSeconds; import static org.elasticsearch.test.ESTestCase.randomLongBetween; +import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.supportsDateNanos; +import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.supportsUnsignedLong; +import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.supportsVersionType; final class JdbcTestUtils { @@ -43,7 +45,9 @@ private JdbcTestUtils() {} static final LocalDate EPOCH = LocalDate.of(1970, 1, 1); static final String UNSIGNED_LONG_TYPE_NAME = "UNSIGNED_LONG"; - static final BigInteger UNSIGNED_LONG_MAX = BigInteger.ONE.shiftLeft(Long.SIZE).subtract(BigInteger.ONE); + + // Build's version is always a SemVer in JDBC tests + public static final SqlVersion CURRENT = SqlVersion.fromString(Build.current().version()); /* * The version of the driver that the QA (bwc-)tests run against. @@ -59,12 +63,11 @@ private JdbcTestUtils() {} * } * */ - static final Version JDBC_DRIVER_VERSION; + static final SqlVersion JDBC_DRIVER_VERSION; static { - // master's version is x.0.0-SNAPSHOT, tho Version#fromString() won't accept that back for recent versions - String jdbcDriverVersion = System.getProperty(DRIVER_VERSION_PROPERTY_NAME, "").replace("-SNAPSHOT", ""); - JDBC_DRIVER_VERSION = Version.fromString(jdbcDriverVersion); // takes empty and null strings, resolves them to CURRENT + String jdbcDriverVersion = System.getProperty(DRIVER_VERSION_PROPERTY_NAME, ""); + JDBC_DRIVER_VERSION = jdbcDriverVersion.isEmpty() ? CURRENT : SqlVersion.fromString(jdbcDriverVersion); // Note: keep in sync with org.elasticsearch.xpack.sql.jdbc.TypeUtils#CLASS_TO_TYPE Map, EsType> aMap = new LinkedHashMap<>(); @@ -150,15 +153,14 @@ static int extractNanosOnly(long nanos) { } static boolean versionSupportsDateNanos() { - return JDBC_DRIVER_VERSION.onOrAfter(Version.V_7_12_0); + return supportsDateNanos(JDBC_DRIVER_VERSION); } public static boolean isUnsignedLongSupported() { - return JDBC_DRIVER_VERSION.onOrAfter(V_8_2_0); + return supportsUnsignedLong(JDBC_DRIVER_VERSION); } public static boolean isVersionFieldTypeSupported() { - return JDBC_DRIVER_VERSION.onOrAfter(V_8_4_0); + return supportsVersionType(JDBC_DRIVER_VERSION); } - } diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcWarningsTestCase.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcWarningsTestCase.java index 663f2bffae2fa..d0ec4a13de2fa 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcWarningsTestCase.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcWarningsTestCase.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.sql.qa.jdbc; -import org.elasticsearch.Version; import org.junit.Before; import java.io.IOException; @@ -20,14 +19,13 @@ import java.util.List; import java.util.Properties; +import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.INTRODUCING_WARNING_HANDLING; import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.JDBC_DRIVER_VERSION; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; public abstract class JdbcWarningsTestCase extends JdbcIntegrationTestCase { - private static final Version WARNING_HANDLING_ADDED_VERSION = Version.V_8_2_0; - @Before public void setupData() throws IOException { index("test_data", b -> b.field("foo", 1)); @@ -89,7 +87,7 @@ public void testClearWarnings() throws SQLException { } private void assumeWarningHandlingDriverVersion() { - assumeTrue("Driver does not yet handle deprecation warnings", JDBC_DRIVER_VERSION.onOrAfter(WARNING_HANDLING_ADDED_VERSION)); + assumeTrue("Driver does not yet handle deprecation warnings", JDBC_DRIVER_VERSION.onOrAfter(INTRODUCING_WARNING_HANDLING)); } } diff --git a/x-pack/plugin/sql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/mixed_node/SqlSearchIT.java b/x-pack/plugin/sql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/mixed_node/SqlSearchIT.java index f05eccb737ca2..78dce9d128cab 100644 --- a/x-pack/plugin/sql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/mixed_node/SqlSearchIT.java +++ b/x-pack/plugin/sql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/mixed_node/SqlSearchIT.java @@ -9,7 +9,6 @@ import org.apache.http.HttpHost; import org.apache.lucene.sandbox.document.HalfFloatPoint; -import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; @@ -19,6 +18,7 @@ import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.xpack.ql.TestNode; import org.elasticsearch.xpack.ql.TestNodes; +import org.elasticsearch.xpack.sql.proto.SqlVersion; import org.junit.After; import org.junit.Before; @@ -36,13 +36,14 @@ import static java.util.Collections.unmodifiableMap; import static org.elasticsearch.xpack.ql.TestUtils.buildNodeAndVersions; import static org.elasticsearch.xpack.ql.TestUtils.readResource; +import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.INTRODUCING_VERSION_FIELD_TYPE; public class SqlSearchIT extends ESRestTestCase { private static final String BWC_NODES_VERSION = System.getProperty("tests.bwc_nodes_version"); - // TODO[lor]: replace this with feature-based checks when we have one - private static final boolean SUPPORTS_VERSION_FIELD_QL_INTRODUCTION = Version.fromString(BWC_NODES_VERSION).onOrAfter(Version.V_8_4_0); + private static final boolean SUPPORTS_VERSION_FIELD_QL_INTRODUCTION = SqlVersion.fromString(BWC_NODES_VERSION) + .onOrAfter(INTRODUCING_VERSION_FIELD_TYPE); private static final String index = "test_sql_mixed_versions"; private static int numShards; diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/BaseRestSqlTestCase.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/BaseRestSqlTestCase.java index bd43d3d651e52..ece8409a022cd 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/BaseRestSqlTestCase.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/BaseRestSqlTestCase.java @@ -9,7 +9,7 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; -import org.elasticsearch.Version; +import org.elasticsearch.Build; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; @@ -20,6 +20,8 @@ import org.elasticsearch.xcontent.cbor.CborXContent; import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.xpack.sql.proto.Mode; +import org.elasticsearch.xpack.sql.proto.SqlVersion; +import org.elasticsearch.xpack.sql.proto.SqlVersions; import org.elasticsearch.xpack.sql.proto.StringUtils; import java.io.IOException; @@ -54,6 +56,11 @@ public abstract class BaseRestSqlTestCase extends RemoteClusterAwareSqlRestTestC private static final String TEST_INDEX = "test"; private static final String DATA_STREAM_TEMPLATE = "test-ds-index-template"; + /** + * What's the version of the server that the clients should be compatible with? + * This will be either the stack version, or SqlVersions.getLatestVersion() if the stack version is not available. + */ + private static final SqlVersion SERVER_COMPAT_VERSION = getServerCompatVersion(); public static class RequestObjectBuilder { private StringBuilder request; @@ -83,7 +90,7 @@ public RequestObjectBuilder mode(Object m) { if (isQuery) { Mode mode = (m instanceof Mode) ? (Mode) m : Mode.fromString(modeString); if (Mode.isDedicatedClient(mode)) { - version(Version.CURRENT.toString()); + version(SERVER_COMPAT_VERSION.toString()); } } return this; @@ -301,4 +308,12 @@ public static Tuple runSqlAsText(RequestObjectBuilder requestObj response.getHeader("Cursor") ); } + + private static SqlVersion getServerCompatVersion() { + try { + return SqlVersion.fromString(Build.current().version()); + } catch (Exception e) { + return SqlVersions.getLatestVersion(); + } + } } diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java index ca9532d8dc7d0..aa1cabe17161a 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java @@ -10,7 +10,6 @@ import org.apache.http.HttpEntity; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; -import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; @@ -23,12 +22,13 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; import org.elasticsearch.test.NotEqualMessageBuilder; -import org.elasticsearch.test.VersionUtils; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.json.JsonStringEncoder; import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.xpack.sql.proto.CoreProtocol; import org.elasticsearch.xpack.sql.proto.Mode; +import org.elasticsearch.xpack.sql.proto.SqlVersion; +import org.elasticsearch.xpack.sql.proto.SqlVersions; import org.elasticsearch.xpack.sql.proto.StringUtils; import org.elasticsearch.xpack.sql.qa.ErrorsTestCase; import org.hamcrest.Matcher; @@ -60,7 +60,6 @@ import static java.util.Collections.unmodifiableMap; import static org.elasticsearch.common.Strings.hasText; import static org.elasticsearch.xpack.ql.TestUtils.getNumberOfSearchContexts; -import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.INTRODUCING_UNSIGNED_LONG; import static org.elasticsearch.xpack.sql.proto.CoreProtocol.COLUMNS_NAME; import static org.elasticsearch.xpack.sql.proto.CoreProtocol.HEADER_NAME_ASYNC_ID; import static org.elasticsearch.xpack.sql.proto.CoreProtocol.HEADER_NAME_ASYNC_PARTIAL; @@ -76,6 +75,7 @@ import static org.elasticsearch.xpack.sql.proto.CoreProtocol.URL_PARAM_DELIMITER; import static org.elasticsearch.xpack.sql.proto.CoreProtocol.URL_PARAM_FORMAT; import static org.elasticsearch.xpack.sql.proto.CoreProtocol.WAIT_FOR_COMPLETION_TIMEOUT_NAME; +import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.INTRODUCING_UNSIGNED_LONG; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.lessThanOrEqualTo; @@ -1159,7 +1159,8 @@ public void testBinaryFieldFiltering() throws IOException { public void testPreventedUnsignedLongMaskedAccess() throws IOException { loadUnsignedLongTestData(); - Version version = VersionUtils.randomVersionBetween(random(), null, VersionUtils.getPreviousVersion(INTRODUCING_UNSIGNED_LONG)); + var preVersionCompat = SqlVersions.getAllVersions().stream().filter(v -> v.onOrAfter(INTRODUCING_UNSIGNED_LONG) == false).toList(); + SqlVersion version = preVersionCompat.get(random().nextInt(preVersionCompat.size())); String query = query("SELECT unsigned_long::STRING FROM " + indexPattern("test")).version(version.toString()).toString(); expectBadRequest( () -> runSql(new StringEntity(query, ContentType.APPLICATION_JSON), "", randomMode()), diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlQueryRequest.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlQueryRequest.java index d711538ad1d09..46308790dc91a 100644 --- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlQueryRequest.java +++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlQueryRequest.java @@ -6,9 +6,7 @@ */ package org.elasticsearch.xpack.sql.action; -import org.elasticsearch.Build; import org.elasticsearch.TransportVersions; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.CompositeIndicesRequest; import org.elasticsearch.common.Strings; @@ -30,7 +28,8 @@ import org.elasticsearch.xpack.sql.proto.Mode; import org.elasticsearch.xpack.sql.proto.RequestInfo; import org.elasticsearch.xpack.sql.proto.SqlTypedParamValue; -import org.elasticsearch.xpack.sql.proto.SqlVersion; +import org.elasticsearch.xpack.sql.proto.SqlVersions; +import org.elasticsearch.xpack.sql.proto.VersionCompatibility; import java.io.IOException; import java.time.ZoneId; @@ -288,15 +287,15 @@ public ActionRequestValidationException validate() { validationException ); } - } else if (SqlVersion.isClientCompatible(SqlVersion.fromId(Version.CURRENT.id), requestInfo().version()) == false) { + } else if (VersionCompatibility.isClientCompatible(SqlVersions.SERVER_COMPAT_VERSION, requestInfo().version()) == false) { validationException = addValidationError( "The [" + requestInfo().version() + "] version of the [" + mode.toString() + "] " - + "client is not compatible with Elasticsearch version [" - + Build.current().version() + + "client is not compatible with Elasticsearch server compatibility version [" + + SqlVersions.SERVER_COMPAT_VERSION + "]", validationException ); diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryResponse.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryResponse.java index 725eb0a2e3b01..978f42f7f50dc 100644 --- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryResponse.java +++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryResponse.java @@ -9,7 +9,6 @@ import com.fasterxml.jackson.core.JsonGenerator; import org.elasticsearch.TransportVersions; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -21,6 +20,7 @@ import org.elasticsearch.xpack.sql.proto.ColumnInfo; import org.elasticsearch.xpack.sql.proto.Mode; import org.elasticsearch.xpack.sql.proto.SqlVersion; +import org.elasticsearch.xpack.sql.proto.SqlVersions; import org.elasticsearch.xpack.sql.proto.StringUtils; import java.io.IOException; @@ -33,8 +33,7 @@ import static org.elasticsearch.xpack.sql.action.AbstractSqlQueryRequest.CURSOR; import static org.elasticsearch.xpack.sql.proto.Mode.CLI; import static org.elasticsearch.xpack.sql.proto.Mode.JDBC; -import static org.elasticsearch.xpack.sql.proto.SqlVersion.fromId; -import static org.elasticsearch.xpack.sql.proto.SqlVersion.isClientCompatible; +import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.isClientCompatible; /** * Response to perform an sql query @@ -108,7 +107,7 @@ public SqlQueryResponse( ) { this.cursor = cursor; this.mode = mode; - this.sqlVersion = sqlVersion != null ? sqlVersion : fromId(Version.CURRENT.id); + this.sqlVersion = sqlVersion != null ? sqlVersion : SqlVersions.SERVER_COMPAT_VERSION; this.columnar = columnar; this.columns = columns; this.rows = rows; @@ -276,7 +275,7 @@ private static XContentBuilder toXContent(ColumnInfo info, XContentBuilder build public static XContentBuilder value(XContentBuilder builder, Mode mode, SqlVersion sqlVersion, Object value) throws IOException { if (value instanceof ZonedDateTime zdt) { // use the ISO format - if (mode == JDBC && isClientCompatible(SqlVersion.fromId(Version.CURRENT.id), sqlVersion)) { + if (mode == JDBC && isClientCompatible(SqlVersions.SERVER_COMPAT_VERSION, sqlVersion)) { builder.value(StringUtils.toString(zdt, sqlVersion)); } else { builder.value(StringUtils.toString(zdt)); diff --git a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryResponseTests.java b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryResponseTests.java index 78c4ebf378770..e8637d8bd80c2 100644 --- a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryResponseTests.java +++ b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryResponseTests.java @@ -33,7 +33,7 @@ import static org.elasticsearch.xpack.sql.action.Protocol.ID_NAME; import static org.elasticsearch.xpack.sql.action.Protocol.IS_PARTIAL_NAME; import static org.elasticsearch.xpack.sql.action.Protocol.IS_RUNNING_NAME; -import static org.elasticsearch.xpack.sql.proto.SqlVersion.DATE_NANOS_SUPPORT_VERSION; +import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.INTRODUCING_DATE_NANOS; import static org.hamcrest.Matchers.hasSize; public class SqlQueryResponseTests extends AbstractXContentSerializingTestCase { @@ -118,7 +118,7 @@ public static SqlQueryResponse createRandomInstance( rows.add(row); } } - return new SqlQueryResponse(cursor, mode, DATE_NANOS_SUPPORT_VERSION, false, columns, rows, asyncExecutionId, isPartial, isRunning); + return new SqlQueryResponse(cursor, mode, INTRODUCING_DATE_NANOS, false, columns, rows, asyncExecutionId, isPartial, isRunning); } public void testToXContent() throws IOException { @@ -177,7 +177,7 @@ protected SqlQueryResponse doParseInstance(XContentParser parser) throws IOExcep return new SqlQueryResponse( protoResponse.cursor(), Mode.JDBC, - DATE_NANOS_SUPPORT_VERSION, + INTRODUCING_DATE_NANOS, false, protoResponse.columns(), protoResponse.rows(), diff --git a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlRequestParsersTests.java b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlRequestParsersTests.java index 871e559eedf75..42229faa2981c 100644 --- a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlRequestParsersTests.java +++ b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlRequestParsersTests.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.sql.action; -import org.elasticsearch.Version; import org.elasticsearch.core.Strings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.test.ESTestCase; @@ -18,6 +17,7 @@ import org.elasticsearch.xpack.sql.proto.CoreProtocol; import org.elasticsearch.xpack.sql.proto.Mode; import org.elasticsearch.xpack.sql.proto.SqlTypedParamValue; +import org.elasticsearch.xpack.sql.proto.SqlVersions; import java.io.IOException; import java.util.ArrayList; @@ -154,7 +154,9 @@ public void testQueryRequestParser() throws IOException { String params; List list = new ArrayList<>(1); - final String clientVersion = Mode.isDedicatedClient(randomMode) ? "\"version\": \"" + Version.CURRENT.toString() + "\"," : ""; + final String clientVersion = Mode.isDedicatedClient(randomMode) + ? "\"version\": \"" + SqlVersions.SERVER_COMPAT_VERSION + "\"," + : ""; if (Mode.isDriver(randomMode)) { params = "{\"value\":123, \"type\":\"whatever\"}"; list.add(new SqlTypedParamValue("whatever", 123, true)); @@ -178,7 +180,7 @@ public void testQueryRequestParser() throws IOException { assertNull(request.clientId()); assertEquals(randomMode, request.mode()); if (Mode.isDedicatedClient(randomMode)) { - assertEquals(Version.CURRENT.toString(), request.version().toString()); + assertEquals(SqlVersions.SERVER_COMPAT_VERSION.toString(), request.version().toString()); } assertEquals("whatever", request.cursor()); assertEquals("select", request.query()); diff --git a/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/CliSessionTests.java b/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/CliSessionTests.java index bcb7fefec4559..b4fb2ed419310 100644 --- a/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/CliSessionTests.java +++ b/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/CliSessionTests.java @@ -6,19 +6,19 @@ */ package org.elasticsearch.xpack.sql.cli; -import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.common.UUIDs; -import org.elasticsearch.test.VersionUtils; import org.elasticsearch.xpack.sql.cli.command.CliSession; import org.elasticsearch.xpack.sql.client.ClientException; import org.elasticsearch.xpack.sql.client.ClientVersion; import org.elasticsearch.xpack.sql.client.HttpClient; import org.elasticsearch.xpack.sql.proto.MainResponse; import org.elasticsearch.xpack.sql.proto.SqlVersion; +import org.elasticsearch.xpack.sql.proto.SqlVersions; import java.sql.SQLException; +import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.INTRODUCING_VERSION_COMPATIBILITY; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -55,8 +55,11 @@ public void testConnection() throws Exception { public void testWrongServerVersion() throws Exception { HttpClient httpClient = mock(HttpClient.class); - Version v = VersionUtils.randomVersionBetween(random(), null, VersionUtils.getPreviousVersion(Version.V_7_7_0)); - SqlVersion version = new SqlVersion(v.major, v.minor, v.revision); + var preVersionCompat = SqlVersions.getAllVersions() + .stream() + .filter(v -> v.onOrAfter(INTRODUCING_VERSION_COMPATIBILITY) == false) + .toList(); + SqlVersion version = preVersionCompat.get(random().nextInt(preVersionCompat.size())); when(httpClient.serverInfo()).thenReturn( new MainResponse(randomAlphaOfLength(5), version.toString(), ClusterName.DEFAULT.value(), UUIDs.randomBase64UUID()) ); @@ -66,7 +69,7 @@ public void testWrongServerVersion() throws Exception { "This version of the CLI is only compatible with Elasticsearch version " + ClientVersion.CURRENT.majorMinorToString() + " or newer; attempting to connect to a server version " - + version.toString(), + + version, throwable.getMessage() ); verify(httpClient, times(1)).serverInfo(); @@ -75,8 +78,8 @@ public void testWrongServerVersion() throws Exception { public void testHigherServerVersion() throws Exception { HttpClient httpClient = mock(HttpClient.class); - Version v = VersionUtils.randomVersionBetween(random(), Version.V_7_7_0, null); - SqlVersion version = new SqlVersion(v.major, v.minor, v.revision); + var postVersionCompat = SqlVersions.getAllVersions().stream().filter(v -> v.onOrAfter(INTRODUCING_VERSION_COMPATIBILITY)).toList(); + SqlVersion version = postVersionCompat.get(random().nextInt(postVersionCompat.size())); when(httpClient.serverInfo()).thenReturn( new MainResponse(randomAlphaOfLength(5), version.toString(), ClusterName.DEFAULT.value(), UUIDs.randomBase64UUID()) ); diff --git a/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/VersionTests.java b/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/VersionTests.java index 9677a08c337e1..fc46da074147a 100644 --- a/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/VersionTests.java +++ b/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/VersionTests.java @@ -6,16 +6,15 @@ */ package org.elasticsearch.xpack.sql.cli; -import org.elasticsearch.Version; +import org.elasticsearch.Build; import org.elasticsearch.xpack.sql.client.ClientVersion; +import org.elasticsearch.xpack.sql.proto.SqlVersion; public class VersionTests extends SqlCliTestCase { public void testVersionIsCurrent() { /* This test will only work properly in gradle because in gradle we run the tests * using the jar. */ - assertEquals(Version.CURRENT.major, ClientVersion.CURRENT.major); - assertEquals(Version.CURRENT.minor, ClientVersion.CURRENT.minor); - assertEquals(Version.CURRENT.revision, ClientVersion.CURRENT.revision); + assertEquals(SqlVersion.fromString(Build.current().version()), ClientVersion.CURRENT); } } diff --git a/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/ClientVersion.java b/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/ClientVersion.java index 63e31e6c9a107..1e3ccbf22b718 100644 --- a/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/ClientVersion.java +++ b/x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/ClientVersion.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.sql.client; import org.elasticsearch.xpack.sql.proto.SqlVersion; +import org.elasticsearch.xpack.sql.proto.VersionCompatibility; import java.io.IOException; import java.net.JarURLConnection; @@ -119,7 +120,7 @@ static SqlVersion extractVersion(URL url) { // as well. public static boolean isServerCompatible(SqlVersion server) { // Starting with this version, the compatibility logic moved from the client to the server. - return SqlVersion.hasVersionCompatibility(server); + return VersionCompatibility.hasVersionCompatibility(server); } public static int jdbcMajorVersion() { diff --git a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlVersion.java b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlVersion.java index 09a931dc7204a..856ff2bcb1ea1 100644 --- a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlVersion.java +++ b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlVersion.java @@ -30,10 +30,6 @@ public class SqlVersion implements Comparable { public static final int MINOR_MULTIPLIER = REVISION_MULTIPLIER * REVISION_MULTIPLIER; public static final int MAJOR_MULTIPLIER = REVISION_MULTIPLIER * MINOR_MULTIPLIER; - public static final SqlVersion V_7_7_0 = new SqlVersion(7, 7, 0); - public static final SqlVersion V_7_12_0 = new SqlVersion(7, 12, 0); - public static final SqlVersion DATE_NANOS_SUPPORT_VERSION = V_7_12_0; // TODO: move to VersionCompatibilityChecks - public SqlVersion(byte major, byte minor, byte revision) { this(toString(major, minor, revision), major, minor, revision); } @@ -148,29 +144,7 @@ public int compareTo(SqlVersion o) { return id - o.id; } - public static int majorMinorId(SqlVersion v) { - return v.major * MAJOR_MULTIPLIER + v.minor * MINOR_MULTIPLIER; - } - - public int compareToMajorMinor(SqlVersion o) { - return majorMinorId(this) - majorMinorId(o); - } - - public static boolean hasVersionCompatibility(SqlVersion version) { - return version.compareTo(V_7_7_0) >= 0; - } - - // A client is version-compatible with the server if: - // - it supports version compatibility (past or on 7.7.0); and - // - it's not on a version newer than server's; and - // - it's major version is at most one unit behind server's. - public static boolean isClientCompatible(SqlVersion server, SqlVersion client) { - // ES's Version.CURRENT not available (core not a dependency), so it needs to be passed in as a parameter. - return hasVersionCompatibility(client) && server.compareTo(client) >= 0 && server.major - client.major <= 1; - } - - // TODO: move to VersionCompatibilityChecks - public static boolean supportsDateNanos(SqlVersion version) { - return DATE_NANOS_SUPPORT_VERSION.compareTo(version) <= 0; + public boolean onOrAfter(SqlVersion other) { + return this.compareTo(other) >= 0; } } diff --git a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlVersions.java b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlVersions.java new file mode 100644 index 0000000000000..7e0960550e634 --- /dev/null +++ b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlVersions.java @@ -0,0 +1,219 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.sql.proto; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.elasticsearch.xpack.sql.proto.SqlVersion.fromId; + +public final class SqlVersions { + + public static final SqlVersion V_7_0_0 = fromId(7_00_00_99); + public static final SqlVersion V_7_0_1 = fromId(7_00_01_99); + public static final SqlVersion V_7_1_0 = fromId(7_01_00_99); + public static final SqlVersion V_7_1_1 = fromId(7_01_01_99); + public static final SqlVersion V_7_2_0 = fromId(7_02_00_99); + public static final SqlVersion V_7_2_1 = fromId(7_02_01_99); + public static final SqlVersion V_7_3_0 = fromId(7_03_00_99); + public static final SqlVersion V_7_3_1 = fromId(7_03_01_99); + public static final SqlVersion V_7_3_2 = fromId(7_03_02_99); + public static final SqlVersion V_7_4_0 = fromId(7_04_00_99); + public static final SqlVersion V_7_4_1 = fromId(7_04_01_99); + public static final SqlVersion V_7_4_2 = fromId(7_04_02_99); + public static final SqlVersion V_7_5_0 = fromId(7_05_00_99); + public static final SqlVersion V_7_5_1 = fromId(7_05_01_99); + public static final SqlVersion V_7_5_2 = fromId(7_05_02_99); + public static final SqlVersion V_7_6_0 = fromId(7_06_00_99); + public static final SqlVersion V_7_6_1 = fromId(7_06_01_99); + public static final SqlVersion V_7_6_2 = fromId(7_06_02_99); + public static final SqlVersion V_7_7_0 = fromId(7_07_00_99); + public static final SqlVersion V_7_7_1 = fromId(7_07_01_99); + public static final SqlVersion V_7_8_0 = fromId(7_08_00_99); + public static final SqlVersion V_7_8_1 = fromId(7_08_01_99); + public static final SqlVersion V_7_9_0 = fromId(7_09_00_99); + public static final SqlVersion V_7_9_1 = fromId(7_09_01_99); + public static final SqlVersion V_7_9_2 = fromId(7_09_02_99); + public static final SqlVersion V_7_9_3 = fromId(7_09_03_99); + public static final SqlVersion V_7_10_0 = fromId(7_10_00_99); + public static final SqlVersion V_7_10_1 = fromId(7_10_01_99); + public static final SqlVersion V_7_10_2 = fromId(7_10_02_99); + public static final SqlVersion V_7_11_0 = fromId(7_11_00_99); + public static final SqlVersion V_7_11_1 = fromId(7_11_01_99); + public static final SqlVersion V_7_11_2 = fromId(7_11_02_99); + public static final SqlVersion V_7_12_0 = fromId(7_12_00_99); + public static final SqlVersion V_7_12_1 = fromId(7_12_01_99); + public static final SqlVersion V_7_13_0 = fromId(7_13_00_99); + public static final SqlVersion V_7_13_1 = fromId(7_13_01_99); + public static final SqlVersion V_7_13_2 = fromId(7_13_02_99); + public static final SqlVersion V_7_13_3 = fromId(7_13_03_99); + public static final SqlVersion V_7_13_4 = fromId(7_13_04_99); + public static final SqlVersion V_7_14_0 = fromId(7_14_00_99); + public static final SqlVersion V_7_14_1 = fromId(7_14_01_99); + public static final SqlVersion V_7_14_2 = fromId(7_14_02_99); + public static final SqlVersion V_7_15_0 = fromId(7_15_00_99); + public static final SqlVersion V_7_15_1 = fromId(7_15_01_99); + public static final SqlVersion V_7_15_2 = fromId(7_15_02_99); + public static final SqlVersion V_7_16_0 = fromId(7_16_00_99); + public static final SqlVersion V_7_16_1 = fromId(7_16_01_99); + public static final SqlVersion V_7_16_2 = fromId(7_16_02_99); + public static final SqlVersion V_7_16_3 = fromId(7_16_03_99); + public static final SqlVersion V_7_17_0 = fromId(7_17_00_99); + public static final SqlVersion V_7_17_1 = fromId(7_17_01_99); + public static final SqlVersion V_7_17_2 = fromId(7_17_02_99); + public static final SqlVersion V_7_17_3 = fromId(7_17_03_99); + public static final SqlVersion V_7_17_4 = fromId(7_17_04_99); + public static final SqlVersion V_7_17_5 = fromId(7_17_05_99); + public static final SqlVersion V_7_17_6 = fromId(7_17_06_99); + public static final SqlVersion V_7_17_7 = fromId(7_17_07_99); + public static final SqlVersion V_7_17_8 = fromId(7_17_08_99); + public static final SqlVersion V_7_17_9 = fromId(7_17_09_99); + public static final SqlVersion V_7_17_10 = fromId(7_17_10_99); + public static final SqlVersion V_7_17_11 = fromId(7_17_11_99); + public static final SqlVersion V_7_17_12 = fromId(7_17_12_99); + public static final SqlVersion V_7_17_13 = fromId(7_17_13_99); + public static final SqlVersion V_7_17_14 = fromId(7_17_14_99); + public static final SqlVersion V_7_17_15 = fromId(7_17_15_99); + public static final SqlVersion V_7_17_16 = fromId(7_17_16_99); + public static final SqlVersion V_7_17_17 = fromId(7_17_17_99); + public static final SqlVersion V_7_17_18 = fromId(7_17_18_99); + public static final SqlVersion V_7_17_19 = fromId(7_17_19_99); + public static final SqlVersion V_7_17_20 = fromId(7_17_20_99); + public static final SqlVersion V_7_17_21 = fromId(7_17_21_99); + public static final SqlVersion V_7_17_22 = fromId(7_17_22_99); + public static final SqlVersion V_7_17_23 = fromId(7_17_23_99); + public static final SqlVersion V_7_17_24 = fromId(7_17_24_99); + + public static final SqlVersion V_8_0_0 = fromId(8_00_00_99); + public static final SqlVersion V_8_0_1 = fromId(8_00_01_99); + public static final SqlVersion V_8_1_0 = fromId(8_01_00_99); + public static final SqlVersion V_8_1_1 = fromId(8_01_01_99); + public static final SqlVersion V_8_1_2 = fromId(8_01_02_99); + public static final SqlVersion V_8_1_3 = fromId(8_01_03_99); + public static final SqlVersion V_8_2_0 = fromId(8_02_00_99); + public static final SqlVersion V_8_2_1 = fromId(8_02_01_99); + public static final SqlVersion V_8_2_2 = fromId(8_02_02_99); + public static final SqlVersion V_8_2_3 = fromId(8_02_03_99); + public static final SqlVersion V_8_3_0 = fromId(8_03_00_99); + public static final SqlVersion V_8_3_1 = fromId(8_03_01_99); + public static final SqlVersion V_8_3_2 = fromId(8_03_02_99); + public static final SqlVersion V_8_3_3 = fromId(8_03_03_99); + public static final SqlVersion V_8_4_0 = fromId(8_04_00_99); + public static final SqlVersion V_8_4_1 = fromId(8_04_01_99); + public static final SqlVersion V_8_4_2 = fromId(8_04_02_99); + public static final SqlVersion V_8_4_3 = fromId(8_04_03_99); + public static final SqlVersion V_8_5_0 = fromId(8_05_00_99); + public static final SqlVersion V_8_5_1 = fromId(8_05_01_99); + public static final SqlVersion V_8_5_2 = fromId(8_05_02_99); + public static final SqlVersion V_8_5_3 = fromId(8_05_03_99); + public static final SqlVersion V_8_6_0 = fromId(8_06_00_99); + public static final SqlVersion V_8_6_1 = fromId(8_06_01_99); + public static final SqlVersion V_8_6_2 = fromId(8_06_02_99); + public static final SqlVersion V_8_7_0 = fromId(8_07_00_99); + public static final SqlVersion V_8_7_1 = fromId(8_07_01_99); + public static final SqlVersion V_8_8_0 = fromId(8_08_00_99); + public static final SqlVersion V_8_8_1 = fromId(8_08_01_99); + public static final SqlVersion V_8_8_2 = fromId(8_08_02_99); + public static final SqlVersion V_8_9_0 = fromId(8_09_00_99); + public static final SqlVersion V_8_9_1 = fromId(8_09_01_99); + public static final SqlVersion V_8_9_2 = fromId(8_09_02_99); + public static final SqlVersion V_8_10_0 = fromId(8_10_00_99); + public static final SqlVersion V_8_10_1 = fromId(8_10_01_99); + public static final SqlVersion V_8_10_2 = fromId(8_10_02_99); + public static final SqlVersion V_8_10_3 = fromId(8_10_03_99); + public static final SqlVersion V_8_10_4 = fromId(8_10_04_99); + public static final SqlVersion V_8_11_0 = fromId(8_11_00_99); + public static final SqlVersion V_8_11_1 = fromId(8_11_01_99); + public static final SqlVersion V_8_11_2 = fromId(8_11_02_99); + public static final SqlVersion V_8_11_3 = fromId(8_11_03_99); + public static final SqlVersion V_8_11_4 = fromId(8_11_04_99); + public static final SqlVersion V_8_12_0 = fromId(8_12_00_99); + public static final SqlVersion V_8_12_1 = fromId(8_12_01_99); + public static final SqlVersion V_8_12_2 = fromId(8_12_02_99); + public static final SqlVersion V_8_13_0 = fromId(8_13_00_99); + public static final SqlVersion V_8_13_1 = fromId(8_13_01_99); + public static final SqlVersion V_8_13_2 = fromId(8_13_02_99); + public static final SqlVersion V_8_13_3 = fromId(8_13_03_99); + public static final SqlVersion V_8_13_4 = fromId(8_13_04_99); + public static final SqlVersion V_8_14_0 = fromId(8_14_00_99); + public static final SqlVersion V_8_14_1 = fromId(8_14_01_99); + public static final SqlVersion V_8_14_2 = fromId(8_14_02_99); + public static final SqlVersion V_8_14_3 = fromId(8_14_03_99); + public static final SqlVersion V_8_15_0 = fromId(8_15_00_99); + public static final SqlVersion V_8_15_1 = fromId(8_15_01_99); + public static final SqlVersion V_8_16_0 = fromId(8_16_00_99); + + static final List DECLARED_VERSIONS = getDeclaredVersions(); + + /** + * What's the version of the server that the clients should be compatible with? + */ + public static final SqlVersion SERVER_COMPAT_VERSION = getLatestVersion(); + + public static SqlVersion getFirstVersion() { + return DECLARED_VERSIONS.get(0); + } + + public static SqlVersion getLatestVersion() { + return DECLARED_VERSIONS.get(DECLARED_VERSIONS.size() - 1); + } + + public static SqlVersion getPreviousVersion(SqlVersion version) { + int index = Collections.binarySearch(DECLARED_VERSIONS, version); + if (index < 1) { + throw new IllegalArgumentException("couldn't find any released versions before [" + version + "]"); + } + return DECLARED_VERSIONS.get(index - 1); + } + + public static SqlVersion getNextVersion(SqlVersion version) { + int index = Collections.binarySearch(DECLARED_VERSIONS, version); + if (index >= DECLARED_VERSIONS.size() - 1) { + throw new IllegalArgumentException("couldn't find any released versions before [" + version + "]"); + } + return DECLARED_VERSIONS.get(index + 1); + } + + public static List getAllVersions() { + return DECLARED_VERSIONS; + } + + // lifted from org.elasticsearch.Version#getDeclaredVersions + private static List getDeclaredVersions() { + final Field[] fields = SqlVersions.class.getFields(); + final List versions = new ArrayList<>(fields.length); + for (final Field field : fields) { + final int mod = field.getModifiers(); + if (false == (Modifier.isStatic(mod) && Modifier.isFinal(mod) && Modifier.isPublic(mod))) { + continue; + } + if (field.getType() != SqlVersion.class) { + continue; + } + switch (field.getName()) { + case "LATEST": + case "SERVER_COMPAT_VERSION": + continue; + } + assert field.getName().matches("V(_\\d+){3}?") : field.getName(); + try { + if (field.get(null) == null) { + throw new IllegalStateException("field " + field.getName() + " is null"); + } + versions.add(((SqlVersion) field.get(null))); + } catch (final IllegalAccessException e) { + throw new RuntimeException(e); + } + } + Collections.sort(versions); + return versions; + } +} diff --git a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/StringUtils.java b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/StringUtils.java index cf0a189a44b96..75bab66e5b23e 100644 --- a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/StringUtils.java +++ b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/StringUtils.java @@ -24,7 +24,7 @@ import static java.time.temporal.ChronoField.MINUTE_OF_HOUR; import static java.time.temporal.ChronoField.NANO_OF_SECOND; import static java.time.temporal.ChronoField.SECOND_OF_MINUTE; -import static org.elasticsearch.xpack.sql.proto.SqlVersion.DATE_NANOS_SUPPORT_VERSION; +import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.INTRODUCING_DATE_NANOS; public final class StringUtils { @@ -82,7 +82,7 @@ private StringUtils() {} // This method doesn't support compatibility with older JDBC drivers public static String toString(Object value) { - return toString(value, DATE_NANOS_SUPPORT_VERSION); + return toString(value, INTRODUCING_DATE_NANOS); } public static String toString(Object value, SqlVersion sqlVersion) { @@ -91,14 +91,14 @@ public static String toString(Object value, SqlVersion sqlVersion) { } if (value instanceof ZonedDateTime) { - if (SqlVersion.supportsDateNanos(sqlVersion)) { + if (VersionCompatibility.supportsDateNanos(sqlVersion)) { return ((ZonedDateTime) value).format(ISO_DATETIME_WITH_NANOS); } else { return ((ZonedDateTime) value).format(ISO_DATETIME_WITH_MILLIS); } } if (value instanceof OffsetTime) { - if (SqlVersion.supportsDateNanos(sqlVersion)) { + if (VersionCompatibility.supportsDateNanos(sqlVersion)) { return ((OffsetTime) value).format(ISO_TIME_WITH_NANOS); } else { return ((OffsetTime) value).format(ISO_TIME_WITH_MILLIS); diff --git a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/VersionCompatibility.java b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/VersionCompatibility.java new file mode 100644 index 0000000000000..691f140111c8f --- /dev/null +++ b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/VersionCompatibility.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.sql.proto; + +public class VersionCompatibility { + + public static final SqlVersion INTRODUCING_VERSION_COMPATIBILITY = SqlVersions.V_7_7_0; + public static final SqlVersion INTRODUCING_DATE_NANOS = SqlVersions.V_7_12_0; + public static final SqlVersion INTRODUCING_UNSIGNED_LONG = SqlVersions.V_8_2_0; + public static final SqlVersion INTRODUCING_WARNING_HANDLING = SqlVersions.V_8_2_0; + public static final SqlVersion INTRODUCING_VERSION_FIELD_TYPE = SqlVersions.V_8_4_0; + public static final SqlVersion INTRODUCING_VERSIONING_INDEPENDENT_FEATURES = SqlVersions.V_8_16_0; + + public static boolean hasVersionCompatibility(SqlVersion version) { + return version.onOrAfter(INTRODUCING_VERSION_COMPATIBILITY); + } + + /** Is the client on or past a version that is SQL-specific, independent of stack versioning? */ + public static boolean hasVersioningIndependentFeatures(SqlVersion version) { + return version.onOrAfter(INTRODUCING_VERSIONING_INDEPENDENT_FEATURES); + } + + // A client is version-compatible with the server if: + // - it is on or past the version-independent feature gating version; OR + // - it supports version compatibility (past or on 7.7.0); AND + // - it's not on a version newer than server's; AND + // - it's major version is at most one unit behind server's. + public static boolean isClientCompatible(SqlVersion server, SqlVersion client) { + if (hasVersioningIndependentFeatures(client)) { + return true; + } + // ES's CURRENT not available (core not a dependency), so it needs to be passed in as a parameter. + return hasVersionCompatibility(client) && server.onOrAfter(client) && server.major - client.major <= 1; + } + + // TODO: move to VersionCompatibilityChecks + public static boolean supportsDateNanos(SqlVersion version) { + return version.onOrAfter(INTRODUCING_DATE_NANOS); + } + + /** + * Does the provided {@code version} support the unsigned_long type (PR#60050)? + */ + public static boolean supportsUnsignedLong(SqlVersion version) { + return version.onOrAfter(INTRODUCING_UNSIGNED_LONG); + } + + /** + * Does the provided {@code version} support the version type (PR#85502)? + */ + public static boolean supportsVersionType(SqlVersion version) { + return version.onOrAfter(INTRODUCING_VERSION_FIELD_TYPE); + } +} diff --git a/x-pack/plugin/sql/sql-proto/src/test/java/org/elasticsearch/xpack/sql/proto/SqlVersionTests.java b/x-pack/plugin/sql/sql-proto/src/test/java/org/elasticsearch/xpack/sql/proto/SqlVersionTests.java index 65fccf9fc4c44..563802c836b89 100644 --- a/x-pack/plugin/sql/sql-proto/src/test/java/org/elasticsearch/xpack/sql/proto/SqlVersionTests.java +++ b/x-pack/plugin/sql/sql-proto/src/test/java/org/elasticsearch/xpack/sql/proto/SqlVersionTests.java @@ -13,8 +13,9 @@ import static org.elasticsearch.xpack.sql.proto.SqlVersion.MAJOR_MULTIPLIER; import static org.elasticsearch.xpack.sql.proto.SqlVersion.MINOR_MULTIPLIER; import static org.elasticsearch.xpack.sql.proto.SqlVersion.REVISION_MULTIPLIER; -import static org.elasticsearch.xpack.sql.proto.SqlVersion.V_7_7_0; -import static org.elasticsearch.xpack.sql.proto.SqlVersion.isClientCompatible; +import static org.elasticsearch.xpack.sql.proto.SqlVersions.V_7_7_0; +import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.INTRODUCING_VERSIONING_INDEPENDENT_FEATURES; +import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.isClientCompatible; public class SqlVersionTests extends ESTestCase { public void test123FromString() { @@ -37,6 +38,17 @@ public void test123AlphaFromString() { assertEquals("1.2.3-Alpha", ver.version); } + public void test123AlphaWithIdFromString() { + String version = "1.2.3-Alpha[" + randomIntBetween(0, 100_000_000) + "]"; + SqlVersion ver = SqlVersion.fromString(version); + assertEquals(1, ver.major); + assertEquals(2, ver.minor); + assertEquals(3, ver.revision); + assertEquals(REVISION_MULTIPLIER - 1, ver.build); + assertEquals(1 * MAJOR_MULTIPLIER + 2 * MINOR_MULTIPLIER + 3 * REVISION_MULTIPLIER + REVISION_MULTIPLIER - 1, ver.id); + assertEquals(version, ver.version); + } + public void test123AlphaSnapshotFromString() { SqlVersion ver = SqlVersion.fromString("1.2.3-Alpha-SNAPSHOT"); assertEquals(1, ver.major); @@ -84,16 +96,28 @@ public void testVersionCompatibilityClientWithNoCompatibility() { } public void testVersionCompatibilityClientNewer() { - int major = randomIntBetween(7, 99); - SqlVersion server = new SqlVersion(major, randomIntBetween(major > 7 ? 0 : 7, 99), randomIntBetween(0, 98)); + SqlVersion server = randomReleasedVersion(false); SqlVersion client = new SqlVersion(server.major, server.minor, (byte) (server.revision + 1)); assertFalse(isClientCompatible(server, client)); } + public void testVersionCompatibilityClientVersionIndependentFeatures() { + SqlVersion server = new SqlVersion( + randomIntBetween(INTRODUCING_VERSIONING_INDEPENDENT_FEATURES.major, 99), + randomIntBetween(INTRODUCING_VERSIONING_INDEPENDENT_FEATURES.minor, 99), + randomIntBetween(INTRODUCING_VERSIONING_INDEPENDENT_FEATURES.revision, 99) + ); + SqlVersion client = new SqlVersion( + randomIntBetween(INTRODUCING_VERSIONING_INDEPENDENT_FEATURES.major, 99), + randomIntBetween(INTRODUCING_VERSIONING_INDEPENDENT_FEATURES.minor, 99), + randomIntBetween(INTRODUCING_VERSIONING_INDEPENDENT_FEATURES.revision, 99) + ); + assertTrue(server + " vs. " + client, isClientCompatible(server, client)); + } + public void testVersionCompatibilityClientTooOld() { - int major = randomIntBetween(9, 99); - SqlVersion server = new SqlVersion(major, randomIntBetween(0, 99), randomIntBetween(0, 99)); - SqlVersion client = new SqlVersion(major - 2, randomIntBetween(0, 99), randomIntBetween(0, 99)); + SqlVersion server = randomReleasedVersion(false); + SqlVersion client = new SqlVersion(server.major - 2, randomIntBetween(0, 99), randomIntBetween(0, 99)); assertFalse(isClientCompatible(server, client)); } @@ -109,4 +133,9 @@ public void testVersionCompatibile() { SqlVersion server = new SqlVersion(serverMajor, serverMinor, serverRevision); assertTrue(isClientCompatible(server, client)); } + + private static SqlVersion randomReleasedVersion(boolean includeVersioningIndependent) { + var allVersions = SqlVersions.getAllVersions(); + return allVersions.get(randomIntBetween(0, allVersions.size() - 1 - (includeVersioningIndependent ? 0 : 1))); + } } diff --git a/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/SqlActionIT.java b/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/SqlActionIT.java index 3b7da36eebe9e..3f7c74f09f6da 100644 --- a/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/SqlActionIT.java +++ b/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/SqlActionIT.java @@ -6,11 +6,11 @@ */ package org.elasticsearch.xpack.sql.action; -import org.elasticsearch.Version; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.xpack.sql.proto.ColumnInfo; import org.elasticsearch.xpack.sql.proto.Mode; +import org.elasticsearch.xpack.sql.proto.SqlVersions; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; @@ -31,7 +31,7 @@ public void testSqlAction() { String columns = dataBeforeCount ? "data, count" : "count, data"; SqlQueryResponse response = new SqlQueryRequestBuilder(client()).query("SELECT " + columns + " FROM test ORDER BY count") .mode(Mode.JDBC) - .version(Version.CURRENT.toString()) + .version(SqlVersions.SERVER_COMPAT_VERSION.toString()) .get(); assertThat(response.size(), equalTo(2L)); assertThat(response.columns(), hasSize(2)); @@ -50,7 +50,7 @@ public void testSqlAction() { public void testSqlActionCurrentVersion() { SqlQueryResponse response = new SqlQueryRequestBuilder(client()).query("SELECT true") .mode(randomFrom(Mode.CLI, Mode.JDBC)) - .version(Version.CURRENT.toString()) + .version(SqlVersions.SERVER_COMPAT_VERSION.toString()) .get(); assertThat(response.size(), equalTo(1L)); assertEquals(true, response.rows().get(0).get(0)); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java index 4c6d06738e16f..3625733227da5 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.sql.analysis.analyzer; -import org.elasticsearch.Version; import org.elasticsearch.core.Tuple; import org.elasticsearch.xpack.ql.capabilities.Unresolvable; import org.elasticsearch.xpack.ql.common.Failure; @@ -77,11 +76,11 @@ import static java.util.stream.Collectors.toMap; import static org.elasticsearch.xpack.ql.analyzer.VerifierChecks.checkFilterConditionType; import static org.elasticsearch.xpack.ql.common.Failure.fail; -import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.isTypeSupportedInVersion; -import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.versionIntroducingType; import static org.elasticsearch.xpack.ql.type.DataTypes.BINARY; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.util.CollectionUtils.combine; +import static org.elasticsearch.xpack.sql.index.VersionCompatibilityChecks.isTypeSupportedInVersion; +import static org.elasticsearch.xpack.sql.index.VersionCompatibilityChecks.versionIntroducingType; import static org.elasticsearch.xpack.sql.stats.FeatureMetric.COMMAND; import static org.elasticsearch.xpack.sql.stats.FeatureMetric.GROUPBY; import static org.elasticsearch.xpack.sql.stats.FeatureMetric.HAVING; @@ -1000,9 +999,8 @@ private static void checkCastOnInexact(LogicalPlan p, Set localFailures } private static void checkClientSupportsDataTypes(LogicalPlan p, Set localFailures, SqlVersion version) { - Version ver = Version.fromId(version.id); p.output().forEach(e -> { - if (e.resolved() && isTypeSupportedInVersion(e.dataType(), ver) == false) { + if (e.resolved() && isTypeSupportedInVersion(e.dataType(), version) == false) { localFailures.add( fail( e, diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java index 9cf60ec3bb2e4..b2ce91140de76 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.search.TotalHits; import org.apache.lucene.util.PriorityQueue; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DelegatingActionListener; import org.elasticsearch.action.search.ClosePointInTimeRequest; @@ -100,7 +101,7 @@ import static org.elasticsearch.action.ActionListener.wrap; import static org.elasticsearch.xpack.ql.execution.search.extractor.AbstractFieldHitExtractor.MultiValueSupport.LENIENT; import static org.elasticsearch.xpack.ql.execution.search.extractor.AbstractFieldHitExtractor.MultiValueSupport.NONE; -import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.INTRODUCING_UNSIGNED_LONG; +import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.INTRODUCING_UNSIGNED_LONG; // TODO: add retry/back-off public class Querier { @@ -201,7 +202,7 @@ public static void closePointInTime(Client client, BytesReference pointInTimeId, public static SearchRequest prepareRequest(SearchSourceBuilder source, SqlConfiguration cfg, boolean includeFrozen, String... indices) { source.timeout(cfg.requestTimeout()); - SearchRequest searchRequest = new SearchRequest(INTRODUCING_UNSIGNED_LONG); + SearchRequest searchRequest = new SearchRequest(Version.fromId(INTRODUCING_UNSIGNED_LONG.id)); if (source.pointInTimeBuilder() == null) { searchRequest.indices(indices); searchRequest.indicesOptions( diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractor.java index 45bb7623f815a..bf54dca279dd0 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractor.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractor.java @@ -22,11 +22,11 @@ import java.util.Map; import java.util.Objects; -import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.INTRODUCING_UNSIGNED_LONG_TRANSPORT; import static org.elasticsearch.xpack.ql.type.DataTypeConverter.toUnsignedLong; import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME; import static org.elasticsearch.xpack.ql.type.DataTypes.NULL; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; +import static org.elasticsearch.xpack.sql.index.VersionCompatibilityChecks.INTRODUCING_UNSIGNED_LONG_TRANSPORT; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.isDateBased; public class CompositeKeyExtractor implements BucketExtractor { diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexCompatibility.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/index/IndexCompatibility.java similarity index 78% rename from x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexCompatibility.java rename to x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/index/IndexCompatibility.java index 2f99854679d8f..dbdedd5b49cdb 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexCompatibility.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/index/IndexCompatibility.java @@ -5,22 +5,24 @@ * 2.0. */ -package org.elasticsearch.xpack.ql.index; +package org.elasticsearch.xpack.sql.index; -import org.elasticsearch.Version; +import org.elasticsearch.xpack.ql.index.EsIndex; +import org.elasticsearch.xpack.ql.index.IndexResolution; import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.EsField; import org.elasticsearch.xpack.ql.type.UnsupportedEsField; +import org.elasticsearch.xpack.sql.proto.SqlVersion; import java.util.Map; -import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.isTypeSupportedInVersion; import static org.elasticsearch.xpack.ql.type.DataTypes.isPrimitive; import static org.elasticsearch.xpack.ql.type.Types.propagateUnsupportedType; +import static org.elasticsearch.xpack.sql.index.VersionCompatibilityChecks.isTypeSupportedInVersion; public final class IndexCompatibility { - public static Map compatible(Map mapping, Version version) { + public static Map compatible(Map mapping, SqlVersion version) { for (Map.Entry entry : mapping.entrySet()) { EsField esField = entry.getValue(); DataType dataType = esField.getDataType(); @@ -35,12 +37,12 @@ public static Map compatible(Map mapping, Vers return mapping; } - public static EsIndex compatible(EsIndex esIndex, Version version) { + public static EsIndex compatible(EsIndex esIndex, SqlVersion version) { compatible(esIndex.mapping(), version); return esIndex; } - public static IndexResolution compatible(IndexResolution indexResolution, Version version) { + public static IndexResolution compatible(IndexResolution indexResolution, SqlVersion version) { if (indexResolution.isValid()) { compatible(indexResolution.get(), version); } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/VersionCompatibilityChecks.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/index/VersionCompatibilityChecks.java similarity index 58% rename from x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/VersionCompatibilityChecks.java rename to x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/index/VersionCompatibilityChecks.java index e194f385d1606..bd98c40ac1674 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/VersionCompatibilityChecks.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/index/VersionCompatibilityChecks.java @@ -5,28 +5,28 @@ * 2.0. */ -package org.elasticsearch.xpack.ql.index; +package org.elasticsearch.xpack.sql.index; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; -import org.elasticsearch.Version; import org.elasticsearch.core.Nullable; import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.sql.proto.SqlVersion; -import static org.elasticsearch.Version.V_8_2_0; -import static org.elasticsearch.Version.V_8_4_0; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.type.DataTypes.VERSION; +import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.INTRODUCING_UNSIGNED_LONG; +import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.INTRODUCING_VERSION_FIELD_TYPE; +import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.supportsUnsignedLong; +import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.supportsVersionType; public final class VersionCompatibilityChecks { - public static final Version INTRODUCING_UNSIGNED_LONG = V_8_2_0; public static final TransportVersion INTRODUCING_UNSIGNED_LONG_TRANSPORT = TransportVersions.V_8_2_0; - public static final Version INTRODUCING_VERSION_FIELD_TYPE = V_8_4_0; private VersionCompatibilityChecks() {} - public static boolean isTypeSupportedInVersion(DataType dataType, Version version) { + public static boolean isTypeSupportedInVersion(DataType dataType, SqlVersion version) { if (dataType == UNSIGNED_LONG) { return supportsUnsignedLong(version); } @@ -36,21 +36,7 @@ public static boolean isTypeSupportedInVersion(DataType dataType, Version versio return true; } - /** - * Does the provided {@code version} support the unsigned_long type (PR#60050)? - */ - public static boolean supportsUnsignedLong(Version version) { - return INTRODUCING_UNSIGNED_LONG.compareTo(version) <= 0; - } - - /** - * Does the provided {@code version} support the version type (PR#85502)? - */ - public static boolean supportsVersionType(Version version) { - return INTRODUCING_VERSION_FIELD_TYPE.compareTo(version) <= 0; - } - - public static @Nullable Version versionIntroducingType(DataType dataType) { + public static @Nullable SqlVersion versionIntroducingType(DataType dataType) { if (dataType == UNSIGNED_LONG) { return INTRODUCING_UNSIGNED_LONG; } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java index 516aaa961b5b3..72f9cf2543d53 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java @@ -6,18 +6,17 @@ */ package org.elasticsearch.xpack.sql.plan.logical.command; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.expression.FieldAttribute; import org.elasticsearch.xpack.ql.expression.predicate.regex.LikePattern; -import org.elasticsearch.xpack.ql.index.IndexCompatibility; import org.elasticsearch.xpack.ql.index.IndexResolver; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.EsField; import org.elasticsearch.xpack.ql.type.KeywordEsField; +import org.elasticsearch.xpack.sql.index.IndexCompatibility; import org.elasticsearch.xpack.sql.session.Cursor.Page; import org.elasticsearch.xpack.sql.session.SqlSession; import org.elasticsearch.xpack.sql.type.SqlDataTypes; @@ -91,8 +90,11 @@ public void execute(SqlSession session, ActionListener listener) { List> rows = emptyList(); if (indexResult.isValid()) { rows = new ArrayList<>(); - Version version = Version.fromId(session.configuration().version().id); - fillInRows(IndexCompatibility.compatible(indexResult, version).get().mapping(), null, rows); + fillInRows( + IndexCompatibility.compatible(indexResult, session.configuration().version()).get().mapping(), + null, + rows + ); } l.onResponse(of(session, rows)); }) diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java index 9f57df5190dcc..b78dea5353733 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java @@ -7,21 +7,21 @@ package org.elasticsearch.xpack.sql.plan.logical.command.sys; import org.apache.lucene.util.Counter; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.Strings; import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.expression.predicate.regex.LikePattern; import org.elasticsearch.xpack.ql.index.EsIndex; -import org.elasticsearch.xpack.ql.index.IndexCompatibility; import org.elasticsearch.xpack.ql.index.IndexResolver; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.EsField; import org.elasticsearch.xpack.ql.util.StringUtils; +import org.elasticsearch.xpack.sql.index.IndexCompatibility; import org.elasticsearch.xpack.sql.plan.logical.command.Command; import org.elasticsearch.xpack.sql.proto.Mode; +import org.elasticsearch.xpack.sql.proto.SqlVersion; import org.elasticsearch.xpack.sql.session.Cursor.Page; import org.elasticsearch.xpack.sql.session.ListCursor; import org.elasticsearch.xpack.sql.session.Rows; @@ -156,7 +156,7 @@ public void execute(SqlSession session, ActionListener listener) { tableCat = cluster; } - Version version = Version.fromId(session.configuration().version().id); + SqlVersion version = session.configuration().version(); // special case for '%' (translated to *) if ("*".equals(idx)) { session.indexResolver() diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypes.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypes.java index 66efd1a4ee879..15d1bed5b9a86 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypes.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypes.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.sql.plan.logical.command.sys; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.tree.NodeInfo; @@ -24,12 +23,12 @@ import static java.util.Arrays.asList; import static java.util.stream.Collectors.toList; -import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.isTypeSupportedInVersion; import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN; import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER; import static org.elasticsearch.xpack.ql.type.DataTypes.SHORT; import static org.elasticsearch.xpack.ql.type.DataTypes.isSigned; import static org.elasticsearch.xpack.ql.type.DataTypes.isString; +import static org.elasticsearch.xpack.sql.index.VersionCompatibilityChecks.isTypeSupportedInVersion; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.metaSqlDataType; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.metaSqlDateTimeSub; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.metaSqlMaximumScale; @@ -85,9 +84,7 @@ public List output() { @Override public final void execute(SqlSession session, ActionListener listener) { - Stream values = SqlDataTypes.types() - .stream() - .filter(t -> isTypeSupportedInVersion(t, Version.fromId(session.configuration().version().id))); + Stream values = SqlDataTypes.types().stream().filter(t -> isTypeSupportedInVersion(t, session.configuration().version())); if (type.intValue() != 0) { values = values.filter(t -> type.equals(sqlType(t).getVendorTypeNumber())); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlConfiguration.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlConfiguration.java index 8f66170c0842b..395f12ce3a66a 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlConfiguration.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlConfiguration.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.sql.session; -import org.elasticsearch.Version; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.query.QueryBuilder; @@ -14,6 +13,7 @@ import org.elasticsearch.xpack.sql.action.SqlQueryTask; import org.elasticsearch.xpack.sql.proto.Mode; import org.elasticsearch.xpack.sql.proto.SqlVersion; +import org.elasticsearch.xpack.sql.proto.SqlVersions; import java.time.ZoneId; import java.util.Map; @@ -73,7 +73,7 @@ public SqlConfiguration( this.runtimeMappings = runtimeMappings; this.mode = mode == null ? Mode.PLAIN : mode; this.clientId = clientId; - this.version = version != null ? version : SqlVersion.fromId(Version.CURRENT.id); + this.version = version != null ? version : SqlVersions.SERVER_COMPAT_VERSION; this.multiValueFieldLeniency = multiValueFieldLeniency; this.includeFrozenIndices = includeFrozen; this.taskId = taskId; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java index 040807981a389..c32155f6daeb6 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.sql.session; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.ParentTaskAssigningClient; @@ -15,7 +14,6 @@ import org.elasticsearch.xpack.ql.analyzer.PreAnalyzer.PreAnalysis; import org.elasticsearch.xpack.ql.analyzer.TableInfo; import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry; -import org.elasticsearch.xpack.ql.index.IndexCompatibility; import org.elasticsearch.xpack.ql.index.IndexResolution; import org.elasticsearch.xpack.ql.index.IndexResolver; import org.elasticsearch.xpack.ql.index.MappingException; @@ -26,6 +24,7 @@ import org.elasticsearch.xpack.sql.analysis.analyzer.AnalyzerContext; import org.elasticsearch.xpack.sql.analysis.analyzer.Verifier; import org.elasticsearch.xpack.sql.execution.PlanExecutor; +import org.elasticsearch.xpack.sql.index.IndexCompatibility; import org.elasticsearch.xpack.sql.optimizer.Optimizer; import org.elasticsearch.xpack.sql.parser.SqlParser; import org.elasticsearch.xpack.sql.plan.physical.PhysicalPlan; @@ -119,7 +118,7 @@ public void analyzedPlan(LogicalPlan parsed, boolean verify, ActionListener plan(sql)); assertThat(ex.getMessage(), containsString("Found 1 problem\nline 1:8: Cannot use field [unsigned_long]")); - for (Version v : List.of(INTRODUCING_UNSIGNED_LONG, postUnsignedLong)) { + for (SqlVersion v : List.of(INTRODUCING_UNSIGNED_LONG, POST_UNSIGNED_LONG)) { analyzer = analyzer( - SqlTestUtils.randomConfiguration(SqlVersion.fromId(v.id)), + SqlTestUtils.randomConfiguration(v), functionRegistry, loadCompatibleIndexResolution("mapping-numeric.json", v), verifier @@ -357,23 +358,21 @@ public void testVersionTypeVersionCompatibility() { String queryWithAlias = "SELECT version_number AS version_number FROM test"; String queryWithCast = "SELECT CONCAT(version_number::string, '-SNAPSHOT')::version AS version_number FROM test"; - Version preVersion = Version.fromId(INTRODUCING_VERSION_FIELD_TYPE.id - SqlVersion.MINOR_MULTIPLIER); - Version postVersion = Version.fromId(INTRODUCING_VERSION_FIELD_TYPE.id + SqlVersion.MINOR_MULTIPLIER); - SqlConfiguration sqlConfig = SqlTestUtils.randomConfiguration(SqlVersion.fromId(preVersion.id)); + SqlConfiguration sqlConfig = SqlTestUtils.randomConfiguration(PRE_VERSION_FIELD); for (String sql : List.of(query, queryWithCastLiteral, queryWithAlias, queryWithCast)) { analyzer = analyzer( sqlConfig, functionRegistry, - loadCompatibleIndexResolution("mapping-version.json", preVersion), + loadCompatibleIndexResolution("mapping-version.json", PRE_VERSION_FIELD), new Verifier(new Metrics()) ); VerificationException ex = expectThrows(VerificationException.class, () -> plan(sql)); assertThat(ex.getMessage(), containsString("Cannot use field [version_number]")); - for (Version v : List.of(INTRODUCING_VERSION_FIELD_TYPE, postVersion)) { + for (SqlVersion v : List.of(INTRODUCING_VERSION_FIELD_TYPE, POST_VERSION_FIELD)) { analyzer = analyzer( - SqlTestUtils.randomConfiguration(SqlVersion.fromId(v.id)), + SqlTestUtils.randomConfiguration(v), functionRegistry, loadCompatibleIndexResolution("mapping-version.json", v), verifier @@ -391,12 +390,10 @@ public void testVersionTypeVersionCompatibility() { } public void testNonProjectedUnsignedLongVersionCompatibility() { - Version preUnsignedLong = Version.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER); - SqlConfiguration sqlConfig = SqlTestUtils.randomConfiguration(SqlVersion.fromId(preUnsignedLong.id)); analyzer = analyzer( - sqlConfig, + SqlTestUtils.randomConfiguration(PRE_UNSIGNED_LONG), functionRegistry, - loadCompatibleIndexResolution("mapping-numeric.json", preUnsignedLong), + loadCompatibleIndexResolution("mapping-numeric.json", PRE_UNSIGNED_LONG), new Verifier(new Metrics()) ); @@ -426,24 +423,17 @@ public void testNestedUnsignedLongVersionCompatibility() { """; String sql = "SELECT container.ul as unsigned_long FROM test"; - Version preUnsignedLong = Version.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER); analyzer = analyzer( - SqlTestUtils.randomConfiguration(SqlVersion.fromId(preUnsignedLong.id)), + SqlTestUtils.randomConfiguration(PRE_UNSIGNED_LONG), functionRegistry, - compatibleIndexResolution(props, preUnsignedLong), + compatibleIndexResolution(props, PRE_UNSIGNED_LONG), new Verifier(new Metrics()) ); VerificationException ex = expectThrows(VerificationException.class, () -> plan(sql)); assertThat(ex.getMessage(), containsString("Cannot use field [container.ul] with unsupported type [UNSIGNED_LONG]")); - Version postUnsignedLong = Version.fromId(INTRODUCING_UNSIGNED_LONG.id + SqlVersion.MINOR_MULTIPLIER); - for (Version v : List.of(INTRODUCING_UNSIGNED_LONG, postUnsignedLong)) { - analyzer = analyzer( - SqlTestUtils.randomConfiguration(SqlVersion.fromId(v.id)), - functionRegistry, - compatibleIndexResolution(props, v), - verifier - ); + for (SqlVersion v : List.of(INTRODUCING_UNSIGNED_LONG, POST_UNSIGNED_LONG)) { + analyzer = analyzer(SqlTestUtils.randomConfiguration(v), functionRegistry, compatibleIndexResolution(props, v), verifier); LogicalPlan plan = plan(sql); assertThat(plan, instanceOf(Project.class)); Project p = (Project) plan; @@ -456,17 +446,15 @@ public void testNestedUnsignedLongVersionCompatibility() { } public void testUnsignedLongStarExpandedVersionControlled() { - SqlVersion preUnsignedLong = SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER); - SqlVersion postUnsignedLong = SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id + SqlVersion.MINOR_MULTIPLIER); String query = "SELECT * FROM test"; - for (SqlVersion version : List.of(preUnsignedLong, SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id), postUnsignedLong)) { + for (SqlVersion version : List.of(PRE_UNSIGNED_LONG, INTRODUCING_UNSIGNED_LONG, POST_UNSIGNED_LONG)) { SqlConfiguration config = SqlTestUtils.randomConfiguration(version); // the mapping is mutated when making it "compatible", so it needs to be reloaded inside the loop. analyzer = analyzer( config, functionRegistry, - loadCompatibleIndexResolution("mapping-numeric.json", Version.fromId(version.id)), + loadCompatibleIndexResolution("mapping-numeric.json", version), new Verifier(new Metrics()) ); @@ -475,7 +463,7 @@ public void testUnsignedLongStarExpandedVersionControlled() { Project p = (Project) plan; List projectedDataTypes = p.projections().stream().map(Expression::dataType).toList(); - assertEquals(isTypeSupportedInVersion(UNSIGNED_LONG, Version.fromId(version.id)), projectedDataTypes.contains(UNSIGNED_LONG)); + assertEquals(isTypeSupportedInVersion(UNSIGNED_LONG, version), projectedDataTypes.contains(UNSIGNED_LONG)); } } @@ -517,11 +505,11 @@ private static IndexResolution loadIndexResolution(String mappingName) { return IndexResolution.valid(index); } - private static IndexResolution loadCompatibleIndexResolution(String mappingName, Version version) { + private static IndexResolution loadCompatibleIndexResolution(String mappingName, SqlVersion version) { return IndexCompatibility.compatible(loadIndexResolution(mappingName), version); } - private static IndexResolution compatibleIndexResolution(String properties, Version version) { + private static IndexResolution compatibleIndexResolution(String properties, SqlVersion version) { Map mapping = Types.fromEs( DefaultDataTypeRegistry.INSTANCE, XContentHelper.convertToMap(JsonXContent.jsonXContent, properties, randomBoolean()) diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java index 6d7822e5619cc..8388dabe23592 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java @@ -150,7 +150,7 @@ public void testMetaFieldsAreIgnored() throws Exception { addFieldCaps(fieldCaps, "text", "keyword", true, true); String wildcard = "*"; - IndexResolution resolution = mergedMappings(wildcard, new String[] { "index" }, fieldCaps); + IndexResolution resolution = mergedMappings(wildcard, new String[] { "org/elasticsearch/xpack/sql/index" }, fieldCaps); assertTrue(resolution.isValid()); EsIndex esIndex = resolution.get(); @@ -160,7 +160,7 @@ public void testMetaFieldsAreIgnored() throws Exception { assertNull(esIndex.mapping().get("_doc_count")); assertEquals(INTEGER, esIndex.mapping().get("_not_meta_field").getDataType()); assertEquals(KEYWORD, esIndex.mapping().get("text").getDataType()); - assertEquals(Set.of("index"), resolution.get().concreteIndices()); + assertEquals(Set.of("org/elasticsearch/xpack/sql/index"), resolution.get().concreteIndices()); } public void testFlattenedHiddenSubfield() throws Exception { @@ -175,12 +175,12 @@ public void testFlattenedHiddenSubfield() throws Exception { addFieldCaps(fieldCaps, "text", "keyword", true, true); String wildcard = "*"; - IndexResolution resolution = mergedMappings(wildcard, new String[] { "index" }, fieldCaps); + IndexResolution resolution = mergedMappings(wildcard, new String[] { "org/elasticsearch/xpack/sql/index" }, fieldCaps); assertTrue(resolution.isValid()); EsIndex esIndex = resolution.get(); assertEquals(wildcard, esIndex.name()); - assertEquals(Set.of("index"), resolution.get().concreteIndices()); + assertEquals(Set.of("org/elasticsearch/xpack/sql/index"), resolution.get().concreteIndices()); assertEquals(UNSUPPORTED, esIndex.mapping().get("some_field").getDataType()); assertEquals(UNSUPPORTED, esIndex.mapping().get("some_field").getProperties().get("_keyed").getDataType()); assertEquals(OBJECT, esIndex.mapping().get("nested_field").getDataType()); @@ -205,12 +205,12 @@ public void testPropagateUnsupportedTypeToSubFields() throws Exception { addFieldCaps(fieldCaps, "a.b.c.e", "foo", true, true); String wildcard = "*"; - IndexResolution resolution = mergedMappings(wildcard, new String[] { "index" }, fieldCaps); + IndexResolution resolution = mergedMappings(wildcard, new String[] { "org/elasticsearch/xpack/sql/index" }, fieldCaps); assertTrue(resolution.isValid()); EsIndex esIndex = resolution.get(); assertEquals(wildcard, esIndex.name()); - assertEquals(Set.of("index"), resolution.get().concreteIndices()); + assertEquals(Set.of("org/elasticsearch/xpack/sql/index"), resolution.get().concreteIndices()); assertEquals(TEXT, esIndex.mapping().get("a").getDataType()); assertEquals(UNSUPPORTED, esIndex.mapping().get("a").getProperties().get("b").getDataType()); assertEquals(UNSUPPORTED, esIndex.mapping().get("a").getProperties().get("b").getProperties().get("c").getDataType()); @@ -241,12 +241,12 @@ public void testRandomMappingFieldTypeMappedAsUnsupported() throws Exception { addFieldCaps(fieldCaps, "text", "keyword", true, true); String wildcard = "*"; - IndexResolution resolution = mergedMappings(wildcard, new String[] { "index" }, fieldCaps); + IndexResolution resolution = mergedMappings(wildcard, new String[] { "org/elasticsearch/xpack/sql/index" }, fieldCaps); assertTrue(resolution.isValid()); EsIndex esIndex = resolution.get(); assertEquals(wildcard, esIndex.name()); - assertEquals(Set.of("index"), resolution.get().concreteIndices()); + assertEquals(Set.of("org/elasticsearch/xpack/sql/index"), resolution.get().concreteIndices()); assertEquals(UNSUPPORTED, esIndex.mapping().get("some_field").getDataType()); assertEquals(OBJECT, esIndex.mapping().get("nested_field").getDataType()); assertEquals(UNSUPPORTED, esIndex.mapping().get("nested_field").getProperties().get("sub_field1").getDataType()); @@ -363,7 +363,7 @@ public void testMultipleCompatibleIndicesWithDifferentFields() { Map mapping = Maps.newMapWithExpectedSize(1); String fieldName = "field" + (i + 1); mapping.put(fieldName, new KeywordEsField(fieldName)); - expectedIndices[i] = new EsIndex("index" + (i + 1), mapping); + expectedIndices[i] = new EsIndex("org/elasticsearch/xpack/sql/index" + (i + 1), mapping); } Arrays.sort(expectedIndices, Comparator.comparing(EsIndex::name)); @@ -385,7 +385,7 @@ public void testMergeConcreteIndices() { Map mapping = Maps.newMapWithExpectedSize(1); String fieldName = "field" + (i + 1); mapping.put(fieldName, new KeywordEsField(fieldName)); - String indexName = "index" + (i + 1); + String indexName = "org/elasticsearch/xpack/sql/index" + (i + 1); expectedIndices[i] = new EsIndex(indexName, mapping, Set.of(indexName)); indexNames.add(indexName); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumnsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumnsTests.java index 8e0edc54e6fbf..48fce8dfbdb0f 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumnsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumnsTests.java @@ -7,10 +7,9 @@ package org.elasticsearch.xpack.sql.plan.logical.command; -import org.elasticsearch.Version; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.ql.index.IndexCompatibility; import org.elasticsearch.xpack.ql.type.EsField; +import org.elasticsearch.xpack.sql.index.IndexCompatibility; import org.elasticsearch.xpack.sql.proto.SqlVersion; import java.sql.JDBCType; @@ -19,8 +18,6 @@ import java.util.Map; import static java.util.Arrays.asList; -import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.supportsUnsignedLong; -import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.supportsVersionType; import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN; import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME; import static org.elasticsearch.xpack.ql.type.DataTypes.FLOAT; @@ -32,11 +29,13 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSUPPORTED; import static org.elasticsearch.xpack.ql.type.DataTypes.VERSION; -import static org.elasticsearch.xpack.sql.plan.logical.command.sys.SysColumnsTests.UNSIGNED_LONG_TEST_VERSIONS; -import static org.elasticsearch.xpack.sql.plan.logical.command.sys.SysColumnsTests.VERSION_FIELD_TEST_VERSIONS; +import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.supportsUnsignedLong; +import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.supportsVersionType; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.GEO_POINT; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.GEO_SHAPE; import static org.elasticsearch.xpack.sql.types.SqlTypesTests.loadMapping; +import static org.elasticsearch.xpack.sql.util.SqlVersionUtils.UNSIGNED_LONG_TEST_VERSIONS; +import static org.elasticsearch.xpack.sql.util.SqlVersionUtils.VERSION_FIELD_TEST_VERSIONS; public class ShowColumnsTests extends ESTestCase { @@ -94,8 +93,8 @@ public void testUnsignedLongFiltering() { List> rows = new ArrayList<>(); // mapping's mutated by IndexCompatibility.compatible, needs to stay in the loop Map mapping = loadMapping("mapping-multi-field-variation.json", true); - ShowColumns.fillInRows(IndexCompatibility.compatible(mapping, Version.fromId(version.id)), null, rows); - assertTrue((supportsUnsignedLong(Version.fromId(version.id)) && rows.contains(rowSupported)) || rows.contains(rowUnsupported)); + ShowColumns.fillInRows(IndexCompatibility.compatible(mapping, version), null, rows); + assertTrue((supportsUnsignedLong(version) && rows.contains(rowSupported)) || rows.contains(rowUnsupported)); } } @@ -106,8 +105,8 @@ public void testVersionFieldFiltering() { List> rows = new ArrayList<>(); // mapping's mutated by IndexCompatibility.compatible, needs to stay in the loop Map mapping = loadMapping("mapping-multi-field-variation.json", true); - ShowColumns.fillInRows(IndexCompatibility.compatible(mapping, Version.fromId(version.id)), null, rows); - assertTrue((supportsVersionType(Version.fromId(version.id)) && rows.contains(rowSupported)) || rows.contains(rowUnsupported)); + ShowColumns.fillInRows(IndexCompatibility.compatible(mapping, version), null, rows); + assertTrue((supportsVersionType(version) && rows.contains(rowSupported)) || rows.contains(rowUnsupported)); } } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java index e0af40bbc4ce5..a44041ce04610 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java @@ -6,23 +6,23 @@ */ package org.elasticsearch.xpack.sql.plan.logical.command.sys; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.core.Tuple; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ql.index.EsIndex; -import org.elasticsearch.xpack.ql.index.IndexCompatibility; import org.elasticsearch.xpack.ql.index.IndexResolution; import org.elasticsearch.xpack.ql.index.IndexResolver; import org.elasticsearch.xpack.ql.type.EsField; import org.elasticsearch.xpack.sql.action.Protocol; import org.elasticsearch.xpack.sql.analysis.analyzer.Analyzer; +import org.elasticsearch.xpack.sql.index.IndexCompatibility; import org.elasticsearch.xpack.sql.parser.SqlParser; import org.elasticsearch.xpack.sql.plan.logical.command.Command; import org.elasticsearch.xpack.sql.proto.Mode; import org.elasticsearch.xpack.sql.proto.SqlTypedParamValue; import org.elasticsearch.xpack.sql.proto.SqlVersion; +import org.elasticsearch.xpack.sql.proto.SqlVersions; import org.elasticsearch.xpack.sql.session.Cursor; import org.elasticsearch.xpack.sql.session.SchemaRowSet; import org.elasticsearch.xpack.sql.session.SqlConfiguration; @@ -42,14 +42,14 @@ import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; import static org.elasticsearch.xpack.ql.TestUtils.UTC; -import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.INTRODUCING_UNSIGNED_LONG; -import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.INTRODUCING_VERSION_FIELD_TYPE; -import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.isTypeSupportedInVersion; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.type.DataTypes.VERSION; import static org.elasticsearch.xpack.sql.analysis.analyzer.AnalyzerTestUtils.analyzer; +import static org.elasticsearch.xpack.sql.index.VersionCompatibilityChecks.isTypeSupportedInVersion; import static org.elasticsearch.xpack.sql.proto.Mode.isDriver; import static org.elasticsearch.xpack.sql.types.SqlTypesTests.loadMapping; +import static org.elasticsearch.xpack.sql.util.SqlVersionUtils.UNSIGNED_LONG_TEST_VERSIONS; +import static org.elasticsearch.xpack.sql.util.SqlVersionUtils.VERSION_FIELD_TEST_VERSIONS; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.eq; @@ -59,20 +59,6 @@ public class SysColumnsTests extends ESTestCase { - public static List UNSIGNED_LONG_TEST_VERSIONS = List.of( - SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER), - SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id), - SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id + SqlVersion.MINOR_MULTIPLIER), - SqlVersion.fromId(Version.CURRENT.id) - ); - - public static List VERSION_FIELD_TEST_VERSIONS = List.of( - SqlVersion.fromId(INTRODUCING_VERSION_FIELD_TYPE.id - SqlVersion.MINOR_MULTIPLIER), - SqlVersion.fromId(INTRODUCING_VERSION_FIELD_TYPE.id), - SqlVersion.fromId(INTRODUCING_VERSION_FIELD_TYPE.id + SqlVersion.MINOR_MULTIPLIER), - SqlVersion.fromId(Version.CURRENT.id) - ); - private static final String CLUSTER_NAME = "cluster"; private static final Map MAPPING1 = loadMapping("mapping-multi-field-with-nested.json", true); private static final Map MAPPING2 = loadMapping("mapping-multi-field-variation.json", true); @@ -84,7 +70,7 @@ public class SysColumnsTests extends ESTestCase { private void sysColumnsInMode(Mode mode) { Class typeClass = mode == Mode.ODBC ? Short.class : Integer.class; List> rows = new ArrayList<>(); - SysColumns.fillInRows("test", "index", MAPPING2, null, rows, null, mode); + SysColumns.fillInRows("test", "org/elasticsearch/xpack/sql/index", MAPPING2, null, rows, null, mode); assertEquals(FIELD_COUNT2, rows.size()); assertEquals(24, rows.get(0).size()); @@ -162,8 +148,8 @@ public void testUnsignedLongFiltering() { Map mapping = loadMapping("mapping-multi-field-variation.json", true); SysColumns.fillInRows( "test", - "index", - IndexCompatibility.compatible(mapping, Version.fromId(version.id)), + "org/elasticsearch/xpack/sql/index", + IndexCompatibility.compatible(mapping, version), null, rows, null, @@ -171,7 +157,7 @@ public void testUnsignedLongFiltering() { ); List types = rows.stream().map(row -> name(row).toString()).collect(Collectors.toList()); assertEquals( - isTypeSupportedInVersion(UNSIGNED_LONG, Version.fromId(version.id)), + isTypeSupportedInVersion(UNSIGNED_LONG, version), types.contains(UNSIGNED_LONG.toString().toLowerCase(Locale.ROOT)) ); } @@ -186,18 +172,15 @@ public void testVersionTypeFiltering() { Map mapping = loadMapping("mapping-multi-field-variation.json", true); SysColumns.fillInRows( "test", - "index", - IndexCompatibility.compatible(mapping, Version.fromId(version.id)), + "org/elasticsearch/xpack/sql/index", + IndexCompatibility.compatible(mapping, version), null, rows, null, mode ); List types = rows.stream().map(row -> name(row).toString()).collect(Collectors.toList()); - assertEquals( - isTypeSupportedInVersion(VERSION, Version.fromId(version.id)), - types.contains(VERSION.toString().toLowerCase(Locale.ROOT)) - ); + assertEquals(isTypeSupportedInVersion(VERSION, version), types.contains(VERSION.toString().toLowerCase(Locale.ROOT))); } } } @@ -313,7 +296,7 @@ private int executeCommandInOdbcModeAndCountRows(String sql) { null, Mode.ODBC, null, - SqlVersion.fromId(Version.CURRENT.id), + SqlVersions.SERVER_COMPAT_VERSION, null, null, false, @@ -360,7 +343,7 @@ private void executeCommand( null, mode, null, - SqlVersion.fromId(Version.CURRENT.id), + SqlVersions.SERVER_COMPAT_VERSION, null, null, false, diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java index d2b2e99d566c8..789d03efbc3f9 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java @@ -6,8 +6,8 @@ */ package org.elasticsearch.xpack.sql.plan.logical.command.sys; -import org.elasticsearch.Version; import org.elasticsearch.action.support.ActionTestUtils; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ql.index.EsIndex; @@ -34,19 +34,20 @@ import java.util.Set; import static java.util.Arrays.asList; -import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.isTypeSupportedInVersion; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.type.DataTypes.VERSION; import static org.elasticsearch.xpack.sql.analysis.analyzer.AnalyzerTestUtils.analyzer; -import static org.elasticsearch.xpack.sql.plan.logical.command.sys.SysColumnsTests.UNSIGNED_LONG_TEST_VERSIONS; -import static org.elasticsearch.xpack.sql.plan.logical.command.sys.SysColumnsTests.VERSION_FIELD_TEST_VERSIONS; +import static org.elasticsearch.xpack.sql.index.VersionCompatibilityChecks.isTypeSupportedInVersion; +import static org.elasticsearch.xpack.sql.proto.SqlVersions.SERVER_COMPAT_VERSION; +import static org.elasticsearch.xpack.sql.util.SqlVersionUtils.UNSIGNED_LONG_TEST_VERSIONS; +import static org.elasticsearch.xpack.sql.util.SqlVersionUtils.VERSION_FIELD_TEST_VERSIONS; import static org.mockito.Mockito.mock; public class SysTypesTests extends ESTestCase { private final SqlParser parser = new SqlParser(); - private Tuple sql(String sql, Mode mode, SqlVersion version) { + private Tuple sql(String sql, Mode mode, @Nullable SqlVersion version) { SqlConfiguration configuration = new SqlConfiguration( DateUtils.UTC, null, @@ -76,7 +77,7 @@ private Tuple sql(String sql, Mode mode, SqlVersion version } private Tuple sql(String sql) { - return sql(sql, randomFrom(Mode.values()), randomBoolean() ? null : SqlVersion.fromId(Version.CURRENT.id)); + return sql(sql, randomFrom(Mode.values()), randomBoolean() ? null : SERVER_COMPAT_VERSION); } public void testSysTypes() { @@ -154,7 +155,7 @@ public void testUnsignedLongFiltering() { List types = new ArrayList<>(); r.forEachRow(rv -> types.add((String) rv.column(0))); assertEquals( - isTypeSupportedInVersion(UNSIGNED_LONG, Version.fromId(cmd.v2().configuration().version().id)), + isTypeSupportedInVersion(UNSIGNED_LONG, cmd.v2().configuration().version()), types.contains(UNSIGNED_LONG.toString()) ); })); @@ -173,10 +174,7 @@ public void testVersionTypeFiltering() { SchemaRowSet r = (SchemaRowSet) p.rowSet(); List types = new ArrayList<>(); r.forEachRow(rv -> types.add((String) rv.column(0))); - assertEquals( - isTypeSupportedInVersion(VERSION, Version.fromId(cmd.v2().configuration().version().id)), - types.contains(VERSION.toString()) - ); + assertEquals(isTypeSupportedInVersion(VERSION, cmd.v2().configuration().version()), types.contains(VERSION.toString())); })); } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plugin/TextFormatTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plugin/TextFormatTests.java index 7224b357fbd7d..10e8496b12304 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plugin/TextFormatTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plugin/TextFormatTests.java @@ -28,7 +28,7 @@ import static org.elasticsearch.xpack.sql.plugin.TextFormat.CSV; import static org.elasticsearch.xpack.sql.plugin.TextFormat.PLAIN_TEXT; import static org.elasticsearch.xpack.sql.plugin.TextFormat.TSV; -import static org.elasticsearch.xpack.sql.proto.SqlVersion.DATE_NANOS_SUPPORT_VERSION; +import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.INTRODUCING_DATE_NANOS; import static org.elasticsearch.xpack.sql.proto.formatter.SimpleFormatter.FormatOption.TEXT; public class TextFormatTests extends ESTestCase { @@ -179,7 +179,7 @@ public void testPlainTextEmptyCursorWithoutColumns() { PLAIN_TEXT.format( req(), new BasicFormatter(emptyList(), emptyList(), TEXT), - new SqlQueryResponse(StringUtils.EMPTY, Mode.JDBC, DATE_NANOS_SUPPORT_VERSION, false, null, emptyList()) + new SqlQueryResponse(StringUtils.EMPTY, Mode.JDBC, INTRODUCING_DATE_NANOS, false, null, emptyList()) ).v1() ); } @@ -188,9 +188,9 @@ private static SqlQueryResponse emptyData() { return new SqlQueryResponse( StringUtils.EMPTY, Mode.JDBC, - DATE_NANOS_SUPPORT_VERSION, + INTRODUCING_DATE_NANOS, false, - singletonList(new ColumnInfo("index", "name", "keyword")), + singletonList(new ColumnInfo("org/elasticsearch/xpack/sql/index", "name", "keyword")), emptyList() ); } @@ -198,29 +198,29 @@ private static SqlQueryResponse emptyData() { private static SqlQueryResponse regularData() { // headers List headers = new ArrayList<>(); - headers.add(new ColumnInfo("index", "string", "keyword")); - headers.add(new ColumnInfo("index", "number", "integer")); + headers.add(new ColumnInfo("org/elasticsearch/xpack/sql/index", "string", "keyword")); + headers.add(new ColumnInfo("org/elasticsearch/xpack/sql/index", "number", "integer")); // values List> values = new ArrayList<>(); values.add(asList("Along The River Bank", 11 * 60 + 48)); values.add(asList("Mind Train", 4 * 60 + 40)); - return new SqlQueryResponse(null, Mode.JDBC, DATE_NANOS_SUPPORT_VERSION, false, headers, values); + return new SqlQueryResponse(null, Mode.JDBC, INTRODUCING_DATE_NANOS, false, headers, values); } private static SqlQueryResponse escapedData() { // headers List headers = new ArrayList<>(); - headers.add(new ColumnInfo("index", "first", "keyword")); - headers.add(new ColumnInfo("index", "\"special\"", "keyword")); + headers.add(new ColumnInfo("org/elasticsearch/xpack/sql/index", "first", "keyword")); + headers.add(new ColumnInfo("org/elasticsearch/xpack/sql/index", "\"special\"", "keyword")); // values List> values = new ArrayList<>(); values.add(asList("normal", "\"quo\"ted\",\n")); values.add(asList("commas", "a,b,c,\n,d,e,\t\n")); - return new SqlQueryResponse(null, Mode.JDBC, DATE_NANOS_SUPPORT_VERSION, false, headers, values); + return new SqlQueryResponse(null, Mode.JDBC, INTRODUCING_DATE_NANOS, false, headers, values); } private static RestRequest req() { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/util/SqlVersionUtils.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/util/SqlVersionUtils.java new file mode 100644 index 0000000000000..016ee661c5054 --- /dev/null +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/util/SqlVersionUtils.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.sql.util; + +import org.elasticsearch.xpack.sql.proto.SqlVersion; +import org.elasticsearch.xpack.sql.proto.SqlVersions; + +import java.util.List; + +import static org.elasticsearch.xpack.sql.proto.SqlVersions.SERVER_COMPAT_VERSION; +import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.INTRODUCING_UNSIGNED_LONG; +import static org.elasticsearch.xpack.sql.proto.VersionCompatibility.INTRODUCING_VERSION_FIELD_TYPE; + +public final class SqlVersionUtils { + + public static final SqlVersion PRE_UNSIGNED_LONG = SqlVersions.getPreviousVersion(INTRODUCING_UNSIGNED_LONG); + public static final SqlVersion POST_UNSIGNED_LONG = SqlVersions.getNextVersion(INTRODUCING_UNSIGNED_LONG); + public static final SqlVersion PRE_VERSION_FIELD = SqlVersions.getPreviousVersion(INTRODUCING_VERSION_FIELD_TYPE); + public static final SqlVersion POST_VERSION_FIELD = SqlVersions.getNextVersion(INTRODUCING_VERSION_FIELD_TYPE); + + public static List UNSIGNED_LONG_TEST_VERSIONS = List.of( + PRE_UNSIGNED_LONG, + INTRODUCING_UNSIGNED_LONG, + POST_UNSIGNED_LONG, + SERVER_COMPAT_VERSION + ); + + public static List VERSION_FIELD_TEST_VERSIONS = List.of( + PRE_VERSION_FIELD, + INTRODUCING_VERSION_FIELD_TYPE, + POST_VERSION_FIELD, + SERVER_COMPAT_VERSION + ); +} diff --git a/x-pack/plugin/src/yamlRestTest/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java b/x-pack/plugin/src/yamlRestTest/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java index 556a417fb5e79..988ee93bda6b4 100644 --- a/x-pack/plugin/src/yamlRestTest/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java +++ b/x-pack/plugin/src/yamlRestTest/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java @@ -43,6 +43,7 @@ public class XPackRestIT extends AbstractXPackRestTest { .setting("xpack.searchable.snapshot.shared_cache.region_size", "256KB") .user("x_pack_rest_user", "x-pack-test-password") .feature(FeatureFlag.TIME_SERIES_MODE) + .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED) .configFile("testnode.pem", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem")) .configFile("testnode.crt", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt")) .configFile("service_tokens", Resource.fromClasspath("service_tokens")) diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/140_metadata.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/140_metadata.yml index 33c9cc7558672..83234901ae8f2 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/140_metadata.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/140_metadata.yml @@ -155,3 +155,19 @@ setup: esql.query: body: query: 'FROM test [metadata _source] | STATS COUNT_DISTINCT(_source)' + +--- +"sort on _source not allowed": + - requires: + test_runner_features: [capabilities] + capabilities: + - method: POST + path: /_query + parameters: [] + capabilities: [sorting_on_source_and_counters_forbidden] + reason: "Sorting on _source shouldn't have been possible" + - do: + catch: /cannot sort on _source/ + esql.query: + body: + query: 'FROM test metadata _source | sort _source' diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml index 642407ac6d45b..ebf464ba667db 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml @@ -178,11 +178,19 @@ cast counter then filter: --- sort on counter without cast: + - requires: + test_runner_features: [capabilities] + capabilities: + - method: POST + path: /_query + parameters: [] + capabilities: [sorting_on_source_and_counters_forbidden] + reason: "Sorting on counters shouldn't have been possible" - do: - catch: bad_request + catch: /cannot sort on counter_long/ esql.query: body: - query: 'from test | KEEP k8s.pod.network.tx | sort @k8s.pod.network.tx | limit 1' + query: 'from test | KEEP k8s.pod.network.tx | sort k8s.pod.network.tx | limit 1' --- cast then sort on counter: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/inference/inference_crud.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/inference/inference_crud.yml index 11be68cc764e2..b1f640a40b34e 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/inference/inference_crud.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/inference/inference_crud.yml @@ -41,6 +41,10 @@ --- "Test get all": + - requires: + cluster_features: "semantic_text.default_elser_2" + reason: semantic_text default ELSER 2 inference ID introduced in 8.16.0 + - do: inference.get: inference_id: "*" diff --git a/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java b/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java index fe62d4e2d2639..0b40828b8e86c 100644 --- a/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java +++ b/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java @@ -48,6 +48,7 @@ public class CoreWithSecurityClientYamlTestSuiteIT extends ESClientYamlSuiteTest .setting("xpack.security.autoconfiguration.enabled", "false") .user(USER, PASS) .feature(FeatureFlag.TIME_SERIES_MODE) + .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED) .build(); public CoreWithSecurityClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { diff --git a/x-pack/qa/runtime-fields/build.gradle b/x-pack/qa/runtime-fields/build.gradle index 43d6d9463e0d1..986baf867b501 100644 --- a/x-pack/qa/runtime-fields/build.gradle +++ b/x-pack/qa/runtime-fields/build.gradle @@ -44,6 +44,7 @@ subprojects { setting 'xpack.security.enabled', 'false' requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0") + requiresFeature 'es.sub_objects_auto_feature_flag_enabled', Version.fromString("8.16.0") } tasks.named("yamlRestTest").configure {