Skip to content

Commit

Permalink
Merge branch 'main' into fix/113148-unmute
Browse files Browse the repository at this point in the history
  • Loading branch information
elasticmachine authored Oct 22, 2024
2 parents a2a41f1 + 36c45c1 commit 71c7908
Show file tree
Hide file tree
Showing 1,112 changed files with 20,764 additions and 6,390 deletions.
4 changes: 2 additions & 2 deletions .buildkite/hooks/pre-command
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@ export COMPOSE_HTTP_TIMEOUT
JOB_BRANCH="$BUILDKITE_BRANCH"
export JOB_BRANCH

GRADLEW="./gradlew --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/"
GRADLEW="./gradlew --console=plain --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/"
export GRADLEW

GRADLEW_BAT="./gradlew.bat --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/"
GRADLEW_BAT="./gradlew.bat --console=plain --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/"
export GRADLEW_BAT

export $(cat .ci/java-versions.properties | grep '=' | xargs)
Expand Down
2 changes: 1 addition & 1 deletion .buildkite/pipelines/intake.yml
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ steps:
timeout_in_minutes: 300
matrix:
setup:
BWC_VERSION: ["8.15.3", "8.16.0", "8.17.0", "9.0.0"]
BWC_VERSION: ["8.15.4", "8.16.0", "8.17.0", "9.0.0"]
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
Expand Down
1 change: 0 additions & 1 deletion .buildkite/pipelines/lucene-snapshot/run-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,6 @@ steps:
matrix:
setup:
BWC_VERSION:
- 7.17.13
- 8.9.1
- 8.10.0
agents:
Expand Down
6 changes: 3 additions & 3 deletions .buildkite/pipelines/periodic-packaging.yml
Original file line number Diff line number Diff line change
Expand Up @@ -272,8 +272,8 @@ steps:
env:
BWC_VERSION: 8.14.3

- label: "{{matrix.image}} / 8.15.3 / packaging-tests-upgrade"
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.3
- label: "{{matrix.image}} / 8.15.4 / packaging-tests-upgrade"
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.4
timeout_in_minutes: 300
matrix:
setup:
Expand All @@ -286,7 +286,7 @@ steps:
machineType: custom-16-32768
buildDirectory: /dev/shm/bk
env:
BWC_VERSION: 8.15.3
BWC_VERSION: 8.15.4

- label: "{{matrix.image}} / 8.16.0 / packaging-tests-upgrade"
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.0
Expand Down
10 changes: 5 additions & 5 deletions .buildkite/pipelines/periodic.yml
Original file line number Diff line number Diff line change
Expand Up @@ -287,8 +287,8 @@ steps:
- signal_reason: agent_stop
limit: 3

- label: 8.15.3 / bwc
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.3#bwcTest
- label: 8.15.4 / bwc
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.4#bwcTest
timeout_in_minutes: 300
agents:
provider: gcp
Expand All @@ -297,7 +297,7 @@ steps:
buildDirectory: /dev/shm/bk
preemptible: true
env:
BWC_VERSION: 8.15.3
BWC_VERSION: 8.15.4
retry:
automatic:
- exit_status: "-1"
Expand Down Expand Up @@ -429,7 +429,7 @@ steps:
setup:
ES_RUNTIME_JAVA:
- openjdk21
BWC_VERSION: ["8.15.3", "8.16.0", "8.17.0", "9.0.0"]
BWC_VERSION: ["8.15.4", "8.16.0", "8.17.0", "9.0.0"]
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
Expand Down Expand Up @@ -471,7 +471,7 @@ steps:
ES_RUNTIME_JAVA:
- openjdk21
- openjdk23
BWC_VERSION: ["8.15.3", "8.16.0", "8.17.0", "9.0.0"]
BWC_VERSION: ["8.15.4", "8.16.0", "8.17.0", "9.0.0"]
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
Expand Down
2 changes: 1 addition & 1 deletion .ci/bwcVersions
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ BWC_VERSION:
- "8.12.2"
- "8.13.4"
- "8.14.3"
- "8.15.3"
- "8.15.4"
- "8.16.0"
- "8.17.0"
- "9.0.0"
2 changes: 1 addition & 1 deletion .ci/scripts/packaging-test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -78,5 +78,5 @@ sudo -E env \
--unset=JAVA_HOME \
SYSTEM_JAVA_HOME=`readlink -f -n $BUILD_JAVA_HOME` \
DOCKER_CONFIG="${HOME}/.docker" \
./gradlew -g $HOME/.gradle --scan --parallel --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ --continue $@
./gradlew -g $HOME/.gradle --console=plain --scan --parallel --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ --continue $@

2 changes: 1 addition & 1 deletion .ci/snapshotBwcVersions
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
BWC_VERSION:
- "8.15.3"
- "8.15.4"
- "8.16.0"
- "8.17.0"
- "9.0.0"
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ public abstract class AbstractDocValuesForUtilBenchmark {
protected final int blockSize;

public AbstractDocValuesForUtilBenchmark() {
this.forUtil = new DocValuesForUtil();
this.forUtil = new DocValuesForUtil(ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE);
this.blockSize = ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ByteArrayDataOutput;
import org.apache.lucene.store.DataOutput;
import org.elasticsearch.index.codec.tsdb.DocValuesForUtil;
import org.openjdk.jmh.infra.Blackhole;

import java.io.IOException;
Expand Down Expand Up @@ -44,7 +43,7 @@ public void setupInvocation(int bitsPerValue) {

@Override
public void benchmark(int bitsPerValue, Blackhole bh) throws IOException {
DocValuesForUtil.decode(bitsPerValue, this.dataInput, this.output);
forUtil.decode(bitsPerValue, this.dataInput, this.output);
bh.consume(this.output);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
import org.apache.lucene.store.MMapDirectory;
import org.apache.lucene.util.hnsw.RandomVectorScorer;
import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
import org.apache.lucene.util.quantization.RandomAccessQuantizedByteVectorValues;
import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
import org.apache.lucene.util.quantization.ScalarQuantizer;
import org.elasticsearch.common.logging.LogConfigurator;
import org.elasticsearch.core.IOUtils;
Expand Down Expand Up @@ -217,19 +217,17 @@ public float squareDistanceScalar() {
return 1 / (1f + adjustedDistance);
}

RandomAccessQuantizedByteVectorValues vectorValues(int dims, int size, IndexInput in, VectorSimilarityFunction sim) throws IOException {
QuantizedByteVectorValues vectorValues(int dims, int size, IndexInput in, VectorSimilarityFunction sim) throws IOException {
var sq = new ScalarQuantizer(0.1f, 0.9f, (byte) 7);
var slice = in.slice("values", 0, in.length());
return new OffHeapQuantizedByteVectorValues.DenseOffHeapVectorValues(dims, size, sq, false, sim, null, slice);
}

RandomVectorScorerSupplier luceneScoreSupplier(RandomAccessQuantizedByteVectorValues values, VectorSimilarityFunction sim)
throws IOException {
RandomVectorScorerSupplier luceneScoreSupplier(QuantizedByteVectorValues values, VectorSimilarityFunction sim) throws IOException {
return new Lucene99ScalarQuantizedVectorScorer(null).getRandomVectorScorerSupplier(sim, values);
}

RandomVectorScorer luceneScorer(RandomAccessQuantizedByteVectorValues values, VectorSimilarityFunction sim, float[] queryVec)
throws IOException {
RandomVectorScorer luceneScorer(QuantizedByteVectorValues values, VectorSimilarityFunction sim, float[] queryVec) throws IOException {
return new Lucene99ScalarQuantizedVectorScorer(null).getRandomVectorScorer(sim, values, queryVec);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public enum DockerBase {
// Chainguard based wolfi image with latest jdk
// This is usually updated via renovatebot
// spotless:off
WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:277ebb42c458ef39cb4028f9204f0b3d51d8cd628ea737a65696a1143c3e42fe",
WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:bf163e1977002301f7b9fd28fe6837a8cb2dd5c83e4cd45fb67fb28d15d5d40f",
"-wolfi",
"apk"
),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,6 @@ org.apache.lucene.util.Version#parseLeniently(java.lang.String)

org.apache.lucene.index.NoMergePolicy#INSTANCE @ explicit use of NoMergePolicy risks forgetting to configure NoMergeScheduler; use org.elasticsearch.common.lucene.Lucene#indexWriterConfigWithNoMerging() instead.

@defaultMessage Spawns a new thread which is solely under lucenes control use ThreadPool#relativeTimeInMillis instead
org.apache.lucene.search.TimeLimitingCollector#getGlobalTimerThread()
org.apache.lucene.search.TimeLimitingCollector#getGlobalCounter()

@defaultMessage Don't interrupt threads use FutureUtils#cancel(Future<T>) instead
java.util.concurrent.Future#cancel(boolean)

Expand Down
2 changes: 1 addition & 1 deletion build-tools-internal/version.properties
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
elasticsearch = 9.0.0
lucene = 9.12.0
lucene = 10.0.0

bundled_jdk_vendor = openjdk
bundled_jdk = 22.0.1+8@c7ec1332f7bb44aeba2eb341ae18aca4
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

package org.elasticsearch.gradle.util;

import java.util.stream.Collectors;

public class PlatformUtils {

public static String normalize(String input) {
return input.lines()
.map(it -> it.replace('\\', '/'))
.map(it -> it.replaceAll("\\d+\\.\\d\\ds", "0.00s"))
.map(it -> it.replace("file:/./", "file:./"))
.collect(Collectors.joining("\n"));
}
}
3 changes: 3 additions & 0 deletions distribution/src/config/jvm.options
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,9 @@
23:-XX:CompileCommand=dontinline,java/lang/invoke/MethodHandle.setAsTypeCache
23:-XX:CompileCommand=dontinline,java/lang/invoke/MethodHandle.asTypeUncached

# Lucene 10: apply MADV_NORMAL advice to enable more aggressive readahead
-Dorg.apache.lucene.store.defaultReadAdvice=normal

## heap dumps

# generate a heap dump when an allocation from the Java heap fails; heap dumps
Expand Down
6 changes: 1 addition & 5 deletions distribution/tools/entitlement-runtime/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -11,16 +11,12 @@ apply plugin: 'elasticsearch.publish'

dependencies {
compileOnly project(':libs:elasticsearch-core') // For @SuppressForbidden
compileOnly project(":libs:elasticsearch-x-content") // for parsing policy files
compileOnly project(':server') // To access the main server module for special permission checks
compileOnly project(':distribution:tools:entitlement-bridge')

testImplementation project(":test:framework")
}

tasks.named('forbiddenApisMain').configure {
replaceSignatureFiles 'jdk-signatures'
}

tasks.named('forbiddenApisMain').configure {
replaceSignatureFiles 'jdk-signatures'
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

module org.elasticsearch.entitlement.runtime {
requires org.elasticsearch.entitlement.bridge;
requires org.elasticsearch.xcontent;
requires org.elasticsearch.server;

exports org.elasticsearch.entitlement.runtime.api;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

package org.elasticsearch.entitlement.runtime.policy;

/**
* Marker interface to ensure that only {@link Entitlement} are
* part of a {@link Policy}. All entitlement classes should implement
* this.
*/
public interface Entitlement {

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

package org.elasticsearch.entitlement.runtime.policy;

import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;

/**
* This annotation indicates an {@link Entitlement} is available
* to "external" classes such as those used in plugins. Any {@link Entitlement}
* using this annotation is considered parseable as part of a policy file
* for entitlements.
*/
@Target(ElementType.CONSTRUCTOR)
@Retention(RetentionPolicy.RUNTIME)
public @interface ExternalEntitlement {

/**
* This is the list of parameter names that are
* parseable in {@link PolicyParser#parseEntitlement(String, String)}.
* The number and order of parameter names much match the number and order
* of constructor parameters as this is how the parser will pass in the
* parsed values from a policy file. However, the names themselves do NOT
* have to match the parameter names of the constructor.
*/
String[] parameterNames() default {};
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

package org.elasticsearch.entitlement.runtime.policy;

import java.util.List;
import java.util.Objects;

/**
* Describes a file entitlement with a path and actions.
*/
public class FileEntitlement implements Entitlement {

public static final int READ_ACTION = 0x1;
public static final int WRITE_ACTION = 0x2;

private final String path;
private final int actions;

@ExternalEntitlement(parameterNames = { "path", "actions" })
public FileEntitlement(String path, List<String> actionsList) {
this.path = path;
int actionsInt = 0;

for (String actionString : actionsList) {
if ("read".equals(actionString)) {
if ((actionsInt & READ_ACTION) == READ_ACTION) {
throw new IllegalArgumentException("file action [read] specified multiple times");
}
actionsInt |= READ_ACTION;
} else if ("write".equals(actionString)) {
if ((actionsInt & WRITE_ACTION) == WRITE_ACTION) {
throw new IllegalArgumentException("file action [write] specified multiple times");
}
actionsInt |= WRITE_ACTION;
} else {
throw new IllegalArgumentException("unknown file action [" + actionString + "]");
}
}

this.actions = actionsInt;
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
FileEntitlement that = (FileEntitlement) o;
return actions == that.actions && Objects.equals(path, that.path);
}

@Override
public int hashCode() {
return Objects.hash(path, actions);
}

@Override
public String toString() {
return "FileEntitlement{" + "path='" + path + '\'' + ", actions=" + actions + '}';
}
}
Loading

0 comments on commit 71c7908

Please sign in to comment.