Skip to content

Commit 40b8f0b

Browse files
chrisparrinellochrisparrinello
committed
Merge remote-tracking branch 'upstream/main' into profile-report-target-query
2 parents 86e6b7a + 8160933 commit 40b8f0b

141 files changed

Lines changed: 6442 additions & 1482 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎benchmarks/src/main/java/org/elasticsearch/benchmark/vector/scorer/VectorScorerOSQBenchmark.java‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.elasticsearch.index.codec.vectors.diskbbq.es94.ES940DiskBBQVectorsFormat;
2323
import org.elasticsearch.simdvec.ES940OSQVectorsScorer;
2424
import org.elasticsearch.simdvec.internal.vectorization.ESVectorizationProvider;
25+
import org.elasticsearch.simdvec.internal.vectorization.PanamaESVectorizationProvider;
2526
import org.elasticsearch.simdvec.internal.vectorization.VectorScorerTestUtils;
2627
import org.elasticsearch.xpack.searchablesnapshots.store.SearchableSnapshotDirectoryFactory;
2728
import org.openjdk.jmh.annotations.Benchmark;
@@ -71,7 +72,8 @@ public enum DirectoryType {
7172

7273
public enum VectorImplementation {
7374
SCALAR,
74-
VECTORIZED
75+
PANAMA,
76+
NATIVE
7577
}
7678

7779
@Param({ "96", "128", "192", "256", "384", "768", "1024" })
@@ -302,7 +304,16 @@ void setup(VectorData data) throws IOException {
302304
ES940OSQVectorsScorer.BULK_SIZE,
303305
resolvedEncoding
304306
);
305-
case VECTORIZED -> ESVectorizationProvider.getInstance()
307+
case PANAMA -> new PanamaESVectorizationProvider(false).newES940OSQVectorsScorer(
308+
input,
309+
(byte) queryBits,
310+
(byte) docBits,
311+
dims,
312+
data.binaryIndexLength,
313+
BULK_SIZE,
314+
resolvedEncoding
315+
);
316+
case NATIVE -> ESVectorizationProvider.getInstance()
306317
.newES940OSQVectorsScorer(
307318
input,
308319
(byte) queryBits,

‎benchmarks/src/test/java/org/elasticsearch/benchmark/vector/scorer/VectorScorerOSQBenchmarkTests.java‎

Lines changed: 52 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -46,92 +46,68 @@ public VectorScorerOSQBenchmarkTests(
4646
this.similarityFunction = similarityFunction;
4747
}
4848

49-
public void testSingleScalarVsVectorized() throws Exception {
49+
public void testSingle() throws Exception {
5050
for (int i = 0; i < REPETITIONS; i++) {
5151
var seed = randomLong();
5252

53-
var scalar = new VectorScorerOSQBenchmark();
54-
var vectorized = new VectorScorerOSQBenchmark();
55-
try {
56-
var data = VectorScorerOSQBenchmark.generateRandomVectorData(
57-
new Random(seed),
58-
dims,
59-
bits,
60-
int4Encoding,
61-
similarityFunction
62-
);
63-
64-
scalar.implementation = VectorScorerOSQBenchmark.VectorImplementation.SCALAR;
65-
scalar.dims = dims;
66-
scalar.bits = bits;
67-
scalar.directoryType = directoryType;
68-
scalar.int4Encoding = int4Encoding;
69-
scalar.similarityFunction = similarityFunction;
70-
scalar.setup(data);
71-
72-
float[] expected = scalar.score();
73-
74-
vectorized.implementation = VectorScorerOSQBenchmark.VectorImplementation.VECTORIZED;
75-
vectorized.dims = dims;
76-
vectorized.bits = bits;
77-
vectorized.directoryType = directoryType;
78-
vectorized.int4Encoding = int4Encoding;
79-
vectorized.similarityFunction = similarityFunction;
80-
vectorized.setup(data);
81-
82-
float[] result = vectorized.score();
83-
84-
assertArrayEqualsPercent("single scoring, scalar VS vectorized", expected, result, deltaPercent, DEFAULT_DELTA);
85-
} finally {
86-
scalar.teardown();
87-
vectorized.teardown();
88-
IOUtils.rm(scalar.tempDir);
89-
IOUtils.rm(vectorized.tempDir);
53+
var data = VectorScorerOSQBenchmark.generateRandomVectorData(new Random(seed), dims, bits, int4Encoding, similarityFunction);
54+
55+
float[] expected = null;
56+
for (var impl : VectorScorerOSQBenchmark.VectorImplementation.values()) {
57+
VectorScorerOSQBenchmark bench = new VectorScorerOSQBenchmark();
58+
bench.implementation = impl;
59+
bench.dims = dims;
60+
bench.bits = bits;
61+
bench.directoryType = directoryType;
62+
bench.int4Encoding = int4Encoding;
63+
bench.similarityFunction = similarityFunction;
64+
bench.setup(data);
65+
66+
try {
67+
float[] result = bench.score();
68+
// just check against the first one - they should all be identical to each other
69+
if (expected == null) {
70+
expected = result;
71+
continue;
72+
}
73+
assertArrayEqualsPercent(impl.toString(), expected, result, deltaPercent, DEFAULT_DELTA);
74+
} finally {
75+
bench.teardown();
76+
IOUtils.rm(bench.tempDir);
77+
}
9078
}
9179
}
9280
}
9381

94-
public void testBulkScalarVsVectorized() throws Exception {
82+
public void testBulk() throws Exception {
9583
for (int i = 0; i < REPETITIONS; i++) {
9684
var seed = randomLong();
9785

98-
var scalar = new VectorScorerOSQBenchmark();
99-
var vectorized = new VectorScorerOSQBenchmark();
100-
try {
101-
var data = VectorScorerOSQBenchmark.generateRandomVectorData(
102-
new Random(seed),
103-
dims,
104-
bits,
105-
int4Encoding,
106-
similarityFunction
107-
);
108-
109-
scalar.implementation = VectorScorerOSQBenchmark.VectorImplementation.SCALAR;
110-
scalar.dims = dims;
111-
scalar.bits = bits;
112-
scalar.directoryType = directoryType;
113-
scalar.int4Encoding = int4Encoding;
114-
scalar.similarityFunction = similarityFunction;
115-
scalar.setup(data);
116-
117-
float[] expected = scalar.bulkScore();
118-
119-
vectorized.implementation = VectorScorerOSQBenchmark.VectorImplementation.VECTORIZED;
120-
vectorized.dims = dims;
121-
vectorized.bits = bits;
122-
vectorized.directoryType = directoryType;
123-
vectorized.int4Encoding = int4Encoding;
124-
vectorized.similarityFunction = similarityFunction;
125-
vectorized.setup(data);
126-
127-
float[] result = vectorized.bulkScore();
128-
129-
assertArrayEqualsPercent("bulk scoring, scalar VS vectorized", expected, result, deltaPercent, DEFAULT_DELTA);
130-
} finally {
131-
scalar.teardown();
132-
vectorized.teardown();
133-
IOUtils.rm(scalar.tempDir);
134-
IOUtils.rm(vectorized.tempDir);
86+
var data = VectorScorerOSQBenchmark.generateRandomVectorData(new Random(seed), dims, bits, int4Encoding, similarityFunction);
87+
88+
float[] expected = null;
89+
for (var impl : VectorScorerOSQBenchmark.VectorImplementation.values()) {
90+
VectorScorerOSQBenchmark bench = new VectorScorerOSQBenchmark();
91+
bench.implementation = impl;
92+
bench.dims = dims;
93+
bench.bits = bits;
94+
bench.directoryType = directoryType;
95+
bench.int4Encoding = int4Encoding;
96+
bench.similarityFunction = similarityFunction;
97+
bench.setup(data);
98+
99+
try {
100+
float[] result = bench.bulkScore();
101+
// just check against the first one - they should all be identical to each other
102+
if (expected == null) {
103+
expected = result;
104+
continue;
105+
}
106+
assertArrayEqualsPercent(impl.toString(), expected, result, deltaPercent, DEFAULT_DELTA);
107+
} finally {
108+
bench.teardown();
109+
IOUtils.rm(bench.tempDir);
110+
}
135111
}
136112
}
137113
}

‎client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java‎

Lines changed: 3 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,7 @@
3030

3131
import java.io.IOException;
3232
import java.io.InputStream;
33-
import java.security.AccessController;
3433
import java.security.NoSuchAlgorithmException;
35-
import java.security.PrivilegedAction;
3634
import java.util.List;
3735
import java.util.Locale;
3836
import java.util.Objects;
@@ -104,12 +102,7 @@ public final class RestClientBuilder {
104102

105103
VersionInfo httpClientVersion = null;
106104
try {
107-
httpClientVersion = AccessController.doPrivileged(
108-
(PrivilegedAction<VersionInfo>) () -> VersionInfo.loadVersionInfo(
109-
"org.apache.http.nio.client",
110-
HttpAsyncClientBuilder.class.getClassLoader()
111-
)
112-
);
105+
httpClientVersion = VersionInfo.loadVersionInfo("org.apache.http.nio.client", HttpAsyncClientBuilder.class.getClassLoader());
113106
} catch (Exception e) {
114107
// Keep unknown
115108
}
@@ -285,9 +278,7 @@ public RestClient build() {
285278
if (failureListener == null) {
286279
failureListener = new RestClient.FailureListener();
287280
}
288-
CloseableHttpAsyncClient httpClient = AccessController.doPrivileged(
289-
(PrivilegedAction<CloseableHttpAsyncClient>) this::createHttpClient
290-
);
281+
CloseableHttpAsyncClient httpClient = createHttpClient();
291282
RestClient restClient = new RestClient(
292283
httpClient,
293284
defaultHeaders,
@@ -344,8 +335,7 @@ private CloseableHttpAsyncClient createHttpClient() {
344335
httpClientBuilder = httpClientConfigCallback.customizeHttpClient(httpClientBuilder);
345336
}
346337

347-
final HttpAsyncClientBuilder finalBuilder = httpClientBuilder;
348-
return AccessController.doPrivileged((PrivilegedAction<CloseableHttpAsyncClient>) finalBuilder::build);
338+
return httpClientBuilder.build();
349339
} catch (NoSuchAlgorithmException e) {
350340
throw new IllegalStateException("could not create the default ssl context", e);
351341
}

‎client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,8 @@
3535
import java.net.InetSocketAddress;
3636
import java.nio.file.Files;
3737
import java.nio.file.Paths;
38-
import java.security.AccessController;
3938
import java.security.KeyFactory;
4039
import java.security.KeyStore;
41-
import java.security.PrivilegedAction;
4240
import java.security.cert.Certificate;
4341
import java.security.cert.CertificateFactory;
4442
import java.security.spec.PKCS8EncodedKeySpec;
@@ -190,7 +188,7 @@ private static SSLContext getSslContext() throws Exception {
190188
* 12.0.1 so we pin to TLSv1.2 when running on an earlier JDK
191189
*/
192190
private static String getProtocol() {
193-
String version = AccessController.doPrivileged((PrivilegedAction<String>) () -> System.getProperty("java.version"));
191+
String version = System.getProperty("java.version");
194192
String[] parts = version.split("-");
195193
String[] numericComponents;
196194
if (parts.length == 1) {

‎client/sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java‎

Lines changed: 5 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,6 @@
2626

2727
import java.io.Closeable;
2828
import java.io.IOException;
29-
import java.security.AccessController;
30-
import java.security.PrivilegedAction;
3129
import java.util.Collection;
3230
import java.util.List;
3331
import java.util.concurrent.Executors;
@@ -298,25 +296,15 @@ static class SnifferThreadFactory implements ThreadFactory {
298296

299297
private SnifferThreadFactory(String namePrefix) {
300298
this.namePrefix = namePrefix;
301-
this.originalThreadFactory = AccessController.doPrivileged(new PrivilegedAction<ThreadFactory>() {
302-
@Override
303-
public ThreadFactory run() {
304-
return Executors.defaultThreadFactory();
305-
}
306-
});
299+
this.originalThreadFactory = Executors.defaultThreadFactory();
307300
}
308301

309302
@Override
310303
public Thread newThread(final Runnable r) {
311-
return AccessController.doPrivileged(new PrivilegedAction<Thread>() {
312-
@Override
313-
public Thread run() {
314-
Thread t = originalThreadFactory.newThread(r);
315-
t.setName(namePrefix + "[T#" + threadNumber.getAndIncrement() + "]");
316-
t.setDaemon(true);
317-
return t;
318-
}
319-
});
304+
Thread t = originalThreadFactory.newThread(r);
305+
t.setName(namePrefix + "[T#" + threadNumber.getAndIncrement() + "]");
306+
t.setDaemon(true);
307+
return t;
320308
}
321309
}
322310
}

‎docs/changelog/144648.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
area: Ingest Node
2+
issues: []
3+
pr: 144648
4+
summary: Fix pipeline resolution cache for bulk requests
5+
type: bug

‎docs/changelog/146063.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
area: ES|QL
2+
issues: []
3+
pr: 146063
4+
summary: Add cluster state metadata for data sources and datasets
5+
type: enhancement

‎docs/changelog/146199.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
area: Search
2+
issues:
3+
- 126665
4+
pr: 146199
5+
summary: Fix `terminate_after` not honored for aggs when size=0
6+
type: bug

‎docs/changelog/146503.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
area: "ES|QL"
2+
issues: []
3+
pr: 146503
4+
summary: Heap attack tests and fixes for SPARKLINE
5+
type: bug

‎docs/changelog/146567.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
area: Mapping
2+
issues:
3+
- 146538
4+
pr: 146567
5+
summary: Fix match only text decoding surrogate pairs
6+
type: bug

0 commit comments

Comments
 (0)