Skip to content

Commit a4a2714

Browse files
afoucretafoucret
authored
Adding ES|QL RERANK command in snapshot builds (#123074)
1 parent 8f38b13 commit a4a2714

File tree

73 files changed

+5535
-2417
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

73 files changed

+5535
-2417
lines changed

Diff for: ‎docs/changelog/123074.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 123074
2+
summary: Adding ES|QL Reranker command in snapshot builds
3+
area: Ranking
4+
type: feature
5+
issues: []

Diff for: ‎muted-tests.yml

+3
Original file line numberDiff line numberDiff line change
@@ -369,6 +369,9 @@ tests:
369369
- class: org.elasticsearch.snapshots.SharedClusterSnapshotRestoreIT
370370
method: testDeletionOfFailingToRecoverIndexShouldStopRestore
371371
issue: https://github.com/elastic/elasticsearch/issues/126204
372+
- class: org.elasticsearch.xpack.esql.inference.RerankOperatorTests
373+
method: testSimpleCircuitBreaking
374+
issue: https://github.com/elastic/elasticsearch/issues/124337
372375
- class: org.elasticsearch.index.engine.ThreadPoolMergeSchedulerTests
373376
method: testSchedulerCloseWaitsForRunningMerge
374377
issue: https://github.com/elastic/elasticsearch/issues/125236

Diff for: ‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/InferenceAction.java

+4
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,10 @@ public static class Request extends BaseInferenceActionRequest {
6464
public static final ParseField TOP_N = new ParseField("top_n");
6565
public static final ParseField TIMEOUT = new ParseField("timeout");
6666

67+
public static Builder builder(String inferenceEntityId, TaskType taskType) {
68+
return new Builder().setInferenceEntityId(inferenceEntityId).setTaskType(taskType);
69+
}
70+
6771
static final ObjectParser<Request.Builder, Void> PARSER = new ObjectParser<>(NAME, Request.Builder::new);
6872
static {
6973
PARSER.declareStringArray(Request.Builder::setInput, INPUT);

Diff for: ‎x-pack/plugin/esql/gen/EsqlBaseLexer.java

+153
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
// Generated from /Users/afoucret/git/elasticsearch/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 by ANTLR 4.13.2
2+
3+
/*
4+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
5+
* or more contributor license agreements. Licensed under the Elastic License
6+
* 2.0; you may not use this file except in compliance with the Elastic License
7+
* 2.0.
8+
*/
9+
10+
import org.antlr.v4.runtime.Lexer;
11+
import org.antlr.v4.runtime.CharStream;
12+
import org.antlr.v4.runtime.Token;
13+
import org.antlr.v4.runtime.TokenStream;
14+
import org.antlr.v4.runtime.*;
15+
import org.antlr.v4.runtime.atn.*;
16+
import org.antlr.v4.runtime.dfa.DFA;
17+
import org.antlr.v4.runtime.misc.*;
18+
19+
@SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast", "CheckReturnValue", "this-escape"})
20+
public class EsqlBaseLexer extends LexerConfig {
21+
static { RuntimeMetaData.checkVersion("4.13.2", RuntimeMetaData.VERSION); }
22+
23+
protected static final DFA[] _decisionToDFA;
24+
protected static final PredictionContextCache _sharedContextCache =
25+
new PredictionContextCache();
26+
public static final int
27+
LINE_COMMENT=1, MULTILINE_COMMENT=2, WS=3;
28+
public static String[] channelNames = {
29+
"DEFAULT_TOKEN_CHANNEL", "HIDDEN"
30+
};
31+
32+
public static String[] modeNames = {
33+
"DEFAULT_MODE"
34+
};
35+
36+
private static String[] makeRuleNames() {
37+
return new String[] {
38+
"LINE_COMMENT", "MULTILINE_COMMENT", "WS"
39+
};
40+
}
41+
public static final String[] ruleNames = makeRuleNames();
42+
43+
private static String[] makeLiteralNames() {
44+
return new String[] {
45+
};
46+
}
47+
private static final String[] _LITERAL_NAMES = makeLiteralNames();
48+
private static String[] makeSymbolicNames() {
49+
return new String[] {
50+
null, "LINE_COMMENT", "MULTILINE_COMMENT", "WS"
51+
};
52+
}
53+
private static final String[] _SYMBOLIC_NAMES = makeSymbolicNames();
54+
public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
55+
56+
/**
57+
* @deprecated Use {@link #VOCABULARY} instead.
58+
*/
59+
@Deprecated
60+
public static final String[] tokenNames;
61+
static {
62+
tokenNames = new String[_SYMBOLIC_NAMES.length];
63+
for (int i = 0; i < tokenNames.length; i++) {
64+
tokenNames[i] = VOCABULARY.getLiteralName(i);
65+
if (tokenNames[i] == null) {
66+
tokenNames[i] = VOCABULARY.getSymbolicName(i);
67+
}
68+
69+
if (tokenNames[i] == null) {
70+
tokenNames[i] = "<INVALID>";
71+
}
72+
}
73+
}
74+
75+
@Override
76+
@Deprecated
77+
public String[] getTokenNames() {
78+
return tokenNames;
79+
}
80+
81+
@Override
82+
83+
public Vocabulary getVocabulary() {
84+
return VOCABULARY;
85+
}
86+
87+
88+
public EsqlBaseLexer(CharStream input) {
89+
super(input);
90+
_interp = new LexerATNSimulator(this,_ATN,_decisionToDFA,_sharedContextCache);
91+
}
92+
93+
@Override
94+
public String getGrammarFileName() { return "EsqlBaseLexer.g4"; }
95+
96+
@Override
97+
public String[] getRuleNames() { return ruleNames; }
98+
99+
@Override
100+
public String getSerializedATN() { return _serializedATN; }
101+
102+
@Override
103+
public String[] getChannelNames() { return channelNames; }
104+
105+
@Override
106+
public String[] getModeNames() { return modeNames; }
107+
108+
@Override
109+
public ATN getATN() { return _ATN; }
110+
111+
public static final String _serializedATN =
112+
"\u0004\u0000\u0003.\u0006\uffff\uffff\u0002\u0000\u0007\u0000\u0002\u0001"+
113+
"\u0007\u0001\u0002\u0002\u0007\u0002\u0001\u0000\u0001\u0000\u0001\u0000"+
114+
"\u0001\u0000\u0005\u0000\f\b\u0000\n\u0000\f\u0000\u000f\t\u0000\u0001"+
115+
"\u0000\u0003\u0000\u0012\b\u0000\u0001\u0000\u0003\u0000\u0015\b\u0000"+
116+
"\u0001\u0000\u0001\u0000\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+
117+
"\u0001\u0001\u0005\u0001\u001e\b\u0001\n\u0001\f\u0001!\t\u0001\u0001"+
118+
"\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0002\u0004"+
119+
"\u0002)\b\u0002\u000b\u0002\f\u0002*\u0001\u0002\u0001\u0002\u0001\u001f"+
120+
"\u0000\u0003\u0001\u0001\u0003\u0002\u0005\u0003\u0001\u0000\u0002\u0002"+
121+
"\u0000\n\n\r\r\u0003\u0000\t\n\r\r 3\u0000\u0001\u0001\u0000\u0000\u0000"+
122+
"\u0000\u0003\u0001\u0000\u0000\u0000\u0000\u0005\u0001\u0000\u0000\u0000"+
123+
"\u0001\u0007\u0001\u0000\u0000\u0000\u0003\u0018\u0001\u0000\u0000\u0000"+
124+
"\u0005(\u0001\u0000\u0000\u0000\u0007\b\u0005/\u0000\u0000\b\t\u0005/"+
125+
"\u0000\u0000\t\r\u0001\u0000\u0000\u0000\n\f\b\u0000\u0000\u0000\u000b"+
126+
"\n\u0001\u0000\u0000\u0000\f\u000f\u0001\u0000\u0000\u0000\r\u000b\u0001"+
127+
"\u0000\u0000\u0000\r\u000e\u0001\u0000\u0000\u0000\u000e\u0011\u0001\u0000"+
128+
"\u0000\u0000\u000f\r\u0001\u0000\u0000\u0000\u0010\u0012\u0005\r\u0000"+
129+
"\u0000\u0011\u0010\u0001\u0000\u0000\u0000\u0011\u0012\u0001\u0000\u0000"+
130+
"\u0000\u0012\u0014\u0001\u0000\u0000\u0000\u0013\u0015\u0005\n\u0000\u0000"+
131+
"\u0014\u0013\u0001\u0000\u0000\u0000\u0014\u0015\u0001\u0000\u0000\u0000"+
132+
"\u0015\u0016\u0001\u0000\u0000\u0000\u0016\u0017\u0006\u0000\u0000\u0000"+
133+
"\u0017\u0002\u0001\u0000\u0000\u0000\u0018\u0019\u0005/\u0000\u0000\u0019"+
134+
"\u001a\u0005*\u0000\u0000\u001a\u001f\u0001\u0000\u0000\u0000\u001b\u001e"+
135+
"\u0003\u0003\u0001\u0000\u001c\u001e\t\u0000\u0000\u0000\u001d\u001b\u0001"+
136+
"\u0000\u0000\u0000\u001d\u001c\u0001\u0000\u0000\u0000\u001e!\u0001\u0000"+
137+
"\u0000\u0000\u001f \u0001\u0000\u0000\u0000\u001f\u001d\u0001\u0000\u0000"+
138+
"\u0000 \"\u0001\u0000\u0000\u0000!\u001f\u0001\u0000\u0000\u0000\"#\u0005"+
139+
"*\u0000\u0000#$\u0005/\u0000\u0000$%\u0001\u0000\u0000\u0000%&\u0006\u0001"+
140+
"\u0000\u0000&\u0004\u0001\u0000\u0000\u0000\')\u0007\u0001\u0000\u0000"+
141+
"(\'\u0001\u0000\u0000\u0000)*\u0001\u0000\u0000\u0000*(\u0001\u0000\u0000"+
142+
"\u0000*+\u0001\u0000\u0000\u0000+,\u0001\u0000\u0000\u0000,-\u0006\u0002"+
143+
"\u0000\u0000-\u0006\u0001\u0000\u0000\u0000\u0007\u0000\r\u0011\u0014"+
144+
"\u001d\u001f*\u0001\u0000\u0001\u0000";
145+
public static final ATN _ATN =
146+
new ATNDeserializer().deserialize(_serializedATN.toCharArray());
147+
static {
148+
_decisionToDFA = new DFA[_ATN.getNumberOfDecisions()];
149+
for (int i = 0; i < _ATN.getNumberOfDecisions(); i++) {
150+
_decisionToDFA[i] = new DFA(_ATN.getDecisionState(i), i);
151+
}
152+
}
153+
}

Diff for: ‎x-pack/plugin/esql/gen/EsqlBaseLexer.tokens

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
LINE_COMMENT=1
2+
MULTILINE_COMMENT=2
3+
WS=3

Diff for: ‎x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java

+3
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V12;
5353
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_PLANNING_V1;
5454
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.METADATA_FIELDS_REMOTE_TEST;
55+
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.RERANK;
5556
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.UNMAPPED_FIELDS;
5657
import static org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase.Mode.SYNC;
5758
import static org.mockito.ArgumentMatchers.any;
@@ -130,6 +131,8 @@ protected void shouldSkipTest(String testName) throws IOException {
130131
assumeFalse("LOOKUP JOIN not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_LOOKUP_V12.capabilityName()));
131132
// Unmapped fields require a coorect capability response from every cluster, which isn't currently implemented.
132133
assumeFalse("UNMAPPED FIELDS not yet supported in CCS", testCase.requiredCapabilities.contains(UNMAPPED_FIELDS.capabilityName()));
134+
// Need to do additional developmnet to get CSS support for the rerank coammnd
135+
assumeFalse("RERANK not yet supported in CCS", testCase.requiredCapabilities.contains(RERANK.capabilityName()));
133136
}
134137

135138
@Override

Diff for: ‎x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java

+8
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,11 @@
6666
import static org.elasticsearch.xpack.esql.CsvTestUtils.loadCsvSpecValues;
6767
import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.availableDatasetsForEs;
6868
import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.clusterHasInferenceEndpoint;
69+
import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.clusterHasRerankInferenceEndpoint;
6970
import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.createInferenceEndpoint;
71+
import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.createRerankInferenceEndpoint;
7072
import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.deleteInferenceEndpoint;
73+
import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.deleteRerankInferenceEndpoint;
7174
import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.loadDataSetIntoEs;
7275
import static org.elasticsearch.xpack.esql.EsqlTestUtils.classpathResources;
7376
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.METRICS_COMMAND;
@@ -134,6 +137,10 @@ public void setup() throws IOException {
134137
createInferenceEndpoint(client());
135138
}
136139

140+
if (supportsInferenceTestService() && clusterHasRerankInferenceEndpoint(client()) == false) {
141+
createRerankInferenceEndpoint(client());
142+
}
143+
137144
boolean supportsLookup = supportsIndexModeLookup();
138145
boolean supportsSourceMapping = supportsSourceFieldMapping();
139146
if (indexExists(availableDatasetsForEs(client(), supportsLookup, supportsSourceMapping).iterator().next().indexName()) == false) {
@@ -153,6 +160,7 @@ public static void wipeTestData() throws IOException {
153160
}
154161

155162
deleteInferenceEndpoint(client());
163+
deleteRerankInferenceEndpoint(client());
156164
}
157165

158166
public boolean logResults() {

0 commit comments

Comments
 (0)