Skip to content

Commit 48f29ad

Browse files
tlrxtlrx
committed
Merge branch 'master' into feature/searchable-snapshots
2 parents 524fb76 + ec1afb9 commit 48f29ad

File tree

84 files changed

+4072
-2648
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

84 files changed

+4072
-2648
lines changed

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -655,6 +655,7 @@ class BuildPlugin implements Plugin<Project> {
655655
project.mkdir(testOutputDir)
656656
project.mkdir(heapdumpDir)
657657
project.mkdir(test.workingDir)
658+
project.mkdir(test.workingDir.toPath().resolve('temp'))
658659

659660
//TODO remove once jvm.options are added to test system properties
660661
test.systemProperty ('java.locale.providers','SPI,COMPAT')
@@ -686,16 +687,13 @@ class BuildPlugin implements Plugin<Project> {
686687
test.jvmArgs '-ea', '-esa'
687688
}
688689

689-
// we use './temp' since this is per JVM and tests are forbidden from writing to CWD
690-
test.systemProperties 'java.io.tmpdir': './temp',
691-
'java.awt.headless': 'true',
690+
test.systemProperties 'java.awt.headless': 'true',
692691
'tests.gradle': 'true',
693692
'tests.artifact': project.name,
694693
'tests.task': test.path,
695694
'tests.security.manager': 'true',
696695
'jna.nosys': 'true'
697696

698-
699697
// ignore changing test seed when build is passed -Dignore.tests.seed for cacheability experimentation
700698
if (System.getProperty('ignore.tests.seed') != null) {
701699
nonInputProperties.systemProperty('tests.seed', BuildParams.testSeed)
@@ -707,6 +705,8 @@ class BuildPlugin implements Plugin<Project> {
707705
nonInputProperties.systemProperty('gradle.dist.lib', new File(project.class.location.toURI()).parent)
708706
nonInputProperties.systemProperty('gradle.worker.jar', "${project.gradle.getGradleUserHomeDir()}/caches/${project.gradle.gradleVersion}/workerMain/gradle-worker.jar")
709707
nonInputProperties.systemProperty('gradle.user.home', project.gradle.getGradleUserHomeDir())
708+
// we use 'temp' relative to CWD since this is per JVM and tests are forbidden from writing to CWD
709+
nonInputProperties.systemProperty('java.io.tmpdir', test.workingDir.toPath().resolve('temp'))
710710

711711
nonInputProperties.systemProperty('compiler.java', "${-> BuildParams.compilerJavaVersion.majorVersion}")
712712

‎distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java

Lines changed: 25 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@
5151
*/
5252
final class JvmOptionsParser {
5353

54-
private static class JvmOptionsFileParserException extends Exception {
54+
static class JvmOptionsFileParserException extends Exception {
5555

5656
private final Path jvmOptionsFile;
5757

@@ -127,6 +127,29 @@ private List<String> jvmOptions(final Path config, final String esJavaOpts, fina
127127
throws InterruptedException,
128128
IOException,
129129
JvmOptionsFileParserException {
130+
131+
final List<String> jvmOptions = readJvmOptionsFiles(config);
132+
133+
if (esJavaOpts != null) {
134+
jvmOptions.addAll(
135+
Arrays.stream(esJavaOpts.split("\\s+")).filter(Predicate.not(String::isBlank)).collect(Collectors.toUnmodifiableList())
136+
);
137+
}
138+
139+
final List<String> substitutedJvmOptions = substitutePlaceholders(jvmOptions, Collections.unmodifiableMap(substitutions));
140+
final List<String> ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions);
141+
final List<String> systemJvmOptions = SystemJvmOptions.systemJvmOptions();
142+
final List<String> finalJvmOptions = new ArrayList<>(
143+
systemJvmOptions.size() + substitutedJvmOptions.size() + ergonomicJvmOptions.size()
144+
);
145+
finalJvmOptions.addAll(systemJvmOptions); // add the system JVM options first so that they can be overridden
146+
finalJvmOptions.addAll(substitutedJvmOptions);
147+
finalJvmOptions.addAll(ergonomicJvmOptions);
148+
149+
return finalJvmOptions;
150+
}
151+
152+
List<String> readJvmOptionsFiles(final Path config) throws IOException, JvmOptionsFileParserException {
130153
final ArrayList<Path> jvmOptionsFiles = new ArrayList<>();
131154
jvmOptionsFiles.add(config.resolve("jvm.options"));
132155

@@ -154,24 +177,7 @@ private List<String> jvmOptions(final Path config, final String esJavaOpts, fina
154177
throw new JvmOptionsFileParserException(jvmOptionsFile, invalidLines);
155178
}
156179
}
157-
158-
if (esJavaOpts != null) {
159-
jvmOptions.addAll(
160-
Arrays.stream(esJavaOpts.split("\\s+")).filter(Predicate.not(String::isBlank)).collect(Collectors.toUnmodifiableList())
161-
);
162-
}
163-
164-
final List<String> substitutedJvmOptions = substitutePlaceholders(jvmOptions, Collections.unmodifiableMap(substitutions));
165-
final List<String> ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions);
166-
final List<String> systemJvmOptions = SystemJvmOptions.systemJvmOptions();
167-
final List<String> finalJvmOptions = new ArrayList<>(
168-
systemJvmOptions.size() + substitutedJvmOptions.size() + ergonomicJvmOptions.size()
169-
);
170-
finalJvmOptions.addAll(systemJvmOptions); // add the system JVM options first so that they can be overridden
171-
finalJvmOptions.addAll(substitutedJvmOptions);
172-
finalJvmOptions.addAll(ergonomicJvmOptions);
173-
174-
return finalJvmOptions;
180+
return jvmOptions;
175181
}
176182

177183
static List<String> substitutePlaceholders(final List<String> jvmOptions, final Map<String, String> substitutions) {

‎distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/JvmOptionsParserTests.java

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@
2222
import java.io.BufferedReader;
2323
import java.io.IOException;
2424
import java.io.StringReader;
25+
import java.nio.file.Files;
26+
import java.nio.file.NoSuchFileException;
27+
import java.nio.file.Path;
28+
import java.nio.file.StandardOpenOption;
2529
import java.util.Arrays;
2630
import java.util.Collections;
2731
import java.util.HashMap;
@@ -31,7 +35,10 @@
3135
import java.util.concurrent.atomic.AtomicBoolean;
3236

3337
import static org.hamcrest.Matchers.contains;
38+
import static org.hamcrest.Matchers.empty;
3439
import static org.hamcrest.Matchers.equalTo;
40+
import static org.hamcrest.Matchers.hasKey;
41+
import static org.hamcrest.Matchers.hasSize;
3542
import static org.junit.Assert.assertFalse;
3643
import static org.junit.Assert.assertNull;
3744
import static org.junit.Assert.assertThat;
@@ -149,6 +156,108 @@ public void testComplexOptions() throws IOException {
149156
}
150157
}
151158

159+
public void testMissingRootJvmOptions() throws IOException, JvmOptionsParser.JvmOptionsFileParserException {
160+
final Path config = newTempDir();
161+
try {
162+
final JvmOptionsParser parser = new JvmOptionsParser();
163+
parser.readJvmOptionsFiles(config);
164+
fail("expected no such file exception, the root jvm.options file does not exist");
165+
} catch (final NoSuchFileException expected) {
166+
// this is expected, the root JVM options file must exist
167+
}
168+
}
169+
170+
public void testReadRootJvmOptions() throws IOException, JvmOptionsParser.JvmOptionsFileParserException {
171+
final Path config = newTempDir();
172+
final Path rootJvmOptions = config.resolve("jvm.options");
173+
Files.write(rootJvmOptions, List.of("# comment", "-Xms256m", "-Xmx256m"), StandardOpenOption.CREATE_NEW, StandardOpenOption.APPEND);
174+
if (randomBoolean()) {
175+
// an empty jvm.options.d directory should be irrelevant
176+
Files.createDirectory(config.resolve("jvm.options.d"));
177+
}
178+
final JvmOptionsParser parser = new JvmOptionsParser();
179+
final List<String> jvmOptions = parser.readJvmOptionsFiles(config);
180+
assertThat(jvmOptions, contains("-Xms256m", "-Xmx256m"));
181+
}
182+
183+
public void testReadJvmOptionsDirectory() throws IOException, JvmOptionsParser.JvmOptionsFileParserException {
184+
final Path config = newTempDir();
185+
Files.createDirectory(config.resolve("jvm.options.d"));
186+
Files.write(
187+
config.resolve("jvm.options"),
188+
List.of("# comment", "-Xms256m", "-Xmx256m"),
189+
StandardOpenOption.CREATE_NEW,
190+
StandardOpenOption.APPEND
191+
);
192+
Files.write(
193+
config.resolve("jvm.options.d").resolve("heap.options"),
194+
List.of("# comment", "-Xms384m", "-Xmx384m"),
195+
StandardOpenOption.CREATE_NEW,
196+
StandardOpenOption.APPEND
197+
);
198+
final JvmOptionsParser parser = new JvmOptionsParser();
199+
final List<String> jvmOptions = parser.readJvmOptionsFiles(config);
200+
assertThat(jvmOptions, contains("-Xms256m", "-Xmx256m", "-Xms384m", "-Xmx384m"));
201+
}
202+
203+
public void testReadJvmOptionsDirectoryInOrder() throws IOException, JvmOptionsParser.JvmOptionsFileParserException {
204+
final Path config = newTempDir();
205+
Files.createDirectory(config.resolve("jvm.options.d"));
206+
Files.write(
207+
config.resolve("jvm.options"),
208+
List.of("# comment", "-Xms256m", "-Xmx256m"),
209+
StandardOpenOption.CREATE_NEW,
210+
StandardOpenOption.APPEND
211+
);
212+
Files.write(
213+
config.resolve("jvm.options.d").resolve("first.options"),
214+
List.of("# comment", "-Xms384m", "-Xmx384m"),
215+
StandardOpenOption.CREATE_NEW,
216+
StandardOpenOption.APPEND
217+
);
218+
Files.write(
219+
config.resolve("jvm.options.d").resolve("second.options"),
220+
List.of("# comment", "-Xms512m", "-Xmx512m"),
221+
StandardOpenOption.CREATE_NEW,
222+
StandardOpenOption.APPEND
223+
);
224+
final JvmOptionsParser parser = new JvmOptionsParser();
225+
final List<String> jvmOptions = parser.readJvmOptionsFiles(config);
226+
assertThat(jvmOptions, contains("-Xms256m", "-Xmx256m", "-Xms384m", "-Xmx384m", "-Xms512m", "-Xmx512m"));
227+
}
228+
229+
public void testReadJvmOptionsDirectoryIgnoresFilesNotNamedOptions() throws IOException,
230+
JvmOptionsParser.JvmOptionsFileParserException {
231+
final Path config = newTempDir();
232+
Files.createFile(config.resolve("jvm.options"));
233+
Files.createDirectory(config.resolve("jvm.options.d"));
234+
Files.write(
235+
config.resolve("jvm.options.d").resolve("heap.not-named-options"),
236+
List.of("# comment", "-Xms256m", "-Xmx256m"),
237+
StandardOpenOption.CREATE_NEW,
238+
StandardOpenOption.APPEND
239+
);
240+
final JvmOptionsParser parser = new JvmOptionsParser();
241+
final List<String> jvmOptions = parser.readJvmOptionsFiles(config);
242+
assertThat(jvmOptions, empty());
243+
}
244+
245+
public void testFileContainsInvalidLinesThrowsParserException() throws IOException {
246+
final Path config = newTempDir();
247+
final Path rootJvmOptions = config.resolve("jvm.options");
248+
Files.write(rootJvmOptions, List.of("XX:+UseG1GC"), StandardOpenOption.CREATE_NEW, StandardOpenOption.APPEND);
249+
try {
250+
final JvmOptionsParser parser = new JvmOptionsParser();
251+
parser.readJvmOptionsFiles(config);
252+
fail("expected JVM options file parser exception, XX:+UseG1GC is improperly formatted");
253+
} catch (final JvmOptionsParser.JvmOptionsFileParserException expected) {
254+
assertThat(expected.jvmOptionsFile(), equalTo(rootJvmOptions));
255+
assertThat(expected.invalidLines().entrySet(), hasSize(1));
256+
assertThat(expected.invalidLines(), hasKey(1));
257+
assertThat(expected.invalidLines().get(1), equalTo("XX:+UseG1GC"));
258+
}
259+
}
260+
152261
private void assertExpectedJvmOptions(final int javaMajorVersion, final BufferedReader br, final List<String> expectedJvmOptions)
153262
throws IOException {
154263
final Map<String, AtomicBoolean> seenJvmOptions = new HashMap<>();

‎docs/Versions.asciidoc

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,16 @@ endif::[]
5858
:javadoc-license: {rest-high-level-client-javadoc}/org/elasticsearch/protocol/xpack/license
5959
:javadoc-watcher: {rest-high-level-client-javadoc}/org/elasticsearch/protocol/xpack/watcher
6060

61+
///////
62+
Permanently unreleased branches (master, n.X)
63+
///////
64+
ifeval::["{source_branch}"=="master"]
65+
:permanently-unreleased-branch:
66+
endif::[]
67+
ifeval::["{source_branch}"=="{major-version}"]
68+
:permanently-unreleased-branch:
69+
endif::[]
70+
6171
///////
6272
Shared attribute values are pulled from elastic/docs
6373
///////

‎docs/reference/index.asciidoc

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ include::ingest.asciidoc[]
4040

4141
include::ilm/index.asciidoc[]
4242

43-
ifeval::["{release-state}"=="unreleased"]
43+
ifdef::permanently-unreleased-branch[]
4444

4545
include::autoscaling/index.asciidoc[]
4646

‎libs/x-content/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@
3737
public abstract class AbstractObjectParser<Value, Context>
3838
implements BiFunction<XContentParser, Context, Value>, ContextParser<Context, Value> {
3939

40+
final List<String[]> requiredFieldSets = new ArrayList<>();
41+
4042
/**
4143
* Declare some field. Usually it is easier to use {@link #declareString(BiConsumer, ParseField)} or
4244
* {@link #declareObject(BiConsumer, ContextParser, ParseField)} rather than call this directly.
@@ -211,6 +213,61 @@ public <T> void declareFieldArray(BiConsumer<Value, List<T>> consumer, ContextPa
211213
declareField(consumer, (p, c) -> parseArray(p, () -> itemParser.parse(p, c)), field, type);
212214
}
213215

216+
/**
217+
* Declares a set of fields that are required for parsing to succeed. Only one of the values
218+
* provided per String[] must be matched.
219+
*
220+
* E.g. <code>declareRequiredFieldSet("foo", "bar");</code> means at least one of "foo" or
221+
* "bar" fields must be present. If neither of those fields are present, an exception will be thrown.
222+
*
223+
* Multiple required sets can be configured:
224+
*
225+
* <pre><code>
226+
* parser.declareRequiredFieldSet("foo", "bar");
227+
* parser.declareRequiredFieldSet("bizz", "buzz");
228+
* </code></pre>
229+
*
230+
* requires that one of "foo" or "bar" fields are present, and also that one of "bizz" or
231+
* "buzz" fields are present.
232+
*
233+
* In JSON, it means any of these combinations are acceptable:
234+
*
235+
* <ul>
236+
* <li><code>{"foo":"...", "bizz": "..."}</code></li>
237+
* <li><code>{"bar":"...", "bizz": "..."}</code></li>
238+
* <li><code>{"foo":"...", "buzz": "..."}</code></li>
239+
* <li><code>{"bar":"...", "buzz": "..."}</code></li>
240+
* <li><code>{"foo":"...", "bar":"...", "bizz": "..."}</code></li>
241+
* <li><code>{"foo":"...", "bar":"...", "buzz": "..."}</code></li>
242+
* <li><code>{"foo":"...", "bizz":"...", "buzz": "..."}</code></li>
243+
* <li><code>{"bar":"...", "bizz":"...", "buzz": "..."}</code></li>
244+
* <li><code>{"foo":"...", "bar":"...", "bizz": "...", "buzz": "..."}</code></li>
245+
* </ul>
246+
*
247+
* The following would however be rejected:
248+
*
249+
* <table>
250+
* <caption>failure cases</caption>
251+
* <tr><th>Provided JSON</th><th>Reason for failure</th></tr>
252+
* <tr><td><code>{"foo":"..."}</code></td><td>Missing "bizz" or "buzz" field</td></tr>
253+
* <tr><td><code>{"bar":"..."}</code></td><td>Missing "bizz" or "buzz" field</td></tr>
254+
* <tr><td><code>{"bizz": "..."}</code></td><td>Missing "foo" or "bar" field</td></tr>
255+
* <tr><td><code>{"buzz": "..."}</code></td><td>Missing "foo" or "bar" field</td></tr>
256+
* <tr><td><code>{"foo":"...", "bar": "..."}</code></td><td>Missing "bizz" or "buzz" field</td></tr>
257+
* <tr><td><code>{"bizz":"...", "buzz": "..."}</code></td><td>Missing "foo" or "bar" field</td></tr>
258+
* <tr><td><code>{"unrelated":"..."}</code></td> <td>Missing "foo" or "bar" field, and missing "bizz" or "buzz" field</td></tr>
259+
* </table>
260+
*
261+
* @param requiredSet
262+
* A set of required fields, where at least one of the fields in the array _must_ be present
263+
*/
264+
public void declareRequiredFieldSet(String... requiredSet) {
265+
if (requiredSet.length == 0) {
266+
return;
267+
}
268+
this.requiredFieldSets.add(requiredSet);
269+
}
270+
214271
private interface IOSupplier<T> {
215272
T get() throws IOException;
216273
}

‎libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import java.util.EnumSet;
2929
import java.util.HashMap;
3030
import java.util.HashSet;
31+
import java.util.Iterator;
3132
import java.util.List;
3233
import java.util.Map;
3334
import java.util.Set;
@@ -272,6 +273,8 @@ public Value parse(XContentParser parser, Value value, Context context) throws I
272273
FieldParser fieldParser = null;
273274
String currentFieldName = null;
274275
XContentLocation currentPosition = null;
276+
List<String[]> requiredFields = new ArrayList<>(this.requiredFieldSets);
277+
275278
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
276279
if (token == XContentParser.Token.FIELD_NAME) {
277280
currentFieldName = parser.currentName();
@@ -285,11 +288,32 @@ public Value parse(XContentParser parser, Value value, Context context) throws I
285288
unknownFieldParser.acceptUnknownField(this, currentFieldName, currentPosition, parser, value, context);
286289
} else {
287290
fieldParser.assertSupports(name, parser, currentFieldName);
291+
292+
// Check to see if this field is a required field, if it is we can
293+
// remove the entry as the requirement is satisfied
294+
Iterator<String[]> iter = requiredFields.iterator();
295+
while (iter.hasNext()) {
296+
String[] requriedFields = iter.next();
297+
for (String field : requriedFields) {
298+
if (field.equals(currentFieldName)) {
299+
iter.remove();
300+
break;
301+
}
302+
}
303+
}
304+
288305
parseSub(parser, fieldParser, currentFieldName, value, context);
289306
}
290307
fieldParser = null;
291308
}
292309
}
310+
if (requiredFields.isEmpty() == false) {
311+
StringBuilder message = new StringBuilder();
312+
for (String[] fields : requiredFields) {
313+
message.append("Required one of fields ").append(Arrays.toString(fields)).append(", but none were specified. ");
314+
}
315+
throw new IllegalArgumentException(message.toString());
316+
}
293317
return value;
294318
}
295319

0 commit comments

Comments
 (0)