Skip to content

Commit 0297612

Browse files
mark-vieiramark-vieira
authored
[7.x] Autodetermine heap settings based on node roles and total system memory (#66428)
This commit expands our JVM egonomics to also automatically determine appropriate heap size based on the total available system memory as well as the roles assigned to the node. Role determination is done via a naive parsing of elasticsearch.yml. No settings validation is done and only the 'node.roles' setting is taken into consideration. For heap purposes a node falls into one of four (4) categories: 1. A 'master-only' node. This is a node with only the 'master' role. 2. A 'ml-only' node. Similarly, a node with only the 'ml' role. 3. A 'data' node. This is basically the 'other' case. A node with any set of roles other than only master or only ml is considered a 'data' node, to include things like coordinating-only or "tie-breaker" nodes. 4. Unknown. This is the case if legacy settings are used. In this scenario we fallback to the old default heap options of 1GB. In all cases we short-circuit if a user provides explicit heap options so we only ever auto-determine heap if no existing heap options exist. Starting with this commit the default heap settings (1GB) are now removed from the default jvm.options which means we'll start auto- setting heap as the new default. (cherry picked from commit a393db9)
1 parent 8c73348 commit 0297612

File tree

18 files changed

+887
-160
lines changed

18 files changed

+887
-160
lines changed

‎distribution/src/config/jvm.options

Lines changed: 7 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,13 @@
2020
## IMPORTANT: JVM heap size
2121
################################################################
2222
##
23-
## You must always set the initial and maximum JVM heap size to
24-
## the same value. For example, to set the heap to 4 GB, create
25-
## a new file in the jvm.options.d directory containing these
26-
## lines:
23+
## The heap size is automatically configured by Elasticsearch
24+
## based on the available memory in your system and the roles
25+
## each node is configured to fulfill. If specifying heap is
26+
## required, it should be done through a file in jvm.options.d,
27+
## and the min and max should be set to the same value. For
28+
## example, to set the heap to 4 GB, create a new file in the
29+
## jvm.options.d directory containing these lines:
2730
##
2831
## -Xms4g
2932
## -Xmx4g
@@ -33,13 +36,6 @@
3336
##
3437
################################################################
3538

36-
# Xms represents the initial size of the JVM heap
37-
# Xmx represents the maximum size of the JVM heap
38-
39-
-Xms${heap.min}
40-
-Xmx${heap.max}
41-
42-
4339

4440
################################################################
4541
## Expert settings

‎distribution/tools/launchers/build.gradle

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ apply plugin: 'elasticsearch.build'
2222

2323
dependencies {
2424
compileOnly project(':distribution:tools:java-version-checker')
25+
compileOnly "org.yaml:snakeyaml:${versions.snakeyaml}"
2526
testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
2627
testImplementation "junit:junit:${versions.junit}"
2728
testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}"
@@ -44,4 +45,4 @@ tasks.named("testingConventions").configure {
4445

4546
["javadoc", "loggerUsageCheck", "jarHell"].each { tsk ->
4647
tasks.named(tsk).configure { enabled = false }
47-
}
48+
}

‎distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/DefaultSystemMemoryInfo.java

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.tools.launchers;
21+
22+
import com.sun.management.OperatingSystemMXBean;
23+
import org.elasticsearch.tools.java_version_checker.JavaVersion;
24+
import org.elasticsearch.tools.java_version_checker.SuppressForbidden;
25+
26+
import java.lang.management.ManagementFactory;
27+
28+
/**
29+
* A {@link SystemMemoryInfo} which delegates to {@link OperatingSystemMXBean}.
30+
*
31+
* <p>Prior to JDK 14 {@link OperatingSystemMXBean} did not take into consideration container memory limits when reporting total system
32+
* memory. Therefore attempts to use this implementation on earlier JDKs will result in an {@link SystemMemoryInfoException}.
33+
*/
34+
@SuppressForbidden(reason = "Using com.sun internals is the only way to query total system memory")
35+
public final class DefaultSystemMemoryInfo implements SystemMemoryInfo {
36+
private final OperatingSystemMXBean operatingSystemMXBean;
37+
38+
public DefaultSystemMemoryInfo() {
39+
this.operatingSystemMXBean = (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
40+
}
41+
42+
@Override
43+
@SuppressWarnings("deprecation")
44+
public long availableSystemMemory() throws SystemMemoryInfoException {
45+
if (JavaVersion.majorVersion(JavaVersion.CURRENT) < 14) {
46+
throw new SystemMemoryInfoException("The minimum required Java version is 14 to use " + this.getClass().getName());
47+
}
48+
49+
return operatingSystemMXBean.getTotalPhysicalMemorySize();
50+
}
51+
}

‎distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmErgonomics.java

Lines changed: 3 additions & 100 deletions
Original file line numberDiff line numberDiff line change
@@ -21,23 +21,13 @@
2121

2222
import org.elasticsearch.tools.java_version_checker.JavaVersion;
2323

24-
import java.io.BufferedReader;
2524
import java.io.IOException;
26-
import java.io.InputStream;
27-
import java.io.InputStreamReader;
28-
import java.nio.charset.StandardCharsets;
29-
import java.nio.file.Paths;
3025
import java.util.ArrayList;
31-
import java.util.Collections;
3226
import java.util.HashMap;
3327
import java.util.List;
34-
import java.util.Locale;
3528
import java.util.Map;
36-
import java.util.Optional;
3729
import java.util.regex.Matcher;
3830
import java.util.regex.Pattern;
39-
import java.util.stream.Collectors;
40-
import java.util.stream.Stream;
4131

4232
/**
4333
* Tunes Elasticsearch JVM settings based on inspection of provided JVM options.
@@ -56,9 +46,9 @@ private JvmErgonomics() {
5646
*/
5747
static List<String> choose(final List<String> userDefinedJvmOptions) throws InterruptedException, IOException {
5848
final List<String> ergonomicChoices = new ArrayList<>();
59-
final Map<String, JvmOption> finalJvmOptions = finalJvmOptions(userDefinedJvmOptions);
60-
final long heapSize = extractHeapSize(finalJvmOptions);
61-
final long maxDirectMemorySize = extractMaxDirectMemorySize(finalJvmOptions);
49+
final Map<String, JvmOption> finalJvmOptions = JvmOption.findFinalOptions(userDefinedJvmOptions);
50+
final long heapSize = JvmOption.extractMaxHeapSize(finalJvmOptions);
51+
final long maxDirectMemorySize = JvmOption.extractMaxDirectMemorySize(finalJvmOptions);
6252

6353
if (System.getProperty("os.name").startsWith("Windows") && JavaVersion.majorVersion(JavaVersion.CURRENT) == 8) {
6454
Launchers.errPrintln("Warning: with JDK 8 on Windows, Elasticsearch may be unable to derive correct");
@@ -96,93 +86,6 @@ static List<String> choose(final List<String> userDefinedJvmOptions) throws Inte
9686
return ergonomicChoices;
9787
}
9888

99-
private static final Pattern OPTION = Pattern.compile(
100-
"^\\s*\\S+\\s+(?<flag>\\S+)\\s+:?=\\s+(?<value>\\S+)?\\s+\\{[^}]+?\\}(\\s+\\{(?<origin>[^}]+)})?"
101-
);
102-
103-
private static class JvmOption {
104-
private final String value;
105-
private final String origin;
106-
107-
JvmOption(String value, String origin) {
108-
this.value = value;
109-
this.origin = origin;
110-
}
111-
112-
public Optional<String> getValue() {
113-
return Optional.ofNullable(value);
114-
}
115-
116-
public String getMandatoryValue() {
117-
return value;
118-
}
119-
120-
public boolean isCommandLineOrigin() {
121-
return "command line".equals(this.origin);
122-
}
123-
}
124-
125-
static Map<String, JvmOption> finalJvmOptions(final List<String> userDefinedJvmOptions) throws InterruptedException, IOException {
126-
return Collections.unmodifiableMap(
127-
flagsFinal(userDefinedJvmOptions).stream()
128-
.map(OPTION::matcher)
129-
.filter(Matcher::matches)
130-
.collect(Collectors.toMap(m -> m.group("flag"), m -> new JvmOption(m.group("value"), m.group("origin"))))
131-
);
132-
}
133-
134-
private static List<String> flagsFinal(final List<String> userDefinedJvmOptions) throws InterruptedException, IOException {
135-
/*
136-
* To deduce the final set of JVM options that Elasticsearch is going to start with, we start a separate Java process with the JVM
137-
* options that we would pass on the command line. For this Java process we will add two additional flags, -XX:+PrintFlagsFinal and
138-
* -version. This causes the Java process that we start to parse the JVM options into their final values, display them on standard
139-
* output, print the version to standard error, and then exit. The JVM itself never bootstraps, and therefore this process is
140-
* lightweight. By doing this, we get the JVM options parsed exactly as the JVM that we are going to execute would parse them
141-
* without having to implement our own JVM option parsing logic.
142-
*/
143-
final String java = Paths.get(System.getProperty("java.home"), "bin", "java").toString();
144-
final List<String> command = Collections.unmodifiableList(
145-
Stream.of(
146-
Stream.of(java),
147-
userDefinedJvmOptions.stream(),
148-
Stream.of("-Xshare:off"),
149-
Stream.of("-XX:+PrintFlagsFinal"),
150-
Stream.of("-version")
151-
).reduce(Stream::concat).get().collect(Collectors.toList())
152-
);
153-
final Process process = new ProcessBuilder().command(command).start();
154-
final List<String> output = readLinesFromInputStream(process.getInputStream());
155-
final List<String> error = readLinesFromInputStream(process.getErrorStream());
156-
final int status = process.waitFor();
157-
if (status != 0) {
158-
final String message = String.format(
159-
Locale.ROOT,
160-
"starting java failed with [%d]\noutput:\n%s\nerror:\n%s",
161-
status,
162-
String.join("\n", output),
163-
String.join("\n", error)
164-
);
165-
throw new RuntimeException(message);
166-
} else {
167-
return output;
168-
}
169-
}
170-
171-
private static List<String> readLinesFromInputStream(final InputStream is) throws IOException {
172-
try (InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8); BufferedReader br = new BufferedReader(isr)) {
173-
return Collections.unmodifiableList(br.lines().collect(Collectors.toList()));
174-
}
175-
}
176-
177-
// package private for testing
178-
static Long extractHeapSize(final Map<String, JvmOption> finalJvmOptions) {
179-
return Long.parseLong(finalJvmOptions.get("MaxHeapSize").getMandatoryValue());
180-
}
181-
182-
static long extractMaxDirectMemorySize(final Map<String, JvmOption> finalJvmOptions) {
183-
return Long.parseLong(finalJvmOptions.get("MaxDirectMemorySize").getMandatoryValue());
184-
}
185-
18689
// Tune G1GC options for heaps < 8GB
18790
static boolean tuneG1GCForSmallHeap(final long heapSize) {
18891
return heapSize < 8L << 30;

‎distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOption.java

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.tools.launchers;
21+
22+
import java.io.BufferedReader;
23+
import java.io.IOException;
24+
import java.io.InputStream;
25+
import java.io.InputStreamReader;
26+
import java.nio.charset.StandardCharsets;
27+
import java.nio.file.Paths;
28+
import java.util.Collections;
29+
import java.util.List;
30+
import java.util.Locale;
31+
import java.util.Map;
32+
import java.util.Optional;
33+
import java.util.regex.Matcher;
34+
import java.util.regex.Pattern;
35+
import java.util.stream.Collectors;
36+
import java.util.stream.Stream;
37+
38+
class JvmOption {
39+
private final String value;
40+
private final String origin;
41+
42+
JvmOption(String value, String origin) {
43+
this.value = value;
44+
this.origin = origin;
45+
}
46+
47+
public Optional<String> getValue() {
48+
return Optional.ofNullable(value);
49+
}
50+
51+
public String getMandatoryValue() {
52+
return value;
53+
}
54+
55+
public boolean isCommandLineOrigin() {
56+
return "command line".equals(this.origin);
57+
}
58+
59+
private static final Pattern OPTION = Pattern.compile(
60+
"^\\s*\\S+\\s+(?<flag>\\S+)\\s+:?=\\s+(?<value>\\S+)?\\s+\\{[^}]+?\\}(\\s+\\{(?<origin>[^}]+)})?"
61+
);
62+
63+
public static Long extractMaxHeapSize(final Map<String, JvmOption> finalJvmOptions) {
64+
return Long.parseLong(finalJvmOptions.get("MaxHeapSize").getMandatoryValue());
65+
}
66+
67+
public static boolean isMaxHeapSpecified(final Map<String, JvmOption> finalJvmOptions) {
68+
JvmOption maxHeapSize = finalJvmOptions.get("MaxHeapSize");
69+
return maxHeapSize != null && maxHeapSize.isCommandLineOrigin();
70+
}
71+
72+
public static boolean isMinHeapSpecified(final Map<String, JvmOption> finalJvmOptions) {
73+
JvmOption minHeapSize = finalJvmOptions.get("MinHeapSize");
74+
return minHeapSize != null && minHeapSize.isCommandLineOrigin();
75+
}
76+
77+
public static boolean isInitialHeapSpecified(final Map<String, JvmOption> finalJvmOptions) {
78+
JvmOption initialHeapSize = finalJvmOptions.get("InitialHeapSize");
79+
return initialHeapSize != null && initialHeapSize.isCommandLineOrigin();
80+
}
81+
82+
public static long extractMaxDirectMemorySize(final Map<String, JvmOption> finalJvmOptions) {
83+
return Long.parseLong(finalJvmOptions.get("MaxDirectMemorySize").getMandatoryValue());
84+
}
85+
86+
/**
87+
* Determine the options present when invoking a JVM with the given user defined options.
88+
*/
89+
public static Map<String, JvmOption> findFinalOptions(final List<String> userDefinedJvmOptions) throws InterruptedException,
90+
IOException {
91+
return Collections.unmodifiableMap(
92+
flagsFinal(userDefinedJvmOptions).stream()
93+
.map(OPTION::matcher)
94+
.filter(Matcher::matches)
95+
.collect(Collectors.toMap(m -> m.group("flag"), m -> new JvmOption(m.group("value"), m.group("origin"))))
96+
);
97+
}
98+
99+
private static List<String> flagsFinal(final List<String> userDefinedJvmOptions) throws InterruptedException, IOException {
100+
/*
101+
* To deduce the final set of JVM options that Elasticsearch is going to start with, we start a separate Java process with the JVM
102+
* options that we would pass on the command line. For this Java process we will add two additional flags, -XX:+PrintFlagsFinal and
103+
* -version. This causes the Java process that we start to parse the JVM options into their final values, display them on standard
104+
* output, print the version to standard error, and then exit. The JVM itself never bootstraps, and therefore this process is
105+
* lightweight. By doing this, we get the JVM options parsed exactly as the JVM that we are going to execute would parse them
106+
* without having to implement our own JVM option parsing logic.
107+
*/
108+
final String java = Paths.get(System.getProperty("java.home"), "bin", "java").toString();
109+
final List<String> command = Collections.unmodifiableList(
110+
Stream.of(
111+
Stream.of(java),
112+
userDefinedJvmOptions.stream(),
113+
Stream.of("-Xshare:off"),
114+
Stream.of("-XX:+PrintFlagsFinal"),
115+
Stream.of("-version")
116+
).reduce(Stream::concat).get().collect(Collectors.toList())
117+
);
118+
final Process process = new ProcessBuilder().command(command).start();
119+
final List<String> output = readLinesFromInputStream(process.getInputStream());
120+
final List<String> error = readLinesFromInputStream(process.getErrorStream());
121+
final int status = process.waitFor();
122+
if (status != 0) {
123+
final String message = String.format(
124+
Locale.ROOT,
125+
"starting java failed with [%d]\noutput:\n%s\nerror:\n%s",
126+
status,
127+
String.join("\n", output),
128+
String.join("\n", error)
129+
);
130+
throw new RuntimeException(message);
131+
} else {
132+
return output;
133+
}
134+
}
135+
136+
private static List<String> readLinesFromInputStream(final InputStream is) throws IOException {
137+
try (InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8); BufferedReader br = new BufferedReader(isr)) {
138+
return Collections.unmodifiableList(br.lines().collect(Collectors.toList()));
139+
}
140+
}
141+
}

‎distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,7 @@ private List<String> jvmOptions(final Path config, Path plugins, final String es
133133
throws InterruptedException, IOException, JvmOptionsFileParserException {
134134

135135
final List<String> jvmOptions = readJvmOptionsFiles(config);
136+
final MachineDependentHeap machineDependentHeap = new MachineDependentHeap(new DefaultSystemMemoryInfo());
136137

137138
if (esJavaOpts != null) {
138139
jvmOptions.addAll(
@@ -141,6 +142,7 @@ private List<String> jvmOptions(final Path config, Path plugins, final String es
141142
}
142143

143144
final List<String> substitutedJvmOptions = substitutePlaceholders(jvmOptions, Collections.unmodifiableMap(substitutions));
145+
substitutedJvmOptions.addAll(machineDependentHeap.determineHeapSettings(config, substitutedJvmOptions));
144146
final List<String> ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions);
145147
final List<String> systemJvmOptions = SystemJvmOptions.systemJvmOptions();
146148
final List<String> bootstrapOptions = BootstrapJvmOptions.bootstrapJvmOptions(plugins);

0 commit comments

Comments
 (0)