Skip to content

Commit c4c0db6

Browse files
authored
Introduce jvm.options.d for customizing JVM options (#51882)
This commit introduces the ability to override JVM options by adding custom JVM options files to a jvm.options.d directory. This simplifies administration of Elasticsearch by not requiring administrators to keep the root jvm.options file in sync with changes that we make to the root jvm.options file. Instead, they are not expected to modify this file but instead supply their own in jvm.options.d. In Docker installations, this means they can bind mount this directory in. In future versions of Elasticsearch, we can consider removing the root jvm.options file (instead, providing all options there as system JVM options).
1 parent 214beed commit c4c0db6

File tree

18 files changed

+239
-109
lines changed

18 files changed

+239
-109
lines changed

distribution/archives/build.gradle

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,11 @@ task createPluginsDir(type: EmptyDirTask) {
4646
dir = "${pluginsDir}"
4747
dirMode = 0755
4848
}
49+
ext.jvmOptionsDir = new File(buildDir, 'jvm-options-hack/jvm.options.d')
50+
task createJvmOptionsDir(type: EmptyDirTask) {
51+
dir = "${jvmOptionsDir}"
52+
dirMode = 0750
53+
}
4954

5055
CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, boolean oss, boolean jdk) {
5156
return copySpec {
@@ -57,6 +62,10 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
5762
dirMode 0750
5863
fileMode 0660
5964
with configFiles(distributionType, oss, jdk)
65+
from {
66+
dirMode 0750
67+
jvmOptionsDir.getParent()
68+
}
6069
}
6170
into('bin') {
6271
with binFiles(distributionType, oss, jdk)
@@ -96,7 +105,7 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
96105

97106
// common config across all zip/tar
98107
tasks.withType(AbstractArchiveTask) {
99-
dependsOn createLogsDir, createPluginsDir
108+
dependsOn createLogsDir, createPluginsDir, createJvmOptionsDir
100109
String subdir = it.name.substring('build'.size()).replaceAll(/[A-Z]/) { '-' + it.toLowerCase() }.substring(1)
101110
destinationDir = file("${subdir}/build/distributions")
102111
baseName = "elasticsearch${subdir.contains('oss') ? '-oss' : ''}"

distribution/docker/src/docker/Dockerfile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,8 @@ ${source_elasticsearch}
3030
RUN tar zxf /opt/${elasticsearch} --strip-components=1
3131
RUN grep ES_DISTRIBUTION_TYPE=tar /usr/share/elasticsearch/bin/elasticsearch-env \
3232
&& sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' /usr/share/elasticsearch/bin/elasticsearch-env
33-
RUN mkdir -p config data logs
34-
RUN chmod 0775 config data logs
33+
RUN mkdir -p config config/jvm.options.d data logs
34+
RUN chmod 0775 config config/jvm.options.d data logs
3535
COPY config/elasticsearch.yml config/log4j2.properties config/
3636
RUN chmod 0660 config/elasticsearch.yml config/log4j2.properties
3737

distribution/packages/build.gradle

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,9 @@ void addProcessFilesTask(String type, boolean oss, boolean jdk) {
7979
mkdir "${packagingFiles}/var/lib/elasticsearch"
8080
mkdir "${packagingFiles}/usr/share/elasticsearch/plugins"
8181

82-
// bare empty dir for /etc/elasticsearch
82+
// bare empty dir for /etc/elasticsearch and /etc/elasticsearch/jvm.options.d
8383
mkdir "${packagingFiles}/elasticsearch"
84+
mkdir "${packagingFiles}/elasticsearch/jvm.options.d"
8485
}
8586
}
8687
}
@@ -197,6 +198,7 @@ Closure commonPackageConfig(String type, boolean oss, boolean jdk) {
197198
includeEmptyDirs true
198199
createDirectoryEntry true
199200
include("elasticsearch") // empty dir, just to add directory entry
201+
include("elasticsearch/jvm.options.d") // empty dir, just to add directory entry
200202
}
201203
from("${packagingFiles}/etc/elasticsearch") {
202204
into('/etc/elasticsearch')

distribution/packages/src/common/scripts/postrm

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ else
1717
fi
1818

1919
REMOVE_DIRS=false
20+
REMOVE_JVM_OPTIONS_DIRECTORY=false
2021
REMOVE_USER_AND_GROUP=false
2122

2223
case "$1" in
@@ -27,6 +28,8 @@ case "$1" in
2728
;;
2829

2930
purge)
31+
REMOVE_DIRS=true
32+
REMOVE_JVM_OPTIONS_DIRECTORY=true
3033
REMOVE_USER_AND_GROUP=true
3134
;;
3235
failed-upgrade|abort-install|abort-upgrade|disappear|upgrade|disappear)
@@ -80,6 +83,20 @@ if [ "$REMOVE_DIRS" = "true" ]; then
8083
rmdir --ignore-fail-on-non-empty /var/lib/elasticsearch
8184
fi
8285

86+
# delete the jvm.options.d directory if and only if empty
87+
if [ -d "${ES_PATH_CONF}/jvm.options.d" ]; then
88+
rmdir --ignore-fail-on-non-empty "${ES_PATH_CONF}/jvm.options.d"
89+
fi
90+
91+
# delete the jvm.options.d directory if we are purging
92+
if [ "$REMOVE_JVM_OPTIONS_DIRECTORY" = "true" ]; then
93+
if [ -d "${ES_PATH_CONF}/jvm.options.d" ]; then
94+
echo -n "Deleting jvm.options.d directory..."
95+
rm -rf "${ES_PATH_CONF}/jvm.options.d"
96+
echo " OK"
97+
fi
98+
fi
99+
83100
# delete the conf directory if and only if empty
84101
if [ -d "${ES_PATH_CONF}" ]; then
85102
rmdir --ignore-fail-on-non-empty "${ES_PATH_CONF}"

distribution/packages/src/deb/lintian/elasticsearch

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ missing-dep-on-jarwrapper
1616
# we prefer to not make our config and log files world readable
1717
non-standard-file-perm etc/default/elasticsearch 0660 != 0644
1818
non-standard-dir-perm etc/elasticsearch/ 2750 != 0755
19+
non-standard-dir-perm etc/elasticsearch/jvm.options.d/ 2750 != 0755
1920
non-standard-file-perm etc/elasticsearch/*
2021
non-standard-dir-perm var/lib/elasticsearch/ 2750 != 0755
2122
non-standard-dir-perm var/log/elasticsearch/ 2750 != 0755

distribution/src/bin/elasticsearch

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,7 @@ then
4646
fi
4747
fi
4848

49-
ES_JVM_OPTIONS="$ES_PATH_CONF"/jvm.options
50-
ES_JAVA_OPTS=`export ES_TMPDIR; "$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.JvmOptionsParser "$ES_JVM_OPTIONS"`
49+
ES_JAVA_OPTS=`export ES_TMPDIR; "$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.JvmOptionsParser "$ES_PATH_CONF"`
5150

5251
# manual parsing to find out, if process should be detached
5352
if [[ $DAEMONIZE = false ]]; then

distribution/src/bin/elasticsearch-service.bat

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ set ES_JVM_OPTIONS=%ES_PATH_CONF%\jvm.options
115115
if not "%ES_JAVA_OPTS%" == "" set ES_JAVA_OPTS=%ES_JAVA_OPTS: =;%
116116

117117
@setlocal
118-
for /F "usebackq delims=" %%a in (`"%JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.JvmOptionsParser" "!ES_JVM_OPTIONS!" || echo jvm_options_parser_failed"`) do set ES_JAVA_OPTS=%%a
118+
for /F "usebackq delims=" %%a in (`"%JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.JvmOptionsParser" "!ES_PATH_CONF!" || echo jvm_options_parser_failed"`) do set ES_JAVA_OPTS=%%a
119119
@endlocal & set "MAYBE_JVM_OPTIONS_PARSER_FAILED=%ES_JAVA_OPTS%" & set ES_JAVA_OPTS=%ES_JAVA_OPTS%
120120

121121
if "%MAYBE_JVM_OPTIONS_PARSER_FAILED%" == "jvm_options_parser_failed" (
@@ -167,15 +167,15 @@ for %%a in ("%ES_JAVA_OPTS:;=","%") do (
167167
@endlocal & set JVM_MS=%JVM_MS% & set JVM_MX=%JVM_MX% & set JVM_SS=%JVM_SS% & set OTHER_JAVA_OPTS=%OTHER_JAVA_OPTS%
168168

169169
if "%JVM_MS%" == "" (
170-
echo minimum heap size not set; configure using -Xms via "%ES_JVM_OPTIONS%" or ES_JAVA_OPTS
170+
echo minimum heap size not set; configure using -Xms via "%ES_PATH_CONF%/jvm.options.d", or ES_JAVA_OPTS
171171
goto:eof
172172
)
173173
if "%JVM_MX%" == "" (
174-
echo maximum heap size not set; configure using -Xmx via "%ES_JVM_OPTIONS%" or ES_JAVA_OPTS
174+
echo maximum heap size not set; configure using -Xmx via "%ES_PATH_CONF%/jvm.options.d", or ES_JAVA_OPTS
175175
goto:eof
176176
)
177177
if "%JVM_SS%" == "" (
178-
echo thread stack size not set; configure using -Xss via "%ES_JVM_OPTIONS%" or ES_JAVA_OPTS
178+
echo thread stack size not set; configure using -Xss via "%ES_PATH_CONF%/jvm.options.d", or ES_JAVA_OPTS
179179
goto:eof
180180
)
181181
set OTHER_JAVA_OPTS=%OTHER_JAVA_OPTS:"=%

distribution/src/bin/elasticsearch.bat

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,9 +72,8 @@ if not defined ES_TMPDIR (
7272
for /f "tokens=* usebackq" %%a in (`CALL %JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.TempDirectory"`) do set ES_TMPDIR=%%a
7373
)
7474

75-
set ES_JVM_OPTIONS=%ES_PATH_CONF%\jvm.options
7675
@setlocal
77-
for /F "usebackq delims=" %%a in (`CALL %JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.JvmOptionsParser" "!ES_JVM_OPTIONS!" ^|^| echo jvm_options_parser_failed`) do set ES_JAVA_OPTS=%%a
76+
for /F "usebackq delims=" %%a in (`CALL %JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.JvmOptionsParser" "!ES_PATH_CONF!" ^|^| echo jvm_options_parser_failed`) do set ES_JAVA_OPTS=%%a
7877
@endlocal & set "MAYBE_JVM_OPTIONS_PARSER_FAILED=%ES_JAVA_OPTS%" & set ES_JAVA_OPTS=%ES_JAVA_OPTS%
7978

8079
if "%MAYBE_JVM_OPTIONS_PARSER_FAILED%" == "jvm_options_parser_failed" (

distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java

Lines changed: 74 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,9 @@
2727
import java.io.InputStreamReader;
2828
import java.io.Reader;
2929
import java.nio.charset.StandardCharsets;
30+
import java.nio.file.DirectoryStream;
3031
import java.nio.file.Files;
32+
import java.nio.file.Path;
3133
import java.nio.file.Paths;
3234
import java.util.ArrayList;
3335
import java.util.Arrays;
@@ -42,6 +44,7 @@
4244
import java.util.regex.Matcher;
4345
import java.util.regex.Pattern;
4446
import java.util.stream.Collectors;
47+
import java.util.stream.StreamSupport;
4548

4649
/**
4750
* Parses JVM options from a file and prints a single line with all JVM options to standard output.
@@ -52,80 +55,88 @@ final class JvmOptionsParser {
5255
* The main entry point. The exit code is 0 if the JVM options were successfully parsed, otherwise the exit code is 1. If an improperly
5356
* formatted line is discovered, the line is output to standard error.
5457
*
55-
* @param args the args to the program which should consist of a single option, the path to the JVM options
58+
* @param args the args to the program which should consist of a single option, the path to ES_PATH_CONF
5659
*/
5760
public static void main(final String[] args) throws InterruptedException, IOException {
5861
if (args.length != 1) {
59-
throw new IllegalArgumentException("expected one argument specifying path to jvm.options but was " + Arrays.toString(args));
60-
}
61-
final List<String> jvmOptions = new ArrayList<>();
62-
final SortedMap<Integer, String> invalidLines = new TreeMap<>();
63-
try (
64-
InputStream is = Files.newInputStream(Paths.get(args[0]));
65-
Reader reader = new InputStreamReader(is, StandardCharsets.UTF_8);
66-
BufferedReader br = new BufferedReader(reader)
67-
) {
68-
parse(JavaVersion.majorVersion(JavaVersion.CURRENT), br, new JvmOptionConsumer() {
69-
@Override
70-
public void accept(final String jvmOption) {
71-
jvmOptions.add(jvmOption);
72-
}
73-
}, new InvalidLineConsumer() {
74-
@Override
75-
public void accept(final int lineNumber, final String line) {
76-
invalidLines.put(lineNumber, line);
77-
}
78-
});
62+
throw new IllegalArgumentException("expected one argument specifying path to ES_PATH_CONF but was " + Arrays.toString(args));
7963
}
8064

81-
if (invalidLines.isEmpty()) {
82-
// now append the JVM options from ES_JAVA_OPTS
83-
final String environmentJvmOptions = System.getenv("ES_JAVA_OPTS");
84-
if (environmentJvmOptions != null) {
85-
jvmOptions.addAll(
86-
Arrays.stream(environmentJvmOptions.split("\\s+")).filter(s -> s.trim().isEmpty() == false).collect(Collectors.toList())
87-
);
65+
final ArrayList<Path> jvmOptionsFiles = new ArrayList<>();
66+
jvmOptionsFiles.add(Paths.get(args[0], "jvm.options"));
67+
68+
final Path jvmOptionsDirectory = Paths.get(args[0], "jvm.options.d");
69+
70+
if (Files.isDirectory(jvmOptionsDirectory)) {
71+
try (
72+
DirectoryStream<Path> jvmOptionsDirectoryStream = Files.newDirectoryStream(Paths.get(args[0], "jvm.options.d"), "*.options")
73+
) {
74+
// collect the matching JVM options files after sorting them by Path::compareTo
75+
StreamSupport.stream(jvmOptionsDirectoryStream.spliterator(), false).sorted().forEach(jvmOptionsFiles::add);
8876
}
89-
final Map<String, String> substitutions = new HashMap<>();
90-
substitutions.put("ES_TMPDIR", System.getenv("ES_TMPDIR"));
91-
if (null != System.getenv("ES_PATH_CONF")) {
92-
substitutions.put("ES_PATH_CONF", System.getenv("ES_PATH_CONF"));
77+
}
78+
79+
final List<String> jvmOptions = new ArrayList<>();
80+
81+
for (final Path jvmOptionsFile : jvmOptionsFiles) {
82+
final SortedMap<Integer, String> invalidLines = new TreeMap<>();
83+
try (
84+
InputStream is = Files.newInputStream(jvmOptionsFile);
85+
Reader reader = new InputStreamReader(is, StandardCharsets.UTF_8);
86+
BufferedReader br = new BufferedReader(reader)
87+
) {
88+
parse(JavaVersion.majorVersion(JavaVersion.CURRENT), br, jvmOptions::add, invalidLines::put);
9389
}
94-
final List<String> substitutedJvmOptions = substitutePlaceholders(jvmOptions, Collections.unmodifiableMap(substitutions));
95-
final List<String> ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions);
96-
final List<String> systemJvmOptions = SystemJvmOptions.systemJvmOptions();
97-
final List<String> finalJvmOptions = new ArrayList<>(
98-
systemJvmOptions.size() + substitutedJvmOptions.size() + ergonomicJvmOptions.size()
99-
);
100-
finalJvmOptions.addAll(systemJvmOptions); // add the system JVM options first so that they can be overridden
101-
finalJvmOptions.addAll(substitutedJvmOptions);
102-
finalJvmOptions.addAll(ergonomicJvmOptions);
103-
final String spaceDelimitedJvmOptions = spaceDelimitJvmOptions(finalJvmOptions);
104-
Launchers.outPrintln(spaceDelimitedJvmOptions);
105-
Launchers.exit(0);
106-
} else {
107-
final String errorMessage = String.format(
108-
Locale.ROOT,
109-
"encountered [%d] error%s parsing [%s]",
110-
invalidLines.size(),
111-
invalidLines.size() == 1 ? "" : "s",
112-
args[0]
113-
);
114-
Launchers.errPrintln(errorMessage);
115-
int count = 0;
116-
for (final Map.Entry<Integer, String> entry : invalidLines.entrySet()) {
117-
count++;
118-
final String message = String.format(
90+
if (invalidLines.isEmpty() == false) {
91+
final String errorMessage = String.format(
11992
Locale.ROOT,
120-
"[%d]: encountered improperly formatted JVM option line [%s] on line number [%d]",
121-
count,
122-
entry.getValue(),
123-
entry.getKey()
93+
"encountered [%d] error%s parsing [%s]",
94+
invalidLines.size(),
95+
invalidLines.size() == 1 ? "" : "s",
96+
jvmOptionsFile
12497
);
125-
Launchers.errPrintln(message);
98+
Launchers.errPrintln(errorMessage);
99+
int count = 0;
100+
for (final Map.Entry<Integer, String> entry : invalidLines.entrySet()) {
101+
count++;
102+
final String message = String.format(
103+
Locale.ROOT,
104+
"[%d]: encountered improperly formatted JVM option in [%s] on line number [%d]: [%s]",
105+
count,
106+
jvmOptionsFile,
107+
entry.getKey(),
108+
entry.getValue()
109+
);
110+
Launchers.errPrintln(message);
111+
}
112+
Launchers.exit(1);
126113
}
127-
Launchers.exit(1);
128114
}
115+
116+
// now append the JVM options from ES_JAVA_OPTS
117+
final String environmentJvmOptions = System.getenv("ES_JAVA_OPTS");
118+
if (environmentJvmOptions != null) {
119+
jvmOptions.addAll(
120+
Arrays.stream(environmentJvmOptions.split("\\s+")).filter(s -> s.trim().isEmpty() == false).collect(Collectors.toList())
121+
);
122+
}
123+
final Map<String, String> substitutions = new HashMap<>();
124+
substitutions.put("ES_TMPDIR", System.getenv("ES_TMPDIR"));
125+
if (null != System.getenv("ES_PATH_CONF")) {
126+
substitutions.put("ES_PATH_CONF", System.getenv("ES_PATH_CONF"));
127+
}
128+
final List<String> substitutedJvmOptions = substitutePlaceholders(jvmOptions, Collections.unmodifiableMap(substitutions));
129+
final List<String> ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions);
130+
final List<String> systemJvmOptions = SystemJvmOptions.systemJvmOptions();
131+
final List<String> finalJvmOptions = new ArrayList<>(
132+
systemJvmOptions.size() + substitutedJvmOptions.size() + ergonomicJvmOptions.size()
133+
);
134+
finalJvmOptions.addAll(systemJvmOptions); // add the system JVM options first so that they can be overridden
135+
finalJvmOptions.addAll(substitutedJvmOptions);
136+
finalJvmOptions.addAll(ergonomicJvmOptions);
137+
final String spaceDelimitedJvmOptions = spaceDelimitJvmOptions(finalJvmOptions);
138+
Launchers.outPrintln(spaceDelimitedJvmOptions);
139+
Launchers.exit(0);
129140
}
130141

131142
static List<String> substitutePlaceholders(final List<String> jvmOptions, final Map<String, String> substitutions) {

docs/reference/setup/important-settings/heap-size.asciidoc

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ caches, but the less memory it leaves available for the operating system to use
4949
for the filesystem cache. Also, larger heaps can cause longer garbage
5050
collection pauses.
5151

52-
Here are examples of how to set the heap size via the jvm.options file:
52+
Here is an example of how to set the heap size via a `jvm.options.d/` file:
5353

5454
[source,txt]
5555
------------------
@@ -60,8 +60,7 @@ Here are examples of how to set the heap size via the jvm.options file:
6060
<2> Set the maximum heap size to 2g.
6161

6262
It is also possible to set the heap size via an environment variable. This can
63-
be done by commenting out the `Xms` and `Xmx` settings in the
64-
<<jvm-options,`jvm.options`>> file and setting these values via `ES_JAVA_OPTS`:
63+
be done by setting these values via `ES_JAVA_OPTS`:
6564

6665
[source,sh]
6766
------------------

docs/reference/setup/install/docker.asciidoc

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -272,22 +272,23 @@ unless you are pinning one container per host.
272272
[[docker-set-heap-size]]
273273
===== Set the heap size
274274

275-
Use the `ES_JAVA_OPTS` environment variable to set the heap size.
276-
For example, to use 16GB, specify `-e ES_JAVA_OPTS="-Xms16g -Xmx16g"` with
277-
`docker run`. Note that while the default configuration file `jvm.options`
278-
sets a default heap of 1GB, any value you set in `ES_JAVA_OPTS` will
279-
override it.
275+
To configure the heap size, you can bind mount a <<jvm-options,JVM options>>
276+
file under `/usr/share/elasticsearch/config/jvm.options.d` that includes your
277+
desired <<heap-size,heap size>> settings. Note that while the default root
278+
`jvm.options` file sets a default heap of 1 GB, any value you set in a
279+
bind-mounted JVM options file will override it.
280+
281+
While setting the heap size via bind-mounted JVM options is the recommended
282+
method, you can also configure this by using the `ES_JAVA_OPTS` environment
283+
variable to set the heap size. For example, to use 16 GB, specify
284+
`-e ES_JAVA_OPTS="-Xms16g -Xmx16g"` with `docker run`. Note that while the
285+
default root `jvm.options` file sets a default heap of 1 GB, any value you set
286+
in `ES_JAVA_OPTS` will override it.
280287

281288
IMPORTANT: You must <<heap-size,configure the heap size>> even if you are
282289
https://docs.docker.com/config/containers/resource_constraints/#limit-a-containers-access-to-memory[limiting
283290
memory access] to the container.
284291

285-
While setting the heap size via an environment variable is the recommended
286-
method, you can also configure this by bind-mounting your own `jvm.options`
287-
file under `/usr/share/elasticsearch/config/`. The file that {es} provides
288-
contains some important settings, so you should start by taking a copy of
289-
`jvm.options` from an {es} container and editing it as you require.
290-
291292
===== Pin deployments to a specific image version
292293

293294
Pin your deployments to a specific version of the {es} Docker image. For

0 commit comments

Comments
 (0)