Skip to content

Commit 7be5875

Browse files
Tim-BrooksTim-Brooks
committed
Merge remote-tracking branch 'upstream/master' into reindex_v2
2 parents 5b2e07e + 4a81606 commit 7be5875

File tree

79 files changed

+1162
-268
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

79 files changed

+1162
-268
lines changed

‎.ci/build-cache.gradle

-18
This file was deleted.

‎.ci/init.gradle

+81-37
Original file line numberDiff line numberDiff line change
@@ -1,46 +1,90 @@
1-
if (System.env.ELASTIC_ARTIFACTORY_USERNAME == null || System.env.ELASTIC_ARTIFACTORY_TOKEN == null) {
2-
throw new GradleException("Using init script without configuration")
3-
} else {
4-
logger.info("Using elastic artifactory repos")
5-
settingsEvaluated { settings ->
6-
settings.pluginManagement {
1+
import com.bettercloud.vault.VaultConfig;
2+
import com.bettercloud.vault.Vault;
3+
4+
initscript {
5+
repositories {
6+
mavenCentral()
7+
}
8+
dependencies {
9+
classpath 'com.bettercloud:vault-java-driver:4.1.0'
10+
}
11+
}
12+
13+
['VAULT_ADDR', 'VAULT_ROLE_ID', 'VAULT_SECRET_ID'].each {
14+
if (System.env."$it" == null) {
15+
throw new GradleException("$it must be set!")
16+
17+
}
18+
}
19+
20+
final String vaultToken = new Vault(
21+
new VaultConfig()
22+
.address(System.env.VAULT_ADDR)
23+
.engineVersion(1)
24+
.build()
25+
)
26+
.auth()
27+
.loginByAppRole("approle", System.env.VAULT_ROLE_ID, System.env.VAULT_SECRET_ID)
28+
.getAuthClientToken();
29+
30+
final Vault vault = new Vault(
31+
new VaultConfig()
32+
.address(System.env.VAULT_ADDR)
33+
.engineVersion(1)
34+
.token(vaultToken)
35+
.build()
36+
)
37+
38+
final Map<String,String> artifactoryCredentials = vault.logical()
39+
.read("secret/elasticsearch-ci/artifactory.elstc.co")
40+
.getData();
41+
42+
logger.info("Using elastic artifactory repos")
43+
Closure configCache = {
44+
return {
45+
name "artifactory-gradle-release"
46+
url "https://artifactory.elstc.co/artifactory/gradle-release"
47+
credentials {
48+
username artifactoryCredentials.get("username")
49+
password artifactoryCredentials.get("token")
50+
}
51+
}
52+
}
53+
settingsEvaluated { settings ->
54+
settings.pluginManagement {
55+
repositories {
56+
maven configCache()
57+
}
58+
}
59+
}
60+
projectsLoaded {
61+
allprojects {
62+
buildscript {
763
repositories {
8-
maven {
9-
name "artifactory-gradle-plugins"
10-
url "https://artifactory.elstc.co/artifactory/gradle-plugins"
11-
credentials {
12-
username System.env.ELASTIC_ARTIFACTORY_USERNAME
13-
password System.env.ELASTIC_ARTIFACTORY_TOKEN
14-
}
15-
}
16-
gradlePluginPortal()
64+
maven configCache()
1765
}
1866
}
67+
repositories {
68+
maven configCache()
69+
}
1970
}
20-
projectsLoaded {
21-
allprojects {
22-
buildscript {
23-
repositories {
24-
maven {
25-
name "artifactory-gradle-release"
26-
url "https://artifactory.elstc.co/artifactory/gradle-release/"
27-
credentials {
28-
username System.env.ELASTIC_ARTIFACTORY_USERNAME
29-
password System.env.ELASTIC_ARTIFACTORY_TOKEN
30-
}
31-
}
32-
}
33-
}
34-
repositories {
35-
maven {
36-
name "artifactory-gradle-release"
37-
url "https://artifactory.elstc.co/artifactory/gradle-release/"
38-
credentials {
39-
username System.env.ELASTIC_ARTIFACTORY_USERNAME
40-
password System.env.ELASTIC_ARTIFACTORY_TOKEN
41-
}
71+
}
72+
73+
if (System.env.GRADLE_BUILD_CACHE_URL != null) {
74+
final Map<String,String> buildCacheCredentials = vault.logical()
75+
.read("secret/elasticsearch-ci/gradle-build-cache")
76+
.getData();
77+
gradle.settingsEvaluated { settings ->
78+
settings.buildCache {
79+
remote(HttpBuildCache) {
80+
url = System.getenv('GRADLE_BUILD_CACHE_URL')
81+
push = Boolean.valueOf(System.getenv('GRADLE_BUILD_CACHE_PUSH') ?: 'false')
82+
credentials {
83+
username = buildCacheCredentials.get("username")
84+
password = buildCacheCredentials.get("password")
4285
}
4386
}
4487
}
4588
}
4689
}
90+

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

+2-18
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,6 @@ import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
2424
import groovy.transform.CompileDynamic
2525
import groovy.transform.CompileStatic
2626
import org.apache.commons.io.IOUtils
27-
import org.eclipse.jgit.lib.Constants
28-
import org.eclipse.jgit.lib.RepositoryBuilder
2927
import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin
3028
import org.elasticsearch.gradle.info.GlobalInfoExtension
3129
import org.elasticsearch.gradle.info.JavaHome
@@ -711,26 +709,12 @@ class BuildPlugin implements Plugin<Project> {
711709
// after the doFirst added by the info plugin, and we can override attributes
712710
JavaVersion compilerJavaVersion = ext.get('compilerJavaVersion') as JavaVersion
713711
jarTask.manifest.attributes(
712+
'Change': ext.get('gitRevision'),
714713
'X-Compile-Elasticsearch-Version': VersionProperties.elasticsearch,
715714
'X-Compile-Lucene-Version': VersionProperties.lucene,
716715
'X-Compile-Elasticsearch-Snapshot': VersionProperties.isElasticsearchSnapshot(),
717-
'Build-Date': ZonedDateTime.now(ZoneOffset.UTC),
716+
'Build-Date': ext.get('buildDate'),
718717
'Build-Java-Version': compilerJavaVersion)
719-
if (jarTask.manifest.attributes.containsKey('Change') == false) {
720-
jarTask.logger.warn('Building without git revision id.')
721-
jarTask.manifest.attributes('Change': 'Unknown')
722-
} else {
723-
/*
724-
* The info-scm plugin assumes that if GIT_COMMIT is set it was set by Jenkins to the commit hash for this build.
725-
* However, that assumption is wrong as this build could be a sub-build of another Jenkins build for which GIT_COMMIT
726-
* is the commit hash for that build. Therefore, if GIT_COMMIT is set we calculate the commit hash ourselves.
727-
*/
728-
if (System.getenv("GIT_COMMIT") != null) {
729-
final String hash = new RepositoryBuilder().findGitDir(project.buildDir).build().resolve(Constants.HEAD).name
730-
final String shortHash = hash?.substring(0, 7)
731-
jarTask.manifest.attributes('Change': shortHash)
732-
}
733-
}
734718
}
735719

736720
// add license/notice files

‎buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java

+1
Original file line numberDiff line numberDiff line change
@@ -273,4 +273,5 @@ private String runJavaAsScript(File javaHome, String script) {
273273
}
274274
return stdout.toString(UTF_8).trim();
275275
}
276+
276277
}

‎buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java

+25
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import org.gradle.api.Project;
88
import org.gradle.api.plugins.ExtraPropertiesExtension;
99
import org.gradle.internal.jvm.Jvm;
10+
import org.gradle.process.ExecResult;
1011

1112
import java.io.BufferedReader;
1213
import java.io.ByteArrayOutputStream;
@@ -15,13 +16,17 @@
1516
import java.io.IOException;
1617
import java.io.InputStreamReader;
1718
import java.io.UncheckedIOException;
19+
import java.time.ZoneOffset;
20+
import java.time.ZonedDateTime;
1821
import java.util.ArrayList;
1922
import java.util.Arrays;
2023
import java.util.HashMap;
2124
import java.util.List;
2225
import java.util.Map;
2326
import java.util.stream.Collectors;
2427

28+
import static java.nio.charset.StandardCharsets.UTF_8;
29+
2530
public class GlobalBuildInfoPlugin implements Plugin<Project> {
2631
private static final String GLOBAL_INFO_EXTENSION_NAME = "globalInfo";
2732
private static Integer _defaultParallel = null;
@@ -87,6 +92,8 @@ public void apply(Project project) {
8792
ext.set("minimumCompilerVersion", minimumCompilerVersion);
8893
ext.set("minimumRuntimeVersion", minimumRuntimeVersion);
8994
ext.set("gradleJavaVersion", Jvm.current().getJavaVersion());
95+
ext.set("gitRevision", gitRevision(project));
96+
ext.set("buildDate", ZonedDateTime.now(ZoneOffset.UTC));
9097
});
9198
}
9299

@@ -195,4 +202,22 @@ private static int findDefaultParallel(Project project) {
195202

196203
return _defaultParallel;
197204
}
205+
206+
private String gitRevision(final Project project) {
207+
final ByteArrayOutputStream stdout = new ByteArrayOutputStream();
208+
final ByteArrayOutputStream stderr = new ByteArrayOutputStream();
209+
final ExecResult result = project.exec(spec -> {
210+
spec.setExecutable("git");
211+
spec.setArgs(Arrays.asList("rev-parse", "HEAD"));
212+
spec.setStandardOutput(stdout);
213+
spec.setErrorOutput(stderr);
214+
spec.setIgnoreExitValue(true);
215+
});
216+
217+
if (result.getExitValue() != 0) {
218+
return "unknown";
219+
}
220+
return stdout.toString(UTF_8).trim();
221+
}
222+
198223
}

‎buildSrc/version.properties

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
elasticsearch = 8.0.0
22
lucene = 8.2.0
33

4-
bundled_jdk = 12.0.1+12@69cfe15208a647278a19ef0990eea691
4+
bundled_jdk = 12.0.2+10@e482c34c86bd4bf8b56c0b35558996b9
55

66
# optional dependencies
77
spatial4j = 0.7

‎client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java

+4
Original file line numberDiff line numberDiff line change
@@ -1045,6 +1045,8 @@ public void testForceMerge() throws IOException {
10451045
assertThat(forceMergeResponse.getSuccessfulShards(), equalTo(1));
10461046
assertThat(forceMergeResponse.getFailedShards(), equalTo(0));
10471047
assertThat(forceMergeResponse.getShardFailures(), equalTo(BroadcastResponse.EMPTY));
1048+
1049+
assertThat(forceMergeRequest.getDescription(), containsString(index));
10481050
}
10491051
{
10501052
String nonExistentIndex = "non_existent_index";
@@ -1053,6 +1055,8 @@ public void testForceMerge() throws IOException {
10531055
ElasticsearchException exception = expectThrows(ElasticsearchException.class,
10541056
() -> execute(forceMergeRequest, highLevelClient().indices()::forcemerge, highLevelClient().indices()::forcemergeAsync));
10551057
assertEquals(RestStatus.NOT_FOUND, exception.status());
1058+
1059+
assertThat(forceMergeRequest.getDescription(), containsString(nonExistentIndex));
10561060
}
10571061
}
10581062

‎client/rest-high-level/src/test/java/org/elasticsearch/client/core/MainResponseTests.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ protected void assertInstances(org.elasticsearch.action.main.MainResponse server
5858
assertThat(serverTestInstance.getNodeName(), equalTo(clientInstance.getNodeName()));
5959
assertThat("You Know, for Search", equalTo(clientInstance.getTagline()));
6060

61-
assertThat(serverTestInstance.getBuild().shortHash(), equalTo(clientInstance.getVersion().getBuildHash()));
61+
assertThat(serverTestInstance.getBuild().hash(), equalTo(clientInstance.getVersion().getBuildHash()));
6262
assertThat(serverTestInstance.getVersion().toString(), equalTo(clientInstance.getVersion().getNumber()));
6363
assertThat(serverTestInstance.getBuild().date(), equalTo(clientInstance.getVersion().getBuildDate()));
6464
assertThat(serverTestInstance.getBuild().flavor().displayName(), equalTo(clientInstance.getVersion().getBuildFlavor()));

‎client/rest/src/main/java/org/elasticsearch/client/Response.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,9 @@ public HttpEntity getEntity() {
102102

103103
private static final Pattern WARNING_HEADER_PATTERN = Pattern.compile(
104104
"299 " + // warn code
105-
"Elasticsearch-\\d+\\.\\d+\\.\\d+(?:-(?:alpha|beta|rc)\\d+)?(?:-SNAPSHOT)?-(?:[a-f0-9]{7}|Unknown) " + // warn agent
105+
"Elasticsearch-" + // warn agent
106+
"\\d+\\.\\d+\\.\\d+(?:-(?:alpha|beta|rc)\\d+)?(?:-SNAPSHOT)?-" + // warn agent
107+
"(?:[a-f0-9]{7}(?:[a-f0-9]{33})?|unknown) " + // warn agent
106108
"\"((?:\t| |!|[\\x23-\\x5B]|[\\x5D-\\x7E]|[\\x80-\\xFF]|\\\\|\\\\\")*)\"( " + // quoted warning value, captured
107109
// quoted RFC 1123 date format
108110
"\"" + // opening quote

‎client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@
6262
import java.net.SocketTimeoutException;
6363
import java.net.URI;
6464
import java.net.URISyntaxException;
65+
import java.nio.charset.StandardCharsets;
6566
import java.util.Arrays;
6667
import java.util.Collections;
6768
import java.util.HashSet;
@@ -526,7 +527,8 @@ private void assertDeprecationWarnings(List<String> warningHeaderTexts, List<Str
526527
* cases. We don't have that available because we're testing against 1.7.
527528
*/
528529
private static String formatWarning(String warningBody) {
529-
return "299 Elasticsearch-1.2.2-SNAPSHOT-eeeeeee \"" + warningBody + "\" \"Mon, 01 Jan 2001 00:00:00 GMT\"";
530+
final String hash = new String(new byte[40], StandardCharsets.UTF_8).replace('\0', 'e');
531+
return "299 Elasticsearch-1.2.2-SNAPSHOT-" + hash + " \"" + warningBody + "\" \"Mon, 01 Jan 2001 00:00:00 GMT\"";
530532
}
531533

532534
private HttpUriRequest performRandomRequest(String method) throws Exception {

‎distribution/docker/build.gradle

+3-1
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,10 @@ ext.expansions = { oss, local ->
2424
final String classifier = 'linux-x86_64'
2525
final String elasticsearch = oss ? "elasticsearch-oss-${VersionProperties.elasticsearch}-${classifier}.tar.gz" : "elasticsearch-${VersionProperties.elasticsearch}-${classifier}.tar.gz"
2626
return [
27+
'build_date' : project.ext.buildDate,
2728
'elasticsearch' : elasticsearch,
28-
'license' : oss ? 'Apache-2.0' : 'Elastic License',
29+
'git_revision' : project.ext.gitRevision,
30+
'license' : oss ? 'Apache-2.0' : 'Elastic-License',
2931
'source_elasticsearch': local ? "COPY $elasticsearch /opt/" : "RUN cd /opt && curl --retry 8 -s -L -O https://artifacts.elastic.co/downloads/elasticsearch/${elasticsearch} && cd -",
3032
'version' : VersionProperties.elasticsearch
3133
]

‎distribution/docker/src/docker/Dockerfile

+17-5
Original file line numberDiff line numberDiff line change
@@ -69,13 +69,25 @@ RUN chgrp 0 /usr/local/bin/docker-entrypoint.sh && \
6969

7070
EXPOSE 9200 9300
7171

72-
LABEL org.label-schema.schema-version="1.0" \
73-
org.label-schema.vendor="Elastic" \
74-
org.label-schema.name="elasticsearch" \
75-
org.label-schema.version="${version}" \
72+
LABEL org.label-schema.build-date="${build_date}" \
73+
org.label-schema.license="${license}" \
74+
org.label-schema.name="Elasticsearch" \
75+
org.label-schema.schema-version="1.0" \
7676
org.label-schema.url="https://www.elastic.co/products/elasticsearch" \
77+
org.label-schema.usage="https://www.elastic.co/guide/en/elasticsearch/reference/index.html" \
78+
org.label-schema.vcs-ref="${git_revision}" \
7779
org.label-schema.vcs-url="https://github.com/elastic/elasticsearch" \
78-
license="${license}"
80+
org.label-schema.vendor="Elastic" \
81+
org.label-schema.version="${version}" \
82+
org.opencontainers.image.created="${build_date}" \
83+
org.opencontainers.image.documentation="https://www.elastic.co/guide/en/elasticsearch/reference/index.html" \
84+
org.opencontainers.image.licenses="${license}" \
85+
org.opencontainers.image.revision="${git_revision}" \
86+
org.opencontainers.image.source="https://github.com/elastic/elasticsearch" \
87+
org.opencontainers.image.title="Elasticsearch" \
88+
org.opencontainers.image.url="https://www.elastic.co/products/elasticsearch" \
89+
org.opencontainers.image.vendor="Elastic" \
90+
org.opencontainers.image.version="${version}"
7991

8092
ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
8193
# Dummy overridable parameter parsed by entrypoint

‎docs/reference/aggregations/pipeline/movfn-aggregation.asciidoc

+15-2
Original file line numberDiff line numberDiff line change
@@ -24,14 +24,15 @@ A `moving_fn` aggregation looks like this in isolation:
2424
--------------------------------------------------
2525
// NOTCONSOLE
2626

27-
[[moving-avg-params]]
28-
.`moving_avg` Parameters
27+
[[moving-fn-params]]
28+
.`moving_fn` Parameters
2929
[options="header"]
3030
|===
3131
|Parameter Name |Description |Required |Default Value
3232
|`buckets_path` |Path to the metric of interest (see <<buckets-path-syntax, `buckets_path` Syntax>> for more details |Required |
3333
|`window` |The size of window to "slide" across the histogram. |Required |
3434
|`script` |The script that should be executed on each window of data |Required |
35+
|`shift` |<<shift-parameter, Shift>> of window position. |Optional | 0
3536
|===
3637

3738
`moving_fn` aggregations must be embedded inside of a `histogram` or `date_histogram` aggregation. They can be
@@ -169,6 +170,18 @@ POST /_search
169170
// CONSOLE
170171
// TEST[setup:sales]
171172

173+
[[shift-parameter]]
174+
==== shift parameter
175+
176+
By default (with `shift = 0`), the window that is offered for calculation is the last `n` values excluding the current bucket.
177+
Increasing `shift` by 1 moves starting window position by `1` to the right.
178+
179+
- To include current bucket to the window, use `shift = 1`.
180+
- For center alignment (`n / 2` values before and after the current bucket), use `shift = window / 2`.
181+
- For right alignment (`n` values after the current bucket), use `shift = window`.
182+
183+
If either of window edges moves outside the borders of data series, the window shrinks to include available values only.
184+
172185
==== Pre-built Functions
173186

174187
For convenience, a number of functions have been prebuilt and are available inside the `moving_fn` script context:

0 commit comments

Comments
 (0)