Skip to content

Commit db25634

Browse files
weizijunweizijun
committed
Merge branch 'upstream/master' into reuse-date-stream-timestamp
* upstream/master: Validate tsdb's routing_path (elastic#79384) Adjust the BWC version for the return200ForClusterHealthTimeout field (elastic#79436) API for adding and removing indices from a data stream (elastic#79279) Exposing the ability to log deprecated settings at non-critical level (elastic#79107) Convert operator privilege license object to LicensedFeature (elastic#79407) Mute SnapshotBasedIndexRecoveryIT testSeqNoBasedRecoveryIsUsedAfterPrimaryFailOver (elastic#79456) Create cache files with CREATE_NEW & SPARSE options (elastic#79371) Revert "[ML] Use a new annotations index for future annotations (elastic#79151)" [ML] Use a new annotations index for future annotations (elastic#79151) [ML] Removing legacy code from ML/transform auditor (elastic#79434) Fix rate agg with custom `_doc_count` (elastic#79346) Optimize SLM Policy Queries (elastic#79341) Fix execution of exists query within nested queries on field with doc_values disabled (elastic#78841) Stricter UpdateSettingsRequest parsing on the REST layer (elastic#79227) Do not release snapshot file download permit during recovery retries (elastic#79409) Preserve request headers in a mixed version cluster (elastic#79412) Adjust versions after elastic#79044 backport to 7.x (elastic#79424) Mute BulkByScrollUsesAllScrollDocumentsAfterConflictsIntegTests (elastic#79429) Fail on SSPL licensed x-pack sources (elastic#79348) # Conflicts: # server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java
2 parents 93beacb + 7a32158 commit db25634

File tree

103 files changed

+3038
-696
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

103 files changed

+3038
-696
lines changed

‎build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/LicenseHeadersTask.java

Lines changed: 35 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@
3636
import org.w3c.dom.Element;
3737
import org.w3c.dom.NodeList;
3838
import org.xml.sax.SAXException;
39-
39+
import org.gradle.api.model.ObjectFactory;
4040
import javax.xml.parsers.DocumentBuilderFactory;
4141
import javax.xml.parsers.ParserConfigurationException;
4242
import java.io.BufferedWriter;
@@ -51,6 +51,8 @@
5151
import java.util.List;
5252
import java.util.Map;
5353
import java.util.stream.Collectors;
54+
import javax.inject.Inject;
55+
import java.io.Serializable;
5456

5557
/**
5658
* Checks files for license headers..
@@ -95,17 +97,18 @@ public List<String> getExcludes() {
9597
return excludes;
9698
}
9799

98-
public Map<String, String> getAdditionalLicenses() {
99-
return additionalLicenses;
100-
}
101-
102100
public void setExcludes(List<String> excludes) {
103101
this.excludes = excludes;
104102
}
105103

106104
@OutputFile
107105
private File reportFile = new File(getProject().getBuildDir(), "reports/licenseHeaders/rat.xml");
108106

107+
private static List<License> conventionalLicenses = Arrays.asList(
108+
// Dual SSPLv1 and Elastic
109+
new License("DUAL", "SSPL+Elastic License", "the Elastic License 2.0 or the Server")
110+
);
111+
109112
/**
110113
* Allowed license families for this project.
111114
*/
@@ -118,13 +121,17 @@ public void setExcludes(List<String> excludes) {
118121
*/
119122
@Input
120123
private List<String> excludes = new ArrayList<String>();
124+
125+
private ListProperty<License> additionalLicenses;
126+
121127
/**
122128
* Additional license families that may be found. The key is the license category name (5 characters),
123129
* followed by the family name and the value list of patterns to search for.
124130
*/
125131
@Input
126-
protected Map<String, String> additionalLicenses = new HashMap<String, String>();
127-
132+
public ListProperty<License> getAdditionalLicenses() {
133+
return additionalLicenses;
134+
}
128135
/**
129136
* Add a new license type.
130137
* <p>
@@ -139,7 +146,12 @@ public void additionalLicense(final String categoryName, String familyName, Stri
139146
throw new IllegalArgumentException("License category name must be exactly 5 characters, got " + categoryName);
140147
}
141148

142-
additionalLicenses.put(categoryName + familyName, pattern);
149+
additionalLicenses.add(new License(categoryName, familyName, pattern));
150+
}
151+
152+
@Inject
153+
public LicenseHeadersTask(ObjectFactory objectFactory) {
154+
additionalLicenses = objectFactory.listProperty(License.class).convention(conventionalLicenses);
143155
}
144156

145157
@TaskAction
@@ -160,14 +172,10 @@ public void runRat() {
160172
matchers.add(subStringMatcher("GEN ", "Generated", "ANTLR GENERATED CODE"));
161173
// Vendored Code
162174
matchers.add(subStringMatcher("VEN ", "Vendored", "@notice"));
163-
// Dual SSPLv1 and Elastic
164-
matchers.add(subStringMatcher("DUAL", "SSPL+Elastic License", "the Elastic License 2.0 or the Server"));
165175

166-
for (Map.Entry<String, String> additional : additionalLicenses.entrySet()) {
167-
String category = additional.getKey().substring(0, 5);
168-
String family = additional.getKey().substring(5);
169-
matchers.add(subStringMatcher(category, family, additional.getValue()));
170-
}
176+
additionalLicenses.get().forEach(l ->
177+
matchers.add(subStringMatcher(l.licenseFamilyCategory, l.licenseFamilyName, l.substringPattern))
178+
);
171179

172180
reportConfiguration.setHeaderMatcher(new HeaderMatcherMultiplexer(matchers.toArray(IHeaderMatcher[]::new)));
173181
reportConfiguration.setApprovedLicenseNames(approvedLicenses.stream().map(license -> {
@@ -190,7 +198,6 @@ private IHeaderMatcher subStringMatcher(String licenseFamilyCategory, String lic
190198
SubstringLicenseMatcher substringLicenseMatcher = new SubstringLicenseMatcher();
191199
substringLicenseMatcher.setLicenseFamilyCategory(licenseFamilyCategory);
192200
substringLicenseMatcher.setLicenseFamilyName(licenseFamilyName);
193-
194201
SubstringLicenseMatcher.Pattern pattern = new SubstringLicenseMatcher.Pattern();
195202
pattern.setSubstring(substringPattern);
196203
substringLicenseMatcher.addConfiguredPattern(pattern);
@@ -249,4 +256,16 @@ private static List<Element> elementList(NodeList resourcesNodes) {
249256
}
250257
return nodeList;
251258
}
259+
260+
static class License implements Serializable {
261+
private String licenseFamilyCategory;
262+
private String licenseFamilyName;
263+
private String substringPattern;
264+
265+
public License(String licenseFamilyCategory, String licenseFamilyName, String substringPattern) {
266+
this.licenseFamilyCategory = licenseFamilyCategory;
267+
this.licenseFamilyName = licenseFamilyName;
268+
this.substringPattern = substringPattern;
269+
}
270+
}
252271
}

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/LicenseHeadersPrecommitPluginFuncTest.groovy

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,45 @@ class LicenseHeadersPrecommitPluginFuncTest extends AbstractGradleFuncTest {
5959
result.task(":licenseHeaders").outcome == TaskOutcome.SUCCESS
6060
}
6161

62+
def "supports sspl by convention"() {
63+
given:
64+
buildFile << """
65+
plugins {
66+
id 'java'
67+
id 'elasticsearch.internal-licenseheaders'
68+
}
69+
"""
70+
dualLicensedFile()
71+
72+
when:
73+
def result = gradleRunner("licenseHeaders").build()
74+
75+
then:
76+
result.task(":licenseHeaders").outcome == TaskOutcome.SUCCESS
77+
}
78+
79+
def "sspl default additional license can be overridden"() {
80+
given:
81+
buildFile << """
82+
plugins {
83+
id 'java'
84+
id 'elasticsearch.internal-licenseheaders'
85+
}
86+
87+
tasks.named("licenseHeaders").configure {
88+
additionalLicense 'ELAST', 'Elastic License 2.0', '2.0; you may not use this file except in compliance with the Elastic License'
89+
}
90+
"""
91+
elasticLicensed()
92+
dualLicensedFile()
93+
94+
when:
95+
def result = gradleRunner("licenseHeaders").buildAndFail()
96+
97+
then:
98+
result.task(":licenseHeaders").outcome == TaskOutcome.FAILED
99+
}
100+
62101
private File unapprovedSourceFile(String filePath = "src/main/java/org/acme/UnapprovedLicensed.java") {
63102
File sourceFile = file(filePath);
64103
sourceFile << """
@@ -115,6 +154,21 @@ class LicenseHeadersPrecommitPluginFuncTest extends AbstractGradleFuncTest {
115154
"""
116155
}
117156

157+
private File elasticLicensed() {
158+
file("src/main/java/org/acme/ElasticLicensed.java") << """
159+
/*
160+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
161+
* or more contributor license agreements. Licensed under the Elastic License
162+
* 2.0; you may not use this file except in compliance with the Elastic License
163+
* 2.0.
164+
*/
165+
166+
package org.acme;
167+
public class ElasticLicensed {
168+
}
169+
"""
170+
}
171+
118172
private String packageString(File sourceFile) {
119173
String normalizedPath = normalized(sourceFile.getPath())
120174
(normalizedPath.substring(normalizedPath.indexOf("src/main/java")) - "src/main/java/" - ("/" + sourceFile.getName())).replaceAll("/", ".")

‎client/rest/src/main/java/org/elasticsearch/client/Response.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -136,14 +136,14 @@ public HttpEntity getEntity() {
136136

137137
/**
138138
* Tests if a string matches the RFC 7234 specification for warning headers.
139-
* This assumes that the warn code is always 299 and the warn agent is always
140-
* Elasticsearch.
139+
* This assumes that the warn code is always 299 or 300 and the warn agent is
140+
* always Elasticsearch.
141141
*
142142
* @param s the value of a warning header formatted according to RFC 7234
143143
* @return {@code true} if the input string matches the specification
144144
*/
145145
private static boolean matchWarningHeaderPatternByPrefix(final String s) {
146-
return s.startsWith("299 Elasticsearch-");
146+
return s.startsWith("299 Elasticsearch-") || s.startsWith("300 Elasticsearch-");
147147
}
148148

149149
/**

‎docs/reference/aggregations/metrics/rate-aggregation.asciidoc

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
++++
88

99
A `rate` metrics aggregation can be used only inside a `date_histogram` or `composite` aggregation. It calculates a rate of documents
10-
or a field in each bucket. The field values can be generated extracted from specific numeric or
10+
or a field in each bucket. The field values can be extracted from specific numeric or
1111
<<histogram,histogram fields>> in the documents.
1212

1313
NOTE: For `composite` aggregations, there must be exactly one `date_histogram` source for the `rate` aggregation to be supported.
@@ -27,7 +27,7 @@ A `rate` aggregation looks like this in isolation:
2727
--------------------------------------------------
2828
// NOTCONSOLE
2929

30-
The following request will group all sales records into monthly bucket and than convert the number of sales transaction in each bucket
30+
The following request will group all sales records into monthly buckets and then convert the number of sales transactions in each bucket
3131
into per annual sales rate.
3232

3333
[source,console]
@@ -56,8 +56,8 @@ GET sales/_search
5656
<1> Histogram is grouped by month.
5757
<2> But the rate is converted into annual rate.
5858

59-
The response will return the annual rate of transaction in each bucket. Since there are 12 months per year, the annual rate will
60-
be automatically calculated by multiplying monthly rate by 12.
59+
The response will return the annual rate of transactions in each bucket. Since there are 12 months per year, the annual rate will
60+
be automatically calculated by multiplying the monthly rate by 12.
6161

6262
[source,console-result]
6363
--------------------------------------------------

‎libs/x-content/src/test/java/org/elasticsearch/xcontent/ConstructingObjectParserTests.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -626,8 +626,8 @@ public void testCompatibleFieldDeclarations() throws IOException {
626626
RestApiVersion.minimumSupported());
627627
StructWithCompatibleFields o = StructWithCompatibleFields.PARSER.parse(parser, null);
628628
assertEquals(1, o.intField);
629-
assertWarnings(false, "[struct_with_compatible_fields][1:14] " +
630-
"Deprecated field [old_name] used, expected [new_name] instead");
629+
assertWarnings(false, new DeprecationWarning(DeprecationLogger.CRITICAL, "[struct_with_compatible_fields][1:14] " +
630+
"Deprecated field [old_name] used, expected [new_name] instead"));
631631
}
632632
}
633633

‎libs/x-content/src/test/java/org/elasticsearch/xcontent/ObjectParserTests.java

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
*/
88
package org.elasticsearch.xcontent;
99

10+
import org.elasticsearch.common.logging.DeprecationLogger;
1011
import org.elasticsearch.common.xcontent.XContentParserUtils;
1112
import org.elasticsearch.core.CheckedFunction;
1213
import org.elasticsearch.core.RestApiVersion;
@@ -211,7 +212,8 @@ class TestStruct {
211212
objectParser.declareField((i, v, c) -> v.test = i.text(), new ParseField("test", "old_test"), ObjectParser.ValueType.STRING);
212213
objectParser.parse(parser, s, null);
213214
assertEquals("foo", s.test);
214-
assertWarnings(false, "[foo][1:15] Deprecated field [old_test] used, expected [test] instead");
215+
assertWarnings(false, new DeprecationWarning(DeprecationLogger.CRITICAL, "[foo][1:15] Deprecated field [old_test] used, " +
216+
"expected [test] instead"));
215217
}
216218

217219
public void testFailOnValueType() throws IOException {
@@ -1072,8 +1074,8 @@ public void testCompatibleFieldDeclarations() throws IOException {
10721074
RestApiVersion.minimumSupported());
10731075
StructWithCompatibleFields o = StructWithCompatibleFields.PARSER.parse(parser, null);
10741076
assertEquals(1, o.intField);
1075-
assertWarnings(false, "[struct_with_compatible_fields][1:14] " +
1076-
"Deprecated field [old_name] used, expected [new_name] instead");
1077+
assertWarnings(false, new DeprecationWarning(DeprecationLogger.CRITICAL, "[struct_with_compatible_fields][1:14] " +
1078+
"Deprecated field [old_name] used, expected [new_name] instead"));
10771079

10781080
}
10791081
}

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -470,8 +470,8 @@ public Processor create(
470470

471471
boolean valid = metadata.isValid(currentState.metadata().settings());
472472
if (valid && metadata.isCloseToExpiration()) {
473-
HeaderWarning.addWarning("database [{}] was not updated for over 25 days, geoip processor will stop working if there " +
474-
"is no update for 30 days", databaseFile);
473+
HeaderWarning.addWarning(DeprecationLogger.CRITICAL, "database [{}] was not updated for over 25 days, geoip processor" +
474+
" will stop working if there is no update for 30 days", databaseFile);
475475
}
476476

477477
return valid;

‎modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/BulkByScrollUsesAllScrollDocumentsAfterConflictsIntegTests.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ public void setUpCluster() {
9898
internalCluster().startDataOnlyNode(Settings.builder().put("thread_pool.write.size", 1).build());
9999
}
100100

101+
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/79300")
101102
public void testUpdateByQuery() throws Exception {
102103
final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
103104
final boolean scriptEnabled = randomBoolean();
@@ -111,7 +112,7 @@ public void testUpdateByQuery() throws Exception {
111112
});
112113
}
113114

114-
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/79342")
115+
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/79300")
115116
public void testReindex() throws Exception {
116117
final String sourceIndex = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
117118
final String targetIndex = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
@@ -134,6 +135,7 @@ public void testReindex() throws Exception {
134135
});
135136
}
136137

138+
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/79300")
137139
public void testDeleteByQuery() throws Exception {
138140
final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
139141
executeConcurrentUpdatesOnSubsetOfDocs(indexName,

‎plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImplTests.java

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,10 @@
1414
import com.amazonaws.auth.AWSCredentialsProvider;
1515
import com.amazonaws.auth.BasicSessionCredentials;
1616
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
17+
18+
import org.elasticsearch.common.logging.DeprecationLogger;
1719
import org.elasticsearch.common.settings.MockSecureSettings;
20+
import org.elasticsearch.common.settings.Setting;
1821
import org.elasticsearch.common.settings.Settings;
1922
import org.elasticsearch.common.settings.SettingsException;
2023
import org.elasticsearch.test.ESTestCase;
@@ -59,8 +62,9 @@ public void testDeprecationOfLoneAccessKey() {
5962
Ec2ClientSettings.getClientSettings(Settings.builder().setSecureSettings(secureSettings).build())).getCredentials();
6063
assertThat(credentials.getAWSAccessKeyId(), is("aws_key"));
6164
assertThat(credentials.getAWSSecretKey(), is(""));
62-
assertSettingDeprecationsAndWarnings(new String[]{},
63-
"Setting [discovery.ec2.access_key] is set but [discovery.ec2.secret_key] is not, which will be unsupported in future");
65+
assertSettingDeprecationsAndWarnings(new Setting<?>[]{},
66+
new DeprecationWarning(DeprecationLogger.CRITICAL, "Setting [discovery.ec2.access_key] is set but " +
67+
"[discovery.ec2.secret_key] is not, which will be unsupported in future"));
6468
}
6569

6670
public void testDeprecationOfLoneSecretKey() {
@@ -70,8 +74,9 @@ public void testDeprecationOfLoneSecretKey() {
7074
Ec2ClientSettings.getClientSettings(Settings.builder().setSecureSettings(secureSettings).build())).getCredentials();
7175
assertThat(credentials.getAWSAccessKeyId(), is(""));
7276
assertThat(credentials.getAWSSecretKey(), is("aws_secret"));
73-
assertSettingDeprecationsAndWarnings(new String[]{},
74-
"Setting [discovery.ec2.secret_key] is set but [discovery.ec2.access_key] is not, which will be unsupported in future");
77+
assertSettingDeprecationsAndWarnings(new Setting<?>[]{},
78+
new DeprecationWarning(DeprecationLogger.CRITICAL, "Setting [discovery.ec2.secret_key] is set but " +
79+
"[discovery.ec2.access_key] is not, which will be unsupported in future"));
7580
}
7681

7782
public void testRejectionOfLoneSessionToken() {

‎plugins/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,8 @@ public void testEnforcedCooldownPeriod() throws IOException {
156156
SnapshotState.SUCCESS,
157157
SnapshotsService.SHARD_GEN_IN_REPO_DATA_VERSION.minimumCompatibilityVersion(),
158158
0L, // -1 would refresh RepositoryData and find the real version
159-
0L // -1 would refresh RepositoryData and find the real version
159+
0L, // -1 would refresh RepositoryData and find the real version,
160+
"" // null would refresh RepositoryData and find the real version
160161
)));
161162
final BytesReference serialized = BytesReference.bytes(modifiedRepositoryData.snapshotsToXContent(XContentFactory.jsonBuilder(),
162163
SnapshotsService.OLD_SNAPSHOT_FORMAT));

‎qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ public void testDeprecationWarnMessage() throws IOException {
100100
);
101101
}
102102

103-
assertWarnings("deprecated warn message1");
103+
assertWarnings(true, new DeprecationWarning(Level.WARN, "deprecated warn message1")) ;
104104
}
105105

106106
public void testDeprecatedMessageWithoutXOpaqueId() throws IOException {

‎rest-api-spec/src/main/resources/rest-api-spec/api/indices.modify_data_stream.json

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
{
2+
"indices.modify_data_stream":{
3+
"documentation":{
4+
"url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams.html",
5+
"description":"Modifies a data stream"
6+
},
7+
"stability":"stable",
8+
"visibility":"public",
9+
"headers":{
10+
"accept": [ "application/json"],
11+
"content_type": ["application/json"]
12+
},
13+
"url":{
14+
"paths":[
15+
{
16+
"path":"/_data_stream/_modify",
17+
"methods":["POST"]
18+
}
19+
]
20+
},
21+
"params":{
22+
},
23+
"body":{
24+
"description":"The data stream modifications",
25+
"required":true
26+
}
27+
}
28+
}

0 commit comments

Comments
 (0)