Skip to content

Commit 0895a5c

Browse files
jasontedorjasontedor
committed
Merge branch 'master' into replication-tracker-primary-term
* master: Remove types from watcher docs (#38002) Add test coverage for Painless general casting of boolean and Boolean (#37780) Fixed test bug, lastFollowTime is null if there are no follower indices. Add ECS schema for user-agent ingest processor (#37727) (#37984) Extract TransportRequestDeduplication from ShardStateAction (#37870) Expose retention leases in shard stats (#37991)
2 parents 5d11aab + 6a78b6a commit 0895a5c

File tree

38 files changed

+902
-382
lines changed

38 files changed

+902
-382
lines changed

‎docs/reference/ingest/processors/user-agent.asciidoc

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -60,13 +60,13 @@ Which returns
6060
"agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36",
6161
"user_agent": {
6262
"name": "Chrome",
63-
"major": "51",
64-
"minor": "0",
65-
"patch": "2704",
66-
"os_name": "Mac OS X",
67-
"os": "Mac OS X 10.10.5",
68-
"os_major": "10",
69-
"os_minor": "10",
63+
"original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36",
64+
"version": "51.0.2704",
65+
"os": {
66+
"name": "Mac OS X",
67+
"version": "10.10.5",
68+
"full": "Mac OS X 10.10.5"
69+
},
7070
"device": "Other"
7171
}
7272
}

‎docs/reference/migration/migrate_7_0/settings.asciidoc

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,3 +182,9 @@ could have lead to dropping audit events while the operations on the system
182182
were allowed to continue as usual. The recommended replacement is the
183183
use of the `logfile` audit output type and using other components from the
184184
Elastic Stack to handle the indexing part.
185+
186+
[float]
187+
[[ingest-user-agent-ecs-always]]
188+
==== Ingest User Agent processor always uses `ecs` output format
189+
The deprecated `ecs` setting for the user agent ingest processor has been
190+
removed. https://github.com/elastic/ecs[ECS] format is now the default.

‎modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java

Lines changed: 55 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919

2020
package org.elasticsearch.ingest.useragent;
2121

22+
import org.apache.logging.log4j.LogManager;
23+
import org.elasticsearch.common.logging.DeprecationLogger;
2224
import org.elasticsearch.ingest.AbstractProcessor;
2325
import org.elasticsearch.ingest.IngestDocument;
2426
import org.elasticsearch.ingest.Processor;
@@ -40,6 +42,8 @@
4042

4143
public class UserAgentProcessor extends AbstractProcessor {
4244

45+
private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(UserAgentProcessor.class));
46+
4347
public static final String TYPE = "user_agent";
4448

4549
private final String field;
@@ -63,7 +67,7 @@ boolean isIgnoreMissing() {
6367
}
6468

6569
@Override
66-
public IngestDocument execute(IngestDocument ingestDocument) throws Exception {
70+
public IngestDocument execute(IngestDocument ingestDocument) {
6771
String userAgent = ingestDocument.getFieldValue(field, String.class, ignoreMissing);
6872

6973
if (userAgent == null && ignoreMissing) {
@@ -75,68 +79,64 @@ public IngestDocument execute(IngestDocument ingestDocument) throws Exception {
7579
Details uaClient = parser.parse(userAgent);
7680

7781
Map<String, Object> uaDetails = new HashMap<>();
82+
83+
// Parse the user agent in the ECS (Elastic Common Schema) format
7884
for (Property property : this.properties) {
7985
switch (property) {
86+
case ORIGINAL:
87+
uaDetails.put("original", userAgent);
88+
break;
8089
case NAME:
8190
if (uaClient.userAgent != null && uaClient.userAgent.name != null) {
8291
uaDetails.put("name", uaClient.userAgent.name);
83-
}
84-
else {
92+
} else {
8593
uaDetails.put("name", "Other");
8694
}
8795
break;
88-
case MAJOR:
96+
case VERSION:
97+
StringBuilder version = new StringBuilder();
8998
if (uaClient.userAgent != null && uaClient.userAgent.major != null) {
90-
uaDetails.put("major", uaClient.userAgent.major);
91-
}
92-
break;
93-
case MINOR:
94-
if (uaClient.userAgent != null && uaClient.userAgent.minor != null) {
95-
uaDetails.put("minor", uaClient.userAgent.minor);
96-
}
97-
break;
98-
case PATCH:
99-
if (uaClient.userAgent != null && uaClient.userAgent.patch != null) {
100-
uaDetails.put("patch", uaClient.userAgent.patch);
101-
}
102-
break;
103-
case BUILD:
104-
if (uaClient.userAgent != null && uaClient.userAgent.build != null) {
105-
uaDetails.put("build", uaClient.userAgent.build);
99+
version.append(uaClient.userAgent.major);
100+
if (uaClient.userAgent.minor != null) {
101+
version.append(".").append(uaClient.userAgent.minor);
102+
if (uaClient.userAgent.patch != null) {
103+
version.append(".").append(uaClient.userAgent.patch);
104+
if (uaClient.userAgent.build != null) {
105+
version.append(".").append(uaClient.userAgent.build);
106+
}
107+
}
108+
}
109+
uaDetails.put("version", version.toString());
106110
}
107111
break;
108112
case OS:
109113
if (uaClient.operatingSystem != null) {
110-
uaDetails.put("os", buildFullOSName(uaClient.operatingSystem));
111-
}
112-
else {
113-
uaDetails.put("os", "Other");
114-
}
115-
116-
break;
117-
case OS_NAME:
118-
if (uaClient.operatingSystem != null && uaClient.operatingSystem.name != null) {
119-
uaDetails.put("os_name", uaClient.operatingSystem.name);
120-
}
121-
else {
122-
uaDetails.put("os_name", "Other");
123-
}
124-
break;
125-
case OS_MAJOR:
126-
if (uaClient.operatingSystem != null && uaClient.operatingSystem.major != null) {
127-
uaDetails.put("os_major", uaClient.operatingSystem.major);
128-
}
129-
break;
130-
case OS_MINOR:
131-
if (uaClient.operatingSystem != null && uaClient.operatingSystem.minor != null) {
132-
uaDetails.put("os_minor", uaClient.operatingSystem.minor);
114+
Map<String, String> osDetails = new HashMap<>(3);
115+
if (uaClient.operatingSystem.name != null) {
116+
osDetails.put("name", uaClient.operatingSystem.name);
117+
StringBuilder sb = new StringBuilder();
118+
if (uaClient.operatingSystem.major != null) {
119+
sb.append(uaClient.operatingSystem.major);
120+
if (uaClient.operatingSystem.minor != null) {
121+
sb.append(".").append(uaClient.operatingSystem.minor);
122+
if (uaClient.operatingSystem.patch != null) {
123+
sb.append(".").append(uaClient.operatingSystem.patch);
124+
if (uaClient.operatingSystem.build != null) {
125+
sb.append(".").append(uaClient.operatingSystem.build);
126+
}
127+
}
128+
}
129+
osDetails.put("version", sb.toString());
130+
osDetails.put("full", uaClient.operatingSystem.name + " " + sb.toString());
131+
}
132+
uaDetails.put("os", osDetails);
133+
}
133134
}
134135
break;
135136
case DEVICE:
136137
if (uaClient.device != null && uaClient.device.name != null) {
137138
uaDetails.put("device", uaClient.device.name);
138-
}
139-
else {
139+
} else {
140140
uaDetails.put("device", "Other");
141141
}
142142
break;
@@ -215,6 +215,10 @@ public UserAgentProcessor create(Map<String, Processor.Factory> factories, Strin
215215
String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME);
216216
List<String> propertyNames = readOptionalList(TYPE, processorTag, config, "properties");
217217
boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
218+
Object ecsValue = config.remove("ecs");
219+
if (ecsValue != null) {
220+
deprecationLogger.deprecated("setting [ecs] is deprecated as ECS format is the default and only option");
221+
}
218222

219223
UserAgentParser parser = userAgentParsers.get(regexFilename);
220224
if (parser == null) {
@@ -242,13 +246,16 @@ public UserAgentProcessor create(Map<String, Processor.Factory> factories, Strin
242246

243247
enum Property {
244248

245-
NAME, MAJOR, MINOR, PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD;
249+
NAME,
250+
OS,
251+
DEVICE,
252+
ORIGINAL,
253+
VERSION;
246254

247255
public static Property parseProperty(String propertyName) {
248256
try {
249257
return valueOf(propertyName.toUpperCase(Locale.ROOT));
250-
}
251-
catch (IllegalArgumentException e) {
258+
} catch (IllegalArgumentException e) {
252259
throw new IllegalArgumentException("illegal property value [" + propertyName + "]. valid values are " +
253260
Arrays.toString(EnumSet.allOf(Property.class).toArray()));
254261
}

‎modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -178,8 +178,8 @@ public void testInvalidProperty() throws Exception {
178178
config.put("properties", Collections.singletonList("invalid"));
179179

180180
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config));
181-
assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, MAJOR, MINOR, "
182-
+ "PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD]"));
181+
assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, OS, DEVICE, " +
182+
"ORIGINAL, VERSION]"));
183183
}
184184

185185
public void testInvalidPropertiesType() throws Exception {

‎modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java

Lines changed: 15 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -103,16 +103,13 @@ public void testCommonBrowser() throws Exception {
103103
Map<String, Object> target = (Map<String, Object>) data.get("target_field");
104104

105105
assertThat(target.get("name"), is("Chrome"));
106-
assertThat(target.get("major"), is("33"));
107-
assertThat(target.get("minor"), is("0"));
108-
assertThat(target.get("patch"), is("1750"));
109-
assertNull(target.get("build"));
110-
111-
assertThat(target.get("os"), is("Mac OS X 10.9.2"));
112-
assertThat(target.get("os_name"), is("Mac OS X"));
113-
assertThat(target.get("os_major"), is("10"));
114-
assertThat(target.get("os_minor"), is("9"));
106+
assertThat(target.get("version"), is("33.0.1750"));
115107

108+
Map<String, String> os = new HashMap<>();
109+
os.put("name", "Mac OS X");
110+
os.put("version", "10.9.2");
111+
os.put("full", "Mac OS X 10.9.2");
112+
assertThat(target.get("os"), is(os));
116113
assertThat(target.get("device"), is("Other"));
117114
}
118115

@@ -131,15 +128,13 @@ public void testUncommonDevice() throws Exception {
131128
Map<String, Object> target = (Map<String, Object>) data.get("target_field");
132129

133130
assertThat(target.get("name"), is("Android"));
134-
assertThat(target.get("major"), is("3"));
135-
assertThat(target.get("minor"), is("0"));
136-
assertNull(target.get("patch"));
137-
assertNull(target.get("build"));
131+
assertThat(target.get("version"), is("3.0"));
138132

139-
assertThat(target.get("os"), is("Android 3.0"));
140-
assertThat(target.get("os_name"), is("Android"));
141-
assertThat(target.get("os_major"), is("3"));
142-
assertThat(target.get("os_minor"), is("0"));
133+
Map<String, String> os = new HashMap<>();
134+
os.put("name", "Android");
135+
os.put("version", "3.0");
136+
os.put("full", "Android 3.0");
137+
assertThat(target.get("os"), is(os));
143138

144139
assertThat(target.get("device"), is("Motorola Xoom"));
145140
}
@@ -158,15 +153,9 @@ public void testSpider() throws Exception {
158153
Map<String, Object> target = (Map<String, Object>) data.get("target_field");
159154

160155
assertThat(target.get("name"), is("EasouSpider"));
161-
assertNull(target.get("major"));
162-
assertNull(target.get("minor"));
163-
assertNull(target.get("patch"));
164-
assertNull(target.get("build"));
165156

166-
assertThat(target.get("os"), is("Other"));
167-
assertThat(target.get("os_name"), is("Other"));
168-
assertNull(target.get("os_major"));
169-
assertNull(target.get("os_minor"));
157+
assertNull(target.get("version"));
158+
assertNull(target.get("os"));
170159

171160
assertThat(target.get("device"), is("Spider"));
172161
}
@@ -190,10 +179,7 @@ public void testUnknown() throws Exception {
190179
assertNull(target.get("patch"));
191180
assertNull(target.get("build"));
192181

193-
assertThat(target.get("os"), is("Other"));
194-
assertThat(target.get("os_name"), is("Other"));
195-
assertNull(target.get("os_major"));
196-
assertNull(target.get("os_minor"));
182+
assertNull(target.get("os"));
197183

198184
assertThat(target.get("device"), is("Other"));
199185
}

‎modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml

Lines changed: 5 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -29,13 +29,9 @@
2929
id: 1
3030
- match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
3131
- match: { _source.user_agent.name: "Chrome" }
32-
- match: { _source.user_agent.os: "Mac OS X 10.9.2" }
33-
- match: { _source.user_agent.os_name: "Mac OS X" }
34-
- match: { _source.user_agent.os_major: "10" }
35-
- match: { _source.user_agent.os_minor: "9" }
36-
- match: { _source.user_agent.major: "33" }
37-
- match: { _source.user_agent.minor: "0" }
38-
- match: { _source.user_agent.patch: "1750" }
32+
- match: { _source.user_agent.original: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
33+
- match: { _source.user_agent.os: {"name":"Mac OS X", "version":"10.9.2", "full":"Mac OS X 10.9.2"} }
34+
- match: { _source.user_agent.version: "33.0.1750" }
3935
- match: { _source.user_agent.device: "Other" }
4036

4137
---
@@ -70,13 +66,8 @@
7066
index: test
7167
id: 1
7268
- match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
73-
- match: { _source.field2.os: "Mac OS X 10.9.2" }
69+
- match: { _source.field2.os.full: "Mac OS X 10.9.2" }
7470
- is_false: _source.user_agent
7571
- is_false: _source.field2.name
76-
- is_false: _source.field2.os_name
77-
- is_false: _source.field2.os_major
78-
- is_false: _source.field2.os_minor
79-
- is_false: _source.field2.major
80-
- is_false: _source.field2.minor
81-
- is_false: _source.field2.patch
8272
- is_false: _source.field2.device
73+
- is_false: _source.field2.original

‎modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,6 @@
3030
id: 1
3131
- match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
3232
- match: { _source.user_agent.name: "Test" }
33-
- match: { _source.user_agent.os: "Other" }
34-
- match: { _source.user_agent.os_name: "Other" }
3533
- match: { _source.user_agent.device: "Other" }
36-
- is_false: _source.user_agent.os_major
37-
- is_false: _source.user_agent.os_minor
38-
- is_false: _source.user_agent.major
39-
- is_false: _source.user_agent.minor
40-
- is_false: _source.user_agent.patch
34+
- is_false: _source.user_agent.os
35+
- is_false: _source.user_agent.version

0 commit comments

Comments
 (0)