From e244f4dda3ab772e720627191b0ad5c748c667a9 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 6 May 2019 10:38:27 +0200 Subject: [PATCH] Rename enrich policy index_pattern field to indices. Relates to #32789 --- .../xpack/core/enrich/EnrichPolicy.java | 28 +++++++++---------- .../elasticsearch/xpack/enrich/EnrichIT.java | 2 +- .../rest-api-spec/test/enrich/10_basic.yml | 4 +-- .../xpack/enrich/EnrichPolicyRunner.java | 10 +++---- .../xpack/enrich/EnrichPolicyRunnerTests.java | 5 ++-- .../xpack/enrich/EnrichPolicyTests.java | 4 +-- .../enrich/EnrichProcessorFactoryTests.java | 11 +++++--- .../enrich/ExactMatchProcessorTests.java | 18 ++++++++---- 8 files changed, 46 insertions(+), 36 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java index ba66764208df5..924c11c11877c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java @@ -34,7 +34,7 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment { static final ParseField TYPE = new ParseField("type"); static final ParseField QUERY = new ParseField("query"); - static final ParseField INDEX_PATTERN = new ParseField("index_pattern"); + static final ParseField INDICES = new ParseField("indices"); static final ParseField ENRICH_KEY = new ParseField("enrich_key"); static final ParseField ENRICH_VALUES = new ParseField("enrich_values"); static final ParseField SCHEDULE = new ParseField("schedule"); @@ -45,7 +45,7 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment { return new EnrichPolicy( (String) args[0], (QuerySource) args[1], - (String) args[2], + (List) args[2], (String) args[3], (List) args[4], (String) args[5] @@ -64,7 +64,7 @@ private static void declareParserOptions(ConstructingObjectParser parser) { contentBuilder.generator().copyCurrentStructure(p); return new QuerySource(BytesReference.bytes(contentBuilder), contentBuilder.contentType()); }, QUERY); - parser.declareString(ConstructingObjectParser.constructorArg(), INDEX_PATTERN); + parser.declareStringArray(ConstructingObjectParser.constructorArg(), INDICES); parser.declareString(ConstructingObjectParser.constructorArg(), ENRICH_KEY); parser.declareStringArray(ConstructingObjectParser.constructorArg(), ENRICH_VALUES); parser.declareString(ConstructingObjectParser.constructorArg(), SCHEDULE); @@ -76,7 +76,7 @@ public static EnrichPolicy fromXContent(XContentParser parser) throws IOExceptio private final String type; private final QuerySource query; - private final String indexPattern; + private final List indices; private final String enrichKey; private final List enrichValues; private final String schedule; @@ -85,7 +85,7 @@ public EnrichPolicy(StreamInput in) throws IOException { this( in.readString(), in.readOptionalWriteable(QuerySource::new), - in.readString(), + in.readStringList(), in.readString(), in.readStringList(), in.readString() @@ -94,14 +94,14 @@ public EnrichPolicy(StreamInput in) throws IOException { public EnrichPolicy(String type, QuerySource query, - String indexPattern, + List indices, String enrichKey, List enrichValues, String schedule) { this.type = type; this.query= query; this.schedule = schedule; - this.indexPattern = indexPattern; + this.indices = indices; this.enrichKey = enrichKey; this.enrichValues = enrichValues; } @@ -114,8 +114,8 @@ public QuerySource getQuery() { return query; } - public String getIndexPattern() { - return indexPattern; + public List getIndices() { + return indices; } public String getEnrichKey() { @@ -139,7 +139,7 @@ public String getAliasName(String policyName) { public void writeTo(StreamOutput out) throws IOException { out.writeString(type); out.writeOptionalWriteable(query); - out.writeString(indexPattern); + out.writeStringCollection(indices); out.writeString(enrichKey); out.writeStringCollection(enrichValues); out.writeString(schedule); @@ -151,7 +151,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (query != null) { builder.field(QUERY.getPreferredName(), query.getQueryAsMap()); } - builder.field(INDEX_PATTERN.getPreferredName(), indexPattern); + builder.array(INDICES.getPreferredName(), indices.toArray(new String[0])); builder.field(ENRICH_KEY.getPreferredName(), enrichKey); builder.array(ENRICH_VALUES.getPreferredName(), enrichValues.toArray(new String[0])); builder.field(SCHEDULE.getPreferredName(), schedule); @@ -165,7 +165,7 @@ public boolean equals(Object o) { EnrichPolicy policy = (EnrichPolicy) o; return type.equals(policy.type) && Objects.equals(query, policy.query) && - indexPattern.equals(policy.indexPattern) && + indices.equals(policy.indices) && enrichKey.equals(policy.enrichKey) && enrichValues.equals(policy.enrichValues) && schedule.equals(policy.schedule); @@ -176,7 +176,7 @@ public int hashCode() { return Objects.hash( type, query, - indexPattern, + indices, enrichKey, enrichValues, schedule @@ -244,7 +244,7 @@ public static class NamedPolicy implements Writeable, ToXContent { (String) args[0], new EnrichPolicy((String) args[1], (QuerySource) args[2], - (String) args[3], + (List) args[3], (String) args[4], (List) args[5], (String) args[6]) diff --git a/x-pack/plugin/enrich/qa/rest/src/test/java/org/elasticsearch/xpack/enrich/EnrichIT.java b/x-pack/plugin/enrich/qa/rest/src/test/java/org/elasticsearch/xpack/enrich/EnrichIT.java index 9faf345618d1e..36519d68f0b49 100644 --- a/x-pack/plugin/enrich/qa/rest/src/test/java/org/elasticsearch/xpack/enrich/EnrichIT.java +++ b/x-pack/plugin/enrich/qa/rest/src/test/java/org/elasticsearch/xpack/enrich/EnrichIT.java @@ -41,7 +41,7 @@ private void deletePolicies() throws Exception { public void testBasicFlow() throws Exception { // Create the policy: Request putPolicyRequest = new Request("PUT", "/_enrich/policy/my_policy"); - putPolicyRequest.setJsonEntity("{\"type\": \"exact_match\",\"index_pattern\": \"my-index*\", \"enrich_key\": \"host\", " + + putPolicyRequest.setJsonEntity("{\"type\": \"exact_match\",\"indices\": [\"my-index*\"], \"enrich_key\": \"host\", " + "\"enrich_values\": [\"globalRank\", \"tldRank\", \"tld\"], \"schedule\": \"0 5 * * *\"}"); assertOK(client().performRequest(putPolicyRequest)); diff --git a/x-pack/plugin/enrich/qa/rest/src/test/resources/rest-api-spec/test/enrich/10_basic.yml b/x-pack/plugin/enrich/qa/rest/src/test/resources/rest-api-spec/test/enrich/10_basic.yml index a51a930b6f3d9..48036ea05b74f 100644 --- a/x-pack/plugin/enrich/qa/rest/src/test/resources/rest-api-spec/test/enrich/10_basic.yml +++ b/x-pack/plugin/enrich/qa/rest/src/test/resources/rest-api-spec/test/enrich/10_basic.yml @@ -6,7 +6,7 @@ name: policy-crud body: type: exact_match - index_pattern: "bar*" + indices: ["bar*"] enrich_key: baz enrich_values: ["a", "b"] schedule: "*/120" @@ -17,7 +17,7 @@ - length: { policies: 1 } - match: { policies.0.name: policy-crud } - match: { policies.0.type: exact_match } - - match: { policies.0.index_pattern: "bar*" } + - match: { policies.0.indices: ["bar*"] } - match: { policies.0.enrich_key: baz } - match: { policies.0.enrich_values: ["a", "b"] } - match: { policies.0.schedule: "*/120" } diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java index fd52d643e39f3..cdf09a510ceba 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java @@ -75,9 +75,9 @@ public class EnrichPolicyRunner implements Runnable { public void run() { // Collect the source index information logger.info("Policy [{}]: Running enrich policy", policyName); - final String sourceIndexPattern = policy.getIndexPattern(); - logger.debug("Policy [{}]: Checking source index [{}]", policyName, sourceIndexPattern); - GetIndexRequest getIndexRequest = new GetIndexRequest().indices(sourceIndexPattern); + final String[] sourceIndices = policy.getIndices().toArray(new String[0]); + logger.debug("Policy [{}]: Checking source indices [{}]", policyName, sourceIndices); + GetIndexRequest getIndexRequest = new GetIndexRequest().indices(sourceIndices); client.admin().indices().getIndex(getIndexRequest, new ActionListener<>() { @Override public void onResponse(GetIndexResponse getIndexResponse) { @@ -110,7 +110,7 @@ private void validateMappings(final GetIndexResponse getIndexResponse) { listener.onFailure( new ElasticsearchException( "Enrich policy execution for [{}] failed. Could not read mapping for source [{}] included by pattern [{}]", - policyName, sourceIndex, policy.getIndexPattern())); + policyName, sourceIndex, policy.getIndices())); } if (properties.contains(policy.getEnrichKey()) == false) { listener.onFailure( @@ -193,7 +193,7 @@ private void transferDataToEnrichIndex(final String destinationIndexName) { } ReindexRequest reindexRequest = new ReindexRequest() .setDestIndex(destinationIndexName) - .setSourceIndices(policy.getIndexPattern()); + .setSourceIndices(policy.getIndices().toArray(new String[0])); reindexRequest.getSearchRequest().source(searchSourceBuilder); reindexRequest.getDestination().source(new BytesArray(new byte[0]), XContentType.SMILE); client.execute(ReindexAction.INSTANCE, reindexRequest, new ActionListener<>() { diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java index 3d42239910a92..a3fbf1860991b 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java @@ -87,7 +87,7 @@ public void testRunner() throws Exception { List enrichFields = new ArrayList<>(); enrichFields.add("field2"); enrichFields.add("field5"); - EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, sourceIndex, "field1", enrichFields, ""); + EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, List.of(sourceIndex), "field1", enrichFields, ""); String policyName = "test1"; ActionListener listener = new ActionListener() { @@ -202,7 +202,8 @@ public void testRunnerMultiSource() throws Exception { enrichFields.add("idx"); enrichFields.add("field2"); enrichFields.add("field5"); - EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, sourceIndexPattern, "field1", enrichFields, ""); + EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, List.of(sourceIndexPattern), "field1", + enrichFields, ""); String policyName = "test1"; ActionListener listener = new ActionListener() { diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyTests.java index 72b5cec3be0f9..1e48330e84946 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyTests.java @@ -59,7 +59,7 @@ public static EnrichPolicy randomEnrichPolicy(XContentType xContentType) { return new EnrichPolicy( randomFrom(EnrichPolicy.SUPPORTED_POLICY_TYPES), randomBoolean() ? querySource : null, - randomAlphaOfLength(4), + Arrays.asList(generateRandomStringArray(8, 4, false, false)), randomAlphaOfLength(4), Arrays.asList(generateRandomStringArray(8, 4, false, false)), randomAlphaOfLength(4) @@ -89,7 +89,7 @@ public static void assertEqualPolicies(EnrichPolicy expectedInstance, EnrichPoli } else { assertThat(expectedInstance.getQuery(), nullValue()); } - assertThat(newInstance.getIndexPattern(), equalTo(expectedInstance.getIndexPattern())); + assertThat(newInstance.getIndices(), equalTo(expectedInstance.getIndices())); assertThat(newInstance.getEnrichKey(), equalTo(expectedInstance.getEnrichKey())); assertThat(newInstance.getEnrichValues(), equalTo(expectedInstance.getEnrichValues())); assertThat(newInstance.getSchedule(), equalTo(expectedInstance.getSchedule())); diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java index fed7bfe0309b8..583305ab75ac3 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java @@ -29,7 +29,8 @@ public class EnrichProcessorFactoryTests extends ESTestCase { public void testCreateProcessorInstance() throws Exception { List enrichValues = List.of("globalRank", "tldRank", "tld"); - EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, "source_index", "my_key", enrichValues, "schedule"); + EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, List.of("source_index"), "my_key", + enrichValues, "schedule"); EnrichProcessorFactory factory = new EnrichProcessorFactory(createClusterStateSupplier("majestic", policy), null); Map config = new HashMap<>(); @@ -96,7 +97,8 @@ public void testPolicyDoesNotExist() { public void testPolicyNameMissing() { List enrichValues = List.of("globalRank", "tldRank", "tld"); - EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, "source_index", "my_key", enrichValues, "schedule"); + EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, List.of("source_index"), "my_key", + enrichValues, "schedule"); EnrichProcessorFactory factory = new EnrichProcessorFactory(createClusterStateSupplier("_name", policy), null); Map config = new HashMap<>(); @@ -124,7 +126,7 @@ public void testPolicyNameMissing() { public void testUnsupportedPolicy() { List enrichValues = List.of("globalRank", "tldRank", "tld"); - EnrichPolicy policy = new EnrichPolicy("unsupported", null, "source_index", "my_key", enrichValues, "schedule"); + EnrichPolicy policy = new EnrichPolicy("unsupported", null, List.of("source_index"), "my_key", enrichValues, "schedule"); EnrichProcessorFactory factory = new EnrichProcessorFactory(createClusterStateSupplier("majestic", policy), null); Map config = new HashMap<>(); @@ -153,7 +155,8 @@ public void testUnsupportedPolicy() { public void testNonExistingDecorateField() throws Exception { List enrichValues = List.of("globalRank", "tldRank", "tld"); - EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, "source_index", "my_key", enrichValues, "schedule"); + EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, List.of("source_index"), "my_key", + enrichValues, "schedule"); EnrichProcessorFactory factory = new EnrichProcessorFactory(createClusterStateSupplier("majestic", policy), null); Map config = new HashMap<>(); diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/ExactMatchProcessorTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/ExactMatchProcessorTests.java index a1c1fb5013564..1503f6bbccfa6 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/ExactMatchProcessorTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/ExactMatchProcessorTests.java @@ -51,7 +51,8 @@ public void testBasics() throws Exception { indexWriter.addDocument(createEnrichDocument("eops.nl", Map.of("globalRank", 4567, "tldRank", 80, "tld", "nl"))); indexWriter.commit(); - EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, "majestic_index", "key", List.of(), "schedule"); + EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, List.of("majestic_index"), "key", + List.of(), "schedule"); Function policyLookup = policyName -> policy; try (IndexReader indexReader = DirectoryReader.open(directory)) { @@ -98,7 +99,8 @@ public void testNoMatch() throws Exception { indexWriter.addDocument(createEnrichDocument("eops.nl", Map.of("globalRank", 4567, "tldRank", 80, "tld", "nl"))); indexWriter.commit(); - EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, "majestic_index", "key", List.of(), "schedule"); + EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, List.of("majestic_index"), "key", + List.of(), "schedule"); Function policyLookup = policyName -> policy; try (IndexReader indexReader = DirectoryReader.open(directory)) { @@ -129,7 +131,8 @@ public void testMoreThanOneSegment() throws Exception { indexWriter.addDocument(createEnrichDocument("eops.nl", Map.of("globalRank", 4567, "tldRank", 80, "tld", "nl"))); indexWriter.commit(); - EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, "majestic_index", "key", List.of(), "schedule"); + EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, List.of("majestic_index"), "key", + List.of(), "schedule"); Function policyLookup = policyName -> policy; try (IndexReader indexReader = DirectoryReader.open(directory)) { @@ -156,7 +159,8 @@ public void testEmptyIndex() throws Exception { try (IndexWriter indexWriter = new IndexWriter(directory, iwConfig)) { indexWriter.commit(); - EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, "majestic_index", "key", List.of(), "schedule"); + EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, List.of("majestic_index"), "key", + List.of(), "schedule"); Function policyLookup = policyName -> policy; try (IndexReader indexReader = DirectoryReader.open(directory)) { @@ -187,7 +191,8 @@ public void testEnrichKeyFieldMissing() throws Exception { indexWriter.addDocument(document); indexWriter.commit(); - EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, "majestic_index", "key", List.of(), "schedule"); + EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, List.of("majestic_index"), "key", + List.of(), "schedule"); Function policyLookup = policyName -> policy; try (IndexReader indexReader = DirectoryReader.open(directory)) { @@ -216,7 +221,8 @@ public void testPolicyMissing() { } public void testIgnoreKeyMissing() throws Exception { - EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, "majestic_index", "key", List.of(), "schedule"); + EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, List.of("majestic_index"), "key", List.of(), + "schedule"); Function policyLookup = policyName -> policy; { ExactMatchProcessor processor = new ExactMatchProcessor("_tag", policyLookup, indexExpression -> null, "_name", "domain",