Skip to content

Add enrich policy list API #41553

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
May 1, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentHelper;
Expand Down Expand Up @@ -53,16 +54,20 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment {
);

static {
PARSER.declareString(ConstructingObjectParser.constructorArg(), TYPE);
PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> {
declareParserOptions(PARSER);
}

private static void declareParserOptions(ConstructingObjectParser parser) {
parser.declareString(ConstructingObjectParser.constructorArg(), TYPE);
parser.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> {
XContentBuilder contentBuilder = XContentBuilder.builder(p.contentType().xContent());
contentBuilder.generator().copyCurrentStructure(p);
return new QuerySource(BytesReference.bytes(contentBuilder), contentBuilder.contentType());
}, QUERY);
PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_PATTERN);
PARSER.declareString(ConstructingObjectParser.constructorArg(), ENRICH_KEY);
PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), ENRICH_VALUES);
PARSER.declareString(ConstructingObjectParser.constructorArg(), SCHEDULE);
parser.declareString(ConstructingObjectParser.constructorArg(), INDEX_PATTERN);
parser.declareString(ConstructingObjectParser.constructorArg(), ENRICH_KEY);
parser.declareStringArray(ConstructingObjectParser.constructorArg(), ENRICH_VALUES);
parser.declareString(ConstructingObjectParser.constructorArg(), SCHEDULE);
}

public static EnrichPolicy fromXContent(XContentParser parser) throws IOException {
Expand Down Expand Up @@ -223,4 +228,84 @@ public int hashCode() {
return Objects.hash(query, contentType);
}
}

public static class NamedPolicy implements Writeable, ToXContent {

static final ParseField NAME = new ParseField("name");
@SuppressWarnings("unchecked")
static final ConstructingObjectParser<NamedPolicy, Void> PARSER = new ConstructingObjectParser<>("named_policy",
args -> {
return new NamedPolicy(
(String) args[0],
new EnrichPolicy((String) args[1],
(QuerySource) args[2],
(String) args[3],
(String) args[4],
(List<String>) args[5],
(String) args[6])
);
}
);

static {
PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME);
declareParserOptions(PARSER);
}

private final String name;
private final EnrichPolicy policy;

public NamedPolicy(String name, EnrichPolicy policy) {
this.name = name;
this.policy = policy;
}

public NamedPolicy(StreamInput in) throws IOException {
name = in.readString();
policy = new EnrichPolicy(in);
}

public String getName() {
return name;
}

public EnrichPolicy getPolicy() {
return policy;
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(name);
policy.writeTo(out);
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
{
builder.field(NAME.getPreferredName(), name);
policy.toXContent(builder, params);
}
builder.endObject();
return builder;
}

public static NamedPolicy fromXContent(XContentParser parser) throws IOException {
return PARSER.parse(parser, null);
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
NamedPolicy that = (NamedPolicy) o;
return name.equals(that.name) &&
policy.equals(that.policy);
}

@Override
public int hashCode() {
return Objects.hash(name, policy);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.core.enrich.action;

import org.elasticsearch.action.Action;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.action.support.master.MasterNodeRequest;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.xpack.core.enrich.EnrichPolicy;

import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.TreeMap;
import java.util.stream.Collectors;

public class ListEnrichPolicyAction extends Action<ListEnrichPolicyAction.Response> {

public static final ListEnrichPolicyAction INSTANCE = new ListEnrichPolicyAction();
public static final String NAME = "cluster:admin/xpack/enrich/list";

private ListEnrichPolicyAction() {
super(NAME);
}

@Override
public Response newResponse() {
throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
}
public static class Request extends MasterNodeRequest<ListEnrichPolicyAction.Request> {

public Request() {}

public Request(StreamInput in) throws IOException {
super(in);
}

@Override
public ActionRequestValidationException validate() {
return null;
}
}

public static class Response extends ActionResponse implements ToXContentObject {

private final List<EnrichPolicy.NamedPolicy> policies;

public Response(Map<String, EnrichPolicy> policies) {
Objects.requireNonNull(policies, "policies cannot be null");
// use a treemap to guarantee ordering in the set, then transform it to the list of named policies
this.policies = new TreeMap<>(policies).entrySet().stream()
.map(entry -> new EnrichPolicy.NamedPolicy(entry.getKey(), entry.getValue())).collect(Collectors.toList());
}

public Response(StreamInput in) throws IOException {
policies = in.readList(EnrichPolicy.NamedPolicy::new);
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeList(policies);
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
{
builder.startArray("policies");
{
for (EnrichPolicy.NamedPolicy policy: policies) {
policy.toXContent(builder, params);
}
}
builder.endArray();
}
builder.endObject();

return builder;
}

public List<EnrichPolicy.NamedPolicy> getPolicies() {
return policies;
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Response response = (Response) o;
return policies.equals(response.policies);
}

@Override
public int hashCode() {
return Objects.hash(policies);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ public class PutEnrichPolicyAction extends Action<AcknowledgedResponse> {
public static final PutEnrichPolicyAction INSTANCE = new PutEnrichPolicyAction();
public static final String NAME = "cluster:admin/xpack/enrich/put";

protected PutEnrichPolicyAction() {
private PutEnrichPolicyAction() {
super(NAME);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,14 @@
enrich_key: baz
enrich_values: ["a", "b"]
schedule: "*/120"

- is_true: acknowledged

- do:
enrich.list_policy: {}
- length: { policies: 1 }
- match: { policies.0.name: policy-crud }
- match: { policies.0.type: exact_match }
- match: { policies.0.index_pattern: "bar*" }
- match: { policies.0.enrich_key: baz }
- match: { policies.0.enrich_values: ["a", "b"] }
- match: { policies.0.schedule: "*/120" }
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,11 @@
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.rest.RestController;
import org.elasticsearch.rest.RestHandler;
import org.elasticsearch.xpack.core.enrich.action.ListEnrichPolicyAction;
import org.elasticsearch.xpack.core.enrich.action.PutEnrichPolicyAction;
import org.elasticsearch.xpack.enrich.action.TransportListEnrichPolicyAction;
import org.elasticsearch.xpack.enrich.action.TransportPutEnrichPolicyAction;
import org.elasticsearch.xpack.enrich.rest.RestListEnrichPolicyAction;
import org.elasticsearch.xpack.enrich.rest.RestPutEnrichPolicyAction;

import java.util.Arrays;
Expand Down Expand Up @@ -57,6 +60,7 @@ public Map<String, Processor.Factory> getProcessors(Processor.Parameters paramet
}

return Arrays.asList(
new ActionHandler<>(ListEnrichPolicyAction.INSTANCE, TransportListEnrichPolicyAction.class),
new ActionHandler<>(PutEnrichPolicyAction.INSTANCE, TransportPutEnrichPolicyAction.class)
);
}
Expand All @@ -70,6 +74,7 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
}

return Arrays.asList(
new RestListEnrichPolicyAction(settings, restController),
new RestPutEnrichPolicyAction(settings, restController)
);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,13 @@ public static EnrichPolicy getPolicy(String name, ClusterState state) {
return getPolicies(state).get(name);
}

private static Map<String, EnrichPolicy> getPolicies(ClusterState state) {
/**
* Gets all policies in the cluster.
*
* @param state the cluster state
* @return a Map of <code>policyName, EnrichPolicy</code> of the policies
*/
public static Map<String, EnrichPolicy> getPolicies(ClusterState state) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe also add a unit test for this? (now that it is public)

final Map<String, EnrichPolicy> policies;
final EnrichMetadata enrichMetadata = state.metaData().custom(EnrichMetadata.TYPE);
if (enrichMetadata != null) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.enrich.action;

import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.master.TransportMasterNodeAction;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.block.ClusterBlockException;
import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xpack.core.enrich.EnrichPolicy;
import org.elasticsearch.xpack.core.enrich.action.ListEnrichPolicyAction;
import org.elasticsearch.xpack.enrich.EnrichStore;

import java.util.Map;

public class TransportListEnrichPolicyAction
extends TransportMasterNodeAction<ListEnrichPolicyAction.Request, ListEnrichPolicyAction.Response> {

@Inject
public TransportListEnrichPolicyAction(TransportService transportService,
ClusterService clusterService,
ThreadPool threadPool,
ActionFilters actionFilters,
IndexNameExpressionResolver indexNameExpressionResolver) {
super(ListEnrichPolicyAction.NAME, transportService, clusterService, threadPool, actionFilters,
ListEnrichPolicyAction.Request::new, indexNameExpressionResolver);
}

@Override
protected String executor() {
return ThreadPool.Names.SAME;
}

@Override
protected ListEnrichPolicyAction.Response newResponse() {
throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
}

@Override
protected void masterOperation(ListEnrichPolicyAction.Request request, ClusterState state,
ActionListener<ListEnrichPolicyAction.Response> listener) throws Exception {
Map<String, EnrichPolicy> policies = EnrichStore.getPolicies(clusterService.state());
listener.onResponse(new ListEnrichPolicyAction.Response(policies));
}

@Override
protected ClusterBlockException checkBlock(ListEnrichPolicyAction.Request request, ClusterState state) {
return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
}


}
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.enrich.rest;

import org.elasticsearch.client.node.NodeClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.rest.BaseRestHandler;
import org.elasticsearch.rest.RestController;
import org.elasticsearch.rest.RestRequest;
import org.elasticsearch.rest.action.RestToXContentListener;
import org.elasticsearch.xpack.core.enrich.action.ListEnrichPolicyAction;

import java.io.IOException;

public class RestListEnrichPolicyAction extends BaseRestHandler {

public RestListEnrichPolicyAction(final Settings settings, final RestController controller) {
super(settings);
controller.registerHandler(RestRequest.Method.GET, "/_enrich/policy", this);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

plural? /_enrich/policies?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if w go with this then i suggest we change all of the URIs, which i like, but I assumed we did not want to go against for now. This is how I view it. You are enacting on a single or multiple policies at any time. So, /policies gives you the list, and /policies/foo gives your single foo policy back, but the resource is the same, you are just saying "give me a singlular one of these policies". Otherwise I think we should keep it the same as policy across the board. I prefer the former, but its possible we dont want to go that route until more actual API and versioning conversations happen.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok :) Lets leave this for now and decide whether we want this in a later stage of development.

Btw I just check get mappings api (returns a hash and not a list) and there both singular and plural is supported. But it looks like most apis don't mix plural with singular nouns when it comes returning a single item or multiple (get pipeline, get repository, get tasks etc). Anyhow, language wise (i'm not a native speaker) plural made sense to me.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

IMHO plural makes sesne for all cases. It reads easier to me, a list of policies GET /_enrich/policies, a single policy from the list of policies GET /_enrich/policies/foo

}

@Override
public String getName() {
return "list_enrich_policy";
}

@Override
protected RestChannelConsumer prepareRequest(final RestRequest restRequest, final NodeClient client) throws IOException {
final ListEnrichPolicyAction.Request request = new ListEnrichPolicyAction.Request();
return channel -> client.execute(ListEnrichPolicyAction.INSTANCE, request, new RestToXContentListener<>(channel));
}
}
Loading