[Transform] add new exclude_generated flag to GET transform (#63093)

This adds a new flag `exclude_generated` for GET transform API.

This flag is useful for when a transform needs to be cloned within a cluster or exported/imported between clusters.

It removes certain fields that are not able to be set via the PUT api (e.g. version, create_time).

relates #63055

Author: benwtrent

client/rest-high-level/src/main/java/org/elasticsearch/client/TransformRequestConverters.java

@@ -40,6 +40,7 @@
 import static org.elasticsearch.client.RequestConverters.createEntity;
 import static org.elasticsearch.client.transform.DeleteTransformRequest.FORCE;
 import static org.elasticsearch.client.transform.GetTransformRequest.ALLOW_NO_MATCH;
+import static org.elasticsearch.client.transform.GetTransformRequest.EXCLUDE_GENERATED;
 import static org.elasticsearch.client.transform.PutTransformRequest.DEFER_VALIDATION;
 import static org.elasticsearch.client.transform.StopTransformRequest.WAIT_FOR_CHECKPOINT;
 
@@ -89,6 +90,9 @@ static Request getTransform(GetTransformRequest getRequest) {
         if (getRequest.getAllowNoMatch() != null) {
             request.addParameter(ALLOW_NO_MATCH, getRequest.getAllowNoMatch().toString());
         }
+        if (getRequest.getExcludeGenerated() != null) {
+            request.addParameter(EXCLUDE_GENERATED, getRequest.getExcludeGenerated().toString());
+        }
         return request;
     }
 

client/rest-high-level/src/main/java/org/elasticsearch/client/transform/GetTransformRequest.java

@@ -30,6 +30,7 @@
 
 public class GetTransformRequest implements Validatable {
 
+    public static final String EXCLUDE_GENERATED = "exclude_generated";
     public static final String ALLOW_NO_MATCH = "allow_no_match";
     /**
      * Helper method to create a request that will get ALL Transforms
@@ -42,6 +43,7 @@ public static GetTransformRequest getAllTransformRequest() {
     private final List<String> ids;
     private PageParams pageParams;
     private Boolean allowNoMatch;
+    private Boolean excludeGenerated;
 
     public GetTransformRequest(String... ids) {
         this.ids = Arrays.asList(ids);
@@ -67,6 +69,14 @@ public void setAllowNoMatch(Boolean allowNoMatch) {
         this.allowNoMatch = allowNoMatch;
     }
 
+    public void setExcludeGenerated(boolean excludeGenerated) {
+        this.excludeGenerated = excludeGenerated;
+    }
+
+    public Boolean getExcludeGenerated() {
+        return excludeGenerated;
+    }
+
     @Override
     public Optional<ValidationException> validate() {
         if (ids == null || ids.isEmpty()) {
@@ -80,7 +90,7 @@ public Optional<ValidationException> validate() {
 
     @Override
     public int hashCode() {
-        return Objects.hash(ids, pageParams, allowNoMatch);
+        return Objects.hash(ids, pageParams, excludeGenerated, allowNoMatch);
     }
 
     @Override
@@ -95,6 +105,7 @@ public boolean equals(Object obj) {
         GetTransformRequest other = (GetTransformRequest) obj;
         return Objects.equals(ids, other.ids)
             && Objects.equals(pageParams, other.pageParams)
+            && Objects.equals(excludeGenerated, other.excludeGenerated)
             && Objects.equals(allowNoMatch, other.allowNoMatch);
     }
 }

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/TransformDocumentationIT.java

@@ -687,6 +687,7 @@ public void testGetDataFrameTransform() throws IOException, InterruptedException
             // tag::get-transform-request-options
             request.setPageParams(new PageParams(0, 100)); // <1>
             request.setAllowNoMatch(true); // <2>
+            request.setExcludeGenerated(false); // <3>
             // end::get-transform-request-options
 
             // tag::get-transform-execute

docs/java-rest/high-level/transform/get_transform.asciidoc

@@ -34,6 +34,9 @@ include-tagged::{doc-tests-file}[{api}-request-options]
 {transforms} to skip. `size` specifies the maximum number of
 {transforms} to get. Defaults to `0` and `100` respectively.
 <2> Whether to ignore if a wildcard expression matches no {transforms}.
+<3> Optional boolean value for requesting the {transform} in a format that can
+then be put into another cluster. Certain fields that can only be set when
+the {transform} is created are removed.
 
 
 include::../execution.asciidoc[]

docs/reference/transform/apis/get-transform.asciidoc

@@ -26,12 +26,12 @@ Retrieves configuration information for {transforms}.
 [[get-transform-prereqs]]
 == {api-prereq-title}
 
-If the {es} {security-features} are enabled, you must have the following 
+If the {es} {security-features} are enabled, you must have the following
 privileges:
 
-* `monitor_transform` 
+* `monitor_transform`
 
-The built-in `transform_user` role has this privilege. 
+The built-in `transform_user` role has this privilege.
 
 For more information, see <<security-privileges>> and <<built-in-roles>>.
 
@@ -49,7 +49,7 @@ specifying `*` as the `<transform_id>`, or by omitting the `<transform_id>`.
 `<transform_id>`::
 (Optional, string)
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-id-wildcard]
-  
+
 [[get-transform-query-parms]]
 == {api-query-parms-title}
 
@@ -65,6 +65,12 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=from-transforms]
 (Optional, integer)
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=size-transforms]
 
+`exclude_generated`::
+(Optional, boolean)
+Excludes fields that were automatically added when creating the transform.
+This allows the configuration to be in an acceptable format to be retrieved
+and then added to another cluster. Default is false.
+
 [[get-transform-response]]
 == {api-response-body-title}
 
@@ -79,13 +85,13 @@ This property is informational; you cannot change its value.
 `version`::
 (string) The version of {es} that existed on the node when the {transform} was
 created.
-  
+
 [[get-transform-response-codes]]
 == {api-response-codes-title}
 
 `404` (Missing resources)::
   If `allow_no_match` is `false`, this code indicates that there are no
-  resources that match the request or only partial matches for the request.  
+  resources that match the request or only partial matches for the request.
 
 [[get-transform-example]]
 == {api-examples-title}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java

@@ -84,6 +84,7 @@ public final class TransformField {
      * API's)
      */
     public static final String FOR_INTERNAL_STORAGE = "for_internal_storage";
+    public static final String EXCLUDE_GENERATED = "exclude_generated";
 
     // internal document id
     public static String DOCUMENT_ID_FIELD = "_id";

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfig.java

@@ -16,6 +16,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.license.RemoteClusterLicenseChecker;
+import org.elasticsearch.xpack.core.transform.TransformField;
 import org.elasticsearch.xpack.core.transform.utils.ExceptionsHelper;
 
 import java.io.IOException;
@@ -113,7 +114,11 @@ public void writeTo(StreamOutput out) throws IOException {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         builder.array(INDEX.getPreferredName(), index);
-        builder.field(QUERY.getPreferredName(), queryConfig);
+        if (params.paramAsBoolean(TransformField.EXCLUDE_GENERATED, false) == false) {
+            builder.field(QUERY.getPreferredName(), queryConfig);
+        } else if(queryConfig.equals(QueryConfig.matchAll()) == false) {
+            builder.field(QUERY.getPreferredName(), queryConfig);
+        }
         builder.endObject();
         return builder;
     }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java

@@ -342,9 +342,31 @@ public void writeTo(final StreamOutput out) throws IOException {
 
     @Override
     public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
+        final boolean excludeGenerated = params.paramAsBoolean(TransformField.EXCLUDE_GENERATED, false);
+        final boolean forInternalStorage = params.paramAsBoolean(TransformField.FOR_INTERNAL_STORAGE, false);
+        assert (forInternalStorage && excludeGenerated) == false:
+            "unsupported behavior, exclude_generated is true and for_internal_storage is true";
         builder.startObject();
         builder.field(TransformField.ID.getPreferredName(), id);
-        builder.field(TransformField.SOURCE.getPreferredName(), source);
+        if (excludeGenerated == false) {
+            if (headers.isEmpty() == false && forInternalStorage) {
+                builder.field(HEADERS.getPreferredName(), headers);
+            }
+            if (transformVersion != null) {
+                builder.field(TransformField.VERSION.getPreferredName(), transformVersion);
+            }
+            if (createTime != null) {
+                builder.timeField(
+                    TransformField.CREATE_TIME.getPreferredName(),
+                    TransformField.CREATE_TIME.getPreferredName() + "_string",
+                    createTime.toEpochMilli()
+                );
+            }
+            if (forInternalStorage) {
+                builder.field(TransformField.INDEX_DOC_TYPE.getPreferredName(), NAME);
+            }
+        }
+        builder.field(TransformField.SOURCE.getPreferredName(), source, params);
         builder.field(TransformField.DESTINATION.getPreferredName(), dest);
         if (frequency != null) {
             builder.field(TransformField.FREQUENCY.getPreferredName(), frequency.getStringRep());
@@ -357,26 +379,10 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
         if (pivotConfig != null) {
             builder.field(PIVOT_TRANSFORM.getPreferredName(), pivotConfig);
         }
-        if (params.paramAsBoolean(TransformField.FOR_INTERNAL_STORAGE, false)) {
-            builder.field(TransformField.INDEX_DOC_TYPE.getPreferredName(), NAME);
-        }
-        if (headers.isEmpty() == false && params.paramAsBoolean(TransformField.FOR_INTERNAL_STORAGE, false)) {
-            builder.field(HEADERS.getPreferredName(), headers);
-        }
         if (description != null) {
             builder.field(TransformField.DESCRIPTION.getPreferredName(), description);
         }
         builder.field(TransformField.SETTINGS.getPreferredName(), settings);
-        if (transformVersion != null) {
-            builder.field(TransformField.VERSION.getPreferredName(), transformVersion);
-        }
-        if (createTime != null) {
-            builder.timeField(
-                TransformField.CREATE_TIME.getPreferredName(),
-                TransformField.CREATE_TIME.getPreferredName() + "_string",
-                createTime.toEpochMilli()
-            );
-        }
         builder.endObject();
         return builder;
     }

x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame_transform_deprecated.get_transform.json

@@ -50,6 +50,12 @@
         "type":"boolean",
         "required":false,
         "description":"Whether to ignore if a wildcard expression matches no transforms. (This includes `_all` string or when no transforms have been specified)"
+      },
+      "exclude_generated": {
+        "required": false,
+        "type": "boolean",
+        "default": false,
+        "description": "Omits generated fields. Allows transform configurations to be easily copied between clusters and within the same cluster"
       }
     }
   }

x-pack/plugin/src/test/resources/rest-api-spec/api/transform.get_transform.json

@@ -42,6 +42,12 @@
         "type":"boolean",
         "required":false,
         "description":"Whether to ignore if a wildcard expression matches no transforms. (This includes `_all` string or when no transforms have been specified)"
+      },
+      "exclude_generated": {
+        "required": false,
+        "type": "boolean",
+        "default": false,
+        "description": "Omits fields that are illegal to set on transform PUT"
       }
     }
   }

x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_crud.yml

@@ -670,3 +670,34 @@ setup:
             },
             "description": "yaml test transform on airline-data"
           }
+
+---
+"Test transform for export":
+  - do:
+      transform.put_transform:
+        transform_id: "airline-transform"
+        body: >
+          {
+            "source": { "index": "airline-data" },
+            "dest": { "index": "airline-data-by-airline" },
+            "pivot": {
+              "group_by": { "airline": {"terms": {"field": "airline"}}},
+              "aggs": {"avg_response": {"avg": {"field": "responsetime"}}}
+            },
+            "description": "yaml test transform on airline-data"
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      transform.get_transform:
+        transform_id: "airline-transform"
+        exclude_generated: true
+
+  - match: {transforms.0.source.index: ["airline-data"]}
+  - match: {transforms.0.dest.index: "airline-data-by-airline"}
+  - match: {transforms.0.pivot.group_by.airline.terms.field: "airline"}
+  - match: {transforms.0.pivot.aggregations.avg_response.avg.field: "responsetime"}
+  - match: {transforms.0.description: "yaml test transform on airline-data"}
+  - match: {transforms.0.id: "airline-transform"}
+  - is_false: transforms.0.create_time
+  - is_false: transforms.0.version

x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java

@@ -9,6 +9,7 @@
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
 import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -1686,6 +1687,38 @@ public void testPivotWithFilter() throws Exception {
         assertEquals(3, actual.longValue());
     }
 
+    @SuppressWarnings("unchecked")
+    public void testExportAndImport() throws Exception {
+        String transformId = "export-transform";
+        String transformIndex = "export_reviews";
+        setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformIndex);
+
+        createPivotReviewsTransform(transformId, transformIndex, null, null, BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS);
+
+        Response response = adminClient().performRequest(new Request("GET",
+            getTransformEndpoint() + transformId + "?exclude_generated=true"));
+        Map<String, Object> storedConfig = ((List<Map<String, Object>>) XContentMapValues.extractValue(
+            "transforms",
+            entityAsMap(response)))
+            .get(0);
+        storedConfig.remove("id");
+        try (XContentBuilder builder = jsonBuilder()) {
+            builder.map(storedConfig);
+            Request putTransform = new Request("PUT", getTransformEndpoint() + transformId + "-import");
+            putTransform.setJsonEntity(Strings.toString(builder));
+            adminClient().performRequest(putTransform);
+        }
+
+        response = adminClient().performRequest(new Request("GET",
+            getTransformEndpoint() + transformId + "-import" + "?exclude_generated=true"));
+        Map<String, Object> importConfig = ((List<Map<String, Object>>) XContentMapValues.extractValue(
+            "transforms",
+            entityAsMap(response)))
+            .get(0);
+        importConfig.remove("id");
+        assertThat(storedConfig, equalTo(importConfig));
+    }
+
     private void createDateNanoIndex(String indexName, int numDocs) throws IOException {
         // create mapping
         try (XContentBuilder builder = jsonBuilder()) {

x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestGetTransformAction.java

@@ -14,10 +14,13 @@
 import org.elasticsearch.xpack.core.transform.TransformField;
 import org.elasticsearch.xpack.core.transform.action.GetTransformAction;
 
+import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.xpack.core.transform.TransformField.ALLOW_NO_MATCH;
+import static org.elasticsearch.xpack.core.transform.TransformField.EXCLUDE_GENERATED;
 
 public class RestGetTransformAction extends BaseRestHandler {
 
@@ -47,4 +50,9 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient
     public String getName() {
         return "transform_get_transform_action";
     }
+
+    @Override
+    protected Set<String> responseParams() {
+        return Collections.singleton(EXCLUDE_GENERATED);
+    }
 }

x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/compat/RestGetTransformActionDeprecated.java

@@ -18,9 +18,11 @@
 
 import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.xpack.core.transform.TransformField.ALLOW_NO_MATCH;
+import static org.elasticsearch.xpack.core.transform.TransformField.EXCLUDE_GENERATED;
 
 public class RestGetTransformActionDeprecated extends BaseRestHandler {
     @Override
@@ -56,4 +58,9 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient
     public String getName() {
         return "data_frame_get_transforms_action";
     }
+
+    @Override
+    protected Set<String> responseParams() {
+        return Collections.singleton(EXCLUDE_GENERATED);
+    }
 }