Fix varying responses for <index>/_analyze request

Currently we loose information about whether a token list in an AnalyzeAction
response is null or an empty list, because we write a 0 value to the stream in
both cases and deserialize to a null value on the receiving side. This was fixed
in elastic#44284 by a change in the serialization protocol starting in 7.3. However
this PR fixes the symptoms without changing the wire protocol which we cannot to
easily on 6.8 because we already released incompatible versions in the 7.x line.
This change adds special handling on xcontent output and if getToken() is
callled on either AnalyzeResponse or DetailedAnalyzeResponse to always return
empty lists instead of null values.

Relates to elastic#44078

Author: Christoph Büscher

server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java

@@ -32,6 +32,7 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -233,7 +234,7 @@ public AnalyzeResponse(List<AnalyzeToken> tokens, DetailAnalyzeResponse detail)
     }
 
     public List<AnalyzeToken> getTokens() {
-        return this.tokens;
+        return this.tokens != null ? this.tokens : Collections.emptyList();
     }
 
     public DetailAnalyzeResponse detail() {
@@ -254,6 +255,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 token.toXContent(builder, params);
             }
             builder.endArray();
+        } else if (detail == null) {
+            // at least write an empty list
+            builder.startArray(Fields.TOKENS);
+            builder.endArray();
         }
 
         if (detail != null) {

server/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.action.admin.indices.analyze;
 
 
+import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse.AnalyzeToken;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -284,7 +285,7 @@ public String getName() {
         }
 
         public AnalyzeResponse.AnalyzeToken[] getTokens() {
-            return tokens;
+            return tokens != null ? tokens : new AnalyzeToken[0];
         }
 
         public static AnalyzeTokenList readAnalyzeTokenList(StreamInput in) throws IOException {
@@ -296,8 +297,10 @@ public static AnalyzeTokenList readAnalyzeTokenList(StreamInput in) throws IOExc
         XContentBuilder toXContentWithoutObject(XContentBuilder builder, Params params) throws IOException {
             builder.field(Fields.NAME, this.name);
             builder.startArray(AnalyzeResponse.Fields.TOKENS);
-            for (AnalyzeResponse.AnalyzeToken token : tokens) {
-                token.toXContent(builder, params);
+            if (tokens != null) {
+                for (AnalyzeResponse.AnalyzeToken token : tokens) {
+                    token.toXContent(builder, params);
+                }
             }
             builder.endArray();
             return builder;

server/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java

@@ -21,6 +21,7 @@
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequestBuilder;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -436,5 +437,24 @@ public void testAnalyzeNormalizedKeywordField() throws IOException {
         assertThat(token.getPositionLength(), equalTo(1));
     }
 
+    /**
+     * Input text that doesn't produce tokens should return an empty token list
+     */
+    public void testZeroTokenAnalysis() throws IOException {
+        assertAcked(prepareCreate("test"));
+        ensureGreen("test");
+
+        AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("test", ".").get();
+        assertNotNull(analyzeResponse.getTokens());
+        assertThat(analyzeResponse.getTokens().size(), equalTo(0));
+        assertEquals("{\"tokens\":[]}", Strings.toString(analyzeResponse));
+
+        // also check detailed response
+        analyzeResponse = client().admin().indices().prepareAnalyze("test", ".").setExplain(true).get();
+        assertNotNull(analyzeResponse.detail().analyzer().getTokens());
+        assertThat(analyzeResponse.getTokens().size(), equalTo(0));
+        assertEquals("{\"detail\":{\"custom_analyzer\":false,\"analyzer\":{\"name\":\"default\",\"tokens\":[]}}}",
+                Strings.toString(analyzeResponse));
+    }
 
 }