Remove types from Monitoring plugin "backend" code (#37745)

This PR removes the use of document types from the monitoring exporters and template + watches setup code.

It does not remove the notion of types from the monitoring bulk API endpoint "front end" code as that code will eventually just go away in 8.0 and be replaced with Beats as collectors/shippers directly to the monitoring cluster.

Author: ycombinator

x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExportBulk.java

@@ -143,7 +143,6 @@ private byte[] toBulkBytes(final MonitoringDoc doc) throws IOException {
                     builder.startObject("index");
                     {
                         builder.field("_index", index);
-                        builder.field("_type", "doc");
                         if (id != null) {
                             builder.field("_id", id);
                         }
@@ -163,7 +162,10 @@ private byte[] toBulkBytes(final MonitoringDoc doc) throws IOException {
             // Adds final bulk separator
             out.write(xContent.streamSeparator());
 
-            logger.trace("added index request [index={}, type={}, id={}]", index, doc.getType(), id);
+            logger.trace(
+                "http exporter [{}] - added index request [index={}, id={}, monitoring data type={}]",
+                name, index, id, doc.getType()
+            );
 
             return BytesReference.toBytes(out.bytes());
         } catch (Exception e) {

x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalBulk.java

@@ -66,7 +66,7 @@ public void doAdd(Collection<MonitoringDoc> docs) throws ExportException {
             try {
                 final String index = MonitoringTemplateUtils.indexName(formatter, doc.getSystem(), doc.getTimestamp());
 
-                final IndexRequest request = new IndexRequest(index, "doc");
+                final IndexRequest request = new IndexRequest(index);
                 if (Strings.hasText(doc.getId())) {
                     request.id(doc.getId());
                 }
@@ -82,8 +82,8 @@ public void doAdd(Collection<MonitoringDoc> docs) throws ExportException {
                 requestBuilder.add(request);
 
                 if (logger.isTraceEnabled()) {
-                    logger.trace("local exporter [{}] - added index request [index={}, type={}, id={}, pipeline={}]",
-                                 name, request.index(), request.type(), request.id(), request.getPipeline());
+                    logger.trace("local exporter [{}] - added index request [index={}, id={}, pipeline={}, monitoring data type={}]",
+                                 name, request.index(), request.id(), request.getPipeline(), doc.getType());
                 }
             } catch (Exception e) {
                 if (exception == null) {

x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_cluster_status.json

@@ -152,7 +152,6 @@
     "add_to_alerts_index": {
       "index": {
         "index": ".monitoring-alerts-6",
-        "doc_type": "doc",
         "doc_id": "${monitoring.watch.unique_id}"
       }
     },

x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_nodes.json

@@ -157,8 +157,7 @@
   "actions": {
     "add_to_alerts_index": {
       "index": {
-        "index": ".monitoring-alerts-6",
-        "doc_type": "doc"
+        "index": ".monitoring-alerts-6"
       }
     },
     "send_email_to_admin": {

x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_version_mismatch.json

@@ -148,7 +148,6 @@
     "add_to_alerts_index": {
       "index": {
         "index": ".monitoring-alerts-6",
-        "doc_type": "doc",
         "doc_id": "${monitoring.watch.unique_id}"
       }
     },

x-pack/plugin/monitoring/src/main/resources/monitoring/watches/kibana_version_mismatch.json

@@ -168,7 +168,6 @@
     "add_to_alerts_index": {
       "index": {
         "index": ".monitoring-alerts-6",
-        "doc_type": "doc",
         "doc_id": "${monitoring.watch.unique_id}"
       }
     },

x-pack/plugin/monitoring/src/main/resources/monitoring/watches/logstash_version_mismatch.json

@@ -168,7 +168,6 @@
     "add_to_alerts_index": {
       "index": {
         "index": ".monitoring-alerts-6",
-        "doc_type": "doc",
         "doc_id": "${monitoring.watch.unique_id}"
       }
     },

x-pack/plugin/monitoring/src/main/resources/monitoring/watches/xpack_license_expiration.json

@@ -141,7 +141,6 @@
     "add_to_alerts_index": {
       "index": {
         "index": ".monitoring-alerts-6",
-        "doc_type": "doc",
         "doc_id": "${monitoring.watch.unique_id}"
       }
     },

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringBulkRequestTests.java

@@ -232,7 +232,6 @@ public void testAddRequestContentWithUnrecognizedIndexName() throws IOException
         assertThat(e.getMessage(), containsString("unrecognized index name [" + indexName + "]"));
         //This test's JSON contains outdated references to types
         assertWarnings(RestBulkAction.TYPES_DEPRECATION_MESSAGE);
-        
     }
 
     public void testSerialization() throws IOException {

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java

@@ -275,7 +275,6 @@ private void checkMonitoringDocs() {
             assertTrue("document is missing cluster_uuid field", Strings.hasText((String) source.get("cluster_uuid")));
             assertTrue("document is missing timestamp field", Strings.hasText(timestamp));
             assertTrue("document is missing type field", Strings.hasText(type));
-            assertEquals("document _type is 'doc'", "doc", hit.getType());
 
             @SuppressWarnings("unchecked")
             Map<String, Object> docSource = (Map<String, Object>) source.get("doc");

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java

@@ -81,6 +81,7 @@ private static BytesReference generateTemplateSource(final String name, final In
                 .field("index.number_of_replicas", 0)
             .endObject()
             .startObject("mappings")
+                // Still need use type, RestPutIndexTemplateAction#prepareRequestSource has logic that adds type if missing
                 .startObject("doc")
                     .startObject("_meta")
                         .field("test", true)

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java

@@ -27,7 +27,6 @@
 import org.elasticsearch.license.License;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.rest.action.document.RestBulkAction;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.collapse.CollapseBuilder;
@@ -105,11 +104,11 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
     }
 
     private String createBulkEntity() {
-        return "{\"index\":{\"_type\":\"test\"}}\n" +
+        return "{\"index\":{}}\n" +
                "{\"foo\":{\"bar\":0}}\n" +
-               "{\"index\":{\"_type\":\"test\"}}\n" +
+               "{\"index\":{}}\n" +
                "{\"foo\":{\"bar\":1}}\n" +
-               "{\"index\":{\"_type\":\"test\"}}\n" +
+               "{\"index\":{}}\n" +
                "{\"foo\":{\"bar\":2}}\n" +
                "\n";
     }
@@ -128,7 +127,7 @@ public void testMonitoringBulk() throws Exception {
 
             final MonitoringBulkResponse bulkResponse =
                     new MonitoringBulkRequestBuilder(client())
-                            .add(system, null, new BytesArray(createBulkEntity().getBytes("UTF-8")), XContentType.JSON,
+                            .add(system, "monitoring_data_type", new BytesArray(createBulkEntity().getBytes("UTF-8")), XContentType.JSON,
                                  System.currentTimeMillis(), interval.millis())
                     .get();
 
@@ -179,10 +178,9 @@ public void testMonitoringBulk() throws Exception {
                        equalTo(1L));
 
             for (final SearchHit hit : hits.getHits()) {
-                assertMonitoringDoc(toMap(hit), system, "test", interval);
+                assertMonitoringDoc(toMap(hit), system, "monitoring_data_type", interval);
             }
         });
-        assertWarnings(RestBulkAction.TYPES_DEPRECATION_MESSAGE);
     }
 
     /**
@@ -265,7 +263,6 @@ private void assertMonitoringDoc(final Map<String, Object> document,
 
         final String index = (String) document.get("_index");
         assertThat(index, containsString(".monitoring-" + expectedSystem.getSystem() + "-" + TEMPLATE_VERSION + "-"));
-        assertThat(document.get("_type"), equalTo("doc"));
         assertThat((String) document.get("_id"), not(isEmptyOrNullString()));
 
         final Map<String, Object> source = (Map<String, Object>) document.get("_source");