Route documents to the correct shards in tsdb

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/FormattingPrecommitPlugin.java

@@ -91,6 +91,9 @@ private Object[] getTargets(String projectPath) {
                 "src/*/java/org/elasticsearch/action/admin/cluster/snapshots/**/*.java",
                 "src/test/java/org/elasticsearch/common/xcontent/support/AbstractFilteringTestCase.java",
                 "src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java",
+                "src/*/java/org/elasticsearch/index/IndexMode.java",
+                "src/*/java/org/elasticsearch/index/IndexRouting.java",
+                "src/*/java/org/elasticsearch/index/TimeSeriesModeTests.java",
                 "src/*/java/org/elasticsearch/index/snapshots/**/*.java",
                 "src/*/java/org/elasticsearch/repositories/**/*.java",
                 "src/*/java/org/elasticsearch/search/aggregations/**/*.java",

qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/100_tsdb.yml

@@ -12,6 +12,7 @@ setup:
           settings:
             index:
               mode: time_series
+              routing_path: [metricset, k8s.pod.uid]
               number_of_replicas: 0
               number_of_shards: 2
           mappings:

qa/multi-cluster-search/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml

@@ -166,6 +166,7 @@ tsdb:
           settings:
             index:
               mode: time_series
+              routing_path: [metricset, k8s.pod.uid]
               number_of_replicas: 0
               number_of_shards: 2
           mappings:

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java

@@ -306,7 +306,7 @@ private void bulk(String index, String entity) throws IOException {
         Request bulk = new Request("POST", "/_bulk");
         bulk.addParameter("refresh", "true");
         bulk.setJsonEntity(entity.toString());
-        client().performRequest(bulk);
+        assertThat(EntityUtils.toString(client().performRequest(bulk).getEntity()), containsString("\"errors\":false"));
     }
 
     private void createTsdbIndex() throws IOException {
@@ -318,7 +318,10 @@ private void createTsdbIndex() throws IOException {
             indexSpec.startObject("dim").field("type", "keyword").field("time_series_dimension", true).endObject();
         }
         indexSpec.endObject().endObject();
-        indexSpec.startObject("settings").field("mode", "time_series").endObject();
+        indexSpec.startObject("settings").startObject("index");
+        indexSpec.field("mode", "time_series");
+        indexSpec.array("routing_path", new String[] {"dim"});
+        indexSpec.endObject().endObject();
         createIndex.setJsonEntity(Strings.toString(indexSpec.endObject()));
         client().performRequest(createIndex);
     }

qa/smoke-test-multinode/src/test/resources/rest-api-spec/test/smoke_test_multinode/20_tsdb_consistency.yml

@@ -0,0 +1,107 @@
+# Test the mode:time_series properly groups by _tsid. If we could put this in
+# rest-api-spec we would, but it requires painless.
+
+setup:
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0
+
+  - do:
+      indices.create:
+          index: test
+          body:
+            settings:
+              index:
+                mode: time_series
+                routing_path: [metricset, k8s.pod.uid]
+                number_of_shards: 10
+                number_of_replicas: 1
+            mappings:
+              properties:
+                "@timestamp":
+                  type: date
+                metricset:
+                  type: keyword
+                  time_series_dimension: true
+                k8s:
+                  properties:
+                    pod:
+                      properties:
+                        uid:
+                          type: keyword
+                          time_series_dimension: true
+                        name:
+                          type: keyword
+                        ip:
+                          type: ip
+                        network:
+                          properties:
+                            tx:
+                              type: long
+                            rx:
+                              type: long
+  - do:
+      bulk:
+        refresh: true
+        index: test
+        body:
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:35:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T19:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2005177954, "rx": 801479970}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T17:53:34.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2006223737, "rx": 802337279}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:03:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.2", "network": {"tx": 2012916202, "rx": 803685721}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:35:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434521831, "rx": 530575198}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T19:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434577921, "rx": 530600088}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T17:53:34.467Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434587694, "rx": 530604797}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:03:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434595272, "rx": 530605511}}}}'
+
+---
+"index with replicas and shards is green":
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0 to be backported to 7.16.0
+
+  - do:
+      cluster.health:
+        wait_for_status: green
+  - match:     { status:     green }
+
+---
+"each shard has unique _tsids":
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0 to be backported to 7.16.0
+
+  - do:
+      search:
+        index: test
+        body:
+          size: 0
+          aggs:
+            check:
+              scripted_metric:
+                init_script: "state.timeSeries = new HashSet()"
+                map_script: "state.timeSeries.add(doc['metricset'].value + ':' + doc['k8s.pod.uid'].value)"
+                combine_script: "return state.timeSeries"
+                reduce_script: |
+                  Set timeSeries = new TreeSet();
+                  for (s in states) {
+                    for (ts in s) {
+                      boolean newTs = timeSeries.add(ts);
+                      if (false == newTs) {
+                        throw new IllegalArgumentException(ts + " appeared in two shards");
+                      }
+                    }
+                  }
+                  return timeSeries;
+  - match: {hits.total.value: 8}
+  - match: {aggregations.check.value: ['pod:947e4ced-1786-4e53-9e0c-5c447e959507', 'pod:df3145b3-0563-4d3b-a0f7-897eb2876ea9']}
+
+# todo clone and shrink

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml

@@ -10,6 +10,7 @@ enable:
             settings:
               index:
                 mode: time_series
+                routing_path: [metricset, k8s.pod.uid]
                 number_of_replicas: 0
                 number_of_shards: 2
             mappings:
@@ -117,3 +118,53 @@ no partitioning:
                 mode: time_series
                 shards: 5
                 routing_partition_size: 2
+
+---
+routing_path required:
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0 to be backported to 7.16.0
+
+  - do:
+      catch: /\[index.mode=time_series\] requires \[index.routing_path\]/
+      indices.create:
+          index: test_index
+          body:
+            settings:
+              index:
+                mode: time_series
+                shards: 5
+
+---
+routing_path is not allowed in standard mode:
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0 to be backported to 7.16.0
+
+  - do:
+      catch: /\[index.routing_path\] requires \[index.mode=time_series\]/
+      indices.create:
+          index: test_index
+          body:
+            settings:
+              index:
+                routing_path: foo
+
+---
+routing required:
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0 to be backported to 7.16.0
+
+  - do:
+      catch: /routing is forbidden on CRUD operations that target indices in \[index.mode=time_series\]/
+      indices.create:
+          index: test_index
+          body:
+            settings:
+              index:
+                mode: time_series
+                routing_path: foo
+            mappings:
+              _routing:
+                required: true

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml

@@ -10,6 +10,7 @@ add time series mappings:
             settings:
               index:
                 mode: time_series
+                routing_path: [metricset, k8s.pod.uid]
                 number_of_replicas: 0
                 number_of_shards: 2
             mappings:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml

@@ -29,6 +29,7 @@ teardown:
             settings:
               index:
                 mode: time_series
+                routing_path: [metricset, k8s.pod.uid]
                 number_of_replicas: 0
                 number_of_shards: 2
             mappings:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml

@@ -10,6 +10,7 @@ setup:
             settings:
               index:
                 mode: time_series
+                routing_path: [metricset, k8s.pod.uid]
             mappings:
               properties:
                 "@timestamp":

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml

@@ -10,6 +10,7 @@ setup:
             settings:
               index:
                 mode: time_series
+                routing_path: [metricset, k8s.pod.uid]
             mappings:
               properties:
                 "@timestamp":

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml

@@ -11,6 +11,7 @@ add dimensions with put_mapping:
             settings:
               index:
                 mode: time_series
+                routing_path: [metricset, k8s.pod.uid]
             mappings:
               properties:
                 "@timestamp":
@@ -60,6 +61,7 @@ add dimensions to no dims with dynamic_template over index:
             settings:
               index:
                 mode: time_series
+                routing_path: [metricset]
             mappings:
               dynamic_templates:
                 - keywords:
@@ -104,6 +106,7 @@ add dimensions to no dims with dynamic_template over bulk:
             settings:
               index:
                 mode: time_series
+                routing_path: [metricset]
             mappings:
               dynamic_templates:
                 - keywords:
@@ -148,6 +151,7 @@ add dimensions to some dims with dynamic_template over index:
             settings:
               index:
                 mode: time_series
+                routing_path: [metricset]
             mappings:
               dynamic_templates:
                 - keywords:
@@ -196,6 +200,7 @@ add dimensions to some dims with dynamic_template over bulk:
             settings:
               index:
                 mode: time_series
+                routing_path: [metricset]
             mappings:
               dynamic_templates:
                 - keywords:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml

@@ -11,6 +11,7 @@ keyword dimension:
             settings:
               index:
                 mode: time_series
+                routing_path: [uid]
             mappings:
               properties:
                 "@timestamp":
@@ -58,10 +59,14 @@ long dimension:
             settings:
               index:
                 mode: time_series
+                routing_path: [metricset]
             mappings:
               properties:
                 "@timestamp":
                   type: date
+                metricset:
+                  type: keyword
+                  time_series_dimension: true
                 id:
                   type: long
                   time_series_dimension: true
@@ -72,21 +77,21 @@ long dimension:
         index: test
         body:
           - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:35:24.467Z", "id": 1, "voltage": 7.2}'
+          - '{"@timestamp": "2021-04-28T18:35:24.467Z", "metricset": "aa", "id": 1, "voltage": 7.2}'
           - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:35:34.467Z", "id": "1", "voltage": 7.6}'
+          - '{"@timestamp": "2021-04-28T18:35:34.467Z", "metricset": "aa", "id": "1", "voltage": 7.6}'
           - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:35:44.467Z", "id": 1.0, "voltage": 7.1}'
+          - '{"@timestamp": "2021-04-28T18:35:44.467Z", "metricset": "aa", "id": 1.0, "voltage": 7.1}'
           - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:35:54.467Z", "id": "001", "voltage": 7.3}'
+          - '{"@timestamp": "2021-04-28T18:35:54.467Z", "metricset": "aa", "id": "001", "voltage": 7.3}'
           - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:35:24.467Z", "id": 2, "voltage": 3.2}'
+          - '{"@timestamp": "2021-04-28T18:35:24.467Z", "metricset": "aa", "id": 2, "voltage": 3.2}'
           - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:35:34.467Z", "id": 2, "voltage": 3.6}'
+          - '{"@timestamp": "2021-04-28T18:35:34.467Z", "metricset": "aa", "id": 2, "voltage": 3.6}'
           - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:35:44.467Z", "id": 2, "voltage": 3.1}'
+          - '{"@timestamp": "2021-04-28T18:35:44.467Z", "metricset": "aa", "id": 2, "voltage": 3.1}'
           - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:35:54.467Z", "id": 2, "voltage": 3.3}'
+          - '{"@timestamp": "2021-04-28T18:35:54.467Z", "metricset": "aa", "id": 2, "voltage": 3.3}'
 
   # TODO aggregate on tsid
 
@@ -104,10 +109,14 @@ ip dimension:
             settings:
               index:
                 mode: time_series
+                routing_path: [metricset]
             mappings:
               properties:
                 "@timestamp":
                   type: date
+                metricset:
+                  type: keyword
+                  time_series_dimension: true
                 ip:
                   type: ip
                   time_series_dimension: true
@@ -118,20 +127,20 @@ ip dimension:
         index: test
         body:
           - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:35:24.467Z", "ip": "10.10.1.1", "voltage": 7.2}'
+          - '{"@timestamp": "2021-04-28T18:35:24.467Z", "metricset": "aa", "ip": "10.10.1.1", "voltage": 7.2}'
           - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:35:34.467Z", "ip": "10.10.1.1", "voltage": 7.6}'
+          - '{"@timestamp": "2021-04-28T18:35:34.467Z", "metricset": "aa", "ip": "10.10.1.1", "voltage": 7.6}'
           - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:35:44.467Z", "ip": "10.10.1.1", "voltage": 7.1}'
+          - '{"@timestamp": "2021-04-28T18:35:44.467Z", "metricset": "aa", "ip": "10.10.1.1", "voltage": 7.1}'
           - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:35:54.467Z", "ip": "::ffff:10.10.1.1", "voltage": 7.3}'
+          - '{"@timestamp": "2021-04-28T18:35:54.467Z", "metricset": "aa", "ip": "::ffff:10.10.1.1", "voltage": 7.3}'
           - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:35:24.467Z", "ip": "2001:0db8:85a3:0000:0000:8a2e:0370:7334", "voltage": 3.2}'
+          - '{"@timestamp": "2021-04-28T18:35:24.467Z", "metricset": "aa", "ip": "2001:0db8:85a3:0000:0000:8a2e:0370:7334", "voltage": 3.2}'
           - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:35:34.467Z", "ip": "2001:0db8:85a3:0:0:8a2e:0370:7334", "voltage": 3.6}'
+          - '{"@timestamp": "2021-04-28T18:35:34.467Z", "metricset": "aa", "ip": "2001:0db8:85a3:0:0:8a2e:0370:7334", "voltage": 3.6}'
           - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:35:44.467Z", "ip": "2001:0db8:85a3::8a2e:0370:7334", "voltage": 3.1}'
+          - '{"@timestamp": "2021-04-28T18:35:44.467Z", "metricset": "aa", "ip": "2001:0db8:85a3::8a2e:0370:7334", "voltage": 3.1}'
           - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:35:54.467Z", "ip": "2001:0db8:85a3::8a2e:0370:7334", "voltage": 3.3}'
+          - '{"@timestamp": "2021-04-28T18:35:54.467Z", "metricset": "aa", "ip": "2001:0db8:85a3::8a2e:0370:7334", "voltage": 3.3}'
 
   # TODO aggregate on tsid

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml

@@ -10,6 +10,7 @@ setup:
             settings:
               index:
                 mode: time_series
+                routing_path: [metricset, k8s.pod.uid]
                 number_of_shards: 3
                 number_of_replicas: 0
             mappings: