Route documents to the correct shards in tsdb

build-tools-internal/src/main/groovy/elasticsearch.formatting.gradle

@@ -207,6 +207,7 @@ subprojects {
                    'src/*/java/org/elasticsearch/action/admin/cluster/snapshots/**/*.java',
                    'src/test/java/org/elasticsearch/common/xcontent/support/AbstractFilteringTestCase.java',
                    'src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java',
+                   'src/*/java/org/elasticsearch/index/IndexRouting.java',
                    'src/*/java/org/elasticsearch/index/snapshots/**/*.java',
                    'src/*/java/org/elasticsearch/repositories/**/*.java',
                    'src/*/java/org/elasticsearch/search/aggregations/**/*.java',

qa/smoke-test-multinode/src/test/resources/rest-api-spec/test/smoke_test_multinode/20_tsdb_consistency.yml

@@ -0,0 +1,107 @@
+# Test the mode:time_series properly groups by _tsid. If we could put this in
+# rest-api-spec we would, but it requires painless.
+
+setup:
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0 to be backported to 7.16.0
+
+  - do:
+      indices.create:
+          index: test
+          body:
+            settings:
+              index:
+                mode: time_series
+                routing_path: [metricset, k8s.pod.uid]
+                number_of_shards: 10
+                number_of_replicas: 1
+            mappings:
+              properties:
+                "@timestamp":
+                  type: date
+                metricset:
+                  type: keyword
+                  dimension: true
+                k8s:
+                  properties:
+                    pod:
+                      properties:
+                        uid:
+                          type: keyword
+                          dimension: true
+                        name:
+                          type: keyword
+                        ip:
+                          type: ip
+                        network:
+                          properties:
+                            tx:
+                              type: long
+                            rx:
+                              type: long
+  - do:
+      bulk:
+        refresh: true
+        index: test
+        body:
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:35:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T19:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2005177954, "rx": 801479970}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T17:53:34.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2006223737, "rx": 802337279}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:03:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.2", "network": {"tx": 2012916202, "rx": 803685721}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:35:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434521831, "rx": 530575198}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T19:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434577921, "rx": 530600088}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T17:53:34.467Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434587694, "rx": 530604797}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:03:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434595272, "rx": 530605511}}}}'
+
+---
+"index with replicas and shards is green":
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0 to be backported to 7.16.0
+
+  - do:
+      cluster.health:
+        wait_for_status: green
+  - match:     { status:     green }
+
+---
+"each shard has unique _tsids":
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0 to be backported to 7.16.0
+
+  - do:
+      search:
+        index: test
+        body:
+          size: 0
+          aggs:
+            check:
+              scripted_metric:
+                init_script: "state.timeSeries = new HashSet()"
+                map_script: "state.timeSeries.add(doc['metricset'].value + ':' + doc['k8s.pod.uid'].value)"
+                combine_script: "return state.timeSeries"
+                reduce_script: |
+                  Set timeSeries = new TreeSet();
+                  for (s in states) {
+                    for (ts in s) {
+                      boolean newTs = timeSeries.add(ts);
+                      if (false == newTs) {
+                        throw new IllegalArgumentException(ts + " appeared in two shards");
+                      }
+                    }
+                  }
+                  return timeSeries;
+  - match: {hits.total.value: 8}
+  - match: {aggregations.check.value: ['pod:947e4ced-1786-4e53-9e0c-5c447e959507', 'pod:df3145b3-0563-4d3b-a0f7-897eb2876ea9']}
+
+# todo clone and shrink

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml

@@ -10,6 +10,7 @@ enable:
             settings:
               index:
                 mode: time_series
+                routing_path: [metricset, k8s.pod.uid]
                 number_of_replicas: 0
                 number_of_shards: 2
             mappings:
@@ -117,3 +118,34 @@ no partitioning:
                 mode: time_series
                 shards: 5
                 routing_partition_size: 2
+
+---
+routing_path required:
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0 to be backported to 7.16.0
+
+  - do:
+      catch: /\[index.mode=time_series\] requires \[index.routing_path\]/
+      indices.create:
+          index: test_index
+          body:
+            settings:
+              index:
+                mode: time_series
+                shards: 5
+
+---
+routing_path is not allowed in standard mode:
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0 to be backported to 7.16.0
+
+  - do:
+      catch: /\[index.routing_path\] requires \[index.mode=time_series\]/
+      indices.create:
+          index: test_index
+          body:
+            settings:
+              index:
+                routing_path: foo

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_unsupported_operations.yml

@@ -0,0 +1,239 @@
+setup:
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0 to be backported to 7.16.0
+
+  - do:
+      indices.create:
+          index: test
+          body:
+            settings:
+              index:
+                mode: time_series
+                routing_path: [metricset, k8s.pod.uid]
+                number_of_replicas: 0
+                number_of_shards: 2
+            mappings:
+              properties:
+                "@timestamp":
+                  type: date
+                metricset:
+                  type: keyword
+                  dimension: true
+                k8s:
+                  properties:
+                    pod:
+                      properties:
+                        uid:
+                          type: keyword
+                          dimension: true
+                        name:
+                          type: keyword
+                        ip:
+                          type: ip
+                        network:
+                          properties:
+                            tx:
+                              type: long
+                            rx:
+                              type: long
+
+  - do:
+      bulk:
+        refresh: true
+        index: test
+        body:
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2005177954, "rx": 801479970}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:44.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2006223737, "rx": 802337279}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:51:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.2", "network": {"tx": 2012916202, "rx": 803685721}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434521831, "rx": 530575198}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:23.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434577921, "rx": 530600088}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:53.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434587694, "rx": 530604797}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434595272, "rx": 530605511}}}}'
+
+---
+index with specified id:
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0 to be backported to 7.16.0
+
+  - do:
+      catch: /indexing with a specified id is not supported because the destination index \[test\] is in time series mode/
+      index:
+        index:   test
+        id:      foo
+        body:
+          doc:
+            "@timestamp": "2021-04-28T18:35:24.467Z"
+            metricset: "pod"
+            k8s:
+              pod:
+                name: "cat"
+                uid: "947e4ced-1786-4e53-9e0c-5c447e959507"
+                ip: "10.10.55.1"
+                network:
+                  tx: 2001818691
+                  rx: 802133794
+
+---
+index with specified id over _bulk:
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0 to be backported to 7.16.0
+
+  - do:
+      bulk:
+        refresh: true
+        index: test
+        body:
+          - '{"index": {"_id": "foo"}}'
+          - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}'
+  - match: {items.0.index.error.reason: "indexing with a specified id is not supported because the destination index [test] is in time series mode"}
+
+---
+index with specified routing:
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0 to be backported to 7.16.0
+
+  - do:
+      catch: /indexing with a specified routing is not supported because the destination index \[test\] is in time series mode/
+      index:
+        index:   test
+        routing: foo
+        body:
+          doc:
+            "@timestamp": "2021-04-28T18:35:24.467Z"
+            metricset: "pod"
+            k8s:
+              pod:
+                name: "cat"
+                uid: "947e4ced-1786-4e53-9e0c-5c447e959507"
+                ip: "10.10.55.1"
+                network:
+                  tx: 2001818691
+                  rx: 802133794
+
+---
+index with specified routing over _bulk:
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0 to be backported to 7.16.0
+
+  - do:
+      bulk:
+        refresh: true
+        index: test
+        body:
+          - '{"index": {"routing": "foo"}}'
+          - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}'
+  - match: {items.0.index.error.reason: "indexing with a specified routing is not supported because the destination index [test] is in time series mode"}
+
+---
+delete:
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0 to be backported to 7.16.0
+
+  - do:
+      catch: /delete is not supported because the destination index \[test\] is in time series mode/
+      delete:
+        index:   test
+        id:      1
+
+---
+delete over _bulk:
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0 to be backported to 7.16.0
+
+  - do:
+      bulk:
+        index:   test
+        body:
+          - '{"delete": {"_id": 1}}'
+          - '{"delete": {"_id": 2}}'
+  - match: {items.0.delete.error.reason: "delete is not supported because the destination index [test] is in time series mode"}
+
+---
+noop update:
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0 to be backported to 7.16.0
+
+  - do:
+      search:
+        index: test
+        size: 1
+
+  - length: {hits.hits: 1}
+
+  - do:
+      catch: /update is not supported because the destination index \[test\] is in time series mode/
+      update:
+        index:   test
+        id:      $body.hits.hits.0._id
+        body:
+          doc:
+            $body.hits.hits.0._source
+
+---
+update:
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0 to be backported to 7.16.0
+
+  # We fail even though the document isn't found.
+  - do:
+      catch: /update is not supported because the destination index \[test\] is in time series mode/
+      update:
+        index:   test
+        id:      1
+        body:
+          doc:
+            "@timestamp": "2021-04-28T18:35:24.467Z"
+            metricset: "pod"
+            k8s:
+              pod:
+                name: "cat"
+                uid: "947e4ced-1786-4e53-9e0c-5c447e959507"
+                ip: "10.10.55.1"
+                network:
+                  tx: 2001818691
+                  rx: 802133794
+
+---
+update over _bulk:
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0 to be backported to 7.16.0
+
+  - do:
+      bulk:
+        index:   test
+        body:
+          - '{"update": {"_id": 1}}'
+          - '{"doc":{"@timestamp": "2021-04-28T18:03:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434595272, "rx": 530605511}}}}}'
+  - match: {items.0.update.error.reason: "update is not supported because the destination index [test] is in time series mode"}
+
+---
+search with routing:
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0 to be backported to 7.16.0
+
+  # We fail even though the document isn't found.
+  - do:
+      catch: /searching with a specified routing is not supported because the destination index \[test\] is in time series mode/
+      search:
+        index:   test
+        routing: rrrr