Add track parameter for query index pattern (elastic#57)

With this commit we introduce the track parameter `query_index_prefix`
which determines the prefix of indices which are used in query
benchmarks and set its default to the former value `elasticlogs_q`.

Relates elastic#56

Author: danielmitterdorfer

README.md

@@ -28,6 +28,7 @@ Note: In general, track parameters are only defined for a subset of the challeng
 | Parameter | Explanation | Type | Default Value |
 | --------- | ----------- | ---- | ------------- |
 | `record_raw_event_size` | Adds a new field `_raw_event_size` to the index which contains the size of the raw logging event in bytes. | `bool` | `False` |
+| `query_index_prefix` | Start of the index name(s) used in queries for this track. | `str` | `elasticlogs_q` |
 
 Note: It is recommended to store any track parameters in a json file and pass them to Rally using `--track-params=./params-file.json`. 
 
@@ -126,18 +127,16 @@ The table below shows the track parameters that can be adjusted along with defau
 | `p2_bulk_indexing_clients` | Number of [clients](https://esrally.readthedocs.io/en/stable/track.html?highlight=number%20of%20clients#schedule) used to index during phase 2 | `int` | `16` |
 | `p2_bulk_size` | The [build-size](https://esrally.readthedocs.io/en/stable/track.html?highlight=number%20of%20clients#bulk) for the autogenerated events during phase 2 | `int` | `1000` |
 | `p2_ops` | Number of bulk indexing ops/s for phase 2. A value of `10` with `p2_bulk_size=10` throttles indexing to 10000 docs/s | `int` | `10` |
-| `index_alias` | Specifies default index alias. | `str` | `elasticlogs_q_write` |
 | `rollover_max_size` | Max index size condition for [rollover API](https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-rollover-index.html#indices-rollover-index) | `str` | `30gb` |
 | `rollover_max_age` | Max age condition for [rollover API](https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-rollover-index.html#indices-rollover-index) | `str` | `1d` |
 | `p2_query1_target_interval` | Frequency of execution (every N sec) of Kibana query: `kibana-traffic-country-dashboard_60m` | `int` | `30` |
 | `p2_query2_target_interval` | Frequency of execution (every N sec) of Kibana query: `kibana-discover_30m` | `int` | `30` |
 | `p2_query3_target_interval` | Frequency of execution (every N sec) of Kibana query: `kibana-traffic-dashboard_30m` | `int` | `30` |
 | `p2_query4_target_interval` | Frequency of execution (every N sec) of Kibana query: `kibana-content_issues-dashboard_30m"` | `int` | `30` |
 | `max_rolledover_indices` | Max amount of recently rolled over indices to retain | `int` | `20` |
-| `indices_delete_pattern` | pattern to use for matching and deleting old rolled over indices. See also suffix_separator. | `str` | `elasticlogs_q-*` |
 | `rolledover_indices_suffix_separator` | Separator for extracting suffix to help determining which rolled-over indices to delete  | `str` | `-` |
 
-The indices use the alias `elasticlogs_q_write` and start with `elasticlogs_q-000001`. As an example, for a cluster with rolled over indices:  `elasticlogs-000001`, `elasticlogs-000002`, ... `000010` a value of `max_rolledover_indices=8` results to the removal of `elasticlogs-000001` and `elasticlogs-000002`.
+The indices use the alias `elasticlogs_q_write` (assuming the default value for `query_index_prefix`) and start with `elasticlogs_q-000001`. As an example, for a cluster with rolled over indices:  `elasticlogs-000001`, `elasticlogs-000002`, ... `000010` a value of `max_rolledover_indices=8` results to the removal of `elasticlogs-000001` and `elasticlogs-000002`.
 
 A value of `max_rolledover_indices=20` on a three node bare-metal cluster with the following specifications:
 

eventdata/challenges/combined-indexing-and-querying.json

@@ -8,7 +8,7 @@
 
 {
   "name": "combined-indexing-and-querying",
-  "description": "This challenge simulates a set of Kibana queries against historical data (elasticlogs_q-* indices) as well as against the most recent data currently being indexed. It combined this with rate-limited indexing at varying levels. It assumes one of the challenges creating elasticlogs_q-* indices has been run.",
+  "description": "This challenge simulates a set of Kibana queries against historical data ({{p_query_index_pattern}} indices) as well as against the most recent data currently being indexed. It combined this with rate-limited indexing at varying levels. It assumes one of the challenges creating {{p_query_index_pattern}} indices has been run.",
   "meta": {
     "benchmark_type": "indexing/querying",
     "target_kibana_queries_per_minute": 7

eventdata/challenges/elasticlogs-1bn-load.json

@@ -4,7 +4,7 @@
 
 {
   "name": "elasticlogs-1bn-load",
-  "description": "Indexes 1bn (default) documents into elasticlogs_q-* indices. IDs are autogenerated by Elasticsearch, meaning there are no conflicts.",
+  "description": "Indexes 1bn (default) documents into {{p_query_index_pattern}} indices. IDs are autogenerated by Elasticsearch, meaning there are no conflicts.",
   "default": true,
   "meta": {
     "client_count": {{ p_bulk_indexing_clients }},
@@ -23,10 +23,10 @@
     {
       "operation": {
         "operation-type": "create-index",
-        "index": "elasticlogs_q-000001",
+        "index": "{{p_query_index_prefix}}-000001",
         "body": {
           "aliases" : {
-            "elasticlogs_q_write" : {}
+            "{{p_query_index_write_alias}}" : {}
           }
         }
       }

eventdata/challenges/elasticlogs-continuous-index-and-query.json

@@ -8,7 +8,7 @@
 {% set p2_rate = (p2_ops * (p2_bulk_size | default(1000))) %}
 {
   "name": "elasticlogs-continuous-index-and-query",
-  "description": "Indexes 1bn (default) documents into elasticlogs_q-* indices. IDs are autogenerated by Elasticsearch, meaning there are no conflicts.",
+  "description": "Indexes 1bn (default) documents into {{p_query_index_pattern}} indices. IDs are autogenerated by Elasticsearch, meaning there are no conflicts.",
   "meta": {
     "benchmark_type": "indexing"
   },
@@ -26,10 +26,10 @@
       "operation": {
         "name": "create_elasticlogs_q_write",
         "operation-type": "create-index",
-        "index": "elasticlogs_q-000001",
+        "index": "{{p_query_index_prefix}}-000001",
         "body": {
           "aliases" : {
-            "elasticlogs_q_write" : {}
+            "{{p_query_index_write_alias}}" : {}
           }
         }
       }
@@ -43,7 +43,7 @@
             "name": "index-append-elasticlogs_q_write-phase1",
             "operation": {
               "operation-type": "bulk",
-              "index": "elasticlogs_q_write",
+              "index": "{{p_query_index_write_alias}}",
               "param-source": "elasticlogs_bulk",
               "bulk-size": {{ p1_bulk_size | default(1000) | int }},
               "record_raw_event_size": {{p_record_raw_event_size}}
@@ -79,7 +79,7 @@
             "name": "index-append-elasticlogs_q_write-phase2",
             "operation": {
               "operation-type": "bulk",
-              "index": "elasticlogs_q_write",
+              "index": "{{p_query_index_write_alias}}",
               "param-source": "elasticlogs_bulk",
               "bulk-size": {{ p2_bulk_size | default(1000) | int }},
               "record_raw_event_size": {{p_record_raw_event_size}}

eventdata/challenges/elasticlogs-querying.json

@@ -1,6 +1,6 @@
 {
   "name": "elasticlogs-querying",
-  "description": "This challenge simulates a set of Kibana queries against historical data (elasticlogs_q-* indices) without any indexing taking place for a period of 30 minutes. It assumes one of the challenges creating elasticlogs_q-* indices has been run.",
+  "description": "This challenge simulates a set of Kibana queries against historical data ({{p_query_index_pattern}} indices) without any indexing taking place for a period of 30 minutes. It assumes one of the challenges creating {{p_query_index_pattern}} indices has been run.",
   "meta": {
     "benchmark_type": "querying",
     "target_kibana_queries_per_minute": 5

eventdata/challenges/max-indexing-querying.json

@@ -3,7 +3,7 @@
 
 {
   "name": "max-indexing-querying",
-  "description": "Indexes documents as quickly as possible into elasticlogs_q-* indices, initially for 2 hiours without concurrent querying and then for 2 hours with querying. IDs are autogenerated by Elasticsearch, meaning there are no conflicts.",
+  "description": "Indexes documents as quickly as possible into {{p_query_index_pattern}} indices, initially for 2 hours without concurrent querying and then for 2 hours with querying. IDs are autogenerated by Elasticsearch, meaning there are no conflicts.",
   "meta": {
     "client_count": {{ p_bulk_indexing_clients }},
     "benchmark_type": "indexing"
@@ -21,10 +21,10 @@
     {
       "operation": {
         "operation-type": "create-index",
-        "index": "elasticlogs_q-000001",
+        "index": "{{p_query_index_prefix}}-000001",
         "body": {
           "aliases" : {
-            "elasticlogs_q_write" : {}
+            "{{p_query_index_write_alias}}" : {}
           }
         }
       }

eventdata/elasticlogs-index-template.json

@@ -1,7 +1,7 @@
 {% set p_disk_type = disk_type | default('ssd') | lower %}
 {% set p_translog_sync = translog_sync | default('request') | lower %}
 {
-  "index_patterns": ["elasticlogs*"],
+  "index_patterns": ["{{p_index_prefix}}*"],
   "settings": {
       "index.refresh_interval": "5s",
       "index.codec": "best_compression",

eventdata/operations/indexing.json

@@ -1,8 +1,6 @@
 {# The following track-params are supported:
-index_alias: used by the `rollover_custom_alias` operation. Defaults to `elasticlogs_q_write`.
 rollover_max_age: used by the `rollover_custom_alias` operation. Defaults to `1d`.
 rollover_max_size: used by the `rollover_custom_alias` operation. Defaults to `30gb`.
-indices_delete_pattern: used by the `delete_rolledover_index_pattern` operation. Defaults to `elasticlogs_q-*`.
 max_rolledover_indices: used by the `delete_rolledover_index_pattern` operation. Defaults to `20`.
 rolledover_indices_suffix_separator: used by the `delete_rolledover_index_pattern` operation. Defaults to `-`.
 #}
@@ -33,7 +31,7 @@ rolledover_indices_suffix_separator: used by the `delete_rolledover_index_patter
   "name": "index-append-1000-elasticlogs_q_write",
   "operation-type": "bulk",
   "param-source": "elasticlogs_bulk",
-  "index": "elasticlogs_q_write",
+  "index": "{{p_query_index_write_alias}}",
   "bulk-size": 1000,
   "record_raw_event_size": {{p_record_raw_event_size}}
 },
@@ -48,7 +46,7 @@ rolledover_indices_suffix_separator: used by the `delete_rolledover_index_patter
 {
   "name": "rollover_elasticlogs_q_write_100M",
   "operation-type": "rollover",
-  "alias": "elasticlogs_q_write",
+  "alias": "{{p_query_index_write_alias}}",
   "body": {
     "conditions": {
       "max_age":   "1d",
@@ -70,7 +68,7 @@ rolledover_indices_suffix_separator: used by the `delete_rolledover_index_patter
 {
   "name": "rollover_elasticlogs_q_write_50M",
   "operation-type": "rollover",
-  "alias": "elasticlogs_q_write",
+  "alias": "{{p_query_index_write_alias}}",
   "body": {
     "conditions": {
       "max_age":   "1d",
@@ -92,7 +90,7 @@ rolledover_indices_suffix_separator: used by the `delete_rolledover_index_patter
 {
   "name": "rollover_elasticlogs_q_write_10M",
   "operation-type": "rollover",
-  "alias": "elasticlogs_q_write",
+  "alias": "{{p_query_index_write_alias}}",
   "body": {
     "conditions": {
       "max_age":   "1d",
@@ -114,7 +112,7 @@ rolledover_indices_suffix_separator: used by the `delete_rolledover_index_patter
 {
   "name": "rollover_elasticlogs_q_write_50gb",
   "operation-type": "rollover",
-  "alias": "elasticlogs_q_write",
+  "alias": "{{p_query_index_write_alias}}",
   "body": {
     "conditions": {
       "max_age": "1d",
@@ -125,7 +123,7 @@ rolledover_indices_suffix_separator: used by the `delete_rolledover_index_patter
 {
   "name": "rollover_custom_alias",
   "operation-type": "rollover",
-  "alias": {{ index_alias | default("elasticlogs_q_write") | tojson }},
+  "alias": "{{p_query_index_write_alias}}",
   "body": {
     "conditions": {
       "max_age": {{ rollover_max_age | default("1d") | tojson }},
@@ -141,7 +139,7 @@ rolledover_indices_suffix_separator: used by the `delete_rolledover_index_patter
 {
   "name": "deleteindex_elasticlogs_q-*",
   "operation-type": "delete-index",
-  "index": "elasticlogs_q-*"
+  "index": "{{p_query_index_pattern}}"
 },
 {
   "name": "deleteindex_elasticlogs",
@@ -151,7 +149,7 @@ rolledover_indices_suffix_separator: used by the `delete_rolledover_index_patter
 {
   "name": "delete_rolledover_index_pattern",
   "operation-type": "delete_indices",
-  "index_pattern": {{ indices_delete_pattern | default("elasticlogs_q-*") | tojson }},
+  "index_pattern": "{{ p_query_index_pattern }}",
   "max_indices": {{ max_rolledover_indices | default(20) | int | tojson }},
   "suffix_separator": {{ rolledover_indices_suffix_separator | default("-") | tojson }}
 }

eventdata/operations/querying.json

@@ -103,7 +103,7 @@
   "operation-type": "kibana",
   "param-source": "elasticlogs_kibana",
   "dashboard": "traffic",
-  "index_pattern": "elasticlogs_q-*",
+  "index_pattern": "{{p_query_index_pattern}}",
   "query_string": "query_string_lists/country_code_query_strings.json",
   "window_end": "START+25%,END",
   "window_length": "25%"
@@ -113,7 +113,7 @@
   "operation-type": "kibana",
   "param-source": "elasticlogs_kibana",
   "dashboard": "content_issues",
-  "index_pattern": "elasticlogs_q-*",
+  "index_pattern": "{{p_query_index_pattern}}",
   "query_string": "query_string_lists/country_code_query_strings.json",
   "window_end": "START+25%,END",
   "window_length": "25%"
@@ -123,7 +123,7 @@
   "operation-type": "kibana",
   "param-source": "elasticlogs_kibana",
   "dashboard": "traffic",
-  "index_pattern": "elasticlogs_q-*",
+  "index_pattern": "{{p_query_index_pattern}}",
   "query_string": "query_string_lists/country_code_query_strings.json",
   "window_end": "START+50%,END",
   "window_length": "50%"
@@ -133,7 +133,7 @@
   "operation-type": "kibana",
   "param-source": "elasticlogs_kibana",
   "dashboard": "content_issues",
-  "index_pattern": "elasticlogs_q-*",
+  "index_pattern": "{{p_query_index_pattern}}",
   "query_string": "query_string_lists/country_code_query_strings.json",
   "window_end": "START+50%,END",
   "window_length": "50%"
@@ -143,7 +143,7 @@
   "operation-type": "kibana",
   "param-source": "elasticlogs_kibana",
   "dashboard": "traffic",
-  "index_pattern": "elasticlogs_q-*",
+  "index_pattern": "{{p_query_index_pattern}}",
   "query_string": "query_string_lists/country_code_query_strings.json",
   "window_end": "START+75%,END",
   "window_length": "75%"
@@ -153,7 +153,7 @@
   "operation-type": "kibana",
   "param-source": "elasticlogs_kibana",
   "dashboard": "content_issues",
-  "index_pattern": "elasticlogs_q-*",
+  "index_pattern": "{{p_query_index_pattern}}",
   "query_string": "query_string_lists/country_code_query_strings.json",
   "window_end": "START+75%,END",
   "window_length": "75%"
@@ -163,7 +163,7 @@
   "operation-type": "kibana",
   "param-source": "elasticlogs_kibana",
   "dashboard": "traffic",
-  "index_pattern": "elasticlogs_q-*",
+  "index_pattern": "{{p_query_index_pattern}}",
   "query_string": ["*"],
   "window_end": "now",
   "window_length": "30m"
@@ -173,7 +173,7 @@
   "operation-type": "kibana",
   "param-source": "elasticlogs_kibana",
   "dashboard": "discover",
-  "index_pattern": "elasticlogs_q-*",
+  "index_pattern": "{{p_query_index_pattern}}",
   "query_string": "query_string_lists/country_code_query_strings.json",
   "window_end": "START+50%,END",
   "window_length": "50%"
@@ -183,7 +183,7 @@
   "operation-type": "kibana",
   "param-source": "elasticlogs_kibana",
   "dashboard": "discover",
-  "index_pattern": "elasticlogs_q-*",
+  "index_pattern": "{{p_query_index_pattern}}",
   "query_string": "query_string_lists/uncommon_ip_query_strings.json",
   "window_end": "START+50%,END",
   "window_length": "50%"
@@ -193,7 +193,7 @@
   "operation-type": "kibana",
   "param-source": "elasticlogs_kibana",
   "dashboard": "discover",
-  "index_pattern": "elasticlogs_q-*",
+  "index_pattern": "{{p_query_index_pattern}}",
   "query_string": "query_string_lists/country_code_query_strings.json",
   "window_end": "START+50%,END",
   "window_length": "50%"

eventdata/operations/stats.json

@@ -1,7 +1,7 @@
 {
   "name": "fieldstats_elasticlogs_q-*",
   "operation-type": "fieldstats",
-  "index_pattern": "elasticlogs_q-*"
+  "index_pattern": "{{p_query_index_pattern}}"
 },
 {
   "name": "indicesstats_elasticlogs",
@@ -11,7 +11,7 @@
 {
   "name": "indicesstats_elasticlogs_q-*",
   "operation-type": "indicesstats",
-  "index_pattern": "elasticlogs_q-*"
+  "index_pattern": "{{p_query_index_pattern}}"
 },
 {
   "name": "indicesstats_elasticlogs_i-*",
@@ -21,7 +21,7 @@
 {
   "name": "indicesstats_elasticlogs_q_write",
   "operation-type": "indicesstats",
-  "index_pattern": "elasticlogs_q_write"
+  "index_pattern": "{{p_query_index_write_alias}}"
 },
 {
   "name": "indicesstats_elasticlogs_i_write",

eventdata/track.json

@@ -2,14 +2,18 @@
 
 {% set p_bulk_indexing_clients = (bulk_indexing_clients | default(8)) %}
 {% set p_record_raw_event_size = record_raw_event_size | default(False) | tojson %}
+{% set p_index_prefix = index_prefix | default("elasticlogs") %}
+{% set p_query_index_prefix = query_index_prefix | default(p_index_prefix ~ "_q") %}
+{% set p_query_index_pattern = p_query_index_prefix ~ "-*" %}
+{% set p_query_index_write_alias = p_query_index_prefix ~ "_write" %}
 
 {
   "version": 2,
   "description": "Track for simulating different aspects of event-based use cases.",
   "templates": [
     {
         "name": "elasticlogs-index-template",
-        "index-pattern": "elasticlogs*",
+        "index-pattern": "{{p_index_prefix}}*",
         "delete-matching-indices": false,
         "template": "elasticlogs-index-template.json"
     }