Query more data for daily indices (elastic#71)

With this commit we first fix the index pattern so queries can actually
match data. We also change the queries so we don't run any redundant
ones and modify them to retrieve more data. Finally, we modify the
Kibana runner so it stores some metadata about the query results (number
of total hits and the maximum value of `took` across all msearch
responses).

Author: danielmitterdorfer

eventdata/challenges/daily-log-volume-index-and-query.json

@@ -40,6 +40,15 @@
         "index": "elasticlogs-2999-01-01-throughput-test"
       }
     },
+    {
+      "operation": {
+        "operation-type": "fieldstats",
+        "index_pattern": "elasticlogs-*"
+      },
+      "iterations": 1,
+      {# this needs to be available for all clients, and we issue three concurrent queries #}
+      "clients": {{ p_bulk_indexing_clients + 3}}
+    },
 {% set comma = joiner() %}
 {% for day in range(p_number_of_days) %}
 {% set utilization = (day + 1) / p_number_of_days %}
@@ -73,50 +82,65 @@
             }
           },
           {
-            "name": "current-kibana-traffic-country-dashboard_60m-querying-{{utilization_task_suffix}}",
-            "operation": "current-kibana-traffic-country-dashboard_60m",
+            "name": "traffic-dashboard-50%-{{utilization_task_suffix}}",
+            "operation": {
+              "operation-type": "kibana",
+              "param-source": "elasticlogs_kibana",
+              "debug": {{p_verbose}},
+              "dashboard": "traffic",
+              "index_pattern": "elasticlogs-*",
+              "query_string": "query_string_lists/country_code_query_strings.json",
+              "window_end": "END",
+              "window_length": "50%"
+            },
             "clients": 1,
             "target-interval": {{ query1_target_interval | default(30) | int }},
             "meta": {
               "querying": "yes",
-              "query_type": "current",
+              "query_type": "relative",
               "utilization": {{ utilization }}
             },
             "schedule": "poisson"
           },
           {
-            "name": "current-kibana-discover_30m-querying-{{utilization_task_suffix}}",
-            "operation": "current-kibana-discover_30m",
-            "clients": 1,
-            "target-interval": {{ query2_target_interval | default(30) | int }},
-            "meta": {
-              "querying": "yes",
-              "query_type": "current",
-              "utilization": {{ utilization }}
+            "name": "discover-30m-{{utilization_task_suffix}}",
+            "operation": {
+              "operation-type": "kibana",
+              "param-source": "elasticlogs_kibana",
+              "debug": {{p_verbose}},
+              "dashboard": "discover",
+              "index_pattern": "elasticlogs-*",
+              "query_string": ["*"],
+              "window_end": "END",
+              "window_length": "30m"
             },
-            "schedule": "poisson"
-          },
-          {
-            "name": "current-kibana-traffic-dashboard_30m-querying-{{utilization_task_suffix}}",
-            "operation": "current-kibana-traffic-dashboard_30m",
             "clients": 1,
-            "target-interval": {{ query3_target_interval | default(30) | int }},
+            "target-interval": {{ query2_target_interval | default(30) | int }},
             "meta": {
               "querying": "yes",
-              "query_type": "current",
+              "query_type": "relative",
               "utilization": {{ utilization }}
             },
             "schedule": "poisson"
           },
           {
-            "name": "current-kibana-content_issues-dashboard_30m-querying-{{utilization_task_suffix}}",
+            "name": "content_issues-dashboard-50%-{{utilization_task_suffix}}",
             "#COMMENT": "Looks only for 404s about 1-1.5% of data",
-            "operation": "current-kibana-content_issues-dashboard_30m",
+            "operation": {
+              "operation-type": "kibana",
+              "param-source": "elasticlogs_kibana",
+              "debug": {{p_verbose}},
+              "dashboard": "content_issues",
+              "index_pattern": "elasticlogs-*",
+              "query_string": ["*"],
+              "window_end": "END",
+              "window_length": "50%"
+            },
             "clients": 1,
-            "target-interval": {{ query4_target_interval | default(30) | int }},
+            "target-interval": {{ query3_target_interval | default(30) | int }},
             "meta": {
               "querying": "yes",
-              "query_type": "current",
+              "query_type": "relative",
               "utilization": {{ utilization }}
             },
             "schedule": "poisson"

eventdata/runners/kibana_runner.py

@@ -35,7 +35,7 @@ def kibana(es, params):
     meta_data = params["meta_data"]
 
     if meta_data["debug"]:
-        logger.info("Received request:\n=====\n{}\n=====".format(json.dumps(request)))
+        logger.info("Request:\n=====\n{}\n=====".format(json.dumps(request)))
 
     visualisations = int(len(request) / 2)
 
@@ -53,12 +53,24 @@ def kibana(es, params):
     else:
         result = es.msearch(body=request)
 
+    sum_hits = 0
+    max_took = 0
+    for r in result["responses"]:
+        hits = r["hits"]["total"]
+        if isinstance(hits, dict):
+            sum_hits += hits["value"]
+        else:
+            sum_hits += hits
+        max_took = max(max_took, r["took"])
+
+    response["took"] = max_took
+    response["hits"] = sum_hits
+
     if meta_data["debug"]:
         for r in result["responses"]:
             # clear hits otherwise we'll spam the log
             r["hits"]["hits"] = []
             r["aggregations"] = {}
-        logger.info("Received response (excluding specific hits):\n=====\n{}\n=====".format(json.dumps(result)))
-
+        logger.info("Response (excluding specific hits):\n=====\n{}\n=====".format(json.dumps(result)))
 
     return response

smoke-test.sh

@@ -59,7 +59,7 @@ function run_test {
   for challenge in "${CHALLENGES[@]}"
   do
       info "Testing ${challenge}"
-      esrally --race-id="${RACE_ID}" --test-mode --pipeline=benchmark-only --target-host=127.0.0.1:39200 --track-path="${PWD}/eventdata" --track-params="bulk_indexing_clients:1,number_of_replicas:0,daily_logging_volume:1MB,rate_limit_max:2,rate_limit_duration_secs:5,p1_bulk_indexing_clients:1,p2_bulk_indexing_clients:1,p1_duration_secs:5,p2_duration_secs:5,ops_per_25_gb:20" --challenge="${challenge}" --on-error=abort
+      esrally --race-id="${RACE_ID}" --test-mode --pipeline=benchmark-only --target-host=127.0.0.1:39200 --track-path="${PWD}/eventdata" --track-params="bulk_indexing_clients:1,number_of_replicas:0,daily_logging_volume:1MB,rate_limit_max:2,rate_limit_duration_secs:5,p1_bulk_indexing_clients:1,p2_bulk_indexing_clients:1,p1_duration_secs:5,p2_duration_secs:5,ops_per_25_gb:20,verbose:true" --challenge="${challenge}" --on-error=abort
   done
 }