Measure raw event size

danielmitterdorfer · web-flow · commit 4557e72f6105 · 2019-08-09T11:26:56.000+02:00
With this commit we measure raw event size and add it is a new field in the generated documents. Relates elastic#35
diff --git a/README.md b/README.md
@@ -19,6 +19,14 @@ The track can be run by specifying the following runtime parameters: `--track=ev
 
 Another option is to download the repository and point to it using the `--track-path` command line parameter.
 
+## Track parameters supported by all challenges
+
+Note: In general, track parameters are only defined for a subset of the challenges so please refer to the documentation of the respective challenge for a list of supported track parameters. Only the parameters documented in the table below are guaranteed to work with all challenges as intended.
+
+| Parameter | Explanation | Type | Default Value |
+| --------- | ----------- | ---- | ------------- |
+| `record_raw_event_size` | Adds a new field `_raw_event_size` to the index which contains the size of the raw logging event in bytes. | `bool` | `False` |
+
 ## Available Challenges
 
 ### 1) append-no-conflicts
diff --git a/eventdata/challenges/bulk-size-evaluation.json b/eventdata/challenges/bulk-size-evaluation.json
@@ -138,7 +138,8 @@
       "operation": {
         "operation-type": "bulk",
         "param-source": "elasticlogs_bulk",
-        "bulk-size": 1000
+        "bulk-size": 1000,
+        "record_raw_event_size": {{p_record_raw_event_size}}
       },
       "warmup-time-period": 300,
       "time-period": 600,
@@ -163,7 +164,8 @@
       "operation": {
         "operation-type": "bulk",
         "param-source": "elasticlogs_bulk",
-        "bulk-size": 100
+        "bulk-size": 100,
+        "record_raw_event_size": {{p_record_raw_event_size}}
       },
       "warmup-time-period": 300,
       "time-period": 600,
@@ -188,7 +190,8 @@
       "operation": {
         "operation-type": "bulk",
         "param-source": "elasticlogs_bulk",
-        "bulk-size": 10
+        "bulk-size": 10,
+        "record_raw_event_size": {{p_record_raw_event_size}}
       },
       "warmup-time-period": 300,
       "time-period": 600,
diff --git a/eventdata/challenges/bulk-update.json b/eventdata/challenges/bulk-update.json
@@ -48,6 +48,7 @@
         "id_seq_probability": 0.4,
         "id_seq_low_id_bias": false,
         "bulk-size": {{ bulk_size | default(1000) }},
+        "record_raw_event_size": {{p_record_raw_event_size}},
         "index": "elasticlogs"
       },
       "iterations": {{ p_iterations_per_client }},
diff --git a/eventdata/challenges/document_id_benchmark.json b/eventdata/challenges/document_id_benchmark.json
@@ -47,6 +47,7 @@
         "param-source": "elasticlogs_bulk",
         "index": "elasticlogs-warmup",
         "bulk-size": 1000,
+        "record_raw_event_size": {{p_record_raw_event_size}},
         "id_type": "auto"
       },
       "iterations": 200,
@@ -103,6 +104,7 @@
         "param-source": "elasticlogs_bulk",
         "index": "elasticlogs-{{ id['desc'] }}",
         "bulk-size": 1000,
+        "record_raw_event_size": {{p_record_raw_event_size}},
         "id_type": "{{ id['type'] }}"
       },
       "iterations": {{ p_iterations_per_client }},
@@ -182,6 +184,7 @@
         "param-source": "elasticlogs_bulk",
         "index": "elasticlogs-{{ id['desc'] }}",
         "bulk-size": 1000,
+        "record_raw_event_size": {{p_record_raw_event_size}},
         "id_type": "{{ id['type'] }}",
         "id_delay_probability": 0.1,
         "id_delay_secs": {{ id['delay'] }}
diff --git a/eventdata/challenges/elasticlogs-continuous-index-and-query.json b/eventdata/challenges/elasticlogs-continuous-index-and-query.json
@@ -36,7 +36,8 @@
               "operation-type": "bulk",
               "index": "elasticlogs_q_write",
               "param-source": "elasticlogs_bulk",
-              "bulk-size": {{ p1_bulk_size | default(1000) | int }}
+              "bulk-size": {{ p1_bulk_size | default(1000) | int }},
+              "record_raw_event_size": {{p_record_raw_event_size}}
             },
             "clients": {{ p1_bulk_indexing_clients }},
             "meta": {
@@ -71,7 +72,8 @@
               "operation-type": "bulk",
               "index": "elasticlogs_q_write",
               "param-source": "elasticlogs_bulk",
-              "bulk-size": {{ p2_bulk_size | default(1000) | int }}
+              "bulk-size": {{ p2_bulk_size | default(1000) | int }},
+              "record_raw_event_size": {{p_record_raw_event_size}}
             },
             "target-throughput": {{ p2_ops }},
             "clients": {{ p2_bulk_indexing_clients }},
diff --git a/eventdata/challenges/frozen.json b/eventdata/challenges/frozen.json
@@ -41,7 +41,8 @@
         "operation-type": "bulk",
         "param-source": "elasticlogs_bulk",
         "index": "elasticlogs",
-        "bulk-size": 1000
+        "bulk-size": 1000,
+        "record_raw_event_size": {{p_record_raw_event_size}}
       },
       "iterations": {{p_bulk_idx_iterations}},
       "clients": 8
diff --git a/eventdata/challenges/large-shard-sizing.json b/eventdata/challenges/large-shard-sizing.json
@@ -47,6 +47,7 @@
         "param-source": "elasticlogs_bulk",
         "index": "elasticlogs-auto",
         "bulk-size": 1000,
+        "record_raw_event_size": {{p_record_raw_event_size}},
         "id_type": "auto"
       },
       "iterations": {{ p_ops_per_client }},
@@ -199,6 +200,7 @@
         "param-source": "elasticlogs_bulk",
         "index": "elasticlogs-{{ id_type }}",
         "bulk-size": 1000,
+        "record_raw_event_size": {{p_record_raw_event_size}},
         "id_type": "{{ id_type }}"
       },
       "iterations": {{ p_ops_per_client }},
diff --git a/eventdata/elasticlogs-index.json b/eventdata/elasticlogs-index.json
@@ -12,6 +12,11 @@
       "@timestamp": {
         "type": "date"
       },
+{%- if record_raw_event_size is defined and record_raw_event_size %}
+      "_raw_event_size": {
+        "type": "short"
+      },
+{%- endif %}
       "source": {
         "type": "keyword",
         "ignore_above": 1024
diff --git a/eventdata/operations/generate-historic-data.json b/eventdata/operations/generate-historic-data.json
@@ -24,5 +24,6 @@
   "daily_index": true,
   "starting_point": "2018-05-01:00:00:00",
   "acceleration_factor": 12,
-  "bulk-size": 1000
+  "bulk-size": 1000,
+  "record_raw_event_size": {{p_record_raw_event_size}}
 }
diff --git a/eventdata/operations/indexing.json b/eventdata/operations/indexing.json
@@ -10,82 +10,44 @@ rolledover_indices_suffix_separator: used by the `delete_rolledover_index_patter
 {% set p_translog_sync = translog_sync | default('request') | lower %}
 
 {% set p_disk_type = disk_type | default('ssd') | lower %}
+
+{% set comma = joiner() %}
+{% for bulk_size in [50000, 20000, 10000, 5000, 2000, 1000, 500, 250, 125] %}
+{{comma()}}
 {
-  "name": "index-append-50000",
-  "operation-type": "bulk",
-  "param-source": "elasticlogs_bulk",
-  "bulk-size": 50000
-},
-{
-  "name": "index-append-20000",
-  "operation-type": "bulk",
-  "param-source": "elasticlogs_bulk",
-  "bulk-size": 20000
-},
-{
-  "name": "index-append-10000",
-  "operation-type": "bulk",
-  "param-source": "elasticlogs_bulk",
-  "bulk-size": 10000
-},
-{
-  "name": "index-append-5000",
-  "operation-type": "bulk",
-  "param-source": "elasticlogs_bulk",
-  "bulk-size": 5000
-},
-{
-  "name": "index-append-2000",
-  "operation-type": "bulk",
-  "param-source": "elasticlogs_bulk",
-  "bulk-size": 2000
-},
-{
-  "name": "index-append-1000",
-  "operation-type": "bulk",
-  "param-source": "elasticlogs_bulk",
-  "bulk-size": 1000
-},
-{
-  "name": "index-append-500",
-  "operation-type": "bulk",
-  "param-source": "elasticlogs_bulk",
-  "bulk-size": 500
-},
-{
-  "name": "index-append-250",
-  "operation-type": "bulk",
-  "param-source": "elasticlogs_bulk",
-  "bulk-size": 250
-},
-{
-  "name": "index-append-125",
+  "name": "index-append-{{bulk_size}}",
   "operation-type": "bulk",
   "param-source": "elasticlogs_bulk",
-  "bulk-size": 125
-},
+  "bulk-size": {{bulk_size}},
+  "record_raw_event_size": {{p_record_raw_event_size}}
+}
+{% endfor%}
+,
 {
   "name": "index-append-1000-shard-sizing",
   "operation-type": "bulk",
   "param-source": "elasticlogs_bulk",
   "index": "elasticlogs",
   "starting_point": "2017-01-01:02:00:00",
   "end_point": "2017-01-01:12:00:00",
-  "bulk-size": 1000
+  "bulk-size": 1000,
+  "record_raw_event_size": {{p_record_raw_event_size}}
 },
 {
   "name": "index-append-1000-elasticlogs_q_write",
   "operation-type": "bulk",
   "param-source": "elasticlogs_bulk",
   "index": "elasticlogs_q_write",
-  "bulk-size": 1000
+  "bulk-size": 1000,
+  "record_raw_event_size": {{p_record_raw_event_size}}
 },
 {
   "name": "index-append-1000-elasticlogs_i_write",
   "operation-type": "bulk",
   "param-source": "elasticlogs_bulk",
   "index": "elasticlogs_i_write",
-  "bulk-size": 1000
+  "bulk-size": 1000,
+  "record_raw_event_size": {{p_record_raw_event_size}}
 },
 {
   "name": "create_elasticlogs_q_write",
diff --git a/eventdata/parameter_sources/randomevent.py b/eventdata/parameter_sources/randomevent.py
@@ -260,6 +260,8 @@ def __init__(self, params):
             for d in params['delete_fields']:
                 self._delete_fields.append(d.split('.'))
 
+        self.record_raw_event_size = params.get("record_raw_event_size", False)
+
     def generate_event(self):
         timestruct = self._timestamp_generator.generate_timestamp_struct()
         index_name = self.__generate_index_pattern(timestruct)
@@ -278,25 +280,57 @@ def generate_event(self):
         # set host name
         event["hostname"] = "web-{}-{}.elastic.co".format(event["geoip_continent_code"],random.randrange(1,3))
 
-        line = '{"@timestamp": "%s", ' \
-               '"offset":%s, ' \
-               '"source":"/usr/local/var/log/nginx/access.log","fileset":{"module":"nginx","name":"access"},"input":{"type":"log"},' \
-               '"beat":{"version":"6.3.0","hostname":"%s","name":"%s"},' \
-               '"prospector":{"type":"log"},' \
-               '"nginx":{"access":{"user_name": "-",' \
-               '"agent":"%s","user_agent": {"major": "%s","os": "%s","os_major": "%s","name": "%s","os_name": "%s","device": "%s"},' \
-               '"remote_ip": "%s","remote_ip_list":["%s"],' \
-               '"geoip":{"continent_name": "%s","city_name": "%s","country_name": "%s","country_iso_code": "%s","location":{"lat": %s,"lon": %s} },' \
-               '"referrer":"%s",' \
-               '"url": "%s","body_sent":{"bytes": %s},"method":"%s","response_code":%s,"http_version":"%s"} } }' % \
-               (event["@timestamp"],
-                event["offset"],
-                event["hostname"],event["hostname"],
-                event["agent"], event["useragent_major"], event["useragent_os"], event["useragent_os_major"], event["useragent_name"], event["useragent_os_name"], event["useragent_device"],
-                event["clientip"], event["clientip"],
-                event["geoip_continent_name"], event["geoip_city_name"], event["geoip_country_name"], event["geoip_country_iso_code"], event["geoip_location_lat"], event["geoip_location_lon"],
-                event["referrer"],
-                event["request"], event["bytes"], event["verb"], event["response"], event["httpversion"])
+        # determine the raw event size (as if this were contained in nginx log file. We do not bother to
+        # reformat the timestamp as this is not worth the overhead.
+        if self.record_raw_event_size:
+            raw_event = '%s - - [%s] "%s %s HTTP/%s" %s %s "%s" "%s"' % (event["clientip"], event["@timestamp"],
+                                                                         event["verb"], event["request"],
+                                                                         event["httpversion"], event["response"],
+                                                                         event["bytes"], event["referrer"],
+                                                                         event["agent"])
+            # we are on the hot code path here and thus we want to avoid conditionally creating strings so we duplicate
+            # the event.
+            line = '{"@timestamp": "%s", ' \
+                   '"_raw_event_size":%d, ' \
+                   '"offset":%s, ' \
+                   '"source":"/usr/local/var/log/nginx/access.log","fileset":{"module":"nginx","name":"access"},"input":{"type":"log"},' \
+                   '"beat":{"version":"6.3.0","hostname":"%s","name":"%s"},' \
+                   '"prospector":{"type":"log"},' \
+                   '"nginx":{"access":{"user_name": "-",' \
+                   '"agent":"%s","user_agent": {"major": "%s","os": "%s","os_major": "%s","name": "%s","os_name": "%s","device": "%s"},' \
+                   '"remote_ip": "%s","remote_ip_list":["%s"],' \
+                   '"geoip":{"continent_name": "%s","city_name": "%s","country_name": "%s","country_iso_code": "%s","location":{"lat": %s,"lon": %s} },' \
+                   '"referrer":"%s",' \
+                   '"url": "%s","body_sent":{"bytes": %s},"method":"%s","response_code":%s,"http_version":"%s"} } }' % \
+                   (event["@timestamp"],
+                    len(raw_event),
+                    event["offset"],
+                    event["hostname"],event["hostname"],
+                    event["agent"], event["useragent_major"], event["useragent_os"], event["useragent_os_major"], event["useragent_name"], event["useragent_os_name"], event["useragent_device"],
+                    event["clientip"], event["clientip"],
+                    event["geoip_continent_name"], event["geoip_city_name"], event["geoip_country_name"], event["geoip_country_iso_code"], event["geoip_location_lat"], event["geoip_location_lon"],
+                    event["referrer"],
+                    event["request"], event["bytes"], event["verb"], event["response"], event["httpversion"])
+        else:
+            line = '{"@timestamp": "%s", ' \
+                   '"offset":%s, ' \
+                   '"source":"/usr/local/var/log/nginx/access.log","fileset":{"module":"nginx","name":"access"},"input":{"type":"log"},' \
+                   '"beat":{"version":"6.3.0","hostname":"%s","name":"%s"},' \
+                   '"prospector":{"type":"log"},' \
+                   '"nginx":{"access":{"user_name": "-",' \
+                   '"agent":"%s","user_agent": {"major": "%s","os": "%s","os_major": "%s","name": "%s","os_name": "%s","device": "%s"},' \
+                   '"remote_ip": "%s","remote_ip_list":["%s"],' \
+                   '"geoip":{"continent_name": "%s","city_name": "%s","country_name": "%s","country_iso_code": "%s","location":{"lat": %s,"lon": %s} },' \
+                   '"referrer":"%s",' \
+                   '"url": "%s","body_sent":{"bytes": %s},"method":"%s","response_code":%s,"http_version":"%s"} } }' % \
+                   (event["@timestamp"],
+                    event["offset"],
+                    event["hostname"],event["hostname"],
+                    event["agent"], event["useragent_major"], event["useragent_os"], event["useragent_os_major"], event["useragent_name"], event["useragent_os_name"], event["useragent_device"],
+                    event["clientip"], event["clientip"],
+                    event["geoip_continent_name"], event["geoip_city_name"], event["geoip_country_name"], event["geoip_country_iso_code"], event["geoip_location_lat"], event["geoip_location_lon"],
+                    event["referrer"],
+                    event["request"], event["bytes"], event["verb"], event["response"], event["httpversion"])
 
         return line, index_name, self._type
 
diff --git a/eventdata/track.json b/eventdata/track.json
@@ -1,6 +1,7 @@
 {% import "rally.helpers" as rally with context %}
 
 {% set p_bulk_indexing_clients = (bulk_indexing_clients | default(8)) %}
+{% set p_record_raw_event_size = record_raw_event_size | default(False) | tojson %}
 
 {
   "version": 2,

Original file line number	Diff line number	Diff line change
`@@ -24,5 +24,6 @@`
`24`	`24`	`"daily_index": true,`
`25`	`25`	`"starting_point": "2018-05-01:00:00:00",`
`26`	`26`	`"acceleration_factor": 12,`
`27`		`- "bulk-size": 1000`
	`27`	`+ "bulk-size": 1000,`
	`28`	`+ "record_raw_event_size": {{p_record_raw_event_size}}`
`28`	`29`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,7 @@`
`1`	`1`	`{% import "rally.helpers" as rally with context %}`
`2`	`2`
`3`	`3`	`{% set p_bulk_indexing_clients = (bulk_indexing_clients \| default(8)) %}`
	`4`	`+{% set p_record_raw_event_size = record_raw_event_size \| default(False) \| tojson %}`
`4`	`5`
`5`	`6`	`{`
`6`	`7`	`"version": 2,`