add input and vars in rally scenario spec, make input and corpora man… (#653)

Author: Andrea Spacca

spec/changelog.yml

@@ -4,6 +4,9 @@
 ##
 - version: 3.0.1-next
   changes:
+  - description: Fix spec for rally benchmark scenario
+    type: bugfix
+    link: https://github.com/elastic/package-spec/pull/653
   - description: Using non-GA versions of the spec in GA packages produces a filterable validation error instead of a warning
     type: enhancement
     link: https://github.com/elastic/package-spec/pull/627

spec/integration/_dev/benchmark/rally.scenario.spec.yml

@@ -3,6 +3,7 @@ spec:
   additionalProperties: false
   required:
     - data_stream
+    - corpora
   properties:
     package:
       $ref: "./system.scenario.spec.yml#/definitions/package"
@@ -12,11 +13,5 @@ spec:
       $ref: "../../manifest.spec.yml#/definitions/version"
     data_stream:
       $ref: "./system.scenario.spec.yml#/definitions/data_stream"
-    warmup_time_period:
-      $ref: "./system.scenario.spec.yml#/definitions/warmup_time_period"
-    benchmark_time_period:
-      $ref: "./system.scenario.spec.yml#/definitions/benchmark_time_period"
-    wait_for_data_timeout:
-      $ref: "./system.scenario.spec.yml#/definitions/wait_for_data_timeout"
     corpora:
       $ref: "./system.scenario.spec.yml#/definitions/corpora"

spec/integration/_dev/benchmark/spec.yml

@@ -33,6 +33,27 @@ spec:
           type: folder
           name: deploy
           $ref: "./deploy/spec.yml"
+        - description: A folder containing resources used by one or more benchmarks.
+          type: folder
+          pattern: '^[a-z0-9]+-benchmark$'
+          additionalContents: false
+          required: false
+          contents:
+            - description: A file with the config definition for the corpus generator.
+              type: file
+              name: 'config.yml'
+              contentMediaType: "application/x-yaml"
+              required: false
+            - description: A file with the fields definition for the corpus generator
+              type: file
+              name: 'fields.yml'
+              contentMediaType: "application/x-yaml"
+              required: false
+            - description: A file with the template for the corpus generator
+              type: file
+              name: 'template.ndjson'
+              contentMediaType: "application/json"
+              required: false
         - description: A rally benchmark scenario.
           type: file
           pattern: '^.+\.yml$'

test/packages/integration_benchmarks/_dev/benchmark/rally/20000-benchmark/config.yml

@@ -0,0 +1,23 @@
+- name: timestamp
+  type: date
+- name: container.id
+  type: keyword
+- name: log.file.path
+  example: /var/log/fun-times.log
+  type: keyword
+- name: log.flags
+  type: keyword
+- name: log.offset
+  type: long
+- name: tags
+  type: keyword
+- name: IP
+  type: ip
+- name: StatusCode
+  type: keyword
+- name: Size
+  type: long
+- name: Hostname
+  type: keyword
+- name: Port
+  type: long

test/packages/integration_benchmarks/_dev/benchmark/rally/20000-benchmark/fields.yml

@@ -0,0 +1,18 @@
+- name: IP
+  type: ip
+- name: Day
+  type: long
+- name: Mon
+  type: keyword
+- name: H
+  type: long
+- name: MS
+  type: long
+- name: StatusCode
+  type: keyword
+- name: Size
+  type: long
+- name: Hostname
+  type: keyword
+- name: Port
+  type: long

test/packages/integration_benchmarks/_dev/benchmark/rally/20000-benchmark/template.ndjson

@@ -0,0 +1,14 @@
+{{- $timestamp := generate "timestamp" -}}
+{
+"@timestamp": "{{ $timestamp.Format "2006-01-02T15:04:05.999999Z07:00" }}",
+"data_stream.type": "logs",
+"data_stream.dataset": "rally_benchmarks.testds",
+"data_stream.namespace": "ep",
+"container.id": "{{ generate "container.id" }}",
+"input.type": "filestream",
+"log.file.path": "{{ generate "log.file.path" }}",
+"log.flags": "{{ generate "log.flags" }}",
+"log.offset": {{ generate "log.offset" }},
+"tags": ["rally_benchmark.testds", "forwarded", "{{ generate "tags" }}" ],
+"message": "{{ generate "IP" }} - - [{{ $timestamp.Format "02/Jan/2006:15:04:05.999999 -0700" }}] \"GET /favicon.ico HTTP/1.1\" {{ generate "StatusCode" }} {{ generate "Size" }} \"http://{{ generate "Hostname" }}:{{ generate "Port" }}/\" \"skip-this-one/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36\""
+}

test/packages/integration_benchmarks/_dev/benchmark/rally/deploy/docker/docker-compose.yml

@@ -1,8 +1,14 @@
 ---
-description: Benchmark 20MiB of data ingested
-data_stream.name: test
-warmup_time_period: 10s
-corpora.generator.total_events: 20000
-corpora.generator.template.raw: '{"@timestamp": "2016-12-07T10:04:37.000Z", "message": "a message"}'
-corpora.generator.config.path: ./20mb-track/config.yml
-corpora.generator.fields.path: ./20mb-track/fields.yml
+description: Benchmark 20000 events ingested
+data_stream:
+  name: test
+corpora:
+  generator:
+    total_events: 20000
+    template:
+      type: gotext
+      path: ./20000-benchmark/template.ndjson
+    config:
+      path: ./20000-benchmark/config.yml
+    fields:
+      path: ./20000-benchmark/fields.yml