Skip to content

Commit a9de3d6

Browse files
cdahlqvistcdahlqvist
committed
Added document id challenge and corrected shard size and index size challenges
1 parent 3087fee commit a9de3d6

7 files changed

+310
-7
lines changed

‎eventdata/challenges/combined-indexing-and-querying.json

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
{% set p_rate_limit_duration_secs = (rate_limit_duration_secs | default(1200)) %}
44
{% set p_rate_limit_step = (rate_limit_step | default(2)) %}
55
{% set p_rate_limit_max = (rate_limit_max | default(32)) %}
6+
{% set p_disk_type = disk_type | default('ssd') | lower %}
7+
{% set p_translog_sync = translog_sync | default('request') | lower %}
68

79
{
810
"name": "combined-indexing-and-querying",
@@ -93,7 +95,7 @@
9395
"schedule": "poisson"
9496
},
9597
{
96-
"name": "current-kibana-content_issues-dashboard_30m-{{rate}}",
98+
"name": "current-kibana-content_issues-dashboard_30m-{{rate}}",
9799
"operation": "current-kibana-content_issues-dashboard_30m",
98100
"target-interval": 60,
99101
"clients": 2,

‎eventdata/challenges/document_id_benchmark.json

Lines changed: 232 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,232 @@
1+
{% set p_bulk_indexing_clients = (bulk_indexing_clients | default(20)) %}
2+
{% set p_iterations = bulk_indexing_iterations | default(100000) %}
3+
{% set p_iterations_per_client = (p_iterations / p_bulk_indexing_clients) | int %}
4+
5+
{
6+
"name": "document_id_evaluation",
7+
"description": "Index about 20GB of data into five single shard indices using different document ID types. IDs are autogenerated by Elasticsearch, meaning there are no conflicts.",
8+
"meta": {
9+
"client_count": {{ p_bulk_indexing_clients }},
10+
"benchmark_type": "document-id-evaluation"
11+
},
12+
"schedule": [
13+
{
14+
"name": "deleteindex_elasticlogs-auto",
15+
"operation": {
16+
"operation-type": "delete-index",
17+
"index": "elasticlogs-auto"
18+
}
19+
},
20+
{
21+
"name": "deleteindex_elasticlogs-uuid",
22+
"operation": {
23+
"operation-type": "delete-index",
24+
"index": "elasticlogs-uuid"
25+
}
26+
},
27+
{
28+
"name": "deleteindex_elasticlogs-epoch-no_delay",
29+
"operation": {
30+
"operation-type": "delete-index",
31+
"index": "elasticlogs-epoch-no_delay"
32+
}
33+
},
34+
{
35+
"name": "deleteindex_elasticlogs-epoch-10pct_60s",
36+
"operation": {
37+
"operation-type": "delete-index",
38+
"index": "elasticlogs-epoch-10pct_60s"
39+
}
40+
},
41+
{
42+
"name": "deleteindex_elasticlogs-epoch-10pct_300s",
43+
"operation": {
44+
"operation-type": "delete-index",
45+
"index": "elasticlogs-epoch-10pct_300s"
46+
}
47+
},
48+
{
49+
"name": "create_elasticlogs-auto",
50+
"operation": {
51+
"operation-type": "createindex",
52+
"index_name": "elasticlogs-auto",
53+
"index_template_body": {
54+
"template": "elasticlogs-auto",
55+
"settings": {
56+
"index.refresh_interval": "5s",
57+
"index.codec": "best_compression",
58+
"index.number_of_replicas": 0,
59+
"index.number_of_shards": 1
60+
},
61+
"mappings":
62+
{% include "mappings.json" %}
63+
,
64+
"aliases": {}
65+
},
66+
"index_template_name": "elasticlogs-auto"
67+
}
68+
},
69+
{
70+
"name": "index-append-1000-elasticlogs-auto",
71+
"operation": {
72+
"operation-type": "bulk",
73+
"param-source": "elasticlogs_bulk",
74+
"index": "elasticlogs-auto",
75+
"bulk-size": 1000
76+
},
77+
"iterations": {{ p_iterations_per_client }},
78+
"clients": {{ p_bulk_indexing_clients }},
79+
"meta": {
80+
"id_mode": "auto"
81+
}
82+
},
83+
{
84+
"name": "create_elasticlogs-uuid",
85+
"operation": {
86+
"operation-type": "createindex",
87+
"index_name": "elasticlogs-uuid",
88+
"index_template_body": {
89+
"template": "elasticlogs-uuid",
90+
"settings": {
91+
"index.refresh_interval": "5s",
92+
"index.codec": "best_compression",
93+
"index.number_of_replicas": 0,
94+
"index.number_of_shards": 1
95+
},
96+
"mappings":
97+
{% include "mappings.json" %}
98+
,
99+
"aliases": {}
100+
},
101+
"index_template_name": "elasticlogs-uuid"
102+
}
103+
},
104+
{
105+
"name": "index-append-1000-elasticlogs-uuid",
106+
"operation": {
107+
"operation-type": "bulk",
108+
"param-source": "elasticlogs_bulk",
109+
"index": "elasticlogs-uuid",
110+
"bulk-size": 1000,
111+
"id_type": "uuid"
112+
},
113+
"iterations": {{ p_iterations_per_client }},
114+
"clients": {{ p_bulk_indexing_clients }},
115+
"meta": {
116+
"id_mode": "uuid"
117+
}
118+
},
119+
{
120+
"name": "create_elasticlogs-epoch-no_delay",
121+
"operation": {
122+
"operation-type": "createindex",
123+
"index_name": "elasticlogs-epoch-no_delay",
124+
"index_template_body": {
125+
"template": "elasticlogs-epoch-no_delay",
126+
"settings": {
127+
"index.refresh_interval": "5s",
128+
"index.codec": "best_compression",
129+
"index.number_of_replicas": 0,
130+
"index.number_of_shards": 1
131+
},
132+
"mappings":
133+
{% include "mappings.json" %}
134+
,
135+
"aliases": {}
136+
},
137+
"index_template_name": "elasticlogs-epoch-no_delay"
138+
}
139+
},
140+
{
141+
"name": "index-append-1000-elasticlogs-epoch-no_delay",
142+
"operation": {
143+
"operation-type": "bulk",
144+
"param-source": "elasticlogs_bulk",
145+
"index": "elasticlogs-epoch-no_delay",
146+
"bulk-size": 1000,
147+
"id_type": "epoch_uuid"
148+
},
149+
"iterations": {{ p_iterations_per_client }},
150+
"clients": {{ p_bulk_indexing_clients }},
151+
"meta": {
152+
"id_mode": "epoch_uuid-no_delay"
153+
}
154+
},
155+
{
156+
"name": "create_elasticlogs-epoch-10pct_60s",
157+
"operation": {
158+
"operation-type": "createindex",
159+
"index_name": "elasticlogs-epoch-10pct_60s",
160+
"index_template_body": {
161+
"template": "elasticlogs-epoch-10pct_60s",
162+
"settings": {
163+
"index.refresh_interval": "5s",
164+
"index.codec": "best_compression",
165+
"index.number_of_replicas": 0,
166+
"index.number_of_shards": 1
167+
},
168+
"mappings":
169+
{% include "mappings.json" %}
170+
,
171+
"aliases": {}
172+
},
173+
"index_template_name": "elasticlogs-epoch-10pct_60s"
174+
}
175+
},
176+
{
177+
"name": "index-append-1000-elasticlogs-epoch-10pct_60s",
178+
"operation": {
179+
"operation-type": "bulk",
180+
"param-source": "elasticlogs_bulk",
181+
"index": "elasticlogs-epoch-10pct_60s",
182+
"bulk-size": 1000,
183+
"id_type": "epoch_uuid",
184+
"id_delay_probability": 0.1,
185+
"id_delay_secs": 60
186+
},
187+
"iterations": {{ p_iterations_per_client }},
188+
"clients": {{ p_bulk_indexing_clients }},
189+
"meta": {
190+
"id_mode": "epoch_uuid-10pct/60s"
191+
}
192+
},
193+
{
194+
"name": "create_elasticlogs-epoch-10pct_300s",
195+
"operation": {
196+
"operation-type": "createindex",
197+
"index_name": "elasticlogs-epoch-10pct_300s",
198+
"index_template_body": {
199+
"template": "elasticlogs-epoch-10pct_300s",
200+
"settings": {
201+
"index.refresh_interval": "5s",
202+
"index.codec": "best_compression",
203+
"index.number_of_replicas": 0,
204+
"index.number_of_shards": 1
205+
},
206+
"mappings":
207+
{% include "mappings.json" %}
208+
,
209+
"aliases": {}
210+
},
211+
"index_template_name": "elasticlogs-epoch-10pct_300s"
212+
}
213+
},
214+
{
215+
"name": "index-append-1000-elasticlogs-epoch-10pct_300s",
216+
"operation": {
217+
"operation-type": "bulk",
218+
"param-source": "elasticlogs_bulk",
219+
"index": "elasticlogs-epoch-10pct_300s",
220+
"bulk-size": 1000,
221+
"id_type": "epoch_uuid",
222+
"id_delay_probability": 0.1,
223+
"id_delay_secs": 300
224+
},
225+
"iterations": {{ p_iterations_per_client }},
226+
"clients": {{ p_bulk_indexing_clients }},
227+
"meta": {
228+
"id_mode": "epoch_uuid-10pct/300s"
229+
}
230+
}
231+
]
232+
}

‎eventdata/challenges/elasticlogs-1bn-load.json

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
{% set p_bulk_indexing_clients = (bulk_indexing_clients | default(20)) %}
22
{% set p_iterations = bulk_indexing_iterations | default(1000000) %}
33
{% set p_iterations_per_client = (p_iterations / p_bulk_indexing_clients) | int %}
4+
{% set p_disk_type = disk_type | default('ssd') | lower %}
5+
{% set p_translog_sync = translog_sync | default('request') | lower %}
46

57
{
68
"name": "elasticlogs-1bn-load",

‎eventdata/challenges/shard-size-on-disk.json

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
{% set p_bulk_indexing_clients = (bulk_indexing_clients | default(8)) %}
2+
{% set p_iterations = bulk_indexing_iterations | default(2000) %}
3+
{% set p_iterations_per_client = (p_iterations / p_bulk_indexing_clients) | int %}
24

35
{
46
"name": "shard-size-on-disk",
@@ -17,7 +19,7 @@
1719
{
1820
"name": "index-append-1000-shard-sizing-{{n}}",
1921
"operation": "index-append-1000-shard-sizing",
20-
"iterations": 2000,
22+
"iterations": {{ p_iterations_per_client }},
2123
"clients": {{ p_bulk_indexing_clients }},
2224
"meta": {
2325
"iteration_number": {{ n }}
@@ -34,7 +36,7 @@
3436
{% endfor %}
3537
{
3638
"operation": "index-append-1000-shard-sizing",
37-
"iterations": 2000,
39+
"iterations": {{ p_iterations_per_client }},
3840
"clients": {{ p_bulk_indexing_clients }},
3941
"meta": {
4042
"iteration_number": {{ 50 }}

‎eventdata/challenges/shard-sizing.json

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
{% set p_bulk_indexing_clients = (bulk_indexing_clients | default(8)) %}
2+
{% set p_iterations = bulk_indexing_iterations | default(2000) %}
3+
{% set p_iterations_per_client = (p_iterations / p_bulk_indexing_clients) | int %}
24
{% set p_shard_sizing_iterations = (shard_sizing_iterations | default(25)) %}
35
{% set p_shard_sizing_queries = (shard_sizing_queries | default(20)) %}
46

@@ -19,7 +21,7 @@
1921
{
2022
"name": "index-append-1000-shard-sizing-iteration-{{n}}",
2123
"operation": "index-append-1000-shard-sizing",
22-
"iterations": 2000,
24+
"iterations": {{ p_iterations_per_client }},
2325
"clients": {{ p_bulk_indexing_clients }},
2426
"meta": {
2527
"iteration_number": {{ n }}
@@ -68,7 +70,7 @@
6870
{% endfor %}
6971
{
7072
"operation": "index-append-1000-shard-sizing",
71-
"iterations": 2000,
73+
"iterations": {{ p_iterations_per_client }},
7274
"clients": {{ p_bulk_indexing_clients }},
7375
"meta": {
7476
"iteration_number": {{ p_shard_sizing_iterations }}

‎eventdata/operations/indexing.json

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
{% set p_disk_type = disk_type | default('ssd') | lower %}
2+
{% set p_translog_sync = translog_sync | default('request') | lower %}
23

34
{
45
"name": "index-append-50000",
@@ -87,6 +88,9 @@
8788
"settings": {
8889
"index.refresh_interval": "5s",
8990
"index.codec": "best_compression",
91+
{% if (p_translog_sync != 'request') %}
92+
"index.translog.durability": "async",
93+
{% endif %}
9094
{% if (p_disk_type != 'ssd') %}
9195
"index.merge.scheduler.max_thread_count": 1,
9296
{% endif %}
@@ -110,6 +114,9 @@
110114
"settings": {
111115
"index.refresh_interval": "5s",
112116
"index.codec": "best_compression",
117+
{% if (p_translog_sync != 'request') %}
118+
"index.translog.durability": "async",
119+
{% endif %}
113120
{% if (p_disk_type != 'ssd') %}
114121
"index.merge.scheduler.max_thread_count": 1,
115122
{% endif %}
@@ -132,6 +139,9 @@
132139
"settings": {
133140
"index.refresh_interval": "5s",
134141
"index.codec": "best_compression",
142+
{% if (p_translog_sync != 'request') %}
143+
"index.translog.durability": "async",
144+
{% endif %}
135145
{% if (p_disk_type != 'ssd') %}
136146
"index.merge.scheduler.max_thread_count": 1,
137147
{% endif %}

0 commit comments

Comments
 (0)