Skip to content

Commit 0b8ea07

Browse files
jasontedorjasontedor
committed
Default to one shard
This commit changes the default out-of-the-box configuration for the number of shards from five to one. We think this will help address a common problem of oversharding. For users with time-based indices that need a different default, this can be managed with index templates. For users with non-time-based indices that find they need to re-shard with the shrink API in place they no longer need to resort only to reindexing. Since this has the impact of changing the default number of shards used in REST tests, we want to ensure that we still have coverage for issues that could arise from multiple shards. As such, we randomize (rarely) the default number of shards in REST tests to two. This is managed via a global index template. However, some tests check the templates that are in the cluster state during the test. Since this template is randomly there, we need a way for tests to skip adding the template used to set the number of shards to two. For this we add the default_shards feature skip. To avoid having to write our docs in a complicated way because sometimes they might be behind one shard, and sometimes they might be behind two shards we apply the default_shards feature skip to all docs tests. That is, these tests will always run with the default number of shards (one).
1 parent 15df911 commit 0b8ea07

File tree

34 files changed

+136
-129
lines changed

34 files changed

+136
-129
lines changed

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
225225
* warning every time. */
226226
current.println(" - skip:")
227227
current.println(" features: ")
228+
current.println(" - default_shards")
228229
current.println(" - stash_in_key")
229230
current.println(" - stash_in_path")
230231
current.println(" - stash_path_replace")

‎client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -312,7 +312,7 @@ public void testSearchWithMatrixStats() throws IOException {
312312
MatrixStats matrixStats = searchResponse.getAggregations().get("agg1");
313313
assertEquals(5, matrixStats.getFieldCount("num"));
314314
assertEquals(56d, matrixStats.getMean("num"), 0d);
315-
assertEquals(1830d, matrixStats.getVariance("num"), 0d);
315+
assertEquals(1830.0000000000002, matrixStats.getVariance("num"), 0d);
316316
assertEquals(0.09340198804973046, matrixStats.getSkewness("num"), 0d);
317317
assertEquals(1.2741646510794589, matrixStats.getKurtosis("num"), 0d);
318318
assertEquals(5, matrixStats.getFieldCount("num2"));

‎distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,18 +21,22 @@
2121

2222
import org.apache.http.entity.ContentType;
2323
import org.apache.http.entity.StringEntity;
24+
import org.elasticsearch.client.Request;
2425
import org.elasticsearch.client.Response;
26+
import org.junit.Before;
2527

2628
import java.io.IOException;
2729

2830
import static java.util.Collections.emptyMap;
2931
import static java.util.Collections.singletonMap;
32+
import static org.hamcrest.Matchers.equalTo;
3033
import static org.hamcrest.Matchers.startsWith;
3134

3235
/**
3336
* Tests for the "Location" header returned when returning {@code 201 CREATED}.
3437
*/
3538
public class CreatedLocationHeaderIT extends ESRestTestCase {
39+
3640
public void testCreate() throws IOException {
3741
locationTestCase("PUT", "test/test/1");
3842
}
@@ -54,8 +58,11 @@ public void testUpsert() throws IOException {
5458
private void locationTestCase(String method, String url) throws IOException {
5559
locationTestCase(client().performRequest(method, url, emptyMap(),
5660
new StringEntity("{\"test\": \"test\"}", ContentType.APPLICATION_JSON)));
61+
// we have to delete the index otherwise the second indexing request will route to the single shard and not produce a 201
62+
final Response response = client().performRequest(new Request("DELETE", "test"));
63+
assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
5764
locationTestCase(client().performRequest(method, url + "?routing=cat", emptyMap(),
58-
new StringEntity("{\"test\": \"test\"}", ContentType.APPLICATION_JSON)));
65+
new StringEntity("{\"test\": \"test\"}", ContentType.APPLICATION_JSON)));
5966
}
6067

6168
private void locationTestCase(Response response) throws IOException {
@@ -65,4 +72,5 @@ private void locationTestCase(Response response) throws IOException {
6572
Response getResponse = client().performRequest("GET", location);
6673
assertEquals(singletonMap("test", "test"), entityAsMap(getResponse).get("_source"));
6774
}
75+
6876
}

‎docs/reference/aggregations/bucket/children-aggregation.asciidoc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -137,8 +137,8 @@ Possible response:
137137
"took": 25,
138138
"timed_out": false,
139139
"_shards": {
140-
"total": 5,
141-
"successful": 5,
140+
"total": 1,
141+
"successful": 1,
142142
"skipped" : 0,
143143
"failed": 0
144144
},

‎docs/reference/aggregations/metrics/geocentroid-aggregation.asciidoc

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,8 @@ The response for the above aggregation:
6060
"aggregations": {
6161
"centroid": {
6262
"location": {
63-
"lat": 51.00982963806018,
64-
"lon": 3.9662131061777472
63+
"lat": 51.009829603135586,
64+
"lon": 3.9662130642682314
6565
},
6666
"count": 6
6767
}
@@ -113,8 +113,8 @@ The response for the above aggregation:
113113
"doc_count": 3,
114114
"centroid": {
115115
"location": {
116-
"lat": 52.371655656024814,
117-
"lon": 4.909563297405839
116+
"lat": 52.371655642054975,
117+
"lon": 4.9095632415264845
118118
},
119119
"count": 3
120120
}
@@ -125,7 +125,7 @@ The response for the above aggregation:
125125
"centroid": {
126126
"location": {
127127
"lat": 48.86055548675358,
128-
"lon": 2.3316944623366
128+
"lon": 2.331694420427084
129129
},
130130
"count": 2
131131
}

‎docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -235,8 +235,8 @@ The output from the above is:
235235
"timed_out": false,
236236
"took": $body.took,
237237
"_shards": {
238-
"total": 5,
239-
"successful": 5,
238+
"total": 1,
239+
"successful": 1,
240240
"skipped" : 0,
241241
"failed": 0
242242
},

‎docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -294,8 +294,8 @@ GET my_index/_search
294294
"took": $body.took,
295295
"timed_out": false,
296296
"_shards": {
297-
"total": 5,
298-
"successful": 5,
297+
"total": 1,
298+
"successful": 1,
299299
"skipped" : 0,
300300
"failed": 0
301301
},

‎docs/reference/api-conventions.asciidoc

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -300,11 +300,7 @@ Responds:
300300
"indices": {
301301
"twitter": {
302302
"shards": {
303-
"0": [{"state": "STARTED"}, {"state": "UNASSIGNED"}],
304-
"1": [{"state": "STARTED"}, {"state": "UNASSIGNED"}],
305-
"2": [{"state": "STARTED"}, {"state": "UNASSIGNED"}],
306-
"3": [{"state": "STARTED"}, {"state": "UNASSIGNED"}],
307-
"4": [{"state": "STARTED"}, {"state": "UNASSIGNED"}]
303+
"0": [{"state": "STARTED"}, {"state": "UNASSIGNED"}]
308304
}
309305
}
310306
}

‎docs/reference/cat/allocation.asciidoc

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ Might respond with:
1616
[source,txt]
1717
--------------------------------------------------
1818
shards disk.indices disk.used disk.avail disk.total disk.percent host ip node
19-
5 260b 47.3gb 43.4gb 100.7gb 46 127.0.0.1 127.0.0.1 CSUXak2
19+
1 260b 47.3gb 43.4gb 100.7gb 46 127.0.0.1 127.0.0.1 CSUXak2
2020
--------------------------------------------------
2121
// TESTRESPONSE[s/\d+(\.\d+)?[tgmk]?b/\\d+(\\.\\d+)?[tgmk]?b/ s/46/\\d+/]
2222
// TESTRESPONSE[s/CSUXak2/.+/ _cat]

‎docs/reference/cat/health.asciidoc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ GET /_cat/health?v
1414
[source,txt]
1515
--------------------------------------------------
1616
epoch timestamp cluster status node.total node.data shards pri relo init unassign pending_tasks max_task_wait_time active_shards_percent
17-
1475871424 16:17:04 elasticsearch green 1 1 5 5 0 0 0 0 - 100.0%
17+
1475871424 16:17:04 elasticsearch green 1 1 1 1 0 0 0 0 - 100.0%
1818
--------------------------------------------------
1919
// TESTRESPONSE[s/1475871424 16:17:04/\\d+ \\d+:\\d+:\\d+/]
2020
// TESTRESPONSE[s/elasticsearch/[^ ]+/ s/0 -/\\d+ (-|\\d+(\\.\\d+)?[ms]+)/ _cat]
@@ -33,7 +33,7 @@ which looks like:
3333
[source,txt]
3434
--------------------------------------------------
3535
cluster status node.total node.data shards pri relo init unassign pending_tasks max_task_wait_time active_shards_percent
36-
elasticsearch green 1 1 5 5 0 0 0 0 - 100.0%
36+
elasticsearch green 1 1 1 1 0 0 0 0 - 100.0%
3737
--------------------------------------------------
3838
// TESTRESPONSE[s/elasticsearch/[^ ]+/ s/0 -/\\d+ (-|\\d+(\\.\\d+)?[ms]+)/ _cat]
3939

‎docs/reference/cat/indices.asciidoc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ Might respond with:
1818
--------------------------------------------------
1919
health status index uuid pri rep docs.count docs.deleted store.size pri.store.size
2020
yellow open twitter u8FNjxh8Rfy_awN11oDKYQ 1 1 1200 0 88.1kb 88.1kb
21-
green open twitter2 nYFWZEO7TUiOjLQXBaYJpA 5 0 0 0 260b 260b
21+
green open twitter2 nYFWZEO7TUiOjLQXBaYJpA 1 0 0 0 260b 260b
2222
--------------------------------------------------
2323
// TESTRESPONSE[s/\d+(\.\d+)?[tgmk]?b/\\d+(\\.\\d+)?[tgmk]?b/]
2424
// TESTRESPONSE[s/u8FNjxh8Rfy_awN11oDKYQ|nYFWZEO7TUiOjLQXBaYJpA/.+/ _cat]
@@ -81,7 +81,7 @@ Which looks like:
8181
--------------------------------------------------
8282
health status index uuid pri rep docs.count docs.deleted store.size pri.store.size
8383
yellow open twitter u8FNjxh8Rfy_awN11oDKYQ 1 1 1200 0 88.1kb 88.1kb
84-
green open twitter2 nYFWZEO7TUiOjLQXBaYJpA 5 0 0 0 260b 260b
84+
green open twitter2 nYFWZEO7TUiOjLQXBaYJpA 1 0 0 0 260b 260b
8585
--------------------------------------------------
8686
// TESTRESPONSE[s/\d+(\.\d+)?[tgmk]?b/\\d+(\\.\\d+)?[tgmk]?b/]
8787
// TESTRESPONSE[s/u8FNjxh8Rfy_awN11oDKYQ|nYFWZEO7TUiOjLQXBaYJpA/.+/ _cat]

‎docs/reference/cat/segments.asciidoc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@ might look like:
1717
["source","txt",subs="attributes,callouts"]
1818
--------------------------------------------------
1919
index shard prirep ip segment generation docs.count docs.deleted size size.memory committed searchable version compound
20-
test 4 p 127.0.0.1 _0 0 1 0 3kb 2042 false true {lucene_version} true
21-
test1 4 p 127.0.0.1 _0 0 1 0 3kb 2042 false true {lucene_version} true
20+
test 0 p 127.0.0.1 _0 0 1 0 3kb 2042 false true {lucene_version} true
21+
test1 0 p 127.0.0.1 _0 0 1 0 3kb 2042 false true {lucene_version} true
2222
--------------------------------------------------
2323
// TESTRESPONSE[s/3kb/\\d+(\\.\\d+)?[mk]?b/ s/2042/\\d+/ _cat]
2424

‎docs/reference/cluster/health.asciidoc

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,11 @@ Returns this:
2222
"timed_out" : false,
2323
"number_of_nodes" : 1,
2424
"number_of_data_nodes" : 1,
25-
"active_primary_shards" : 5,
26-
"active_shards" : 5,
25+
"active_primary_shards" : 1,
26+
"active_shards" : 1,
2727
"relocating_shards" : 0,
2828
"initializing_shards" : 0,
29-
"unassigned_shards" : 5,
29+
"unassigned_shards" : 1,
3030
"delayed_unassigned_shards": 0,
3131
"number_of_pending_tasks" : 0,
3232
"number_of_in_flight_fetch": 0,

‎docs/reference/getting-started.asciidoc

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -366,7 +366,7 @@ And the response:
366366
[source,txt]
367367
--------------------------------------------------
368368
health status index uuid pri rep docs.count docs.deleted store.size pri.store.size
369-
yellow open customer 95SQ4TSUT7mWBT7VNHH67A 5 1 0 0 260b 260b
369+
yellow open customer 95SQ4TSUT7mWBT7VNHH67A 1 1 0 0 260b 260b
370370
--------------------------------------------------
371371
// TESTRESPONSE[s/95SQ4TSUT7mWBT7VNHH67A/.+/ s/260b/\\d+\\.?\\d?k?b/ _cat]
372372

‎docs/reference/how-to/recipes/stemming.asciidoc

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -78,31 +78,31 @@ GET index/_search
7878
"took": 2,
7979
"timed_out": false,
8080
"_shards": {
81-
"total": 5,
82-
"successful": 5,
81+
"total": 1,
82+
"successful": 1,
8383
"skipped" : 0,
8484
"failed": 0
8585
},
8686
"hits": {
8787
"total": 2,
88-
"max_score": 0.2876821,
88+
"max_score": 0.18232156,
8989
"hits": [
9090
{
9191
"_index": "index",
9292
"_type": "_doc",
93-
"_id": "2",
94-
"_score": 0.2876821,
93+
"_id": "1",
94+
"_score": 0.18232156,
9595
"_source": {
96-
"body": "A pair of skis"
96+
"body": "Ski resort"
9797
}
9898
},
9999
{
100100
"_index": "index",
101101
"_type": "_doc",
102-
"_id": "1",
103-
"_score": 0.2876821,
102+
"_id": "2",
103+
"_score": 0.18232156,
104104
"_source": {
105-
"body": "Ski resort"
105+
"body": "A pair of skis"
106106
}
107107
}
108108
]
@@ -136,20 +136,20 @@ GET index/_search
136136
"took": 1,
137137
"timed_out": false,
138138
"_shards": {
139-
"total": 5,
140-
"successful": 5,
139+
"total": 1,
140+
"successful": 1,
141141
"skipped" : 0,
142142
"failed": 0
143143
},
144144
"hits": {
145145
"total": 1,
146-
"max_score": 0.2876821,
146+
"max_score": 0.80259144,
147147
"hits": [
148148
{
149149
"_index": "index",
150150
"_type": "_doc",
151151
"_id": "1",
152-
"_score": 0.2876821,
152+
"_score": 0.80259144,
153153
"_source": {
154154
"body": "Ski resort"
155155
}
@@ -193,20 +193,20 @@ GET index/_search
193193
"took": 2,
194194
"timed_out": false,
195195
"_shards": {
196-
"total": 5,
197-
"successful": 5,
196+
"total": 1,
197+
"successful": 1,
198198
"skipped" : 0,
199199
"failed": 0
200200
},
201201
"hits": {
202202
"total": 1,
203-
"max_score": 0.2876821,
203+
"max_score": 0.80259144,
204204
"hits": [
205205
{
206206
"_index": "index",
207207
"_type": "_doc",
208208
"_id": "1",
209-
"_score": 0.2876821,
209+
"_score": 0.80259144,
210210
"_source": {
211211
"body": "Ski resort"
212212
}

‎docs/reference/indices/flush.asciidoc

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -106,11 +106,7 @@ which returns something similar to:
106106
"num_docs" : 0
107107
}
108108
}
109-
],
110-
"1": ...,
111-
"2": ...,
112-
"3": ...,
113-
"4": ...
109+
]
114110
}
115111
}
116112
}
@@ -120,10 +116,6 @@ which returns something similar to:
120116
// TESTRESPONSE[s/"translog_uuid" : "hnOG3xFcTDeoI_kvvvOdNA"/"translog_uuid": $body.indices.twitter.shards.0.0.commit.user_data.translog_uuid/]
121117
// TESTRESPONSE[s/"history_uuid" : "XP7KDJGiS1a2fHYiFL5TXQ"/"history_uuid": $body.indices.twitter.shards.0.0.commit.user_data.history_uuid/]
122118
// TESTRESPONSE[s/"sync_id" : "AVvFY-071siAOuFGEO9P"/"sync_id": $body.indices.twitter.shards.0.0.commit.user_data.sync_id/]
123-
// TESTRESPONSE[s/"1": \.\.\./"1": $body.indices.twitter.shards.1/]
124-
// TESTRESPONSE[s/"2": \.\.\./"2": $body.indices.twitter.shards.2/]
125-
// TESTRESPONSE[s/"3": \.\.\./"3": $body.indices.twitter.shards.3/]
126-
// TESTRESPONSE[s/"4": \.\.\./"4": $body.indices.twitter.shards.4/]
127119
<1> the `sync id` marker
128120

129121
[float]

‎docs/reference/indices/shrink-index.asciidoc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ PUT /my_source_index/_settings
4242
}
4343
--------------------------------------------------
4444
// CONSOLE
45-
// TEST[s/^/PUT my_source_index\n/]
45+
// TEST[s/^/PUT my_source_index\n{"settings":{"index.number_of_shards":2}}\n/]
4646
<1> Forces the relocation of a copy of each shard to the node with name
4747
`shrink_node_name`. See <<shard-allocation-filtering>> for more options.
4848

@@ -110,7 +110,7 @@ POST my_source_index/_shrink/my_target_index
110110
}
111111
--------------------------------------------------
112112
// CONSOLE
113-
// TEST[s/^/PUT my_source_index\n{"settings": {"index.blocks.write": true}}\n/]
113+
// TEST[s/^/PUT my_source_index\n{"settings": {"index.number_of_shards":5,"index.blocks.write": true}}\n/]
114114

115115
<1> The number of shards in the target index. This must be a factor of the
116116
number of shards in the source index.

0 commit comments

Comments
 (0)