Skip to content

Commit 973dba9

Browse files
richardwilly98richardwilly98
committed
Switch to BulkProcessor API
- Ass sugested here http://blog.trifork.com/2013/01/10/how-to-write-an-elasticsearch-river-plugin/ implement BulkkProcessor API - Bulk processor concurrent request: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/modules-threadpool.html#modules-threadpool
1 parent 0da87d2 commit 973dba9

9 files changed

+376
-120
lines changed

Diff for: ‎src/main/java/org/elasticsearch/river/mongodb/Indexer.java

+121-75
Large diffs are not rendered by default.

Diff for: ‎src/main/java/org/elasticsearch/river/mongodb/MongoDBRiver.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131

3232
import org.bson.types.BSONTimestamp;
3333
import org.elasticsearch.ExceptionsHelper;
34-
import org.elasticsearch.action.bulk.BulkRequestBuilder;
34+
import org.elasticsearch.action.bulk.BulkProcessor;
3535
import org.elasticsearch.action.count.CountResponse;
3636
import org.elasticsearch.action.get.GetResponse;
3737
import org.elasticsearch.client.Client;
@@ -399,9 +399,9 @@ public static BSONTimestamp getLastTimestamp(Client client, MongoDBRiverDefiniti
399399
*
400400
* @param bulk
401401
*/
402-
static void updateLastTimestamp(final MongoDBRiverDefinition definition, final BSONTimestamp time, final BulkRequestBuilder bulk) {
402+
static void updateLastTimestamp(final MongoDBRiverDefinition definition, final BSONTimestamp time, final BulkProcessor bulkProcessor) {
403403
try {
404-
bulk.add(indexRequest(definition.getRiverIndexName())
404+
bulkProcessor.add(indexRequest(definition.getRiverIndexName())
405405
.type(definition.getRiverName())
406406
.id(definition.getMongoOplogNamespace())
407407
.source(jsonBuilder().startObject().startObject(TYPE).field(LAST_TIMESTAMP_FIELD, JSON.serialize(time)).endObject()

Diff for: ‎src/main/java/org/elasticsearch/river/mongodb/MongoDBRiverDefinition.java

+114-34
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
import org.elasticsearch.common.collect.Maps;
2323
import org.elasticsearch.common.logging.ESLogger;
2424
import org.elasticsearch.common.logging.Loggers;
25+
import org.elasticsearch.common.unit.ByteSizeUnit;
26+
import org.elasticsearch.common.unit.ByteSizeValue;
2527
import org.elasticsearch.common.unit.TimeValue;
2628
import org.elasticsearch.common.xcontent.support.XContentMapValues;
2729
import org.elasticsearch.river.RiverSettings;
@@ -42,6 +44,10 @@ public class MongoDBRiverDefinition {
4244
// defaults
4345
public final static String DEFAULT_DB_HOST = "localhost";
4446
public final static int DEFAULT_DB_PORT = 27017;
47+
public final static int DEFAULT_CONCURRENT_REQUESTS = 50;
48+
public final static int DEFAULT_BULK_ACTIONS = 500;
49+
public final static TimeValue DEFAULT_FLUSH_INTERVAL = TimeValue.timeValueMillis(10);
50+
public final static ByteSizeValue DEFAULT_BULK_SIZE = new ByteSizeValue(5, ByteSizeUnit.MB);
4551

4652
// fields
4753
public final static String DB_FIELD = "db";
@@ -79,6 +85,13 @@ public class MongoDBRiverDefinition {
7985
public final static String THROTTLE_SIZE_FIELD = "throttle_size";
8086
public final static String BULK_SIZE_FIELD = "bulk_size";
8187
public final static String BULK_TIMEOUT_FIELD = "bulk_timeout";
88+
public final static String CONCURRENT_BULK_REQUESTS_FIELD = "concurrent_bulk_requests";
89+
90+
public final static String BULK_FIELD = "bulk";
91+
public final static String ACTIONS_FIELD = "actions";
92+
public final static String SIZE_FIELD = "size";
93+
public final static String CONCURRENT_REQUESTS_FIELD = "concurrent_requests";
94+
public final static String FLUSH_INTERVAL_FIELD = "flush_interval";
8295

8396
// river
8497
private final String riverName;
@@ -117,9 +130,10 @@ public class MongoDBRiverDefinition {
117130
// index
118131
private final String indexName;
119132
private final String typeName;
120-
private final int bulkSize;
121-
private final TimeValue bulkTimeout;
122133
private final int throttleSize;
134+
135+
// bulk
136+
private final Bulk bulk;
123137

124138
public static class Builder {
125139
// river
@@ -159,10 +173,10 @@ public static class Builder {
159173
// index
160174
private String indexName;
161175
private String typeName;
162-
private int bulkSize;
163-
private TimeValue bulkTimeout;
164176
private int throttleSize;
165-
177+
178+
private Bulk bulk;
179+
166180
public Builder mongoServers(List<ServerAddress> mongoServers) {
167181
this.mongoServers = mongoServers;
168182
return this;
@@ -308,18 +322,13 @@ public Builder typeName(String typeName) {
308322
return this;
309323
}
310324

311-
public Builder bulkSize(int bulkSize) {
312-
this.bulkSize = bulkSize;
313-
return this;
314-
}
315-
316-
public Builder bulkTimeout(TimeValue bulkTimeout) {
317-
this.bulkTimeout = bulkTimeout;
325+
public Builder throttleSize(int throttleSize) {
326+
this.throttleSize = throttleSize;
318327
return this;
319328
}
320329

321-
public Builder throttleSize(int throttleSize) {
322-
this.throttleSize = throttleSize;
330+
public Builder bulk(Bulk bulk) {
331+
this.bulk = bulk;
323332
return this;
324333
}
325334

@@ -328,6 +337,73 @@ public MongoDBRiverDefinition build() {
328337
}
329338
}
330339

340+
static class Bulk {
341+
342+
private final int concurrentRequests;
343+
private final int bulkActions;
344+
private final ByteSizeValue bulkSize;
345+
private final TimeValue flushInterval;
346+
347+
static class Builder {
348+
349+
private int concurrentRequests = DEFAULT_CONCURRENT_REQUESTS;
350+
private int bulkActions = DEFAULT_BULK_ACTIONS;
351+
private ByteSizeValue bulkSize = DEFAULT_BULK_SIZE;
352+
private TimeValue flushInterval = DEFAULT_FLUSH_INTERVAL;
353+
354+
public Builder concurrentRequests(int concurrentRequests) {
355+
this.concurrentRequests = concurrentRequests;
356+
return this;
357+
}
358+
359+
public Builder bulkActions(int bulkActions) {
360+
this.bulkActions = bulkActions;
361+
return this;
362+
}
363+
364+
public Builder bulkSize(ByteSizeValue bulkSize) {
365+
this.bulkSize = bulkSize;
366+
return this;
367+
}
368+
369+
public Builder flushInterval(TimeValue flushInterval) {
370+
this.flushInterval = flushInterval;
371+
return this;
372+
}
373+
374+
/**
375+
* Builds a new bulk processor.
376+
*/
377+
public Bulk build() {
378+
return new Bulk(this);
379+
}
380+
}
381+
382+
public Bulk(final Builder builder) {
383+
this.bulkActions = builder.bulkActions;
384+
this.bulkSize = builder.bulkSize;
385+
this.concurrentRequests = builder.concurrentRequests;
386+
this.flushInterval = builder.flushInterval;
387+
}
388+
389+
public int getConcurrentRequests() {
390+
return concurrentRequests;
391+
}
392+
393+
public int getBulkActions() {
394+
return bulkActions;
395+
}
396+
397+
public ByteSizeValue getBulkSize() {
398+
return bulkSize;
399+
}
400+
401+
public TimeValue getFlushInterval() {
402+
return flushInterval;
403+
}
404+
405+
}
406+
331407
@SuppressWarnings("unchecked")
332408
public synchronized static MongoDBRiverDefinition parseSettings(String riverName, String riverIndexName, RiverSettings settings,
333409
ScriptService scriptService) {
@@ -563,21 +639,30 @@ public synchronized static MongoDBRiverDefinition parseSettings(String riverName
563639
Map<String, Object> indexSettings = (Map<String, Object>) settings.settings().get(INDEX_OBJECT);
564640
builder.indexName(XContentMapValues.nodeStringValue(indexSettings.get(NAME_FIELD), builder.mongoDb));
565641
builder.typeName(XContentMapValues.nodeStringValue(indexSettings.get(TYPE_FIELD), builder.mongoDb));
566-
int bulkSize = XContentMapValues.nodeIntegerValue(indexSettings.get(BULK_SIZE_FIELD), 100);
567-
builder.bulkSize(bulkSize);
568-
if (indexSettings.containsKey(BULK_TIMEOUT_FIELD)) {
569-
builder.bulkTimeout(TimeValue.parseTimeValue(
570-
XContentMapValues.nodeStringValue(indexSettings.get(BULK_TIMEOUT_FIELD), "10ms"), TimeValue.timeValueMillis(10)));
642+
643+
Bulk.Builder bulkBuilder = new Bulk.Builder();
644+
if (indexSettings.containsKey(BULK_FIELD)) {
645+
Map<String, Object> bulkSettings = (Map<String, Object>) indexSettings.get(BULK_FIELD);
646+
int bulkActions = XContentMapValues.nodeIntegerValue(bulkSettings.get(ACTIONS_FIELD), DEFAULT_BULK_ACTIONS);
647+
bulkBuilder.bulkActions(bulkActions);
648+
String size = XContentMapValues.nodeStringValue(bulkSettings.get(SIZE_FIELD), DEFAULT_BULK_SIZE.toString());
649+
bulkBuilder.bulkSize(ByteSizeValue.parseBytesSizeValue(size));
650+
bulkBuilder.concurrentRequests(XContentMapValues.nodeIntegerValue(bulkSettings.get(CONCURRENT_REQUESTS_FIELD), DEFAULT_CONCURRENT_REQUESTS));
651+
bulkBuilder.flushInterval(XContentMapValues.nodeTimeValue(bulkSettings.get(FLUSH_INTERVAL_FIELD), DEFAULT_FLUSH_INTERVAL));
652+
builder.throttleSize(XContentMapValues.nodeIntegerValue(indexSettings.get(THROTTLE_SIZE_FIELD), bulkActions * 5));
571653
} else {
572-
builder.bulkTimeout(TimeValue.timeValueMillis(10));
654+
int bulkActions = XContentMapValues.nodeIntegerValue(indexSettings.get(BULK_SIZE_FIELD), DEFAULT_BULK_ACTIONS);
655+
bulkBuilder.bulkActions(bulkActions);
656+
bulkBuilder.bulkSize(DEFAULT_BULK_SIZE);
657+
bulkBuilder.flushInterval(XContentMapValues.nodeTimeValue(indexSettings.get(BULK_TIMEOUT_FIELD), DEFAULT_FLUSH_INTERVAL));
658+
bulkBuilder.concurrentRequests(XContentMapValues.nodeIntegerValue(indexSettings.get(CONCURRENT_BULK_REQUESTS_FIELD), DEFAULT_CONCURRENT_REQUESTS));
659+
builder.throttleSize(XContentMapValues.nodeIntegerValue(indexSettings.get(THROTTLE_SIZE_FIELD), bulkActions * 5));
573660
}
574-
builder.throttleSize(XContentMapValues.nodeIntegerValue(indexSettings.get(THROTTLE_SIZE_FIELD), bulkSize * 5));
661+
builder.bulk(bulkBuilder.build());
575662
} else {
576663
builder.indexName(builder.mongoDb);
577664
builder.typeName(builder.mongoDb);
578-
builder.bulkSize(100);
579-
builder.bulkTimeout(TimeValue.timeValueMillis(10));
580-
builder.throttleSize(builder.bulkSize * 5);
665+
builder.bulk(new Bulk.Builder().build());
581666
}
582667
return builder.build();
583668
}
@@ -694,10 +779,10 @@ private MongoDBRiverDefinition(final Builder builder) {
694779
// index
695780
this.indexName = builder.indexName;
696781
this.typeName = builder.typeName;
697-
this.bulkSize = builder.bulkSize;
698-
this.bulkTimeout = builder.bulkTimeout;
699782
this.throttleSize = builder.throttleSize;
700783

784+
// bulk
785+
this.bulk = builder.bulk;
701786
}
702787

703788
public List<ServerAddress> getMongoServers() {
@@ -816,14 +901,6 @@ public String getTypeName() {
816901
return typeName;
817902
}
818903

819-
public int getBulkSize() {
820-
return bulkSize;
821-
}
822-
823-
public TimeValue getBulkTimeout() {
824-
return bulkTimeout;
825-
}
826-
827904
public int getThrottleSize() {
828905
return throttleSize;
829906
}
@@ -832,4 +909,7 @@ public String getMongoOplogNamespace() {
832909
return getMongoDb() + "." + getMongoCollection();
833910
}
834911

912+
public Bulk getBulk() {
913+
return bulk;
914+
}
835915
}

Diff for: ‎src/main/java/org/elasticsearch/river/mongodb/Status.java

+1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ public enum Status {
44

55
RUNNING,
66
STOPPED,
7+
IMPORT_FAILED,
78
INITIAL_IMPORT_FAILED;
89

910
}

Diff for: ‎src/test/java/org/elasticsearch/river/mongodb/MongoDBRiverDefinitionTest.java

+71-1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44

55
import org.elasticsearch.common.io.Streams;
66
import org.elasticsearch.common.settings.ImmutableSettings;
7+
import org.elasticsearch.common.unit.ByteSizeValue;
8+
import org.elasticsearch.common.unit.TimeValue;
79
import org.elasticsearch.common.xcontent.XContentHelper;
810
import org.elasticsearch.river.RiverName;
911
import org.elasticsearch.river.RiverSettings;
@@ -17,6 +19,32 @@
1719

1820
public class MongoDBRiverDefinitionTest {
1921

22+
@Test
23+
public void testLoadMongoDBRiverSimpleDefinition() {
24+
try {
25+
RiverName riverName = new RiverName("mongodb", "mongodb-" + System.currentTimeMillis());
26+
InputStream in = getClass().getResourceAsStream("/org/elasticsearch/river/mongodb/test-mongodb-river-simple-definition.json");
27+
RiverSettings riverSettings = new RiverSettings(ImmutableSettings.settingsBuilder().build(), XContentHelper.convertToMap(
28+
Streams.copyToByteArray(in), false).v2());
29+
ScriptService scriptService = null;
30+
MongoDBRiverDefinition definition = MongoDBRiverDefinition.parseSettings(riverName.name(), "my-river-index", riverSettings,
31+
scriptService);
32+
Assert.assertNotNull(definition);
33+
Assert.assertEquals("mydb", definition.getMongoDb());
34+
Assert.assertEquals("mycollection", definition.getMongoCollection());
35+
Assert.assertEquals("myindex", definition.getIndexName());
36+
37+
// Test default bulk values
38+
Assert.assertEquals(MongoDBRiverDefinition.DEFAULT_BULK_ACTIONS, definition.getBulk().getBulkActions());
39+
Assert.assertEquals(MongoDBRiverDefinition.DEFAULT_CONCURRENT_REQUESTS, definition.getBulk().getConcurrentRequests());
40+
Assert.assertEquals(MongoDBRiverDefinition.DEFAULT_BULK_SIZE, definition.getBulk().getBulkSize());
41+
Assert.assertEquals(MongoDBRiverDefinition.DEFAULT_FLUSH_INTERVAL, definition.getBulk().getFlushInterval());
42+
43+
} catch (Throwable t) {
44+
Assert.fail("testLoadMongoDBRiverSimpleDefinition failed", t);
45+
}
46+
}
47+
2048
@Test
2149
public void testLoadMongoDBRiverDefinition() {
2250
try {
@@ -39,7 +67,49 @@ public void testLoadMongoDBRiverDefinition() {
3967
Assert.assertEquals(0, definition.getSocketTimeout());
4068
Assert.assertEquals(11000, definition.getConnectTimeout());
4169
Assert.assertEquals(riverName.getName(), definition.getRiverName());
42-
Assert.assertEquals(500, definition.getBulkSize());
70+
71+
// Test bulk
72+
Assert.assertEquals(500, definition.getBulk().getBulkActions());
73+
Assert.assertEquals(40, definition.getBulk().getConcurrentRequests());
74+
75+
} catch (Throwable t) {
76+
Assert.fail("testLoadMongoDBRiverDefinition failed", t);
77+
}
78+
}
79+
80+
@Test
81+
public void testLoadMongoDBRiverNewDefinition() {
82+
try {
83+
RiverName riverName = new RiverName("mongodb", "mongodb-" + System.currentTimeMillis());
84+
InputStream in = getClass().getResourceAsStream("/org/elasticsearch/river/mongodb/test-mongodb-river-new-definition.json");
85+
RiverSettings riverSettings = new RiverSettings(ImmutableSettings.settingsBuilder().build(), XContentHelper.convertToMap(
86+
Streams.copyToByteArray(in), false).v2());
87+
ScriptService scriptService = null;
88+
MongoDBRiverDefinition definition = MongoDBRiverDefinition.parseSettings(riverName.name(), "my-river-index", riverSettings,
89+
scriptService);
90+
Assert.assertNotNull(definition);
91+
Assert.assertEquals("mycollection", definition.getIncludeCollection());
92+
Assert.assertTrue(definition.getParentTypes().contains("parent1"));
93+
Assert.assertTrue(definition.getParentTypes().contains("parent2"));
94+
Assert.assertFalse(definition.getParentTypes().contains("parent3"));
95+
Assert.assertTrue(definition.isAdvancedTransformation());
96+
Assert.assertEquals("mydatabase", definition.getMongoDb());
97+
Assert.assertEquals("mycollection", definition.getMongoCollection());
98+
Assert.assertEquals("myindex", definition.getIndexName());
99+
Assert.assertEquals(0, definition.getSocketTimeout());
100+
Assert.assertEquals(11000, definition.getConnectTimeout());
101+
Assert.assertEquals(riverName.getName(), definition.getRiverName());
102+
103+
// actions: 500
104+
// size: "20mb",
105+
// concurrent_requests: 40,
106+
// flush_interval: "50ms"
107+
108+
// Test bulk
109+
Assert.assertEquals(500, definition.getBulk().getBulkActions());
110+
Assert.assertEquals(40, definition.getBulk().getConcurrentRequests());
111+
Assert.assertEquals(ByteSizeValue.parseBytesSizeValue("20mb"), definition.getBulk().getBulkSize());
112+
Assert.assertEquals(TimeValue.timeValueMillis(50), definition.getBulk().getFlushInterval());
43113

44114
} catch (Throwable t) {
45115
Assert.fail("testLoadMongoDBRiverDefinition failed", t);

0 commit comments

Comments
 (0)