Skip to content

Commit 0da87d2

Browse files
richardwilly98richardwilly98
committed
Fix for #133
- with ```options/drop_collection``` the river will also track ```dropDatabase``` operation
1 parent 80d1c6e commit 0da87d2

File tree

4 files changed

+65
-16
lines changed

4 files changed

+65
-16
lines changed

Diff for: ‎src/main/java/org/elasticsearch/river/mongodb/Indexer.java

+4-2
Original file line numberDiff line numberDiff line change
@@ -258,8 +258,10 @@ private void updateBulkRequest(final BulkRequestBuilder bulk, DBObject data, Str
258258
}
259259
if (MongoDBRiver.OPLOG_COMMAND_OPERATION.equals(operation)) {
260260
if (definition.isDropCollection()) {
261-
if (data.get(MongoDBRiver.OPLOG_DROP_COMMAND_OPERATION) != null
262-
&& data.get(MongoDBRiver.OPLOG_DROP_COMMAND_OPERATION).equals(definition.getMongoCollection())) {
261+
if ((data.get(MongoDBRiver.OPLOG_DROP_COMMAND_OPERATION) != null
262+
&& data.get(MongoDBRiver.OPLOG_DROP_COMMAND_OPERATION).equals(definition.getMongoCollection()) || (data
263+
.get(MongoDBRiver.OPLOG_DROP_DATABASE_COMMAND_OPERATION) != null && data.get(
264+
MongoDBRiver.OPLOG_DROP_DATABASE_COMMAND_OPERATION).equals(1)))) {
263265
logger.info("Drop collection request [{}], [{}]", index, type);
264266
bulk.request().requests().clear();
265267
client.admin().indices().prepareRefresh(index).execute().actionGet();

Diff for: ‎src/main/java/org/elasticsearch/river/mongodb/MongoDBRiver.java

+1
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ public class MongoDBRiver extends AbstractRiverComponent implements River {
9898
public final static String OPLOG_DELETE_OPERATION = "d";
9999
public final static String OPLOG_COMMAND_OPERATION = "c";
100100
public final static String OPLOG_DROP_COMMAND_OPERATION = "drop";
101+
public final static String OPLOG_DROP_DATABASE_COMMAND_OPERATION = "dropDatabase";
101102
public final static String OPLOG_TIMESTAMP = "ts";
102103
public final static String OPLOG_FROM_MIGRATE = "fromMigrate";
103104
public final static String GRIDFS_FILES_SUFFIX = ".files";

Diff for: ‎src/main/java/org/elasticsearch/river/mongodb/Slurper.java

+5-9
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package org.elasticsearch.river.mongodb;
22

33
import java.util.ArrayList;
4-
import java.util.Collections;
54
import java.util.List;
65
import java.util.NoSuchElementException;
76
import java.util.Set;
@@ -10,6 +9,7 @@
109
import org.bson.types.BSONTimestamp;
1110
import org.bson.types.ObjectId;
1211
import org.elasticsearch.client.Client;
12+
import org.elasticsearch.common.collect.ImmutableList;
1313
import org.elasticsearch.common.logging.ESLogger;
1414
import org.elasticsearch.common.logging.ESLoggerFactory;
1515
import org.elasticsearch.river.mongodb.util.MongoDBHelper;
@@ -346,9 +346,8 @@ private DBObject getOplogFilter(final BSONTimestamp time) {
346346
if (definition.isMongoGridFS()) {
347347
filter.put(MongoDBRiver.OPLOG_NAMESPACE, definition.getMongoOplogNamespace() + MongoDBRiver.GRIDFS_FILES_SUFFIX);
348348
} else {
349-
List<String> namespaceFilter = new ArrayList<String>();
350-
namespaceFilter.add(definition.getMongoOplogNamespace());
351-
namespaceFilter.add(definition.getMongoDb() + "." + MongoDBRiver.OPLOG_NAMESPACE_COMMAND);
349+
List<String> namespaceFilter = ImmutableList.of(definition.getMongoOplogNamespace(), definition.getMongoDb() + "."
350+
+ MongoDBRiver.OPLOG_NAMESPACE_COMMAND);
352351
filter.put(MongoDBRiver.OPLOG_NAMESPACE, new BasicBSONObject(MongoDBRiver.MONGODB_IN_OPERATOR, namespaceFilter));
353352
}
354353
if (definition.getMongoOplogFilter().size() > 0) {
@@ -364,11 +363,8 @@ private DBObject getMongoFilter() {
364363
List<DBObject> filters = new ArrayList<DBObject>();
365364
List<DBObject> filters2 = new ArrayList<DBObject>();
366365

367-
List<String> operationFilter = new ArrayList<String>();
368-
operationFilter.add(MongoDBRiver.OPLOG_DELETE_OPERATION);
369-
operationFilter.add(MongoDBRiver.OPLOG_UPDATE_OPERATION);
370-
operationFilter.add(MongoDBRiver.OPLOG_INSERT_OPERATION);
371-
filters.add(new BasicDBObject(MongoDBRiver.OPLOG_OPERATION, new BasicBSONObject(MongoDBRiver.MONGODB_IN_OPERATOR, operationFilter)));
366+
filters.add(new BasicDBObject(MongoDBRiver.OPLOG_OPERATION, new BasicBSONObject(MongoDBRiver.MONGODB_IN_OPERATOR, ImmutableList.of(
367+
MongoDBRiver.OPLOG_DELETE_OPERATION, MongoDBRiver.OPLOG_UPDATE_OPERATION, MongoDBRiver.OPLOG_INSERT_OPERATION))));
372368

373369
// include custom filter in filters2
374370
filters2.add(definition.getMongoOplogFilter());

Diff for: ‎src/test/java/org/elasticsearch/river/mongodb/simple/RiverMongoDropCollectionTest.java

+55-5
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@
2929
import org.elasticsearch.action.count.CountResponse;
3030
import org.elasticsearch.river.mongodb.RiverMongoDBTestAbstract;
3131
import org.testng.Assert;
32-
import org.testng.annotations.AfterClass;
33-
import org.testng.annotations.BeforeClass;
32+
import org.testng.annotations.AfterMethod;
33+
import org.testng.annotations.BeforeMethod;
3434
import org.testng.annotations.Test;
3535

3636
import com.mongodb.DB;
@@ -56,7 +56,8 @@ protected RiverMongoDropCollectionTest(String river, String database, String col
5656
super(river, database, collection, index);
5757
}
5858

59-
@BeforeClass
59+
// @BeforeClass
60+
@BeforeMethod
6061
public void createDatabase() {
6162
logger.debug("createDatabase {}", getDatabase());
6263
try {
@@ -73,11 +74,12 @@ public void createDatabase() {
7374
}
7475
}
7576

76-
@AfterClass
77+
// @AfterClass
78+
@AfterMethod
7779
public void cleanUp() {
7880
super.deleteRiver();
7981
logger.info("Drop database " + mongoDB.getName());
80-
mongoDB.dropDatabase();
82+
// mongoDB.dropDatabase();
8183
}
8284

8385
@Test
@@ -112,6 +114,8 @@ public void testDropCollection() throws Throwable {
112114
logger.error("testDropCollection failed.", t);
113115
t.printStackTrace();
114116
throw t;
117+
} finally {
118+
mongoDB.dropDatabase();
115119
}
116120
}
117121

@@ -158,6 +162,52 @@ public void testDropCollectionIssue79() throws Throwable {
158162
logger.error("testDropCollectionIssue79 failed.", t);
159163
t.printStackTrace();
160164
throw t;
165+
} finally {
166+
mongoDB.dropDatabase();
161167
}
162168
}
169+
170+
@Test
171+
public void testDropDatabaseIssue133() throws Throwable {
172+
logger.debug("Start testDropDatabaseIssue133");
173+
try {
174+
String mongoDocument = copyToStringFromClasspath(TEST_SIMPLE_MONGODB_DOCUMENT_JSON);
175+
DBObject dbObject = (DBObject) JSON.parse(mongoDocument);
176+
mongoCollection.insert(dbObject);
177+
Thread.sleep(wait);
178+
179+
assertThat(getNode().client().admin().indices().exists(new IndicesExistsRequest(getIndex())).actionGet().isExists(),
180+
equalTo(true));
181+
assertThat(getNode().client().admin().indices().prepareTypesExists(getIndex()).setTypes(getDatabase()).execute().actionGet()
182+
.isExists(), equalTo(true));
183+
long countRequest = getNode().client().count(countRequest(getIndex())).actionGet().getCount();
184+
mongoDB.dropDatabase();
185+
Thread.sleep(wait);
186+
assertThat(databaseExists(database), equalTo(false));
187+
Thread.sleep(wait);
188+
refreshIndex();
189+
190+
if (!dropCollectionOption) {
191+
countRequest = getNode().client().count(countRequest(getIndex())).actionGet().getCount();
192+
assertThat(countRequest, greaterThan(0L));
193+
} else {
194+
countRequest = getNode().client().count(countRequest(getIndex())).actionGet().getCount();
195+
assertThat(countRequest, equalTo(0L));
196+
}
197+
} catch (Throwable t) {
198+
logger.error("testDropDatabaseIssue133 failed.", t);
199+
t.printStackTrace();
200+
throw t;
201+
} finally {
202+
}
203+
}
204+
205+
private boolean databaseExists(String name) {
206+
for(String databaseName :mongo.getDatabaseNames()) {
207+
if (databaseName.equals(name)) {
208+
return true;
209+
}
210+
}
211+
return false;
212+
}
163213
}

0 commit comments

Comments
 (0)