DRILL-6341: Fixed failing tests for mongodb storage plugin by upgrading MongoDB version.

closes #1222

4 files changed, 55 insertions, 23 deletions

diff --git a/contrib/storage-mongo/pom.xml b/contrib/storage-mongo/pom.xml
index 221524d86..80cbf3e85 100644
--- a/contrib/storage-mongo/pom.xml
+++ b/contrib/storage-mongo/pom.xml
@@ -66,7 +66,7 @@
     <dependency>
       <groupId>de.flapdoodle.embed</groupId>
       <artifactId>de.flapdoodle.embed.mongo</artifactId>
-      <version>1.50.5</version>
+      <version>2.0.3</version>
       <scope>test</scope>
     </dependency>
   </dependencies>
diff --git a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestConstants.java b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestConstants.java
index 2320d5316..5aeaf9ff5 100644
--- a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestConstants.java
+++ b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestConstants.java
@@ -21,7 +21,9 @@ public interface MongoTestConstants {
 
   public static final String LOCALHOST = "localhost";
   // TODO: DRILL-3934: add some randomization to this as it fails when running concurrent builds
-  public static final int CONFIG_SERVER_PORT = 27019;
+  int CONFIG_SERVER_1_PORT = 61114;
+  int CONFIG_SERVER_2_PORT = 61215;
+  int CONFIG_SERVER_3_PORT = 61316;
   public static final int MONGOD_1_PORT = 27020;
   public static final int MONGOD_2_PORT = 27021;
   public static final int MONGOD_3_PORT = 27022;
@@ -44,10 +46,12 @@ public interface MongoTestConstants {
   public static final String EMP_DATA = "emp.json";
   public static final String SCHEMA_CHANGE_DATA = "schema_change_int_to_string.json";
 
+  String STORAGE_ENGINE = "wiredTiger";
   String DATATYPE_DB = "datatype";
   String DATATYPE_COLLECTION = "types";
   String DATATYPE_DATA = "datatype-oid.json";
 
+  String CONFIG_REPLICA_SET = "config_replicas";
   public static final String REPLICA_SET_1_NAME = "shard_1_replicas";
   public static final String REPLICA_SET_2_NAME = "shard_2_replicas";
 
diff --git a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestSuit.java b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestSuit.java
index 7e2782c44..b3f0bd137 100644
--- a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestSuit.java
+++ b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestSuit.java
@@ -25,6 +25,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import com.google.common.collect.Lists;
 import org.bson.Document;
 import org.bson.conversions.Bson;
 import org.junit.AfterClass;
@@ -88,9 +89,9 @@ public class MongoTestSuit implements MongoTestConstants {
     private static void setup() throws Exception {
       // creating configServers
       List<IMongodConfig> configServers = new ArrayList<>(1);
-      IMongodConfig configIMongodConfig = crateConfigServerConfig(
-          CONFIG_SERVER_PORT, true);
-      configServers.add(configIMongodConfig);
+      configServers.add(crateConfigServerConfig(CONFIG_SERVER_1_PORT));
+      configServers.add(crateConfigServerConfig(CONFIG_SERVER_2_PORT));
+      configServers.add(crateConfigServerConfig(CONFIG_SERVER_3_PORT));
 
       // creating replicaSets
       Map<String, List<IMongodConfig>> replicaSets = new HashMap<>();
@@ -110,11 +111,12 @@ public class MongoTestSuit implements MongoTestConstants {
       replicaSet2.add(crateIMongodConfig(MONGOD_6_PORT, false,
           REPLICA_SET_2_NAME));
       replicaSets.put(REPLICA_SET_2_NAME, replicaSet2);
+      replicaSets.put(CONFIG_REPLICA_SET, configServers);
 
       // create mongos
       IMongosConfig mongosConfig = createIMongosConfig();
       mongosTestFactory = new MongosSystemForTestFactory(mongosConfig,
-          replicaSets, configServers, EMPLOYEE_DB, EMPINFO_COLLECTION,
+          replicaSets, Lists.newArrayList(), EMPLOYEE_DB, EMPINFO_COLLECTION,
           "employee_id");
       try {
         mongosTestFactory.start();
@@ -128,42 +130,63 @@ public class MongoTestSuit implements MongoTestConstants {
       createDbAndCollections(DATATYPE_DB, DATATYPE_COLLECTION, "_id");
     }
 
-    private static IMongodConfig crateConfigServerConfig(int configServerPort,
-        boolean flag) throws UnknownHostException, IOException {
-      IMongoCmdOptions cmdOptions = new MongoCmdOptionsBuilder().useNoJournal(false).verbose(false)
-          .build();
+    private static IMongodConfig crateConfigServerConfig(int configServerPort) throws UnknownHostException, IOException {
+      IMongoCmdOptions cmdOptions = new MongoCmdOptionsBuilder()
+        .useNoPrealloc(false)
+        .useSmallFiles(false)
+        .useNoJournal(false)
+        .useStorageEngine(STORAGE_ENGINE)
+        .verbose(false)
+        .build();
+
+      Storage replication = new Storage(null, CONFIG_REPLICA_SET, 0);
 
       IMongodConfig mongodConfig = new MongodConfigBuilder()
-          .version(Version.Main.PRODUCTION)
+          .version(Version.Main.V3_4)
           .net(new Net(LOCALHOST, configServerPort, Network.localhostIsIPv6()))
-          .configServer(flag).cmdOptions(cmdOptions).build();
+          .replication(replication)
+          .shardServer(false)
+          .configServer(true).cmdOptions(cmdOptions).build();
       return mongodConfig;
     }
 
     private static IMongodConfig crateIMongodConfig(int mongodPort,
         boolean flag, String replicaName) throws UnknownHostException,
         IOException {
-      IMongoCmdOptions cmdOptions = new MongoCmdOptionsBuilder().useNoJournal(false).verbose(false)
-          .build();
+      IMongoCmdOptions cmdOptions = new MongoCmdOptionsBuilder()
+        .useNoPrealloc(false)
+        .useSmallFiles(false)
+        .useNoJournal(false)
+        .useStorageEngine(STORAGE_ENGINE)
+        .verbose(false)
+        .build();
 
       Storage replication = new Storage(null, replicaName, 0);
       IMongodConfig mongodConfig = new MongodConfigBuilder()
-          .version(Version.Main.PRODUCTION)
+          .version(Version.Main.V3_4)
+        .shardServer(true)
           .net(new Net(LOCALHOST, mongodPort, Network.localhostIsIPv6()))
           .configServer(flag).replication(replication).cmdOptions(cmdOptions)
           .build();
+
       return mongodConfig;
     }
 
     private static IMongosConfig createIMongosConfig()
         throws UnknownHostException, IOException {
-      IMongoCmdOptions cmdOptions = new MongoCmdOptionsBuilder().useNoJournal(false).verbose(false)
-          .build();
+      IMongoCmdOptions cmdOptions = new MongoCmdOptionsBuilder()
+        .useNoPrealloc(false)
+        .useSmallFiles(false)
+        .useNoJournal(false)
+        .useStorageEngine(STORAGE_ENGINE)
+        .verbose(false)
+        .build();
 
       IMongosConfig mongosConfig = new MongosConfigBuilder()
-          .version(Version.Main.PRODUCTION)
+          .version(Version.Main.V3_4)
           .net(new Net(LOCALHOST, MONGOS_PORT, Network.localhostIsIPv6()))
-          .configDB(LOCALHOST + ":" + CONFIG_SERVER_PORT)
+          .replicaSet(CONFIG_REPLICA_SET)
+          .configDB(LOCALHOST + ":" + CONFIG_SERVER_1_PORT)
           .cmdOptions(cmdOptions).build();
       return mongosConfig;
     }
@@ -191,7 +214,7 @@ public class MongoTestSuit implements MongoTestConstants {
           .enableAuth(authEnabled).build();
 
       IMongodConfig mongodConfig = new MongodConfigBuilder()
-          .version(Version.Main.PRODUCTION)
+          .version(Version.Main.V3_4)
           .net(new Net(LOCALHOST, MONGOS_PORT, Network.localhostIsIPv6()))
           .cmdOptions(cmdOptions).build();
 
@@ -247,8 +270,13 @@ public class MongoTestSuit implements MongoTestConstants {
       db.createCollection(collectionName);
       mongoCollection = db.getCollection(collectionName);
     }
-    IndexOptions indexOptions = new IndexOptions().unique(true)
-        .background(false).name(indexFieldName);
+
+    if (indexFieldName.equals("_id")) {
+      // Mongo 3.4 and later already makes an index for a field named _id
+      return;
+    }
+
+    IndexOptions indexOptions = new IndexOptions().unique(true).background(false).name(indexFieldName);
     Bson keys = Indexes.ascending(indexFieldName);
     mongoCollection.createIndex(keys, indexOptions);
   }
diff --git a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestTableGenerator.java b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestTableGenerator.java
index 37b975a76..c0b1e0aa7 100644
--- a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestTableGenerator.java
+++ b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestTableGenerator.java
@@ -53,7 +53,7 @@ public class TestTableGenerator implements MongoTestConstants {
     logger.info("Started importing file {} into collection {} ", jsonFile,
         collection);
     IMongoImportConfig mongoImportConfig = new MongoImportConfigBuilder()
-        .version(Version.Main.PRODUCTION)
+        .version(Version.Main.V3_4)
         .net(new Net(MONGOS_PORT, Network.localhostIsIPv6())).db(dbName)
         .collection(collection).upsert(upsert).dropCollection(drop)
         .jsonArray(jsonArray).importFile(jsonFile).build();