DRILL-5196: Init MongoDB cluster when run a single test case directly through command line or IDE

Other fixes include:
+ Sync mongo-java-driver versions to newer 3.2.0
+ update flapdoodle package to latest accordingly

closes #741

6 files changed, 57 insertions, 24 deletions

diff --git a/contrib/storage-mongo/pom.xml b/contrib/storage-mongo/pom.xml
index 8ce34ef39..c7c2b0d61 100644
--- a/contrib/storage-mongo/pom.xml
+++ b/contrib/storage-mongo/pom.xml
@@ -69,7 +69,7 @@
     <dependency>
       <groupId>de.flapdoodle.embed</groupId>
       <artifactId>de.flapdoodle.embed.mongo</artifactId>
-      <version>1.50.0</version>
+      <version>1.50.5</version>
       <scope>test</scope>
     </dependency>
   </dependencies>
diff --git a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestBase.java b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestBase.java
index 1877a6458..ec9751414 100644
--- a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestBase.java
+++ b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestBase.java
@@ -36,6 +36,7 @@ public class MongoTestBase extends PlanTestBase implements MongoTestConstants {
 
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
+    MongoTestSuit.initMongo();
     initMongoStoragePlugin();
   }
 
@@ -84,6 +85,7 @@ public class MongoTestBase extends PlanTestBase implements MongoTestConstants {
 
   @AfterClass
   public static void tearDownMongoTestBase() throws Exception {
+    MongoTestSuit.tearDownCluster();
     storagePlugin = null;
   }
 
diff --git a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestConstants.java b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestConstants.java
index 0ff031a02..290c3c336 100644
--- a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestConstants.java
+++ b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestConstants.java
@@ -44,6 +44,10 @@ public interface MongoTestConstants {
   public static final String EMP_DATA = "emp.json";
   public static final String SCHEMA_CHANGE_DATA = "schema_change_int_to_string.json";
 
+  String DATATYPE_DB = "datatype";
+  String DATATYPE_COLLECTION = "types";
+  String DATATYPE_DATA = "datatype-oid.json";
+
   public static final String REPLICA_SET_1_NAME = "shard_1_replicas";
   public static final String REPLICA_SET_2_NAME = "shard_2_replicas";
 
diff --git a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestSuit.java b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestSuit.java
index 01ee91597..c34afe360 100644
--- a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestSuit.java
+++ b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestSuit.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.bson.Document;
 import org.bson.conversions.Bson;
@@ -73,6 +74,8 @@ public class MongoTestSuit implements MongoTestConstants {
   private static boolean authEnabled = System.getProperty(
       "drill.mongo.tests.authEnabled", "false").equalsIgnoreCase("true");
 
+  private static volatile AtomicInteger initCount = new AtomicInteger(0);
+
   private static class DistributedMode {
     private static MongosSystemForTestFactory mongosTestFactory;
 
@@ -116,11 +119,12 @@ public class MongoTestSuit implements MongoTestConstants {
       }
       createDbAndCollections(DONUTS_DB, DONUTS_COLLECTION, "id");
       createDbAndCollections(EMPLOYEE_DB, EMPTY_COLLECTION, "field_2");
+      createDbAndCollections(DATATYPE_DB, DATATYPE_COLLECTION, "_id");
     }
 
     private static IMongodConfig crateConfigServerConfig(int configServerPort,
         boolean flag) throws UnknownHostException, IOException {
-      IMongoCmdOptions cmdOptions = new MongoCmdOptionsBuilder().verbose(false)
+      IMongoCmdOptions cmdOptions = new MongoCmdOptionsBuilder().useNoJournal(false).verbose(false)
           .build();
 
       IMongodConfig mongodConfig = new MongodConfigBuilder()
@@ -133,7 +137,7 @@ public class MongoTestSuit implements MongoTestConstants {
     private static IMongodConfig crateIMongodConfig(int mongodPort,
         boolean flag, String replicaName) throws UnknownHostException,
         IOException {
-      IMongoCmdOptions cmdOptions = new MongoCmdOptionsBuilder().verbose(false)
+      IMongoCmdOptions cmdOptions = new MongoCmdOptionsBuilder().useNoJournal(false).verbose(false)
           .build();
 
       Storage replication = new Storage(null, replicaName, 0);
@@ -147,7 +151,7 @@ public class MongoTestSuit implements MongoTestConstants {
 
     private static IMongosConfig createIMongosConfig()
         throws UnknownHostException, IOException {
-      IMongoCmdOptions cmdOptions = new MongoCmdOptionsBuilder().verbose(false)
+      IMongoCmdOptions cmdOptions = new MongoCmdOptionsBuilder().useNoJournal(false).verbose(false)
           .build();
 
       IMongosConfig mongosConfig = new MongosConfigBuilder()
@@ -189,6 +193,7 @@ public class MongoTestSuit implements MongoTestConstants {
       createDbAndCollections(EMPLOYEE_DB, EMPINFO_COLLECTION, "employee_id");
       createDbAndCollections(EMPLOYEE_DB, SCHEMA_CHANGE_COLLECTION, "field_2");
       createDbAndCollections(EMPLOYEE_DB, EMPTY_COLLECTION, "field_2");
+      createDbAndCollections(DATATYPE_DB, DATATYPE_COLLECTION, "_id");
     }
 
     private static void cleanup() {
@@ -204,16 +209,20 @@ public class MongoTestSuit implements MongoTestConstants {
   @BeforeClass
   public static void initMongo() throws Exception {
     synchronized (MongoTestSuit.class) {
-      if (distMode) {
-        logger.info("Executing tests in distributed mode");
-        DistributedMode.setup();
-      } else {
-        logger.info("Executing tests in single mode");
-        SingleMode.setup();
+      if (initCount.get() == 0) {
+        if (distMode) {
+          logger.info("Executing tests in distributed mode");
+          DistributedMode.setup();
+        } else {
+          logger.info("Executing tests in single mode");
+          SingleMode.setup();
+        }
+        TestTableGenerator.importData(EMPLOYEE_DB, EMPINFO_COLLECTION, EMP_DATA);
+        TestTableGenerator.importData(EMPLOYEE_DB, SCHEMA_CHANGE_COLLECTION, SCHEMA_CHANGE_DATA);
+        TestTableGenerator.importData(DONUTS_DB, DONUTS_COLLECTION, DONUTS_DATA);
+        TestTableGenerator.importData(DATATYPE_DB, DATATYPE_COLLECTION, DATATYPE_DATA);
       }
-      TestTableGenerator.importData(EMPLOYEE_DB, EMPINFO_COLLECTION, EMP_DATA);
-      TestTableGenerator.importData(EMPLOYEE_DB, SCHEMA_CHANGE_COLLECTION, SCHEMA_CHANGE_DATA);
-      TestTableGenerator.importData(DONUTS_DB, DONUTS_COLLECTION, DONUTS_DATA);
+      initCount.incrementAndGet();
     }
   }
 
@@ -234,15 +243,25 @@ public class MongoTestSuit implements MongoTestConstants {
 
   @AfterClass
   public static void tearDownCluster() throws Exception {
-    if (mongoClient != null) {
-      mongoClient.dropDatabase(EMPLOYEE_DB);
-      mongoClient.close();
-    }
     synchronized (MongoTestSuit.class) {
-      if (distMode) {
-        DistributedMode.cleanup();
-      } else {
-        SingleMode.cleanup();
+      if (initCount.decrementAndGet() == 0) {
+        try {
+          if (mongoClient != null) {
+            mongoClient.dropDatabase(EMPLOYEE_DB);
+            mongoClient.dropDatabase(DATATYPE_DB);
+            mongoClient.dropDatabase(DONUTS_DB);
+          }
+        }
+        finally {
+          if (mongoClient != null) {
+            mongoClient.close();
+          }
+          if (distMode) {
+            DistributedMode.cleanup();
+          } else {
+            SingleMode.cleanup();
+          }
+        }
       }
     }
   }
diff --git a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestTableGenerator.java b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestTableGenerator.java
index 62e1204ba..ffe37910f 100644
--- a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestTableGenerator.java
+++ b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestTableGenerator.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.store.mongo;
 import java.io.IOException;
 import java.net.UnknownHostException;
 
+import de.flapdoodle.embed.mongo.MongoImportProcess;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -40,7 +41,7 @@ public class TestTableGenerator implements MongoTestConstants {
       .getLogger(TestTableGenerator.class);
 
   public static void importData(String dbName, String collectionName,
-      String fileName) throws IOException {
+      String fileName) throws InterruptedException,IOException {
     String jsonFile = Resources.getResource(fileName).toString();
     jsonFile = jsonFile.replaceFirst("file:", StringUtils.EMPTY);
     generateTable(dbName, collectionName, jsonFile, true, true, false);
@@ -48,7 +49,7 @@ public class TestTableGenerator implements MongoTestConstants {
 
   public static void generateTable(String dbName, String collection,
       String jsonFile, Boolean jsonArray, Boolean upsert, Boolean drop)
-      throws UnknownHostException, IOException {
+      throws InterruptedException, IOException {
     logger.info("Started importing file {} into collection {} ", jsonFile,
         collection);
     IMongoImportConfig mongoImportConfig = new MongoImportConfigBuilder()
@@ -58,7 +59,13 @@ public class TestTableGenerator implements MongoTestConstants {
         .jsonArray(jsonArray).importFile(jsonFile).build();
     MongoImportExecutable importExecutable = MongoImportStarter
         .getDefaultInstance().prepare(mongoImportConfig);
-    importExecutable.start();
+    MongoImportProcess importProcess = importExecutable.start();
+
+    // import is in a separate process, we should wait until the process exit
+    while (importProcess.isProcessRunning()) {
+        Thread.sleep(1000);
+    }
+
     logger.info("Imported file {} into collection {} ", jsonFile, collection);
   }
 
diff --git a/contrib/storage-mongo/src/test/resources/datatype-oid.json b/contrib/storage-mongo/src/test/resources/datatype-oid.json
new file mode 100644
index 000000000..52058e83a
--- /dev/null
+++ b/contrib/storage-mongo/src/test/resources/datatype-oid.json
@@ -0,0 +1 @@
+[{"_id": {"$oid" : "582081d96b69060001fd8938"}, "account" : { "$ref": "contact", "$id": {"$oid": "999cbf116b69060001fd8611"} }}, {"_id" : {"$oid" : "582081d96b69060001fd8939" } } ]
\ No newline at end of file