DRILL-4383: Allow custom configurations to be specified for a FileSystem plugin

add an example s3 plugin, disabled by default

Closes #375

3 files changed, 71 insertions, 0 deletions

diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemConfig.java
index d88750f65..adc390947 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemConfig.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemConfig.java
@@ -19,6 +19,7 @@ package org.apache.drill.exec.store.dfs;
 
 import java.util.Map;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
 import org.apache.drill.common.logical.FormatPluginConfig;
 import org.apache.drill.common.logical.StoragePluginConfig;
 
@@ -29,6 +30,7 @@ public class FileSystemConfig extends StoragePluginConfig {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSystemConfig.class);
   public static final String NAME = "file";
   public String connection;
+  public Map<String, String> config;
   public Map<String, WorkspaceConfig> workspaces;
   public Map<String, FormatPluginConfig> formats;
 
@@ -36,6 +38,7 @@ public class FileSystemConfig extends StoragePluginConfig {
   public int hashCode() {
     final int prime = 31;
     int result = 1;
+    result = prime * result + ((config == null) ? 0 : config.hashCode());
     result = prime * result + ((connection == null) ? 0 : connection.hashCode());
     result = prime * result + ((formats == null) ? 0 : formats.hashCode());
     result = prime * result + ((workspaces == null) ? 0 : workspaces.hashCode());
@@ -75,6 +78,13 @@ public class FileSystemConfig extends StoragePluginConfig {
     } else if (!workspaces.equals(other.workspaces)) {
       return false;
     }
+    if (config == null) {
+      if (other.config != null) {
+        return false;
+      }
+    } else if (!config.equals(other.config)) {
+      return false;
+    }
     return true;
   }
 
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
index 6a9bddb1e..7f2a9c16a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
@@ -69,6 +69,11 @@ public class FileSystemPlugin extends AbstractStoragePlugin{
     try {
 
       fsConf = new Configuration();
+      if (config.config != null) {
+        for (String s : config.config.keySet()) {
+          fsConf.set(s, config.config.get(s));
+        }
+      }
       fsConf.set(FileSystem.FS_DEFAULT_NAME_KEY, config.connection);
       fsConf.set("fs.classpath.impl", ClassPathFileSystem.class.getName());
       fsConf.set("fs.drill-local.impl", LocalSyncableFileSystem.class.getName());
diff --git a/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json b/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json
index f5d388077..13d29ea9e 100644
--- a/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json
+++ b/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json
@@ -51,6 +51,62 @@
         }
       }
     },
+    s3: {
+      type: "file",
+      connection: "s3a://my.bucket.location.com",
+      enabled : false,
+      config : {
+        "fs.s3a.access.key": "ID",
+        "fs.s3a.secret.key": "SECRET"
+      },
+      workspaces: {
+        "root" : {
+          location: "/",
+          writable: false
+        },
+        "tmp" : {
+          location: "/tmp",
+          writable: true
+        }
+      },
+      formats: {
+        "psv" : {
+          type: "text",
+          extensions: [ "tbl" ],
+          delimiter: "|"
+        },
+        "csv" : {
+          type: "text",
+          extensions: [ "csv" ],
+          delimiter: ","
+        },
+        "tsv" : {
+          type: "text",
+          extensions: [ "tsv" ],
+          delimiter: "\t"
+        },
+        "parquet" : {
+          type: "parquet"
+        },
+        "json" : {
+          type: "json",
+          extensions: [ "json" ]
+        },
+        "avro" : {
+          type: "avro"
+        },
+        "sequencefile": {
+          type : "sequencefile",
+          extensions: [ "seq" ]
+        },
+        "csvh" : {
+          type: "text",
+          extensions: [ "csvh" ],
+          delimiter: ",",
+          extractHeader: true
+        }
+      }
+    },
 
     cp: {
       type: "file",