DRILL-7046: Support for loading and parsing new RM config file

closes #1652

10 files changed, 222 insertions, 49 deletions

diff --git a/common/src/main/java/org/apache/drill/common/config/CommonConstants.java b/common/src/main/java/org/apache/drill/common/config/ConfigConstants.java
index e203972b8..3283fe07f 100644
--- a/common/src/main/java/org/apache/drill/common/config/CommonConstants.java
+++ b/common/src/main/java/org/apache/drill/common/config/ConfigConstants.java
@@ -17,21 +17,33 @@
  */
 package org.apache.drill.common.config;
 
-public interface CommonConstants {
+public final class ConfigConstants {
 
   /** Default (base) configuration file name.  (Classpath resource pathname.) */
-  String CONFIG_DEFAULT_RESOURCE_PATHNAME = "drill-default.conf";
+  public static final String CONFIG_DEFAULT_RESOURCE_PATHNAME = "drill-default.conf";
 
   /** Module configuration files name.  (Classpath resource pathname.) */
-  String DRILL_JAR_MARKER_FILE_RESOURCE_PATHNAME = "drill-module.conf";
+  public static final String DRILL_JAR_MARKER_FILE_RESOURCE_PATHNAME = "drill-module.conf";
 
   /** Distribution Specific Override configuration file name.  (Classpath resource pathname.) */
-  String CONFIG_DISTRIBUTION_RESOURCE_PATHNAME = "drill-distrib.conf";
+  public static final String CONFIG_DISTRIBUTION_RESOURCE_PATHNAME = "drill-distrib.conf";
 
   /** Override configuration file name.  (Classpath resource pathname.) */
-  String CONFIG_OVERRIDE_RESOURCE_PATHNAME = "drill-override.conf";
+  public static final String CONFIG_OVERRIDE_RESOURCE_PATHNAME = "drill-override.conf";
 
   /** Override plugins configs file name.  (Classpath resource pathname.) */
-  String STORAGE_PLUGINS_OVERRIDE_CONF = "storage-plugins-override.conf";
+  public static final String STORAGE_PLUGINS_OVERRIDE_CONF = "storage-plugins-override.conf";
 
+  /** Default RM configuration file name. (Classpath resource pathname.) */
+  public static final String RM_CONFIG_DEFAULT_RESOURCE_PATHNAME = "drill-rm-default.conf";
+
+  /** Distribution Specific RM Override configuration file name.  (Classpath resource pathname.) */
+  public static final String RM_CONFIG_DISTRIBUTION_RESOURCE_PATHNAME = "drill-rm-distrib.conf";
+
+  /** RM Override configuration file name. (Classpath resource pathname.) */
+  public static final String RM_CONFIG_OVERRIDE_RESOURCE_PATHNAME = "drill-rm-override.conf";
+
+  // suppress default constructor
+  private ConfigConstants() {
+  }
 }
diff --git a/common/src/main/java/org/apache/drill/common/config/ConfigFileInfo.java b/common/src/main/java/org/apache/drill/common/config/ConfigFileInfo.java
new file mode 100644
index 000000000..1f0fe971b
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/config/ConfigFileInfo.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.config;
+
+/**
+ * Interface that defines implementation to get all the config files names for default, module specific, distribution
+ * specific and override files.
+ */
+public interface ConfigFileInfo {
+
+  String getDefaultFileName();
+
+  String getModuleFileName();
+
+  String getDistributionFileName();
+
+  String getOverrideFileName();
+}
diff --git a/common/src/main/java/org/apache/drill/common/config/DrillConfig.java b/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
index d7d7340a5..6fa09e5a4 100644
--- a/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
+++ b/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
@@ -17,30 +17,30 @@
  */
 package org.apache.drill.common.config;
 
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigRenderOptions;
+import io.netty.util.internal.PlatformDependent;
+import org.apache.drill.common.exceptions.DrillConfigurationException;
+import org.apache.drill.common.exceptions.UserException;
+import org.apache.drill.common.scanner.ClassPathScanner;
+import org.apache.drill.shaded.guava.com.google.common.annotations.VisibleForTesting;
+import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
+import org.apache.drill.shaded.guava.com.google.common.base.Stopwatch;
+import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableList;
+import org.reflections.util.ClasspathHelper;
+
 import java.lang.management.ManagementFactory;
 import java.lang.management.RuntimeMXBean;
 import java.lang.reflect.Constructor;
 import java.net.URL;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map.Entry;
 import java.util.Properties;
 import java.util.concurrent.TimeUnit;
 
-import io.netty.util.internal.PlatformDependent;
-import org.apache.drill.common.exceptions.DrillConfigurationException;
-import org.apache.drill.common.exceptions.UserException;
-import org.apache.drill.common.scanner.ClassPathScanner;
-import org.reflections.util.ClasspathHelper;
-
-import org.apache.drill.shaded.guava.com.google.common.annotations.VisibleForTesting;
-import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
-import org.apache.drill.shaded.guava.com.google.common.base.Stopwatch;
-import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableList;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigRenderOptions;
-
 public class DrillConfig extends NestedConfig {
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillConfig.class);
 
@@ -139,7 +139,7 @@ public class DrillConfig extends NestedConfig {
    * @param overrideFileResourcePathname
    *          the classpath resource pathname of the file to use for
    *          configuration override purposes; {@code null} specifies to use the
-   *          default pathname ({@link CommonConstants#CONFIG_OVERRIDE_RESOURCE_PATHNAME}) (does
+   *          default pathname ({@link ConfigConstants#CONFIG_OVERRIDE_RESOURCE_PATHNAME}) (does
    *          <strong>not</strong> specify to suppress trying to load an
    *          overrides file)
    *  @return A merged Config object.
@@ -153,7 +153,7 @@ public class DrillConfig extends NestedConfig {
    */
   @VisibleForTesting
   public static DrillConfig create(Properties testConfigurations) {
-    return create(null, testConfigurations, true);
+    return create(null, testConfigurations, true, new DrillExecConfigFileInfo());
   }
 
   /**
@@ -161,7 +161,27 @@ public class DrillConfig extends NestedConfig {
    *          see {@link #create(String)}'s {@code overrideFileResourcePathname}
    */
   public static DrillConfig create(String overrideFileResourcePathname, boolean enableServerConfigs) {
-    return create(overrideFileResourcePathname, null, enableServerConfigs);
+    return create(overrideFileResourcePathname, null, enableServerConfigs, new DrillExecConfigFileInfo());
+  }
+
+  /**
+   * Merged DrillConfig object for all the RM Configurations provided through various resource files. The order of
+   * precedence is as follows:
+   * <p>
+   * Configuration values are retrieved as follows:
+   * <ul>
+   * <li>Check a single copy of "drill-rm-override.conf".  If multiple copies are
+   *     on the classpath, which copy is read is indeterminate.</li>
+   * <li>Check a single copy of "drill-rm-distrib.conf". If multiple copies are
+   *     on the classpath, which copy is read is indeterminate. </li>
+   * <li>Check a single copy of "{@code drill-rm-default.conf}".  If multiple
+   *     copies are on the classpath, which copy is read is indeterminate.</li>
+   * </ul>
+   * </p>
+   *  @return A merged Config object.
+   */
+  public static DrillConfig createForRM() {
+    return create(null, null, true, new DrillRMConfigFileInfo());
   }
 
   /**
@@ -181,35 +201,37 @@ public class DrillConfig extends NestedConfig {
    *          is assimilated
    * @param enableServerConfigs
    *          whether to enable server-specific configuration options
-   * @return
+   * @param configInfo
+   *          see {@link ConfigFileInfo}
+   * @return {@link DrillConfig} object with all configs from passed in resource files
    */
   private static DrillConfig create(String overrideFileResourcePathname,
                                     final Properties overriderProps,
-                                    final boolean enableServerConfigs) {
+                                    final boolean enableServerConfigs,
+                                    ConfigFileInfo configInfo) {
     final StringBuilder logString = new StringBuilder();
     final Stopwatch watch = Stopwatch.createStarted();
-    overrideFileResourcePathname =
-        overrideFileResourcePathname == null
-            ? CommonConstants.CONFIG_OVERRIDE_RESOURCE_PATHNAME
-            : overrideFileResourcePathname;
+    overrideFileResourcePathname = overrideFileResourcePathname == null ?
+      configInfo.getOverrideFileName() : overrideFileResourcePathname;
 
     // 1. Load defaults configuration file.
-    Config fallback = null;
+    Config fallback = ConfigFactory.empty();
     final ClassLoader[] classLoaders = ClasspathHelper.classLoaders();
     for (ClassLoader classLoader : classLoaders) {
       final URL url =
-          classLoader.getResource(CommonConstants.CONFIG_DEFAULT_RESOURCE_PATHNAME);
+          classLoader.getResource(configInfo.getDefaultFileName());
       if (null != url) {
         logString.append("Base Configuration:\n\t- ").append(url).append("\n");
         fallback =
-            ConfigFactory.load(classLoader,
-                               CommonConstants.CONFIG_DEFAULT_RESOURCE_PATHNAME);
+            ConfigFactory.load(classLoader, configInfo.getDefaultFileName());
         break;
       }
     }
 
     // 2. Load per-module configuration files.
-    final Collection<URL> urls = ClassPathScanner.getConfigURLs();
+    final String perModuleResourcePathName = configInfo.getModuleFileName();
+    final Collection<URL> urls = (perModuleResourcePathName != null) ?
+      ClassPathScanner.getConfigURLs(perModuleResourcePathName) : new ArrayList<>();
     logString.append("\nIntermediate Configuration and Plugin files, in order of precedence:\n");
     for (URL url : urls) {
       logString.append("\t- ").append(url).append("\n");
@@ -220,12 +242,12 @@ public class DrillConfig extends NestedConfig {
     final ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
 
     // 3. Load distribution specific configuration file.
-    final URL distribConfigFileUrl = classLoader.getResource(CommonConstants.CONFIG_DISTRIBUTION_RESOURCE_PATHNAME);
+    final URL distribConfigFileUrl = classLoader.getResource(configInfo.getDistributionFileName());
     if (null != distribConfigFileUrl ) {
       logString.append("Distribution Specific Configuration File: ").append(distribConfigFileUrl).append("\n");
     }
     fallback =
-      ConfigFactory.load(CommonConstants.CONFIG_DISTRIBUTION_RESOURCE_PATHNAME).withFallback(fallback);
+      ConfigFactory.load(configInfo.getDistributionFileName()).withFallback(fallback);
 
     // 4. Load any specified overrides configuration file along with any
     //    overrides from JVM system properties (e.g., {-Dname=value").
@@ -258,7 +280,7 @@ public class DrillConfig extends NestedConfig {
     return new DrillConfig(effectiveConfig.resolve());
   }
 
-  public <T> Class<T> getClassAt(String location, Class<T> clazz) throws DrillConfigurationException {
+  private <T> Class<T> getClassAt(String location, Class<T> clazz) throws DrillConfigurationException {
     final String className = getString(location);
     if (className == null) {
       throw new DrillConfigurationException(String.format(
@@ -286,8 +308,7 @@ public class DrillConfig extends NestedConfig {
   public <T> T getInstanceOf(String location, Class<T> clazz) throws DrillConfigurationException{
     final Class<T> c = getClassAt(location, clazz);
     try {
-      final T t = c.newInstance();
-      return t;
+      return c.newInstance();
     } catch (Exception ex) {
       throw new DrillConfigurationException(String.format("Failure while instantiating class [%s] located at '%s.", clazz.getCanonicalName(), location), ex);
     }
diff --git a/common/src/main/java/org/apache/drill/common/config/DrillExecConfigFileInfo.java b/common/src/main/java/org/apache/drill/common/config/DrillExecConfigFileInfo.java
new file mode 100644
index 000000000..c95f38b70
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/config/DrillExecConfigFileInfo.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.config;
+
+public class DrillExecConfigFileInfo implements ConfigFileInfo {
+
+  @Override
+  public String getDefaultFileName() {
+    return ConfigConstants.CONFIG_DEFAULT_RESOURCE_PATHNAME;
+  }
+
+  @Override
+  public String getModuleFileName() {
+    return ConfigConstants.DRILL_JAR_MARKER_FILE_RESOURCE_PATHNAME;
+  }
+
+  @Override
+  public String getDistributionFileName() {
+    return ConfigConstants.CONFIG_DISTRIBUTION_RESOURCE_PATHNAME;
+  }
+
+  @Override
+  public String getOverrideFileName() {
+    return ConfigConstants.CONFIG_OVERRIDE_RESOURCE_PATHNAME;
+  }
+}
diff --git a/common/src/main/java/org/apache/drill/common/config/DrillProperties.java b/common/src/main/java/org/apache/drill/common/config/DrillProperties.java
index f7688f569..87b489fbd 100644
--- a/common/src/main/java/org/apache/drill/common/config/DrillProperties.java
+++ b/common/src/main/java/org/apache/drill/common/config/DrillProperties.java
@@ -83,6 +83,8 @@ public final class DrillProperties extends Properties {
   public static final String TLS_PROVIDER = "TLSProvider";
   public static final String USE_SYSTEM_TRUSTSTORE = "useSystemTrustStore";
 
+  public static final String QUERY_TAGS = "queryTags";
+
   // Although all properties from the application are sent to the server (from the client), the following
   // sets of properties are used by the client and server respectively. These are reserved words.
 
@@ -94,14 +96,15 @@ public final class DrillProperties extends Properties {
           SERVICE_PRINCIPAL, SERVICE_NAME, SERVICE_HOST, REALM, KEYTAB, KERBEROS_FROM_SUBJECT,
           ENABLE_TLS, TLS_PROTOCOL, TRUSTSTORE_TYPE, TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD,
           DISABLE_HOST_VERIFICATION, DISABLE_CERT_VERIFICATION, TLS_HANDSHAKE_TIMEOUT, TLS_PROVIDER,
-          USE_SYSTEM_TRUSTSTORE
+          USE_SYSTEM_TRUSTSTORE, QUERY_TAGS
       );
 
   public static final ImmutableSet<String> ACCEPTED_BY_SERVER = ImmutableSet.of(
       USER /** deprecated */, PASSWORD /** deprecated */,
       SCHEMA,
       IMPERSONATION_TARGET,
-      QUOTING_IDENTIFIERS
+      QUOTING_IDENTIFIERS,
+      QUERY_TAGS
   );
 
   private DrillProperties() {
diff --git a/common/src/main/java/org/apache/drill/common/config/DrillRMConfigFileInfo.java b/common/src/main/java/org/apache/drill/common/config/DrillRMConfigFileInfo.java
new file mode 100644
index 000000000..1db06ff7d
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/config/DrillRMConfigFileInfo.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.config;
+
+public class DrillRMConfigFileInfo implements ConfigFileInfo {
+
+  @Override
+  public String getDefaultFileName() {
+    return ConfigConstants.RM_CONFIG_DEFAULT_RESOURCE_PATHNAME;
+  }
+
+  @Override
+  public String getModuleFileName() {
+    return null;
+  }
+
+  @Override
+  public String getDistributionFileName() {
+    return ConfigConstants.RM_CONFIG_DISTRIBUTION_RESOURCE_PATHNAME;
+  }
+
+  @Override
+  public String getOverrideFileName() {
+    return ConfigConstants.RM_CONFIG_OVERRIDE_RESOURCE_PATHNAME;
+  }
+}
diff --git a/common/src/main/java/org/apache/drill/common/scanner/BuildTimeScan.java b/common/src/main/java/org/apache/drill/common/scanner/BuildTimeScan.java
index a8b65cc91..1e52d4fc6 100644
--- a/common/src/main/java/org/apache/drill/common/scanner/BuildTimeScan.java
+++ b/common/src/main/java/org/apache/drill/common/scanner/BuildTimeScan.java
@@ -28,6 +28,7 @@ import java.net.URL;
 import java.util.List;
 import java.util.Set;
 
+import org.apache.drill.common.config.ConfigConstants;
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.exceptions.DrillRuntimeException;
 import org.apache.drill.common.scanner.persistence.ScanResult;
@@ -136,7 +137,7 @@ public class BuildTimeScan {
       basePath = "/" + basePath;
     }
     URL url = new URL("file:" + basePath);
-    Set<URL> markedPaths = ClassPathScanner.getMarkedPaths();
+    Set<URL> markedPaths = ClassPathScanner.getMarkedPaths(ConfigConstants.DRILL_JAR_MARKER_FILE_RESOURCE_PATHNAME);
     if (!markedPaths.contains(url)) {
       throw new IllegalArgumentException(url + " not in " + markedPaths);
     }
diff --git a/common/src/main/java/org/apache/drill/common/scanner/ClassPathScanner.java b/common/src/main/java/org/apache/drill/common/scanner/ClassPathScanner.java
index 353f3af6c..eeec2d69e 100644
--- a/common/src/main/java/org/apache/drill/common/scanner/ClassPathScanner.java
+++ b/common/src/main/java/org/apache/drill/common/scanner/ClassPathScanner.java
@@ -32,7 +32,6 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-import org.apache.drill.common.config.CommonConstants;
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.exceptions.UserException;
 import org.apache.drill.common.scanner.persistence.AnnotationDescriptor;
@@ -304,12 +303,12 @@ public final class ClassPathScanner {
   /**
    * @return paths that have a drill config file in them
    */
-  static Set<URL> getMarkedPaths() {
-    return forResource(CommonConstants.DRILL_JAR_MARKER_FILE_RESOURCE_PATHNAME, true);
+  static Set<URL> getMarkedPaths(String resourcePathName) {
+    return forResource(resourcePathName, true);
   }
 
-  public static Collection<URL> getConfigURLs() {
-    return forResource(CommonConstants.DRILL_JAR_MARKER_FILE_RESOURCE_PATHNAME, false);
+  public static Collection<URL> getConfigURLs(String resourcePathName) {
+    return forResource(resourcePathName, false);
   }
 
   /**
diff --git a/common/src/main/java/org/apache/drill/common/scanner/RunTimeScan.java b/common/src/main/java/org/apache/drill/common/scanner/RunTimeScan.java
index 70b9939f9..cc86c6792 100644
--- a/common/src/main/java/org/apache/drill/common/scanner/RunTimeScan.java
+++ b/common/src/main/java/org/apache/drill/common/scanner/RunTimeScan.java
@@ -22,6 +22,7 @@ import java.util.Collection;
 import java.util.List;
 import java.util.Set;
 
+import org.apache.drill.common.config.ConfigConstants;
 import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.scanner.persistence.ScanResult;
@@ -42,7 +43,7 @@ public class RunTimeScan {
    * @return getMarkedPaths() sans getPrescannedPaths()
    */
   static Collection<URL> getNonPrescannedMarkedPaths() {
-    Collection<URL> markedPaths = ClassPathScanner.getMarkedPaths();
+    Collection<URL> markedPaths = ClassPathScanner.getMarkedPaths(ConfigConstants.DRILL_JAR_MARKER_FILE_RESOURCE_PATHNAME);
     markedPaths.removeAll(BuildTimeScan.getPrescannedPaths());
     return markedPaths;
   }
@@ -69,7 +70,7 @@ public class RunTimeScan {
     } else {
       // scan everything
       return ClassPathScanner.scan(
-          ClassPathScanner.getMarkedPaths(),
+          ClassPathScanner.getMarkedPaths(ConfigConstants.DRILL_JAR_MARKER_FILE_RESOURCE_PATHNAME),
           packagePrefixes,
           scannedBaseClasses,
           scannedAnnotations,
diff --git a/common/src/test/java/org/apache/drill/categories/ResourceManagerTest.java b/common/src/test/java/org/apache/drill/categories/ResourceManagerTest.java
new file mode 100644
index 000000000..fc05e8be0
--- /dev/null
+++ b/common/src/test/java/org/apache/drill/categories/ResourceManagerTest.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.categories;
+
+public interface ResourceManagerTest {
+}