diff options
Diffstat (limited to 'common/src/main/java/org/apache/drill/common')
-rw-r--r-- | common/src/main/java/org/apache/drill/common/config/ConfigConstants.java (renamed from common/src/main/java/org/apache/drill/common/config/CommonConstants.java) | 24 | ||||
-rw-r--r-- | common/src/main/java/org/apache/drill/common/config/ConfigFileInfo.java | 33 | ||||
-rw-r--r-- | common/src/main/java/org/apache/drill/common/config/DrillConfig.java | 87 | ||||
-rw-r--r-- | common/src/main/java/org/apache/drill/common/config/DrillExecConfigFileInfo.java | 41 | ||||
-rw-r--r-- | common/src/main/java/org/apache/drill/common/config/DrillProperties.java | 7 | ||||
-rw-r--r-- | common/src/main/java/org/apache/drill/common/config/DrillRMConfigFileInfo.java | 41 | ||||
-rw-r--r-- | common/src/main/java/org/apache/drill/common/scanner/BuildTimeScan.java | 3 | ||||
-rw-r--r-- | common/src/main/java/org/apache/drill/common/scanner/ClassPathScanner.java | 9 | ||||
-rw-r--r-- | common/src/main/java/org/apache/drill/common/scanner/RunTimeScan.java | 5 |
9 files changed, 201 insertions, 49 deletions
diff --git a/common/src/main/java/org/apache/drill/common/config/CommonConstants.java b/common/src/main/java/org/apache/drill/common/config/ConfigConstants.java index e203972b8..3283fe07f 100644 --- a/common/src/main/java/org/apache/drill/common/config/CommonConstants.java +++ b/common/src/main/java/org/apache/drill/common/config/ConfigConstants.java @@ -17,21 +17,33 @@ */ package org.apache.drill.common.config; -public interface CommonConstants { +public final class ConfigConstants { /** Default (base) configuration file name. (Classpath resource pathname.) */ - String CONFIG_DEFAULT_RESOURCE_PATHNAME = "drill-default.conf"; + public static final String CONFIG_DEFAULT_RESOURCE_PATHNAME = "drill-default.conf"; /** Module configuration files name. (Classpath resource pathname.) */ - String DRILL_JAR_MARKER_FILE_RESOURCE_PATHNAME = "drill-module.conf"; + public static final String DRILL_JAR_MARKER_FILE_RESOURCE_PATHNAME = "drill-module.conf"; /** Distribution Specific Override configuration file name. (Classpath resource pathname.) */ - String CONFIG_DISTRIBUTION_RESOURCE_PATHNAME = "drill-distrib.conf"; + public static final String CONFIG_DISTRIBUTION_RESOURCE_PATHNAME = "drill-distrib.conf"; /** Override configuration file name. (Classpath resource pathname.) */ - String CONFIG_OVERRIDE_RESOURCE_PATHNAME = "drill-override.conf"; + public static final String CONFIG_OVERRIDE_RESOURCE_PATHNAME = "drill-override.conf"; /** Override plugins configs file name. (Classpath resource pathname.) */ - String STORAGE_PLUGINS_OVERRIDE_CONF = "storage-plugins-override.conf"; + public static final String STORAGE_PLUGINS_OVERRIDE_CONF = "storage-plugins-override.conf"; + /** Default RM configuration file name. (Classpath resource pathname.) */ + public static final String RM_CONFIG_DEFAULT_RESOURCE_PATHNAME = "drill-rm-default.conf"; + + /** Distribution Specific RM Override configuration file name. (Classpath resource pathname.) */ + public static final String RM_CONFIG_DISTRIBUTION_RESOURCE_PATHNAME = "drill-rm-distrib.conf"; + + /** RM Override configuration file name. (Classpath resource pathname.) */ + public static final String RM_CONFIG_OVERRIDE_RESOURCE_PATHNAME = "drill-rm-override.conf"; + + // suppress default constructor + private ConfigConstants() { + } } diff --git a/common/src/main/java/org/apache/drill/common/config/ConfigFileInfo.java b/common/src/main/java/org/apache/drill/common/config/ConfigFileInfo.java new file mode 100644 index 000000000..1f0fe971b --- /dev/null +++ b/common/src/main/java/org/apache/drill/common/config/ConfigFileInfo.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.common.config; + +/** + * Interface that defines implementation to get all the config files names for default, module specific, distribution + * specific and override files. + */ +public interface ConfigFileInfo { + + String getDefaultFileName(); + + String getModuleFileName(); + + String getDistributionFileName(); + + String getOverrideFileName(); +} diff --git a/common/src/main/java/org/apache/drill/common/config/DrillConfig.java b/common/src/main/java/org/apache/drill/common/config/DrillConfig.java index d7d7340a5..6fa09e5a4 100644 --- a/common/src/main/java/org/apache/drill/common/config/DrillConfig.java +++ b/common/src/main/java/org/apache/drill/common/config/DrillConfig.java @@ -17,30 +17,30 @@ */ package org.apache.drill.common.config; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import com.typesafe.config.ConfigRenderOptions; +import io.netty.util.internal.PlatformDependent; +import org.apache.drill.common.exceptions.DrillConfigurationException; +import org.apache.drill.common.exceptions.UserException; +import org.apache.drill.common.scanner.ClassPathScanner; +import org.apache.drill.shaded.guava.com.google.common.annotations.VisibleForTesting; +import org.apache.drill.shaded.guava.com.google.common.base.Preconditions; +import org.apache.drill.shaded.guava.com.google.common.base.Stopwatch; +import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableList; +import org.reflections.util.ClasspathHelper; + import java.lang.management.ManagementFactory; import java.lang.management.RuntimeMXBean; import java.lang.reflect.Constructor; import java.net.URL; +import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map.Entry; import java.util.Properties; import java.util.concurrent.TimeUnit; -import io.netty.util.internal.PlatformDependent; -import org.apache.drill.common.exceptions.DrillConfigurationException; -import org.apache.drill.common.exceptions.UserException; -import org.apache.drill.common.scanner.ClassPathScanner; -import org.reflections.util.ClasspathHelper; - -import org.apache.drill.shaded.guava.com.google.common.annotations.VisibleForTesting; -import org.apache.drill.shaded.guava.com.google.common.base.Preconditions; -import org.apache.drill.shaded.guava.com.google.common.base.Stopwatch; -import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableList; -import com.typesafe.config.Config; -import com.typesafe.config.ConfigFactory; -import com.typesafe.config.ConfigRenderOptions; - public class DrillConfig extends NestedConfig { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillConfig.class); @@ -139,7 +139,7 @@ public class DrillConfig extends NestedConfig { * @param overrideFileResourcePathname * the classpath resource pathname of the file to use for * configuration override purposes; {@code null} specifies to use the - * default pathname ({@link CommonConstants#CONFIG_OVERRIDE_RESOURCE_PATHNAME}) (does + * default pathname ({@link ConfigConstants#CONFIG_OVERRIDE_RESOURCE_PATHNAME}) (does * <strong>not</strong> specify to suppress trying to load an * overrides file) * @return A merged Config object. @@ -153,7 +153,7 @@ public class DrillConfig extends NestedConfig { */ @VisibleForTesting public static DrillConfig create(Properties testConfigurations) { - return create(null, testConfigurations, true); + return create(null, testConfigurations, true, new DrillExecConfigFileInfo()); } /** @@ -161,7 +161,27 @@ public class DrillConfig extends NestedConfig { * see {@link #create(String)}'s {@code overrideFileResourcePathname} */ public static DrillConfig create(String overrideFileResourcePathname, boolean enableServerConfigs) { - return create(overrideFileResourcePathname, null, enableServerConfigs); + return create(overrideFileResourcePathname, null, enableServerConfigs, new DrillExecConfigFileInfo()); + } + + /** + * Merged DrillConfig object for all the RM Configurations provided through various resource files. The order of + * precedence is as follows: + * <p> + * Configuration values are retrieved as follows: + * <ul> + * <li>Check a single copy of "drill-rm-override.conf". If multiple copies are + * on the classpath, which copy is read is indeterminate.</li> + * <li>Check a single copy of "drill-rm-distrib.conf". If multiple copies are + * on the classpath, which copy is read is indeterminate. </li> + * <li>Check a single copy of "{@code drill-rm-default.conf}". If multiple + * copies are on the classpath, which copy is read is indeterminate.</li> + * </ul> + * </p> + * @return A merged Config object. + */ + public static DrillConfig createForRM() { + return create(null, null, true, new DrillRMConfigFileInfo()); } /** @@ -181,35 +201,37 @@ public class DrillConfig extends NestedConfig { * is assimilated * @param enableServerConfigs * whether to enable server-specific configuration options - * @return + * @param configInfo + * see {@link ConfigFileInfo} + * @return {@link DrillConfig} object with all configs from passed in resource files */ private static DrillConfig create(String overrideFileResourcePathname, final Properties overriderProps, - final boolean enableServerConfigs) { + final boolean enableServerConfigs, + ConfigFileInfo configInfo) { final StringBuilder logString = new StringBuilder(); final Stopwatch watch = Stopwatch.createStarted(); - overrideFileResourcePathname = - overrideFileResourcePathname == null - ? CommonConstants.CONFIG_OVERRIDE_RESOURCE_PATHNAME - : overrideFileResourcePathname; + overrideFileResourcePathname = overrideFileResourcePathname == null ? + configInfo.getOverrideFileName() : overrideFileResourcePathname; // 1. Load defaults configuration file. - Config fallback = null; + Config fallback = ConfigFactory.empty(); final ClassLoader[] classLoaders = ClasspathHelper.classLoaders(); for (ClassLoader classLoader : classLoaders) { final URL url = - classLoader.getResource(CommonConstants.CONFIG_DEFAULT_RESOURCE_PATHNAME); + classLoader.getResource(configInfo.getDefaultFileName()); if (null != url) { logString.append("Base Configuration:\n\t- ").append(url).append("\n"); fallback = - ConfigFactory.load(classLoader, - CommonConstants.CONFIG_DEFAULT_RESOURCE_PATHNAME); + ConfigFactory.load(classLoader, configInfo.getDefaultFileName()); break; } } // 2. Load per-module configuration files. - final Collection<URL> urls = ClassPathScanner.getConfigURLs(); + final String perModuleResourcePathName = configInfo.getModuleFileName(); + final Collection<URL> urls = (perModuleResourcePathName != null) ? + ClassPathScanner.getConfigURLs(perModuleResourcePathName) : new ArrayList<>(); logString.append("\nIntermediate Configuration and Plugin files, in order of precedence:\n"); for (URL url : urls) { logString.append("\t- ").append(url).append("\n"); @@ -220,12 +242,12 @@ public class DrillConfig extends NestedConfig { final ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); // 3. Load distribution specific configuration file. - final URL distribConfigFileUrl = classLoader.getResource(CommonConstants.CONFIG_DISTRIBUTION_RESOURCE_PATHNAME); + final URL distribConfigFileUrl = classLoader.getResource(configInfo.getDistributionFileName()); if (null != distribConfigFileUrl ) { logString.append("Distribution Specific Configuration File: ").append(distribConfigFileUrl).append("\n"); } fallback = - ConfigFactory.load(CommonConstants.CONFIG_DISTRIBUTION_RESOURCE_PATHNAME).withFallback(fallback); + ConfigFactory.load(configInfo.getDistributionFileName()).withFallback(fallback); // 4. Load any specified overrides configuration file along with any // overrides from JVM system properties (e.g., {-Dname=value"). @@ -258,7 +280,7 @@ public class DrillConfig extends NestedConfig { return new DrillConfig(effectiveConfig.resolve()); } - public <T> Class<T> getClassAt(String location, Class<T> clazz) throws DrillConfigurationException { + private <T> Class<T> getClassAt(String location, Class<T> clazz) throws DrillConfigurationException { final String className = getString(location); if (className == null) { throw new DrillConfigurationException(String.format( @@ -286,8 +308,7 @@ public class DrillConfig extends NestedConfig { public <T> T getInstanceOf(String location, Class<T> clazz) throws DrillConfigurationException{ final Class<T> c = getClassAt(location, clazz); try { - final T t = c.newInstance(); - return t; + return c.newInstance(); } catch (Exception ex) { throw new DrillConfigurationException(String.format("Failure while instantiating class [%s] located at '%s.", clazz.getCanonicalName(), location), ex); } diff --git a/common/src/main/java/org/apache/drill/common/config/DrillExecConfigFileInfo.java b/common/src/main/java/org/apache/drill/common/config/DrillExecConfigFileInfo.java new file mode 100644 index 000000000..c95f38b70 --- /dev/null +++ b/common/src/main/java/org/apache/drill/common/config/DrillExecConfigFileInfo.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.common.config; + +public class DrillExecConfigFileInfo implements ConfigFileInfo { + + @Override + public String getDefaultFileName() { + return ConfigConstants.CONFIG_DEFAULT_RESOURCE_PATHNAME; + } + + @Override + public String getModuleFileName() { + return ConfigConstants.DRILL_JAR_MARKER_FILE_RESOURCE_PATHNAME; + } + + @Override + public String getDistributionFileName() { + return ConfigConstants.CONFIG_DISTRIBUTION_RESOURCE_PATHNAME; + } + + @Override + public String getOverrideFileName() { + return ConfigConstants.CONFIG_OVERRIDE_RESOURCE_PATHNAME; + } +} diff --git a/common/src/main/java/org/apache/drill/common/config/DrillProperties.java b/common/src/main/java/org/apache/drill/common/config/DrillProperties.java index f7688f569..87b489fbd 100644 --- a/common/src/main/java/org/apache/drill/common/config/DrillProperties.java +++ b/common/src/main/java/org/apache/drill/common/config/DrillProperties.java @@ -83,6 +83,8 @@ public final class DrillProperties extends Properties { public static final String TLS_PROVIDER = "TLSProvider"; public static final String USE_SYSTEM_TRUSTSTORE = "useSystemTrustStore"; + public static final String QUERY_TAGS = "queryTags"; + // Although all properties from the application are sent to the server (from the client), the following // sets of properties are used by the client and server respectively. These are reserved words. @@ -94,14 +96,15 @@ public final class DrillProperties extends Properties { SERVICE_PRINCIPAL, SERVICE_NAME, SERVICE_HOST, REALM, KEYTAB, KERBEROS_FROM_SUBJECT, ENABLE_TLS, TLS_PROTOCOL, TRUSTSTORE_TYPE, TRUSTSTORE_PATH, TRUSTSTORE_PASSWORD, DISABLE_HOST_VERIFICATION, DISABLE_CERT_VERIFICATION, TLS_HANDSHAKE_TIMEOUT, TLS_PROVIDER, - USE_SYSTEM_TRUSTSTORE + USE_SYSTEM_TRUSTSTORE, QUERY_TAGS ); public static final ImmutableSet<String> ACCEPTED_BY_SERVER = ImmutableSet.of( USER /** deprecated */, PASSWORD /** deprecated */, SCHEMA, IMPERSONATION_TARGET, - QUOTING_IDENTIFIERS + QUOTING_IDENTIFIERS, + QUERY_TAGS ); private DrillProperties() { diff --git a/common/src/main/java/org/apache/drill/common/config/DrillRMConfigFileInfo.java b/common/src/main/java/org/apache/drill/common/config/DrillRMConfigFileInfo.java new file mode 100644 index 000000000..1db06ff7d --- /dev/null +++ b/common/src/main/java/org/apache/drill/common/config/DrillRMConfigFileInfo.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.common.config; + +public class DrillRMConfigFileInfo implements ConfigFileInfo { + + @Override + public String getDefaultFileName() { + return ConfigConstants.RM_CONFIG_DEFAULT_RESOURCE_PATHNAME; + } + + @Override + public String getModuleFileName() { + return null; + } + + @Override + public String getDistributionFileName() { + return ConfigConstants.RM_CONFIG_DISTRIBUTION_RESOURCE_PATHNAME; + } + + @Override + public String getOverrideFileName() { + return ConfigConstants.RM_CONFIG_OVERRIDE_RESOURCE_PATHNAME; + } +} diff --git a/common/src/main/java/org/apache/drill/common/scanner/BuildTimeScan.java b/common/src/main/java/org/apache/drill/common/scanner/BuildTimeScan.java index a8b65cc91..1e52d4fc6 100644 --- a/common/src/main/java/org/apache/drill/common/scanner/BuildTimeScan.java +++ b/common/src/main/java/org/apache/drill/common/scanner/BuildTimeScan.java @@ -28,6 +28,7 @@ import java.net.URL; import java.util.List; import java.util.Set; +import org.apache.drill.common.config.ConfigConstants; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.scanner.persistence.ScanResult; @@ -136,7 +137,7 @@ public class BuildTimeScan { basePath = "/" + basePath; } URL url = new URL("file:" + basePath); - Set<URL> markedPaths = ClassPathScanner.getMarkedPaths(); + Set<URL> markedPaths = ClassPathScanner.getMarkedPaths(ConfigConstants.DRILL_JAR_MARKER_FILE_RESOURCE_PATHNAME); if (!markedPaths.contains(url)) { throw new IllegalArgumentException(url + " not in " + markedPaths); } diff --git a/common/src/main/java/org/apache/drill/common/scanner/ClassPathScanner.java b/common/src/main/java/org/apache/drill/common/scanner/ClassPathScanner.java index 353f3af6c..eeec2d69e 100644 --- a/common/src/main/java/org/apache/drill/common/scanner/ClassPathScanner.java +++ b/common/src/main/java/org/apache/drill/common/scanner/ClassPathScanner.java @@ -32,7 +32,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import org.apache.drill.common.config.CommonConstants; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.common.scanner.persistence.AnnotationDescriptor; @@ -304,12 +303,12 @@ public final class ClassPathScanner { /** * @return paths that have a drill config file in them */ - static Set<URL> getMarkedPaths() { - return forResource(CommonConstants.DRILL_JAR_MARKER_FILE_RESOURCE_PATHNAME, true); + static Set<URL> getMarkedPaths(String resourcePathName) { + return forResource(resourcePathName, true); } - public static Collection<URL> getConfigURLs() { - return forResource(CommonConstants.DRILL_JAR_MARKER_FILE_RESOURCE_PATHNAME, false); + public static Collection<URL> getConfigURLs(String resourcePathName) { + return forResource(resourcePathName, false); } /** diff --git a/common/src/main/java/org/apache/drill/common/scanner/RunTimeScan.java b/common/src/main/java/org/apache/drill/common/scanner/RunTimeScan.java index 70b9939f9..cc86c6792 100644 --- a/common/src/main/java/org/apache/drill/common/scanner/RunTimeScan.java +++ b/common/src/main/java/org/apache/drill/common/scanner/RunTimeScan.java @@ -22,6 +22,7 @@ import java.util.Collection; import java.util.List; import java.util.Set; +import org.apache.drill.common.config.ConfigConstants; import org.apache.drill.shaded.guava.com.google.common.collect.Lists; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.scanner.persistence.ScanResult; @@ -42,7 +43,7 @@ public class RunTimeScan { * @return getMarkedPaths() sans getPrescannedPaths() */ static Collection<URL> getNonPrescannedMarkedPaths() { - Collection<URL> markedPaths = ClassPathScanner.getMarkedPaths(); + Collection<URL> markedPaths = ClassPathScanner.getMarkedPaths(ConfigConstants.DRILL_JAR_MARKER_FILE_RESOURCE_PATHNAME); markedPaths.removeAll(BuildTimeScan.getPrescannedPaths()); return markedPaths; } @@ -69,7 +70,7 @@ public class RunTimeScan { } else { // scan everything return ClassPathScanner.scan( - ClassPathScanner.getMarkedPaths(), + ClassPathScanner.getMarkedPaths(ConfigConstants.DRILL_JAR_MARKER_FILE_RESOURCE_PATHNAME), packagePrefixes, scannedBaseClasses, scannedAnnotations, |