diff options
| author | Jacques Nadeau <jacques@apache.org> | 2014-02-17 12:06:06 -0800 |
|---|---|---|
| committer | Jacques Nadeau <jacques@apache.org> | 2014-03-03 23:22:17 -0800 |
| commit | 8b90cae3fe3c5aa48b7d82147452fda043305bd4 (patch) | |
| tree | 2e5249d6fd6f3d2adb58327a37604f98b9aad46d /exec | |
| parent | 2dfadc9d9945c347fded28ec90af5d0842f91853 (diff) | |
Integrate new SQL changes with Hive storage engine, moving to automatic file detection. Rename storage engine to storage plugin. Separate storage plugins from format plugins, updating Parquet and JSON to format engines. Refactor distribution logic and enable use within JSON format and all EasyFormatPlugins.
Diffstat (limited to 'exec')
126 files changed, 5104 insertions, 2635 deletions
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml index 5c6fcfe0a..b4c282c03 100644 --- a/exec/java-exec/pom.xml +++ b/exec/java-exec/pom.xml @@ -272,7 +272,6 @@ <plugins> <plugin> <artifactId>maven-resources-plugin</artifactId> - <version>2.6</version> <executions> <execution> <id>copy-resources</id> @@ -369,18 +368,6 @@ </executions> </plugin> - <!-- <plugin> --> - <!-- <groupId>com.github.igor-petruk.protobuf</groupId> --> - <!-- <artifactId>protobuf-maven-plugin</artifactId> --> - <!-- <version>0.6.2</version> --> - <!-- <executions> --> - <!-- <execution> --> - <!-- <goals> --> - <!-- <goal>run</goal> --> - <!-- </goals> --> - <!-- </execution> --> - <!-- </executions> --> - <!-- </plugin> --> </plugins> </build> diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java index 11658e63c..36cfb532d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java @@ -20,18 +20,16 @@ package org.apache.drill.exec.ops; import java.util.Collection; import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.common.expression.FunctionRegistry; -import org.apache.drill.common.logical.StorageEngineConfig; import org.apache.drill.exec.cache.DistributedCache; import org.apache.drill.exec.planner.PhysicalPlanReader; -import org.apache.drill.exec.planner.sql.DrillSchemaFactory; import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; import org.apache.drill.exec.proto.UserBitShared.QueryId; import org.apache.drill.exec.rpc.control.WorkEventBus; import org.apache.drill.exec.rpc.data.DataConnectionCreator; import org.apache.drill.exec.server.DrillbitContext; -import org.apache.drill.exec.store.StorageEngine; +import org.apache.drill.exec.store.StoragePluginRegistry; +import org.apache.drill.exec.store.StoragePluginRegistry.DrillSchemaFactory; public class QueryContext { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(QueryContext.class); @@ -55,10 +53,11 @@ public class QueryContext { return queryId; } - public StorageEngine getStorageEngine(StorageEngineConfig config) throws ExecutionSetupException { - return drillbitContext.getStorageEngine(config); + public StoragePluginRegistry getStorage(){ + return drillbitContext.getStorage(); } - + + public DistributedCache getCache(){ return drillbitContext.getCache(); } @@ -82,14 +81,13 @@ public class QueryContext { public WorkEventBus getWorkBus(){ return workBus; } - - public DrillSchemaFactory getSchemaFactory(){ - return drillbitContext.getSchemaFactory(); - } public FunctionRegistry getFunctionRegistry(){ return drillbitContext.getFunctionRegistry(); - + } + + public DrillSchemaFactory getFactory(){ + return drillbitContext.getSchemaFactory(); } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java index cd59428f3..aed48027a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java @@ -30,9 +30,19 @@ import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.logical.LogicalPlan; import org.apache.drill.common.logical.PlanProperties; -import org.apache.drill.common.logical.StorageEngineConfig; -import org.apache.drill.common.logical.data.*; +import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.common.logical.data.CollapsingAggregate; +import org.apache.drill.common.logical.data.Filter; +import org.apache.drill.common.logical.data.Join; +import org.apache.drill.common.logical.data.JoinCondition; +import org.apache.drill.common.logical.data.NamedExpression; +import org.apache.drill.common.logical.data.Order; import org.apache.drill.common.logical.data.Order.Ordering; +import org.apache.drill.common.logical.data.Project; +import org.apache.drill.common.logical.data.Scan; +import org.apache.drill.common.logical.data.Segment; +import org.apache.drill.common.logical.data.SinkOperator; +import org.apache.drill.common.logical.data.Store; import org.apache.drill.common.logical.data.visitors.AbstractLogicalVisitor; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.TypeProtos.DataMode; @@ -41,13 +51,13 @@ import org.apache.drill.exec.exception.OptimizerException; import org.apache.drill.exec.ops.QueryContext; import org.apache.drill.exec.physical.PhysicalPlan; import org.apache.drill.exec.physical.base.PhysicalOperator; +import org.apache.drill.exec.physical.config.Limit; import org.apache.drill.exec.physical.config.MergeJoinPOP; import org.apache.drill.exec.physical.config.Screen; import org.apache.drill.exec.physical.config.SelectionVectorRemover; import org.apache.drill.exec.physical.config.Sort; -import org.apache.drill.exec.physical.config.Limit; import org.apache.drill.exec.physical.config.StreamingAggregate; -import org.apache.drill.exec.store.StorageEngine; +import org.apache.drill.exec.store.StoragePlugin; import org.eigenbase.rel.RelFieldCollation.Direction; import org.eigenbase.rel.RelFieldCollation.NullDirection; @@ -192,11 +202,11 @@ public class BasicOptimizer extends Optimizer{ @Override public PhysicalOperator visitScan(Scan scan, Object obj) throws OptimizerException { - StorageEngineConfig config = logicalPlan.getStorageEngineConfig(scan.getStorageEngine()); + StoragePluginConfig config = logicalPlan.getStorageEngineConfig(scan.getStorageEngine()); if(config == null) throw new OptimizerException(String.format("Logical plan referenced the storage engine config %s but the logical plan didn't have that available as a config.", scan.getStorageEngine())); - StorageEngine engine; + StoragePlugin engine; try { - engine = context.getStorageEngine(config); + engine = context.getStorage().getEngine(config); return engine.getPhysicalScan(scan); } catch (IOException | ExecutionSetupException e) { throw new OptimizerException("Failure while attempting to retrieve storage engine.", e); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScan.java index 0b486e9a1..343435b86 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScan.java @@ -19,8 +19,6 @@ package org.apache.drill.exec.physical.base; import java.util.Iterator; -import org.apache.drill.exec.physical.ReadEntry; - import com.google.common.collect.Iterators; public abstract class AbstractGroupScan extends AbstractBase implements GroupScan { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractSubScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractSubScan.java new file mode 100644 index 000000000..c6e11d117 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractSubScan.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.physical.base; + +import java.util.Iterator; +import java.util.List; + +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.common.graph.GraphVisitor; +import org.apache.drill.exec.physical.OperatorCost; + +import com.google.common.collect.Iterators; + +public abstract class AbstractSubScan implements SubScan{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractSubScan.class); + + @Override + public OperatorCost getCost() { + throw new UnsupportedOperationException(); + } + + @Override + public Size getSize() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isExecutable() { + return true; + } + + @Override + public <T, X, E extends Throwable> T accept(PhysicalVisitor<T, X, E> physicalVisitor, X value) throws E { + return physicalVisitor.visitSubScan(this, value); + } + + @Override + public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) throws ExecutionSetupException { + assert children == null || children.isEmpty(); + return this; + } + + @Override + public void accept(GraphVisitor<PhysicalOperator> visitor) { + visitor.enter(this); + visitor.visit(this); + visitor.leave(this); + } + + @Override + public Iterator<PhysicalOperator> iterator() { + return Iterators.emptyIterator(); + } + +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java index b4a85dffa..e51ebb67d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java @@ -20,6 +20,7 @@ package org.apache.drill.exec.physical.base; import java.util.List; import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.common.exceptions.PhysicalOperatorSetupException; import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -30,7 +31,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; */ public interface GroupScan extends Scan, HasAffinity{ - public abstract void applyAssignments(List<DrillbitEndpoint> endpoints); + public abstract void applyAssignments(List<DrillbitEndpoint> endpoints) throws PhysicalOperatorSetupException; public abstract SubScan getSpecificScan(int minorFragmentId) throws ExecutionSetupException; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/Scan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/Scan.java index 63af5eb16..979543306 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/Scan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/Scan.java @@ -17,7 +17,6 @@ */ package org.apache.drill.exec.physical.base; -import org.apache.drill.exec.physical.ReadEntry; public interface Scan extends Leaf { } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SubScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SubScan.java index 5fe3b997f..e6c58a3e3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SubScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SubScan.java @@ -17,7 +17,6 @@ */ package org.apache.drill.exec.physical.base; -import org.apache.drill.exec.physical.ReadEntry; /** * A SubScan operator represents the data scanned by a particular major/minor fragment. This is in contrast to diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java index e270b7980..a1001631e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java @@ -37,7 +37,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.module.SimpleModule; import org.apache.drill.exec.server.DrillbitContext; -import org.apache.drill.exec.store.StorageEngineRegistry; +import org.apache.drill.exec.store.StoragePluginRegistry; public class PhysicalPlanReader { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PhysicalPlanReader.class); @@ -48,7 +48,7 @@ public class PhysicalPlanReader { private final ObjectReader logicalPlanReader; public PhysicalPlanReader(DrillConfig config, ObjectMapper mapper, final DrillbitEndpoint endpoint, - final StorageEngineRegistry engineRegistry) { + final StoragePluginRegistry engineRegistry) { // Endpoint serializer/deserializer. SimpleModule deserModule = new SimpleModule("PhysicalOperatorModule") // @@ -61,7 +61,7 @@ public class PhysicalPlanReader { mapper.registerModule(deserModule); mapper.registerSubtypes(PhysicalOperatorUtil.getSubTypes(config)); InjectableValues injectables = new InjectableValues.Std() // - .addValue(StorageEngineRegistry.class, engineRegistry) // + .addValue(StoragePluginRegistry.class, engineRegistry) // .addValue(DrillbitEndpoint.class, endpoint); // this.mapper = mapper; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java index 30dd48de2..02ca0adaf 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java @@ -17,42 +17,31 @@ */ package org.apache.drill.exec.planner.logical; -import java.util.Collections; - import net.hydromatic.optiq.Schema.TableType; import net.hydromatic.optiq.Statistic; import net.hydromatic.optiq.Statistics; import net.hydromatic.optiq.Table; -import org.apache.drill.common.logical.StorageEngineConfig; +import org.apache.drill.common.logical.StoragePluginConfig; import org.eigenbase.rel.RelNode; import org.eigenbase.relopt.RelOptTable; -import org.eigenbase.reltype.RelDataType; -import org.eigenbase.reltype.RelDataTypeFactory; -import org.eigenbase.sql.type.SqlTypeName; -/** Optiq Table used by Drill. */ -public class DrillTable implements Table{ + +public abstract class DrillTable implements Table{ - private final String name; private final String storageEngineName; - public final StorageEngineConfig storageEngineConfig; + public final StoragePluginConfig storageEngineConfig; private Object selection; /** Creates a DrillTable. */ - public DrillTable(String name, String storageEngineName, Object selection, StorageEngineConfig storageEngineConfig) { - this.name = name; + public DrillTable(String storageEngineName, Object selection, StoragePluginConfig storageEngineConfig) { this.selection = selection; this.storageEngineConfig = storageEngineConfig; this.storageEngineName = storageEngineName; } - - public String getName() { - return name; - } - public StorageEngineConfig getStorageEngineConfig(){ + public StoragePluginConfig getStorageEngineConfig(){ return storageEngineConfig; } @@ -76,32 +65,10 @@ public class DrillTable implements Table{ } @Override - public RelDataType getRowType(RelDataTypeFactory typeFactory) { - return new RelDataTypeDrillImpl(typeFactory); - } - - @Override public TableType getJdbcTableType() { - return null; + return TableType.TABLE; } -// /** Factory for custom tables in Optiq schema. */ -// @SuppressWarnings("UnusedDeclaration") -// public static class Factory implements TableFactory<DrillTable> { -// -// @Override -// public DrillTable create(Schema schema, String name, Map<String, Object> operand, RelDataType rowType) { -// -// final ClasspathRSE.ClasspathRSEConfig rseConfig = new ClasspathRSE.ClasspathRSEConfig(); -// final ClasspathInputConfig inputConfig = new ClasspathInputConfig(); -// inputConfig.path = "/" + name.toLowerCase() + ".json"; -// inputConfig.type = DataWriter.ConverterType.JSON; -// return createTable(schema.getTypeFactory(), (MutableSchema) schema, name, "donuts-json", rseConfig, inputConfig); -// } -// } - - - }
\ No newline at end of file diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntryOld.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DynamicDrillTable.java index ef7266c52..c50e67ca6 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntryOld.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DynamicDrillTable.java @@ -15,36 +15,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store.hive; +package org.apache.drill.exec.planner.logical; -import org.apache.drill.exec.physical.OperatorCost; -import org.apache.drill.exec.physical.ReadEntry; -import org.apache.drill.exec.physical.base.Size; -import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.drill.common.logical.StoragePluginConfig; +import org.eigenbase.reltype.RelDataType; +import org.eigenbase.reltype.RelDataTypeFactory; -public class HiveReadEntryOld implements ReadEntry { - private final HiveConf conf; - private final String table; - private Size size; +public class DynamicDrillTable extends DrillTable{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DynamicDrillTable.class); - public HiveReadEntryOld(HiveConf conf, String table) { - this.conf = conf; - this.table = table; + public DynamicDrillTable(String storageEngineName, Object selection, StoragePluginConfig storageEngineConfig) { + super(storageEngineName, selection, storageEngineConfig); } @Override - public OperatorCost getCost() { - // TODO: need to come up with way to calculate the cost for Hive tables - return new OperatorCost(1, 1, 2, 2); - } - - @Override - public Size getSize() { - if (size != null) { - // TODO: contact the metastore and find the size of the data in table - size = new Size(1, 1); - } - - return size; + public RelDataType getRowType(RelDataTypeFactory typeFactory) { + return new RelDataTypeDrillImpl(typeFactory); } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/StorageEngines.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/StorageEngines.java index a020ab392..d29804068 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/StorageEngines.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/StorageEngines.java @@ -23,19 +23,19 @@ import java.util.Map; import java.util.Map.Entry; import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.common.logical.StorageEngineConfig; +import org.apache.drill.common.logical.StoragePluginConfig; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Charsets; import com.google.common.io.Resources; -public class StorageEngines implements Iterable<Map.Entry<String, StorageEngineConfig>>{ +public class StorageEngines implements Iterable<Map.Entry<String, StoragePluginConfig>>{ - private Map<String, StorageEngineConfig> storage; + private Map<String, StoragePluginConfig> storage; @JsonCreator - public StorageEngines(@JsonProperty("storage") Map<String, StorageEngineConfig> storage){ + public StorageEngines(@JsonProperty("storage") Map<String, StoragePluginConfig> storage){ this.storage = storage; } @@ -53,7 +53,7 @@ public class StorageEngines implements Iterable<Map.Entry<String, StorageEngineC } @Override - public Iterator<Entry<String, StorageEngineConfig>> iterator() { + public Iterator<Entry<String, StoragePluginConfig>> iterator() { return storage.entrySet().iterator(); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSchemaFactory.java deleted file mode 100644 index 23d03b81c..000000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSchemaFactory.java +++ /dev/null @@ -1,96 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.planner.sql; - -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; - -import net.hydromatic.linq4j.function.Function1; -import net.hydromatic.optiq.Schema; -import net.hydromatic.optiq.SchemaPlus; - -import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.common.logical.StorageEngineConfig; -import org.apache.drill.exec.exception.SetupException; -import org.apache.drill.exec.planner.logical.StorageEngines; -import org.apache.drill.exec.store.SchemaProvider; -import org.apache.drill.exec.store.SchemaProviderRegistry; - -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; - -public class DrillSchemaFactory implements Function1<SchemaPlus, Schema>{ - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillSchemaFactory.class); - - private final SchemaProviderRegistry registry; - private final Map<String, StorageEngineEntry> preEntries = Maps.newHashMap(); - - public static DrillSchemaFactory createEmpty(){ - return new DrillSchemaFactory(); - } - - private DrillSchemaFactory(){ - this.registry = null; - } - - public DrillSchemaFactory(StorageEngines engines, DrillConfig config) throws SetupException { - super(); - this.registry = new SchemaProviderRegistry(config); - - for (Map.Entry<String, StorageEngineConfig> entry : engines) { - SchemaProvider provider = registry.getSchemaProvider(entry.getValue()); - preEntries.put(entry.getKey(), new StorageEngineEntry(entry.getValue(), provider)); - } - - } - - public Schema apply(SchemaPlus root) { - List<String> schemaNames = Lists.newArrayList(); - Schema defaultSchema = null; - for(Entry<String, StorageEngineEntry> e : preEntries.entrySet()){ - FileSystemSchema schema = new FileSystemSchema(e.getValue().getConfig(), e.getValue().getProvider(), root, e.getKey()); - if(defaultSchema == null) defaultSchema = schema; - root.add(schema); - schemaNames.add(e.getKey()); - } - logger.debug("Registered schemas for {}", schemaNames); - return defaultSchema; - } - - - private class StorageEngineEntry{ - StorageEngineConfig config; - SchemaProvider provider; - - - public StorageEngineEntry(StorageEngineConfig config, SchemaProvider provider) { - super(); - this.config = config; - this.provider = provider; - } - - public StorageEngineConfig getConfig() { - return config; - } - public SchemaProvider getProvider() { - return provider; - } - - } -} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java index 818e892c9..0fdff3c64 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java @@ -24,7 +24,6 @@ import net.hydromatic.optiq.tools.RelConversionException; import net.hydromatic.optiq.tools.RuleSet; import net.hydromatic.optiq.tools.ValidationException; -import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.expression.FunctionRegistry; import org.apache.drill.common.logical.LogicalPlan; import org.apache.drill.common.logical.PlanProperties.Generator.ResultMode; @@ -34,7 +33,7 @@ import org.apache.drill.exec.planner.logical.DrillRel; import org.apache.drill.exec.planner.logical.DrillRuleSets; import org.apache.drill.exec.planner.logical.DrillScreenRel; import org.apache.drill.exec.planner.logical.DrillStoreRel; -import org.apache.drill.exec.planner.logical.StorageEngines; +import org.apache.drill.exec.store.StoragePluginRegistry.DrillSchemaFactory; import org.eigenbase.rel.RelNode; import org.eigenbase.sql.SqlExplain; import org.eigenbase.sql.SqlKind; @@ -43,9 +42,6 @@ import org.eigenbase.sql.SqlNode; import org.eigenbase.sql.fun.SqlStdOperatorTable; import org.eigenbase.sql.parser.SqlParseException; -import com.google.common.base.Charsets; -import com.google.common.io.Resources; - public class DrillSqlWorker { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillSqlWorker.class); @@ -55,9 +51,7 @@ public class DrillSqlWorker { public DrillSqlWorker(DrillSchemaFactory schemaFactory, FunctionRegistry functionRegistry) throws Exception { this.registry = functionRegistry; this.planner = Frameworks.getPlanner(ConnectionConfig.Lex.MYSQL, schemaFactory, SqlStdOperatorTable.instance(), new RuleSet[]{DrillRuleSets.DRILL_BASIC_RULES}); - } - public LogicalPlan getPlan(String sql) throws SqlParseException, ValidationException, RelConversionException{ SqlNode sqlNode = planner.parse(sql); @@ -77,8 +71,6 @@ public class DrillSqlWorker { break; default: } - - } SqlNode validatedNode = planner.validate(sqlNode); @@ -96,28 +88,6 @@ public class DrillSqlWorker { return implementor.getPlan(); } - private void x() throws Exception { - String sqlAgg = "select a, count(1) from parquet.`/Users/jnadeau/region.parquet` group by a"; - String sql = "select * from parquet.`/Users/jnadeau/region.parquet`"; - - - System.out.println(sql); - System.out.println(getPlan(sql).toJsonString(DrillConfig.create())); - System.out.println("///////////"); - System.out.println(sqlAgg); - System.out.println(getPlan(sqlAgg).toJsonString(DrillConfig.create())); - } - - public static void main(String[] args) throws Exception { - DrillConfig config = DrillConfig.create(); - - String enginesData = Resources.toString(Resources.getResource("storage-engines.json"), Charsets.UTF_8); - StorageEngines engines = config.getMapper().readValue(enginesData, StorageEngines.class); - FunctionRegistry fr = new FunctionRegistry(config); - DrillSchemaFactory schemaFactory = new DrillSchemaFactory(engines, config); - DrillSqlWorker worker = new DrillSqlWorker(schemaFactory, fr); - worker.x(); - } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java index 60a7d5c83..0e6b7bffd 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java @@ -24,26 +24,22 @@ import java.util.Collection; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.common.expression.FunctionRegistry; -import org.apache.drill.common.logical.StorageEngineConfig; import org.apache.drill.exec.cache.DistributedCache; import org.apache.drill.exec.coord.ClusterCoordinator; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry; import org.apache.drill.exec.planner.PhysicalPlanReader; -import org.apache.drill.exec.planner.logical.StorageEngines; -import org.apache.drill.exec.planner.sql.DrillSchemaFactory; import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; import org.apache.drill.exec.rpc.control.Controller; import org.apache.drill.exec.rpc.control.WorkEventBus; import org.apache.drill.exec.rpc.data.DataConnectionCreator; -import org.apache.drill.exec.store.StorageEngine; -import org.apache.drill.exec.store.StorageEngineRegistry; +import org.apache.drill.exec.store.StoragePluginRegistry; +import org.apache.drill.exec.store.StoragePluginRegistry.DrillSchemaFactory; +import org.apache.drill.exec.store.StoragePlugin; import com.codahale.metrics.MetricRegistry; -import com.google.common.base.Charsets; import com.google.common.base.Preconditions; -import com.google.common.io.Resources; public class DrillbitContext { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillbitContext.class); @@ -55,13 +51,12 @@ public class DrillbitContext { private final DataConnectionCreator connectionsPool; private final DistributedCache cache; private final DrillbitEndpoint endpoint; - private final StorageEngineRegistry storageEngineRegistry; + private final StoragePluginRegistry storagePlugins; private final OperatorCreatorRegistry operatorCreatorRegistry; private final Controller controller; private final WorkEventBus workBus; private final FunctionImplementationRegistry functionRegistry; private final FunctionRegistry functionRegistryX; - private final DrillSchemaFactory factory; public DrillbitContext(DrillbitEndpoint endpoint, BootStrapContext context, ClusterCoordinator coord, Controller controller, DataConnectionCreator connectionsPool, DistributedCache cache, WorkEventBus workBus) { super(); @@ -76,27 +71,11 @@ public class DrillbitContext { this.connectionsPool = connectionsPool; this.cache = cache; this.endpoint = endpoint; - this.storageEngineRegistry = new StorageEngineRegistry(this); - this.reader = new PhysicalPlanReader(context.getConfig(), context.getConfig().getMapper(), endpoint, storageEngineRegistry); + this.storagePlugins = new StoragePluginRegistry(this); + this.reader = new PhysicalPlanReader(context.getConfig(), context.getConfig().getMapper(), endpoint, storagePlugins); this.operatorCreatorRegistry = new OperatorCreatorRegistry(context.getConfig()); this.functionRegistry = new FunctionImplementationRegistry(context.getConfig()); - - DrillSchemaFactory factory = null; - try{ - String enginesData = Resources.toString(Resources.getResource("storage-engines.json"), Charsets.UTF_8); - StorageEngines engines = context.getConfig().getMapper().readValue(enginesData, StorageEngines.class); - factory = new DrillSchemaFactory(engines, context.getConfig()); - }catch(Exception e){ - logger.error("Failure reading storage engines data. Creating empty list of schemas.", e); - factory = DrillSchemaFactory.createEmpty(); - } - this.factory = factory; this.functionRegistryX = new FunctionRegistry(context.getConfig()); - - } - - public DrillSchemaFactory getSchemaFactory(){ - return factory; } public FunctionRegistry getFunctionRegistry(){ @@ -131,8 +110,8 @@ public class DrillbitContext { return operatorCreatorRegistry; } - public StorageEngine getStorageEngine(StorageEngineConfig config) throws ExecutionSetupException { - return storageEngineRegistry.getEngine(config); + public StoragePluginRegistry getStorage(){ + return this.storagePlugins; } public NioEventLoopGroup getBitLoopGroup(){ @@ -160,5 +139,8 @@ public class DrillbitContext { return reader; } + public DrillSchemaFactory getSchemaFactory(){ + return storagePlugins.getSchemaFactory(); + } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/FileSystemSchema.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractSchema.java index ac29fc3ef..5b8ccff6d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/FileSystemSchema.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractSchema.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.planner.sql; +package org.apache.drill.exec.store; import java.util.Collection; import java.util.Collections; @@ -27,32 +27,26 @@ import net.hydromatic.optiq.Schema; import net.hydromatic.optiq.SchemaPlus; import net.hydromatic.optiq.TableFunction; -import org.apache.drill.common.logical.StorageEngineConfig; import org.apache.drill.exec.planner.logical.DrillTable; -import org.apache.drill.exec.store.SchemaProvider; -public class FileSystemSchema implements Schema, ExpandingConcurrentMap.MapValueFactory<String, DrillTable>{ - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSystemSchema.class); - - private ExpandingConcurrentMap<String, DrillTable> tables = new ExpandingConcurrentMap<String, DrillTable>(this); - - private final SchemaPlus parentSchema; - private final String name; - private final Expression expression = new DefaultExpression(Object.class); - private final SchemaProvider schemaProvider; - private final StorageEngineConfig config; - - public FileSystemSchema(StorageEngineConfig config, SchemaProvider schemaProvider, SchemaPlus parentSchema, String name) { +public abstract class AbstractSchema implements Schema{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractSchema.class); + + private final SchemaHolder parentSchema; + + protected final String name; + private static final Expression EXPRESSION = new DefaultExpression(Object.class); + + public AbstractSchema(SchemaHolder parentSchema, String name) { super(); this.parentSchema = parentSchema; this.name = name; - this.schemaProvider = schemaProvider; - this.config = config; } + @Override - public Schema getSubSchema(String name) { - return null; + public SchemaPlus getParentSchema() { + return parentSchema.getSchema(); } @Override @@ -61,28 +55,20 @@ public class FileSystemSchema implements Schema, ExpandingConcurrentMap.MapValue } @Override - public Expression getExpression() { - return expression; - } - - @Override public Collection<TableFunction> getTableFunctions(String name) { return Collections.emptyList(); } - - @Override - public SchemaPlus getParentSchema() { - return parentSchema; - } @Override - public Set<String> getTableNames() { - return tables.keySet(); + public Set<String> getTableFunctionNames() { + return Collections.emptySet(); } + + @Override - public Set<String> getTableFunctionNames() { - return Collections.emptySet(); + public Schema getSubSchema(String name) { + return null; } @Override @@ -91,27 +77,25 @@ public class FileSystemSchema implements Schema, ExpandingConcurrentMap.MapValue } @Override - public boolean isMutable() { - return true; + public Expression getExpression() { + return EXPRESSION; } - + @Override - public DrillTable getTable(String name) { - return tables.get(name); + public boolean isMutable() { + return false; } @Override - public DrillTable create(String key) { - Object selection = schemaProvider.getSelectionBaseOnName(key); - if(selection == null) return null; - - return new DrillTable(name, this.name, selection, config); + public DrillTable getTable(String name){ + return null; } @Override - public void destroy(DrillTable value) { + public Set<String> getTableNames() { + return Collections.emptySet(); } - + } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStorageEngine.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStoragePlugin.java index 77fad2a8f..8baa72a22 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStorageEngine.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStoragePlugin.java @@ -18,26 +18,18 @@ package org.apache.drill.exec.store; import java.io.IOException; -import java.util.Collection; import java.util.Collections; import java.util.List; import org.apache.drill.common.logical.data.Scan; -import org.apache.drill.exec.ops.FragmentContext; -import org.apache.drill.exec.physical.ReadEntry; import org.apache.drill.exec.physical.base.AbstractGroupScan; -import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; -import com.google.common.collect.ListMultimap; -import com.google.common.collect.Multimap; +public abstract class AbstractStoragePlugin implements StoragePlugin{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractStoragePlugin.class); -public abstract class AbstractStorageEngine implements StorageEngine{ - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractStorageEngine.class); - - protected AbstractStorageEngine(){ + protected AbstractStoragePlugin(){ } - @Override public boolean supportsRead() { return false; @@ -58,32 +50,5 @@ public abstract class AbstractStorageEngine implements StorageEngine{ throw new UnsupportedOperationException(); } - @Override - public ListMultimap<ReadEntry, DrillbitEndpoint> getReadLocations(Collection<ReadEntry> entries) { - throw new UnsupportedOperationException(); - } - - @Override - public RecordReader getReader(FragmentContext context, ReadEntry readEntry) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public RecordRecorder getWriter(FragmentContext context, WriteEntry writeEntry) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public Multimap<DrillbitEndpoint, ReadEntry> getEntryAssignments(List<DrillbitEndpoint> assignments, - Collection<ReadEntry> entries) { - throw new UnsupportedOperationException(); - } - - @Override - public Multimap<DrillbitEndpoint, WriteEntry> getWriteAssignments(List<DrillbitEndpoint> assignments, - Collection<ReadEntry> entries) { - throw new UnsupportedOperationException(); - } - } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AffinityCalculator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AffinityCalculator.java deleted file mode 100644 index 7061c588c..000000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AffinityCalculator.java +++ /dev/null @@ -1,150 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store; - - -import com.google.common.base.Stopwatch; -import com.google.common.collect.ImmutableRangeMap; -import com.google.common.collect.Range; -import com.codahale.metrics.*; -import com.codahale.metrics.Timer; -import org.apache.drill.exec.metrics.DrillMetrics; -import org.apache.drill.exec.store.parquet.ParquetGroupScan; - -import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; -import org.apache.hadoop.fs.BlockLocation; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; - -import java.io.IOException; -import java.util.*; -import java.util.concurrent.TimeUnit; - -public class AffinityCalculator { - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AffinityCalculator.class); - static final MetricRegistry metrics = DrillMetrics.getInstance(); - static final String BLOCK_MAP_BUILDER_TIMER = MetricRegistry.name(AffinityCalculator.class, "blockMapBuilderTimer"); - - - HashMap<String,ImmutableRangeMap<Long,BlockLocation>> blockMapMap = new HashMap<>(); - FileSystem fs; - String fileName; - Collection<DrillbitEndpoint> endpoints; - HashMap<String,DrillbitEndpoint> endPointMap; - - public AffinityCalculator(FileSystem fs, Collection<DrillbitEndpoint> endpoints) { - this.fs = fs; - this.endpoints = endpoints; - buildEndpointMap(); - } - - /** - * Builds a mapping of block locations to file byte range - */ - private void buildBlockMap(String fileName) { - final Timer.Context context = metrics.timer(BLOCK_MAP_BUILDER_TIMER).time(); - BlockLocation[] blocks; - ImmutableRangeMap<Long,BlockLocation> blockMap; - try { - FileStatus file = fs.getFileStatus(new Path(fileName)); - blocks = fs.getFileBlockLocations(file, 0 , file.getLen()); - } catch (IOException ioe) { throw new RuntimeException(ioe); } - ImmutableRangeMap.Builder<Long, BlockLocation> blockMapBuilder = new ImmutableRangeMap.Builder<Long,BlockLocation>(); - for (BlockLocation block : blocks) { - long start = block.getOffset(); - long end = start + block.getLength(); - Range<Long> range = Range.closedOpen(start, end); - blockMapBuilder = blockMapBuilder.put(range, block); - } - blockMap = blockMapBuilder.build(); - blockMapMap.put(fileName, blockMap); - context.stop(); - } - /** - * For a given RowGroup, calculate how many bytes are available on each on drillbit endpoint - * - * @param rowGroup the RowGroup to calculate endpoint bytes for - */ - public void setEndpointBytes(ParquetGroupScan.RowGroupInfo rowGroup) { - Stopwatch watch = new Stopwatch(); - watch.start(); - String fileName = rowGroup.getPath(); - if (!blockMapMap.containsKey(fileName)) { - buildBlockMap(fileName); - } - - ImmutableRangeMap<Long,BlockLocation> blockMap = blockMapMap.get(fileName); - HashMap<String,Long> hostMap = new HashMap<>(); - HashMap<DrillbitEndpoint,Long> endpointByteMap = new HashMap(); - long start = rowGroup.getStart(); - long end = start + rowGroup.getLength(); - Range<Long> rowGroupRange = Range.closedOpen(start, end); - - // Find submap of ranges that intersect with the rowGroup - ImmutableRangeMap<Long,BlockLocation> subRangeMap = blockMap.subRangeMap(rowGroupRange); - - // Iterate through each block in this submap and get the host for the block location - for (Map.Entry<Range<Long>,BlockLocation> block : subRangeMap.asMapOfRanges().entrySet()) { - String[] hosts; - Range<Long> blockRange = block.getKey(); - try { - hosts = block.getValue().getHosts(); - } catch (IOException ioe) { - throw new RuntimeException("Failed to get hosts for block location", ioe); - } - Range<Long> intersection = rowGroupRange.intersection(blockRange); - long bytes = intersection.upperEndpoint() - intersection.lowerEndpoint(); - - // For each host in the current block location, add the intersecting bytes to the corresponding endpoint - for (String host : hosts) { - DrillbitEndpoint endpoint = getDrillBitEndpoint(host); - if (endpointByteMap.containsKey(endpoint)) { - endpointByteMap.put(endpoint, endpointByteMap.get(endpoint) + bytes); - } else { - if (endpoint != null ) endpointByteMap.put(endpoint, bytes); - } - } - } - - rowGroup.setEndpointBytes(endpointByteMap); - rowGroup.setMaxBytes(endpointByteMap.size() > 0 ? Collections.max(endpointByteMap.values()) : 0); - logger.debug("Row group ({},{}) max bytes {}", rowGroup.getPath(), rowGroup.getStart(), rowGroup.getMaxBytes()); - watch.stop(); - logger.debug("Took {} ms to set endpoint bytes", watch.elapsed(TimeUnit.MILLISECONDS)); - } - - private DrillbitEndpoint getDrillBitEndpoint(String hostName) { - return endPointMap.get(hostName); - } - - /** - * Builds a mapping of drillbit endpoints to hostnames - */ - private void buildEndpointMap() { - Stopwatch watch = new Stopwatch(); - watch.start(); - endPointMap = new HashMap<String, DrillbitEndpoint>(); - for (DrillbitEndpoint d : endpoints) { - String hostName = d.getAddress(); - endPointMap.put(hostName, d); - } - watch.stop(); - logger.debug("Took {} ms to build endpoint map", watch.elapsed(TimeUnit.MILLISECONDS)); - } -} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ClassPathFileSystem.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ClassPathFileSystem.java index 5ca575b8e..e39e1a33a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ClassPathFileSystem.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ClassPathFileSystem.java @@ -61,13 +61,27 @@ public class ClassPathFileSystem extends FileSystem{ throw new IOException(ERROR_MSG); } + private String getFileName(Path path){ + String file = path.toString(); + if(file.charAt(0) == '/'){ + file = file.substring(1); + } + return file; + } + @Override public FileStatus getFileStatus(Path arg0) throws IOException { - URL url = Resources.getResource(arg0.toString()); + String file = getFileName(arg0); + + try{ + URL url = Resources.getResource(file); if(url == null){ throw new IOException(String.format("Unable to find path %s.", arg0.toString())); } - return new FileStatus(-1, false, 1, 8096, System.currentTimeMillis(), arg0); + return new FileStatus(1, false, 1, 8096, System.currentTimeMillis(), arg0); + }catch(RuntimeException e){ + throw new IOException(String.format("Failure trying to load file %s", arg0), e); + } } @Override @@ -96,7 +110,8 @@ public class ClassPathFileSystem extends FileSystem{ @Override public FSDataInputStream open(Path arg0, int arg1) throws IOException { - URL url = Resources.getResource(arg0.toString()); + String file = getFileName(arg0); + URL url = Resources.getResource(file); if(url == null){ throw new IOException(String.format("Unable to find path %s.", arg0.getName())); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/NamedStoragePluginConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/NamedStoragePluginConfig.java new file mode 100644 index 000000000..b67c019ca --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/NamedStoragePluginConfig.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store; + +import org.apache.drill.common.logical.StoragePluginConfig; + +import com.fasterxml.jackson.annotation.JsonTypeName; + +@JsonTypeName("named") +public class NamedStoragePluginConfig implements StoragePluginConfig{ + public String name; +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaFactory.java index 7611461c8..a8d053ca8 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaProvider.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaFactory.java @@ -17,9 +17,11 @@ */ package org.apache.drill.exec.store; -public interface SchemaProvider { - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SchemaProvider.class); - - public Object getSelectionBaseOnName(String tableName); +import net.hydromatic.optiq.Schema; +import net.hydromatic.optiq.SchemaPlus; +public interface SchemaFactory { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SchemaFactory.class); + + public Schema add(SchemaPlus parent); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaHolder.java new file mode 100644 index 000000000..faaca2a57 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaHolder.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store; + +import net.hydromatic.optiq.SchemaPlus; + +/** + * Helper class to provide parent schema after initialization given Optiq's backwards schema build model. + */ +public class SchemaHolder { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SchemaHolder.class); + + private SchemaPlus schema; + + public SchemaHolder(){} + + public SchemaHolder(SchemaPlus schema){ + this.schema = schema; + } + + public SchemaPlus getSchema() { + return schema; + } + + public void setSchema(SchemaPlus schema) { + this.schema = schema; + } + +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaProviderRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaProviderRegistry.java deleted file mode 100644 index 3ae694346..000000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaProviderRegistry.java +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store; - -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; - -import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.common.logical.StorageEngineConfig; -import org.apache.drill.common.util.PathScanner; -import org.apache.drill.exec.ExecConstants; -import org.apache.drill.exec.exception.SetupException; - -public class SchemaProviderRegistry { - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SchemaProviderRegistry.class); - - private Map<Object, Constructor<? extends SchemaProvider>> allProviders = new HashMap<Object, Constructor<? extends SchemaProvider>>(); - private Map<StorageEngineConfig, SchemaProvider> activeEngines = new HashMap<StorageEngineConfig, SchemaProvider>(); - - private final DrillConfig config; - - public SchemaProviderRegistry(DrillConfig config){ - init(config); - this.config = config; - } - - @SuppressWarnings("unchecked") - public void init(DrillConfig config){ - Collection<Class<? extends SchemaProvider>> providers = PathScanner.scanForImplementations(SchemaProvider.class, config.getStringList(ExecConstants.STORAGE_ENGINE_SCAN_PACKAGES)); - logger.debug("Loading schema providers {}", providers); - for(Class<? extends SchemaProvider> schema: providers){ - int i =0; - for(Constructor<?> c : schema.getConstructors()){ - Class<?>[] params = c.getParameterTypes(); - if(params.length != 2 || params[1] != DrillConfig.class || !StorageEngineConfig.class.isAssignableFrom(params[0])){ - logger.info("Skipping SchemaProvider constructor {} for provider class {} since it doesn't implement a [constructor(StorageEngineConfig, DrillConfig)]", c, schema); - continue; - } - allProviders.put(params[0], (Constructor<? extends SchemaProvider>) c); - i++; - } - if(i == 0){ - logger.debug("Skipping registration of StorageSchemaProvider {} as it doesn't have a constructor with the parameters of (StorangeEngineConfig, Config)", schema.getName()); - } - } - } - - public SchemaProvider getSchemaProvider(StorageEngineConfig engineConfig) throws SetupException{ - SchemaProvider engine = activeEngines.get(engineConfig); - if(engine != null) return engine; - Constructor<? extends SchemaProvider> c = allProviders.get(engineConfig.getClass()); - if(c == null) throw new SetupException(String.format("Failure finding StorageSchemaProvider constructor for config %s", engineConfig)); - try { - return c.newInstance(engineConfig, config); - } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { - Throwable t = e instanceof InvocationTargetException ? ((InvocationTargetException)e).getTargetException() : e; - if(t instanceof SetupException) throw ((SetupException) t); - throw new SetupException(String.format("Failure setting up new storage engine configuration for config %s", engineConfig), t); - } - } - - - -} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StorageEngine.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StorageEngine.java deleted file mode 100644 index 00574e60b..000000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StorageEngine.java +++ /dev/null @@ -1,122 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store; - -import java.io.IOException; -import java.util.Collection; -import java.util.List; - -import org.apache.drill.common.logical.StorageEngineConfig; -import org.apache.drill.common.logical.data.Scan; -import org.apache.drill.exec.ops.FragmentContext; -import org.apache.drill.exec.physical.ReadEntry; -import org.apache.drill.exec.physical.base.AbstractGroupScan; -import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; - -import com.google.common.collect.ListMultimap; -import com.google.common.collect.Multimap; - -public interface StorageEngine { - public boolean supportsRead(); - - public boolean supportsWrite(); - - public enum PartitionCapabilities { - NONE, HASH, RANGE; - } - - public List<QueryOptimizerRule> getOptimizerRules(); - - /** - * Get the physical scan operator populated with a set of read entries required for the particular GroupScan (read) node. - * This is somewhat analogous to traditional MapReduce. The difference is, this is the most granular split paradigm. - * - * @param scan - * The configured scan entries. - * @return - * @throws IOException - */ - public AbstractGroupScan getPhysicalScan(Scan scan) throws IOException; - - public SchemaProvider getSchemaProvider(); - - - /** - * Get the set of Drillbit endpoints that are available for each read entry. Note that it is possible for a read entry - * to have no Drillbit locations. In that case, the multimap will contain no values for that read entry. - * - * @return Multimap of ReadEntry > List<DrillbitEndpoint> for ReadEntries with available locations. - */ - public ListMultimap<ReadEntry, DrillbitEndpoint> getReadLocations(Collection<ReadEntry> entries); - - /** - * Apply read entry assignments based on the list of actually assigned Endpoints. A storage engine is allowed to - * update or modify the read entries based on the nature of the assignments. For example, if two blocks are initially - * considered separate read entries but then the storage engine realizes that the assignments for those two reads are - * on the same system, the storage engine may decide to collapse those entries into a single read entry that covers - * both original read entries. - * - * @param assignments - * @param entries - * @return - */ - public Multimap<DrillbitEndpoint, ReadEntry> getEntryAssignments(List<DrillbitEndpoint> assignments, - Collection<ReadEntry> entries); - - /** - * Get a particular reader for a fragment context. - * - * @param context - * @param readEntry - * @return - * @throws IOException - */ - public RecordReader getReader(FragmentContext context, ReadEntry readEntry) throws IOException; - - /** - * Apply write entry assignments based on the list of actually assigned endpoints. A storage engine is allowed to - * rewrite the WriteEntries if desired based on the nature of the assignments. For example, a storage engine could - * hold off actually determining the specific level of partitioning required until it finds out exactly the number of - * nodes associated with the operation. - * - * @param assignments - * @param entries - * @return - */ - public Multimap<DrillbitEndpoint, WriteEntry> getWriteAssignments(List<DrillbitEndpoint> assignments, - Collection<ReadEntry> entries); - - /** - * - * @param context - * @param writeEntry - * @return - * @throws IOException - */ - public RecordRecorder getWriter(FragmentContext context, WriteEntry writeEntry) throws IOException; - - public interface WriteEntry { - } - - public static class Cost { - public long disk; - public long network; - public long memory; - public long cpu; - } -} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StorageEngineRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StorageEngineRegistry.java deleted file mode 100644 index 4cc73468b..000000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StorageEngineRegistry.java +++ /dev/null @@ -1,84 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store; - -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; - -import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.common.exceptions.ExecutionSetupException; -import org.apache.drill.common.logical.StorageEngineConfig; -import org.apache.drill.common.util.PathScanner; -import org.apache.drill.exec.ExecConstants; -import org.apache.drill.exec.server.DrillbitContext; - -public class StorageEngineRegistry { - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StorageEngineRegistry.class); - - private Map<Object, Constructor<? extends StorageEngine>> availableEngines = new HashMap<Object, Constructor<? extends StorageEngine>>(); - private Map<StorageEngineConfig, StorageEngine> activeEngines = new HashMap<StorageEngineConfig, StorageEngine>(); - - private DrillbitContext context; - public StorageEngineRegistry(DrillbitContext context){ - init(context.getConfig()); - this.context = context; - } - - @SuppressWarnings("unchecked") - public void init(DrillConfig config){ - Collection<Class<? extends StorageEngine>> engines = PathScanner.scanForImplementations(StorageEngine.class, config.getStringList(ExecConstants.STORAGE_ENGINE_SCAN_PACKAGES)); - logger.debug("Loading storage engines {}", engines); - for(Class<? extends StorageEngine> engine: engines){ - int i =0; - for(Constructor<?> c : engine.getConstructors()){ - Class<?>[] params = c.getParameterTypes(); - if(params.length != 2 || params[1] != DrillbitContext.class || !StorageEngineConfig.class.isAssignableFrom(params[0])){ - logger.info("Skipping StorageEngine constructor {} for engine class {} since it doesn't implement a [constructor(StorageEngineConfig, DrillbitContext)]", c, engine); - continue; - } - availableEngines.put(params[0], (Constructor<? extends StorageEngine>) c); - i++; - } - if(i == 0){ - logger.debug("Skipping registration of StorageEngine {} as it doesn't have a constructor with the parameters of (StorangeEngineConfig, Config)", engine.getCanonicalName()); - } - } - } - - public synchronized StorageEngine getEngine(StorageEngineConfig engineConfig) throws ExecutionSetupException{ - StorageEngine engine = activeEngines.get(engineConfig); - if(engine != null) return engine; - Constructor<? extends StorageEngine> c = availableEngines.get(engineConfig.getClass()); - if(c == null) throw new ExecutionSetupException(String.format("Failure finding StorageEngine constructor for config %s", engineConfig)); - try { - engine = c.newInstance(engineConfig, context); - activeEngines.put(engineConfig, engine); - return engine; - } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { - Throwable t = e instanceof InvocationTargetException ? ((InvocationTargetException)e).getTargetException() : e; - if(t instanceof ExecutionSetupException) throw ((ExecutionSetupException) t); - throw new ExecutionSetupException(String.format("Failure setting up new storage engine configuration for config %s", engineConfig), t); - } - } - - - -} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java new file mode 100644 index 000000000..2e54b0d1d --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store; + +import java.io.IOException; +import java.util.List; + +import net.hydromatic.optiq.Schema; +import net.hydromatic.optiq.SchemaPlus; + +import org.apache.drill.common.logical.data.Scan; +import org.apache.drill.exec.physical.base.AbstractGroupScan; + +public interface StoragePlugin { + public boolean supportsRead(); + + public boolean supportsWrite(); + + public List<QueryOptimizerRule> getOptimizerRules(); + + /** + * Get the physical scan operator for the particular GroupScan (read) node. + * + * @param scan + * The configured scan with a storage engine specific selection. + * @return + * @throws IOException + */ + public AbstractGroupScan getPhysicalScan(Scan scan) throws IOException; + + public Schema createAndAddSchema(SchemaPlus parent); + +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java new file mode 100644 index 000000000..b182abe18 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java @@ -0,0 +1,175 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store; + +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; + +import net.hydromatic.linq4j.function.Function1; +import net.hydromatic.optiq.Schema; +import net.hydromatic.optiq.SchemaPlus; + +import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.common.logical.FormatPluginConfig; +import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.common.util.PathScanner; +import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.planner.logical.StorageEngines; +import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.store.dfs.FileSystemPlugin; +import org.apache.drill.exec.store.dfs.FormatPlugin; + +import com.google.common.base.Charsets; +import com.google.common.collect.ImmutableMap; +import com.google.common.io.Resources; + +public class StoragePluginRegistry implements Iterable<Map.Entry<String, StoragePlugin>>{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StoragePluginRegistry.class); + + private Map<Object, Constructor<? extends StoragePlugin>> availableEngines = new HashMap<Object, Constructor<? extends StoragePlugin>>(); + private final ImmutableMap<String, StoragePlugin> engines; + + private DrillbitContext context; + private final DrillSchemaFactory schemaFactory = new DrillSchemaFactory(); + + public StoragePluginRegistry(DrillbitContext context) { + try{ + this.context = context; + init(context.getConfig()); + this.engines = ImmutableMap.copyOf(createEngines()); + }catch(RuntimeException e){ + logger.error("Failure while loading storage engine registry.", e); + throw new RuntimeException("Faiure while reading and loading storage plugin configuration.", e); + } + } + + @SuppressWarnings("unchecked") + public void init(DrillConfig config){ + Collection<Class<? extends StoragePlugin>> engines = PathScanner.scanForImplementations(StoragePlugin.class, config.getStringList(ExecConstants.STORAGE_ENGINE_SCAN_PACKAGES)); + logger.debug("Loading storage engines {}", engines); + for(Class<? extends StoragePlugin> engine: engines){ + int i =0; + for(Constructor<?> c : engine.getConstructors()){ + Class<?>[] params = c.getParameterTypes(); + if(params.length != 3 || params[1] != DrillbitContext.class || !StoragePluginConfig.class.isAssignableFrom(params[0]) || params[2] != String.class){ + logger.info("Skipping StorageEngine constructor {} for engine class {} since it doesn't implement a [constructor(StorageEngineConfig, DrillbitContext, String)]", c, engine); + continue; + } + availableEngines.put(params[0], (Constructor<? extends StoragePlugin>) c); + i++; + } + if(i == 0){ + logger.debug("Skipping registration of StorageEngine {} as it doesn't have a constructor with the parameters of (StorangeEngineConfig, Config)", engine.getCanonicalName()); + } + } + + } + + private Map<String, StoragePlugin> createEngines(){ + StorageEngines engines = null; + Map<String, StoragePlugin> activeEngines = new HashMap<String, StoragePlugin>(); + try{ + String enginesData = Resources.toString(Resources.getResource("storage-engines.json"), Charsets.UTF_8); + engines = context.getConfig().getMapper().readValue(enginesData, StorageEngines.class); + }catch(IOException e){ + throw new IllegalStateException("Failure while reading storage engines data.", e); + } + + for(Map.Entry<String, StoragePluginConfig> config : engines){ + try{ + StoragePlugin plugin = create(config.getKey(), config.getValue()); + activeEngines.put(config.getKey(), plugin); + }catch(ExecutionSetupException e){ + logger.error("Failure while setting up StoragePlugin with name: '{}'.", config.getKey(), e); + } + } + return activeEngines; + } + + public StoragePlugin getEngine(String registeredStorageEngineName) throws ExecutionSetupException { + return engines.get(registeredStorageEngineName); + } + + public StoragePlugin getEngine(StoragePluginConfig config) throws ExecutionSetupException { + if(config instanceof NamedStoragePluginConfig){ + return engines.get(((NamedStoragePluginConfig) config).name); + }else{ + // TODO: for now, we'll throw away transient configs. we really ought to clean these up. + return create(null, config); + } + } + + public FormatPlugin getFormatPlugin(StoragePluginConfig storageConfig, FormatPluginConfig formatConfig) throws ExecutionSetupException{ + StoragePlugin p = getEngine(storageConfig); + if(!(p instanceof FileSystemPlugin)) throw new ExecutionSetupException(String.format("You tried to request a format plugin for a stroage engine that wasn't of type FileSystemPlugin. The actual type of plugin was %s.", p.getClass().getName())); + FileSystemPlugin storage = (FileSystemPlugin) p; + return storage.getFormatPlugin(formatConfig); + } + + private StoragePlugin create(String name, StoragePluginConfig engineConfig) throws ExecutionSetupException { + StoragePlugin engine = null; + Constructor<? extends StoragePlugin> c = availableEngines.get(engineConfig.getClass()); + if (c == null) + throw new ExecutionSetupException(String.format("Failure finding StorageEngine constructor for config %s", + engineConfig)); + try { + engine = c.newInstance(engineConfig, context, name); + return engine; + } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { + Throwable t = e instanceof InvocationTargetException ? ((InvocationTargetException) e).getTargetException() : e; + if (t instanceof ExecutionSetupException) + throw ((ExecutionSetupException) t); + throw new ExecutionSetupException(String.format( + "Failure setting up new storage engine configuration for config %s", engineConfig), t); + } + } + + @Override + public Iterator<Entry<String, StoragePlugin>> iterator() { + return engines.entrySet().iterator(); + } + + public DrillSchemaFactory getSchemaFactory(){ + return schemaFactory; + } + + public class DrillSchemaFactory implements Function1<SchemaPlus, Schema>{ + + @Override + public Schema apply(SchemaPlus parent) { + Schema defaultSchema = null; + for(Map.Entry<String, StoragePlugin> e : engines.entrySet()){ + Schema s = e.getValue().createAndAddSchema(parent); + if(defaultSchema == null) defaultSchema = s; + } + return defaultSchema; + } + + } + + + + +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/BasicFormatMatcher.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/BasicFormatMatcher.java new file mode 100644 index 000000000..1c391de6a --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/BasicFormatMatcher.java @@ -0,0 +1,130 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.dfs; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.regex.Pattern; + +import org.apache.drill.exec.store.dfs.shim.DrillFileSystem; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileStatus; + +import com.beust.jcommander.internal.Lists; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Range; + +public class BasicFormatMatcher extends FormatMatcher{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BasicFormatMatcher.class); + + private final List<Pattern> patterns; + private final MagicStringMatcher matcher; + protected final DrillFileSystem fs; + protected final FormatPlugin plugin; + + public BasicFormatMatcher(FormatPlugin plugin, DrillFileSystem fs, List<Pattern> patterns, List<MagicString> magicStrings) { + super(); + this.patterns = ImmutableList.copyOf(patterns); + this.matcher = new MagicStringMatcher(magicStrings); + this.fs = fs; + this.plugin = plugin; + } + + public BasicFormatMatcher(FormatPlugin plugin, DrillFileSystem fs, String extension){ + this(plugin, fs, // + Lists.newArrayList(Pattern.compile(".*\\." + extension)), // + (List<MagicString>) Collections.EMPTY_LIST); + } + + @Override + public boolean supportDirectoryReads() { + return false; + } + + @Override + public FormatSelection isReadable(FileSelection file) throws IOException { + if(isReadable(file.getFirstPath(fs))){ + return new FormatSelection(plugin.getConfig(), file); + } + return null; + } + + protected final boolean isReadable(FileStatus status) throws IOException { + for(Pattern p : patterns){ + if(p.matcher(status.getPath().toString()).matches()){ + return true; + } + } + + if(matcher.matches(status)) return true; + return false; + } + + + @Override + @JsonIgnore + public FormatPlugin getFormatPlugin() { + return plugin; + } + + + private class MagicStringMatcher{ + + private List<RangeMagics> ranges; + + public MagicStringMatcher(List<MagicString> magicStrings){ + ranges = Lists.newArrayList(); + for(MagicString ms : magicStrings){ + ranges.add(new RangeMagics(ms)); + } + } + + public boolean matches(FileStatus status) throws IOException{ + if(ranges.isEmpty()) return false; + final Range<Long> fileRange = Range.closedOpen( 0L, status.getLen()); + + try(FSDataInputStream is = fs.open(status.getPath()).getInputStream()){ + for(RangeMagics rMagic : ranges){ + Range<Long> r = rMagic.range; + if(!fileRange.encloses(r)) continue; + int len = (int) (r.upperEndpoint() - r.lowerEndpoint()); + byte[] bytes = new byte[len]; + is.readFully(r.lowerEndpoint(), bytes); + for(byte[] magic : rMagic.magics){ + if(Arrays.equals(magic, bytes)) return true; + } + + } + } + return false; + } + + private class RangeMagics{ + Range<Long> range; + byte[][] magics; + + public RangeMagics(MagicString ms){ + this.range = Range.closedOpen( ms.getOffset(), (long) ms.getBytes().length); + this.magics = new byte[][]{ms.getBytes()}; + } + } + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java new file mode 100644 index 000000000..5ab2c1aa7 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java @@ -0,0 +1,127 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.dfs; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.regex.Pattern; + +import org.apache.drill.exec.store.dfs.shim.DrillFileSystem; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.collect.Lists; + +/** + * Jackson serializable description of a file selection. Maintains an internal set of file statuses. However, also + * serializes out as a list of Strings. All accessing methods first regenerate the FileStatus objects if they are not + * available. This allows internal movement of FileStatus and the ability to serialize if need be. + */ +public class FileSelection { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSelection.class); + + @JsonIgnore + private List<FileStatus> statuses; + + public List<String> files; + + public FileSelection() { + } + + + public FileSelection(List<String> files, boolean dummy){ + this.files = files; + } + + public FileSelection(List<FileStatus> statuses) { + this.statuses = statuses; + this.files = Lists.newArrayList(); + for (FileStatus f : statuses) { + files.add(f.getPath().toString()); + } + } + + public boolean containsDirectories(DrillFileSystem fs) throws IOException { + init(fs); + for (FileStatus p : statuses) { + if (p.isDir()) return true; + } + return false; + } + + public FileSelection minusDirectorries(DrillFileSystem fs) throws IOException { + init(fs); + List<FileStatus> newList = Lists.newArrayList(); + for (FileStatus p : statuses) { + if (p.isDir()) { + List<FileStatus> statuses = fs.list(true, p.getPath()); + for (FileStatus s : statuses) { + newList.add(s); + } + + } else { + newList.add(p); + } + } + return new FileSelection(newList); + } + + public FileStatus getFirstPath(DrillFileSystem fs) throws IOException { + init(fs); + return statuses.get(0); + } + + public List<String> getAsFiles(){ + if(!files.isEmpty()) return files; + if(statuses == null) return Collections.emptyList(); + List<String> files = Lists.newArrayList(); + for(FileStatus s : statuses){ + files.add(s.getPath().toString()); + } + return files; + } + + private void init(DrillFileSystem fs) throws IOException { + if (files != null && statuses == null) { + statuses = Lists.newArrayList(); + for (String p : files) { + statuses.add(fs.getFileStatus(new Path(p))); + } + } + } + + public List<FileStatus> getFileStatusList(DrillFileSystem fs) throws IOException { + init(fs); + return statuses; + } + + public static FileSelection create(DrillFileSystem fs, Path parent, String path) throws IOException { + if ( !(path.contains("*") || path.contains("?")) ) { + Path p = new Path(parent, path); + FileStatus status = fs.getFileStatus(p); + return new FileSelection(Collections.singletonList(status)); + } else { + FileStatus[] status = fs.getUnderlying().globStatus(new Path(parent, path)); + if(status == null || status.length == 0) return null; + return new FileSelection(Lists.newArrayList(status)); + } + } + +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemConfig.java new file mode 100644 index 000000000..455c4b280 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemConfig.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.dfs; + +import java.util.Map; + +import org.apache.drill.common.logical.FormatPluginConfig; +import org.apache.drill.common.logical.StoragePluginConfig; + +import com.fasterxml.jackson.annotation.JsonTypeName; + +@JsonTypeName("file") +public class FileSystemConfig implements StoragePluginConfig{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSystemConfig.class); + + public String connection; + public Map<String, String> workspaces; + public Map<String, FormatPluginConfig> formats; + +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemFormatConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemFormatConfig.java new file mode 100644 index 000000000..be396a869 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemFormatConfig.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.dfs; + +import org.apache.drill.common.logical.FormatPluginConfig; +import org.apache.drill.common.logical.StoragePluginConfig; + +public class FileSystemFormatConfig<T extends FormatPluginConfig> implements StoragePluginConfig{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSystemFormatConfig.class); + + public T getFormatConfig(){ + return null; + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java new file mode 100644 index 000000000..3762c1a26 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.dfs; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import net.hydromatic.optiq.Schema; +import net.hydromatic.optiq.SchemaPlus; + +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.common.logical.FormatPluginConfig; +import org.apache.drill.common.logical.data.Scan; +import org.apache.drill.exec.physical.base.AbstractGroupScan; +import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.store.AbstractStoragePlugin; +import org.apache.drill.exec.store.ClassPathFileSystem; +import org.apache.drill.exec.store.dfs.shim.DrillFileSystem; +import org.apache.drill.exec.store.dfs.shim.FileSystemCreator; +import org.apache.hadoop.conf.Configuration; + +import com.beust.jcommander.internal.Lists; +import com.beust.jcommander.internal.Maps; + +/** + * A Storage engine associated with a Hadoop FileSystem Implementation. Examples include HDFS, MapRFS, QuantacastFileSystem, + * LocalFileSystem, as well Apache Drill specific CachedFileSystem, ClassPathFileSystem and LocalSyncableFileSystem. + * Tables are file names, directories and path patterns. This storage engine delegates to FSFormatEngines but shares + * references to the FileSystem configuration and path management. + */ +public class FileSystemPlugin extends AbstractStoragePlugin{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSystemPlugin.class); + + private final FileSystemSchemaFactory schemaFactory; + private Map<String, FormatPlugin> formatsByName; + private Map<FormatPluginConfig, FormatPlugin> formatPluginsByConfig; + private FileSystemConfig config; + private DrillbitContext context; + private final DrillFileSystem fs; + + public FileSystemPlugin(FileSystemConfig config, DrillbitContext context, String name) throws ExecutionSetupException{ + try{ + this.config = config; + this.context = context; + + Configuration fsConf = new Configuration(); + fsConf.set("fs.default.name", config.connection); + fsConf.set("fs.classpath.impl", ClassPathFileSystem.class.getName()); + this.fs = FileSystemCreator.getFileSystem(context.getConfig(), fsConf); + this.formatsByName = FormatCreator.getFormatPlugins(context, fs, config); + List<FormatMatcher> matchers = Lists.newArrayList(); + formatPluginsByConfig = Maps.newHashMap(); + for(FormatPlugin p : formatsByName.values()){ + matchers.add(p.getMatcher()); + formatPluginsByConfig.put(p.getConfig(), p); + } + + List<WorkspaceSchemaFactory> factories = null; + if(config.workspaces == null || config.workspaces.isEmpty()){ + factories = Collections.singletonList(new WorkspaceSchemaFactory("default", name, fs, "/", matchers)); + }else{ + factories = Lists.newArrayList(); + for(Map.Entry<String, String> space : config.workspaces.entrySet()){ + factories.add(new WorkspaceSchemaFactory(space.getKey(), name, fs, space.getValue(), matchers)); + } + } + this.schemaFactory = new FileSystemSchemaFactory(name, factories); + }catch(IOException e){ + throw new ExecutionSetupException("Failure setting up file system plugin.", e); + } + } + + @Override + public boolean supportsRead() { + return true; + } + + @Override + public AbstractGroupScan getPhysicalScan(Scan scan) throws IOException { + FormatSelection formatSelection = scan.getSelection().getWith(context.getConfig(), FormatSelection.class); + FormatPlugin plugin; + if(formatSelection.getFormat() instanceof NamedFormatPluginConfig){ + plugin = formatsByName.get( ((NamedFormatPluginConfig) formatSelection.getFormat()).name); + }else{ + plugin = formatPluginsByConfig.get(formatSelection.getFormat()); + } + if(plugin == null) throw new IOException(String.format("Failure getting requested format plugin named '%s'. It was not one of the format plugins registered.", formatSelection.getFormat())); + return plugin.getGroupScan(scan.getOutputReference(), formatSelection.getSelection()); + } + + @Override + public Schema createAndAddSchema(SchemaPlus parent) { + return schemaFactory.add(parent); + } + + public FormatPlugin getFormatPlugin(String name){ + return formatsByName.get(name); + } + + public FormatPlugin getFormatPlugin(FormatPluginConfig config){ + if(config instanceof NamedFormatPluginConfig){ + return formatsByName.get(((NamedFormatPluginConfig) config).name); + }else{ + return formatPluginsByConfig.get(config); + } + } + +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java new file mode 100644 index 000000000..fbea81c8a --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.dfs; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import net.hydromatic.optiq.Schema; +import net.hydromatic.optiq.SchemaPlus; +import net.hydromatic.optiq.TableFunction; + +import org.apache.drill.exec.planner.logical.DrillTable; +import org.apache.drill.exec.store.AbstractSchema; +import org.apache.drill.exec.store.SchemaFactory; +import org.apache.drill.exec.store.SchemaHolder; +import org.apache.drill.exec.store.dfs.WorkspaceSchemaFactory.WorkspaceSchema; + +import com.google.common.collect.Maps; + + +/** + * This is the top level schema that responds to root level path requests. Also supports + */ +public class FileSystemSchemaFactory implements SchemaFactory{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSystemSchemaFactory.class); + + private List<WorkspaceSchemaFactory> factories; + private String schemaName; + + + public FileSystemSchemaFactory(String schemaName, List<WorkspaceSchemaFactory> factories) { + super(); + this.schemaName = schemaName; + this.factories = factories; + } + + @Override + public Schema add(SchemaPlus parent) { + FileSystemSchema schema = new FileSystemSchema(parent, schemaName); + schema.setHolder(parent.add(schema)); + return schema; + } + + public class FileSystemSchema extends AbstractSchema{ + + private final WorkspaceSchema defaultSchema; + private final Map<String, WorkspaceSchema> schemaMap = Maps.newHashMap(); + final SchemaHolder selfHolder = new SchemaHolder(); + + public FileSystemSchema(SchemaPlus parentSchema, String name) { + super(new SchemaHolder(parentSchema), name); + for(WorkspaceSchemaFactory f : factories){ + WorkspaceSchema s = f.create(selfHolder); + schemaMap.put(s.getName(), s); + } + + defaultSchema = schemaMap.get("default"); + } + + void setHolder(SchemaPlus plusOfThis){ + selfHolder.setSchema(plusOfThis); + for(WorkspaceSchema s : schemaMap.values()){ + plusOfThis.add(s); + } + } + + @Override + public DrillTable getTable(String name) { + return defaultSchema.getTable(name); + } + + @Override + public Collection<TableFunction> getTableFunctions(String name) { + return defaultSchema.getTableFunctions(name); + } + + @Override + public Set<String> getTableFunctionNames() { + return defaultSchema.getTableFunctionNames(); + } + + @Override + public Schema getSubSchema(String name) { + return defaultSchema.getSubSchema(name); + } + + @Override + public Set<String> getSubSchemaNames() { + return defaultSchema.getSubSchemaNames(); + } + + @Override + public Set<String> getTableNames() { + return defaultSchema.getTableNames(); + } + + } + +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatCreator.java new file mode 100644 index 000000000..7ce8c5054 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatCreator.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.dfs; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.Collection; +import java.util.Map; + +import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.logical.FormatPluginConfig; +import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.common.util.ConstructorChecker; +import org.apache.drill.common.util.PathScanner; +import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.store.dfs.shim.DrillFileSystem; + +import com.google.common.collect.Maps; + +public class FormatCreator { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FormatCreator.class); + + + static final ConstructorChecker FORMAT_BASED = new ConstructorChecker(String.class, DrillbitContext.class, DrillFileSystem.class, StoragePluginConfig.class, FormatPlugin.class); + static final ConstructorChecker DEFAULT_BASED = new ConstructorChecker(String.class, DrillbitContext.class, DrillFileSystem.class, StoragePluginConfig.class); + + static Map<String, FormatPlugin> getFormatPlugins(DrillbitContext context, DrillFileSystem fileSystem, FileSystemConfig storageConfig){ + final DrillConfig config = context.getConfig(); + Map<String, FormatPlugin> plugins = Maps.newHashMap(); + + Collection<Class<? extends FormatPlugin>> pluginClasses = PathScanner.scanForImplementations(FormatPlugin.class, config.getStringList(ExecConstants.STORAGE_ENGINE_SCAN_PACKAGES)); + + + if(storageConfig.formats == null || storageConfig.formats.isEmpty()){ + + for(Class<? extends FormatPlugin> pluginClass: pluginClasses){ + for(Constructor<?> c : pluginClass.getConstructors()){ + try{ + + if(!DEFAULT_BASED.check(c)) continue; + FormatPlugin plugin = (FormatPlugin) c.newInstance(null, context, fileSystem, storageConfig); + plugins.put(plugin.getName(), plugin); + }catch(Exception e){ + logger.warn(String.format("Failure while trying instantiate FormatPlugin %s.", pluginClass.getName()), e); + } + } + } + + }else{ + + Map<Class<?>, Constructor<?>> constructors = Maps.newHashMap(); + for(Class<? extends FormatPlugin> pluginClass: pluginClasses){ + for(Constructor<?> c : pluginClass.getConstructors()){ + try{ + if(!FORMAT_BASED.check(c)) continue; + constructors.put(pluginClass, c); + }catch(Exception e){ + logger.warn(String.format("Failure while trying instantiate FormatPlugin %s.", pluginClass.getName()), e); + } + } + } + + for(Map.Entry<String, FormatPluginConfig> e : storageConfig.formats.entrySet()){ + Constructor<?> c = constructors.get(e.getValue().getClass()); + if(c == null){ + logger.warn("Unable to find constructor for storage config named '{}' of type '{}'.", e.getKey(), e.getValue().getClass().getName()); + continue; + } + try{ + plugins.put(e.getKey(), (FormatPlugin) c.newInstance(e.getKey(), context, fileSystem, storageConfig, e.getValue())); + } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e1) { + logger.warn("Failure initializing storage config named '{}' of type '{}'.", e.getKey(), e.getValue().getClass().getName(), e1); + } + } + + + } + + return plugins; + } + + + +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatMatcher.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatMatcher.java new file mode 100644 index 000000000..e8521e498 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatMatcher.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.dfs; + +import java.io.IOException; + +public abstract class FormatMatcher { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FormatMatcher.class); + + public abstract boolean supportDirectoryReads(); + public abstract FormatSelection isReadable(FileSelection file) throws IOException; + public abstract FormatPlugin getFormatPlugin(); +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java new file mode 100644 index 000000000..a37142ee6 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.dfs; + +import java.io.IOException; +import java.util.List; + +import org.apache.drill.common.expression.FieldReference; +import org.apache.drill.common.logical.FormatPluginConfig; +import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.exec.physical.base.AbstractGroupScan; +import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.store.QueryOptimizerRule; +import org.apache.drill.exec.store.dfs.shim.DrillFileSystem; + +/** + * Similar to a storage engine but built specifically to work within a FileSystem context. + */ +public interface FormatPlugin { + + public boolean supportsRead(); + + public boolean supportsWrite(); + + public FormatMatcher getMatcher(); + + public AbstractGroupScan getGroupScan(FieldReference outputRef, FileSelection selection) throws IOException; + + public List<QueryOptimizerRule> getOptimizerRules(); + + public FormatPluginConfig getConfig(); + public StoragePluginConfig getStorageConfig(); + public DrillFileSystem getFileSystem(); + public DrillbitContext getContext(); + public String getName(); + +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatSelection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatSelection.java new file mode 100644 index 000000000..5cf6ce390 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatSelection.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.dfs; + +import java.util.List; + +import org.apache.drill.common.logical.FormatPluginConfig; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; + + +public class FormatSelection { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FormatSelection.class); + + private FormatPluginConfig format; + private FileSelection selection; + + public FormatSelection(){} + + @JsonCreator + public FormatSelection(@JsonProperty("format") FormatPluginConfig format, @JsonProperty("files") List<String> files){ + this.format = format; + this.selection = new FileSelection(files, true); + } + + public FormatSelection(FormatPluginConfig format, FileSelection selection) { + super(); + this.format = format; + this.selection = selection; + } + + @JsonProperty("format") + public FormatPluginConfig getFormat(){ + return format; + } + + @JsonProperty("files") + public List<String> getAsFiles(){ + return selection.getAsFiles(); + } + + @JsonIgnore + public FileSelection getSelection(){ + return selection; + } + +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/MagicString.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/MagicString.java new file mode 100644 index 000000000..f514388be --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/MagicString.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.dfs; + +public class MagicString { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MagicString.class); + + private long offset; + private byte[] bytes; + + public MagicString(long offset, byte[] bytes) { + super(); + this.offset = offset; + this.bytes = bytes; + } + + public long getOffset() { + return offset; + } + + public byte[] getBytes() { + return bytes; + } + + + +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/NamedFormatPluginConfig.java index f99ad0e74..6b98ea223 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntry.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/NamedFormatPluginConfig.java @@ -15,19 +15,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.physical; +package org.apache.drill.exec.store.dfs; -import org.apache.drill.exec.physical.base.Size; +import org.apache.drill.common.logical.FormatPluginConfig; -import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonTypeName; -/** - * Describes a chunk of read work that will be done. - */ -public interface ReadEntry { - @JsonIgnore - public OperatorCost getCost(); - @JsonIgnore - public Size getSize(); +@JsonTypeName("named") +public class NamedFormatPluginConfig implements FormatPluginConfig{ + public String name; } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntryFromHDFS.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/ReadEntryFromHDFS.java index e748d4c60..97124bce6 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntryFromHDFS.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/ReadEntryFromHDFS.java @@ -15,13 +15,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.physical; +package org.apache.drill.exec.store.dfs; + +import org.apache.drill.exec.store.dfs.easy.FileWork; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.drill.exec.physical.base.Size; -public class ReadEntryFromHDFS extends ReadEntryWithPath { +public class ReadEntryFromHDFS extends ReadEntryWithPath implements FileWork{ private long start; private long length; @@ -33,17 +34,6 @@ public class ReadEntryFromHDFS extends ReadEntryWithPath { this.length = length; } - @Override - public OperatorCost getCost() { - return new OperatorCost(1, 2, 1, 1); - } - - @Override - public Size getSize() { - // TODO - these values are wrong, I cannot know these until after I read a file - return new Size(10, 10); - } - public long getStart() { return start; } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntryWithPath.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/ReadEntryWithPath.java index a24869b5f..bf1d762f6 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntryWithPath.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/ReadEntryWithPath.java @@ -15,11 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.physical; +package org.apache.drill.exec.store.dfs; -import org.apache.drill.exec.physical.base.Size; -public class ReadEntryWithPath implements ReadEntry { +public class ReadEntryWithPath { protected String path; @@ -35,15 +34,4 @@ public class ReadEntryWithPath implements ReadEntry { return path; } - @Override - public OperatorCost getCost() { - throw new UnsupportedOperationException(this.getClass().getCanonicalName() + " is only for extracting path data from " + - "selections inside a scan node from a logical plan, it cannot be used in an executing plan and has no cost."); - } - - @Override - public Size getSize() { - throw new UnsupportedOperationException(this.getClass().getCanonicalName() + " is only for extracting path data from " + - "selections on a scan node from a logical plan, it cannot be used in an executing plan and has no size."); - } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java new file mode 100644 index 000000000..c69edb7e7 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java @@ -0,0 +1,123 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.dfs; + +import java.io.IOException; +import java.util.List; +import java.util.Set; + +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.exec.planner.logical.DrillTable; +import org.apache.drill.exec.planner.logical.DynamicDrillTable; +import org.apache.drill.exec.planner.sql.ExpandingConcurrentMap; +import org.apache.drill.exec.store.AbstractSchema; +import org.apache.drill.exec.store.SchemaHolder; +import org.apache.drill.exec.store.dfs.shim.DrillFileSystem; +import org.apache.hadoop.fs.Path; + +import com.beust.jcommander.internal.Lists; + +public class WorkspaceSchemaFactory implements ExpandingConcurrentMap.MapValueFactory<String, DrillTable> { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(WorkspaceSchemaFactory.class); + + private ExpandingConcurrentMap<String, DrillTable> tables = new ExpandingConcurrentMap<String, DrillTable>(this); + private final List<FormatMatcher> fileMatchers; + private final List<FormatMatcher> dirMatchers; + + private final Path root; + private final DrillFileSystem fs; + private final String storageEngineName; + private final String schemaName; + + public WorkspaceSchemaFactory(String schemaName, String storageEngineName, DrillFileSystem fileSystem, String path, + List<FormatMatcher> formatMatchers) throws ExecutionSetupException { + this.fs = fileSystem; + this.root = new Path(path); + this.fileMatchers = Lists.newArrayList(); + this.dirMatchers = Lists.newArrayList(); + for (FormatMatcher m : formatMatchers) { + if (m.supportDirectoryReads()) { + dirMatchers.add(m); + } + fileMatchers.add(m); + } + this.storageEngineName = storageEngineName; + this.schemaName = schemaName; + } + + public WorkspaceSchema create(SchemaHolder holder) { + return new WorkspaceSchema(holder, schemaName); + } + + @Override + public DrillTable create(String key) { + try { + + FileSelection fileSelection = FileSelection.create(fs, root, key); + if(fileSelection == null) return null; + + if (fileSelection.containsDirectories(fs)) { + for (FormatMatcher m : dirMatchers) { + try { + Object selection = m.isReadable(fileSelection); + if (selection != null) + return new DynamicDrillTable(storageEngineName, selection, m.getFormatPlugin().getStorageConfig()); + } catch (IOException e) { + logger.debug("File read failed.", e); + } + } + fileSelection = fileSelection.minusDirectorries(fs); + } + + for (FormatMatcher m : fileMatchers) { + Object selection = m.isReadable(fileSelection); + if (selection != null) + return new DynamicDrillTable(storageEngineName, selection, m.getFormatPlugin().getStorageConfig()); + } + return null; + + } catch (IOException e) { + logger.debug("Failed to create DrillTable with root {} and name {}", root, key, e); + } + + return null; + } + + @Override + public void destroy(DrillTable value) { + } + + public class WorkspaceSchema extends AbstractSchema { + + public WorkspaceSchema(SchemaHolder parentSchema, String name) { + super(parentSchema, name); + } + + @Override + public Set<String> getTableNames() { + return tables.keySet(); + } + + @Override + public DrillTable getTable(String name) { + return tables.get(name); + } + + } + +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONScanBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyBatchCreator.java index c40cb47fb..d79e5429d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONScanBatchCreator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyBatchCreator.java @@ -15,32 +15,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store.json; +package org.apache.drill.exec.store.dfs.easy; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; +import java.util.List; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.physical.impl.BatchCreator; -import org.apache.drill.exec.physical.impl.ScanBatch; import org.apache.drill.exec.record.RecordBatch; -import org.apache.drill.exec.store.RecordReader; - -import java.util.List; - -public class JSONScanBatchCreator implements BatchCreator<JSONSubScan> { - @Override - public RecordBatch getBatch(FragmentContext context, JSONSubScan config, List<RecordBatch> children) throws ExecutionSetupException { - Preconditions.checkArgument(children.isEmpty()); - List<JSONGroupScan.ScanEntry> entries = config.getReadEntries(); - List<RecordReader> readers = Lists.newArrayList(); - for (JSONGroupScan.ScanEntry e : entries) { - readers.add(new JSONRecordReader(context, e.getPath(), config.getStorageEngine().getFileSystem(), config.getRef(), - config.getColumns())); - } +public class EasyBatchCreator implements BatchCreator<EasySubScan>{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(EasyBatchCreator.class); - return new ScanBatch(context, readers.iterator()); - } + @Override + public RecordBatch getBatch(FragmentContext context, EasySubScan config, List<RecordBatch> children) + throws ExecutionSetupException { + assert children == null || children.isEmpty(); + return config.getFormatPlugin().getBatch(context, config); + } + + } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java new file mode 100644 index 000000000..8a41575f1 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.dfs.easy; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.common.expression.FieldReference; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.logical.FormatPluginConfig; +import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.exec.ops.FragmentContext; +import org.apache.drill.exec.physical.base.AbstractGroupScan; +import org.apache.drill.exec.physical.impl.ScanBatch; +import org.apache.drill.exec.record.RecordBatch; +import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.store.QueryOptimizerRule; +import org.apache.drill.exec.store.RecordReader; +import org.apache.drill.exec.store.dfs.BasicFormatMatcher; +import org.apache.drill.exec.store.dfs.FileSelection; +import org.apache.drill.exec.store.dfs.FormatMatcher; +import org.apache.drill.exec.store.dfs.FormatPlugin; +import org.apache.drill.exec.store.dfs.shim.DrillFileSystem; + +import com.beust.jcommander.internal.Lists; + +public abstract class EasyFormatPlugin<T extends FormatPluginConfig> implements FormatPlugin { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(EasyFormatPlugin.class); + + private final BasicFormatMatcher matcher; + private final DrillbitContext context; + private final boolean readable; + private final boolean writable; + private final boolean blockSplittable; + private final DrillFileSystem fs; + private final StoragePluginConfig storageConfig; + private final FormatPluginConfig formatConfig; + private final String name; + + protected EasyFormatPlugin(String name, DrillbitContext context, DrillFileSystem fs, StoragePluginConfig storageConfig, T formatConfig, boolean readable, boolean writable, boolean blockSplittable, String extension, String defaultName){ + this.matcher = new BasicFormatMatcher(this, fs, extension); + this.readable = readable; + this.writable = writable; + this.context = context; + this.blockSplittable = blockSplittable; + this.fs = fs; + this.storageConfig = storageConfig; + this.formatConfig = formatConfig; + this.name = name == null ? defaultName : name; + } + + @Override + public DrillFileSystem getFileSystem() { + return fs; + } + + @Override + public DrillbitContext getContext() { + return context; + } + + @Override + public String getName() { + return name; + } + + /** + * Whether or not you can split the format based on blocks within file boundaries. If not, the simple format engine will + * only split on file boundaries. + * + * @return True if splittable. + */ + public boolean isBlockSplittable(){ + return blockSplittable; + }; + + public abstract RecordReader getRecordReader(FragmentContext context, FileWork fileWork, FieldReference ref, List<SchemaPath> columns) throws ExecutionSetupException; + + + RecordBatch getBatch(FragmentContext context, EasySubScan scan) throws ExecutionSetupException { + List<RecordReader> readers = Lists.newArrayList(); + for(FileWork work : scan.getWorkUnits()){ + readers.add(getRecordReader(context, work, scan.getRef(), scan.getColumns())); + } + + return new ScanBatch(context, readers.iterator()); + } + + @Override + public AbstractGroupScan getGroupScan(FieldReference outputRef, FileSelection selection) throws IOException { + return new EasyGroupScan(selection, this, outputRef, null); + } + + @Override + public FormatPluginConfig getConfig() { + return formatConfig; + } + + @Override + public StoragePluginConfig getStorageConfig() { + return storageConfig; + } + + @Override + public boolean supportsRead() { + return readable; + } + + @Override + public boolean supportsWrite() { + return writable; + } + + @Override + public FormatMatcher getMatcher() { + return matcher; + } + + @Override + public List<QueryOptimizerRule> getOptimizerRules() { + return Collections.emptyList(); + } + + +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java new file mode 100644 index 000000000..5fcf7a5e4 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java @@ -0,0 +1,183 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.dfs.easy; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.common.expression.FieldReference; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.logical.FormatPluginConfig; +import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.exec.physical.EndpointAffinity; +import org.apache.drill.exec.physical.OperatorCost; +import org.apache.drill.exec.physical.base.AbstractGroupScan; +import org.apache.drill.exec.physical.base.PhysicalOperator; +import org.apache.drill.exec.physical.base.Size; +import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; +import org.apache.drill.exec.store.StoragePluginRegistry; +import org.apache.drill.exec.store.dfs.FileSelection; +import org.apache.drill.exec.store.schedule.AffinityCreator; +import org.apache.drill.exec.store.schedule.AssignmentCreator; +import org.apache.drill.exec.store.schedule.BlockMapBuilder; +import org.apache.drill.exec.store.schedule.CompleteFileWork; +import org.apache.drill.exec.store.schedule.CompleteFileWork.FileWorkImpl; + +import com.beust.jcommander.internal.Lists; +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.google.common.base.Preconditions; +import com.google.common.collect.ListMultimap; + +@JsonTypeName("fs-scan") +public class EasyGroupScan extends AbstractGroupScan{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(EasyGroupScan.class); + + private final FileSelection selection; + private final EasyFormatPlugin<?> formatPlugin; + private final FieldReference ref; + private final int maxWidth; + private final List<SchemaPath> columns; + + private ListMultimap<Integer, CompleteFileWork> mappings; + private List<CompleteFileWork> chunks; + private List<EndpointAffinity> endpointAffinities; + + @JsonCreator + public EasyGroupScan( + @JsonProperty("files") List<String> files, // + @JsonProperty("storage") StoragePluginConfig storageConfig, // + @JsonProperty("format") FormatPluginConfig formatConfig, // + @JacksonInject StoragePluginRegistry engineRegistry, // + @JsonProperty("ref") FieldReference ref, // + @JsonProperty("columns") List<SchemaPath> columns + ) throws IOException, ExecutionSetupException { + + this.formatPlugin = (EasyFormatPlugin<?>) engineRegistry.getFormatPlugin(storageConfig, formatConfig); + this.selection = new FileSelection(files, true); + this.maxWidth = selection.getFileStatusList(formatPlugin.getFileSystem()).size(); + this.ref = ref; + this.columns = columns; + } + + public EasyGroupScan( + FileSelection selection, // + EasyFormatPlugin<?> formatPlugin, // + FieldReference ref, // + List<SchemaPath> columns + ) throws IOException{ + this.selection = selection; + this.maxWidth = selection.getFileStatusList(formatPlugin.getFileSystem()).size(); + this.formatPlugin = formatPlugin; + this.ref = ref; + this.columns = columns; + } + + @Override + public int getMaxParallelizationWidth() { + return maxWidth; + } + + @Override + public OperatorCost getCost() { + return new OperatorCost(1,1,1,1); + } + + @Override + public Size getSize() { + return new Size(1024,1024); + } + + @JsonProperty("files") + public List<String> getFiles() { + return selection.getAsFiles(); + } + + @JsonIgnore + public FileSelection getFileSelection(){ + return selection; + } + + @Override + public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) throws ExecutionSetupException { + assert children == null || children.isEmpty(); + return this; + } + + @JsonProperty("columns") + public List<SchemaPath> getColumns(){ + return columns; + } + + @Override + public List<EndpointAffinity> getOperatorAffinity() { + if (this.endpointAffinities == null) { + try{ + BlockMapBuilder b = new BlockMapBuilder(formatPlugin.getFileSystem().getUnderlying(), formatPlugin.getContext().getBits()); + this.chunks = b.generateFileWork(selection.getFileStatusList(formatPlugin.getFileSystem()), formatPlugin.isBlockSplittable()); + this.endpointAffinities = AffinityCreator.getAffinityMap(chunks); + }catch(IOException e){ + logger.warn("Failure determining endpoint affinity.", e); + this.endpointAffinities = Collections.emptyList(); + } + } + return this.endpointAffinities; + } + + @Override + public void applyAssignments(List<DrillbitEndpoint> incomingEndpoints) { + this.mappings = AssignmentCreator.getMappings(incomingEndpoints, chunks); + } + + @Override + public EasySubScan getSpecificScan(int minorFragmentId) { + assert minorFragmentId < mappings.size() : String.format( + "Mappings length [%d] should be longer than minor fragment id [%d] but it isn't.", mappings.size(), + minorFragmentId); + + List<CompleteFileWork> filesForMinor = mappings.get(minorFragmentId); + + Preconditions.checkArgument(!filesForMinor.isEmpty(), + String.format("MinorFragmentId %d has no read entries assigned", minorFragmentId)); + + return new EasySubScan(convert(filesForMinor), formatPlugin, ref, columns); + } + + private List<FileWorkImpl> convert(List<CompleteFileWork> list){ + List<FileWorkImpl> newList = Lists.newArrayList(); + for(CompleteFileWork f : list){ + newList.add(f.getAsFileWork()); + } + return newList; + } + + @JsonProperty("storage") + public StoragePluginConfig getStorageConfig(){ + return formatPlugin.getStorageConfig(); + } + + @JsonProperty("format") + public FormatPluginConfig getFormatConfig(){ + return formatPlugin.getConfig(); + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasySubScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasySubScan.java new file mode 100644 index 000000000..72d1fe6ed --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasySubScan.java @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.dfs.easy; + +import java.io.IOException; +import java.util.List; + +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.common.expression.FieldReference; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.logical.FormatPluginConfig; +import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.exec.physical.base.AbstractSubScan; +import org.apache.drill.exec.store.StoragePluginRegistry; +import org.apache.drill.exec.store.schedule.CompleteFileWork.FileWorkImpl; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.google.common.base.Preconditions; + +@JsonTypeName("fs-sub-scan") +public class EasySubScan extends AbstractSubScan{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(EasySubScan.class); + + private final List<FileWorkImpl> files; + private final EasyFormatPlugin<?> formatPlugin; + private final FieldReference ref; + private final List<SchemaPath> columns; + + @JsonCreator + public EasySubScan( + @JsonProperty("files") List<FileWorkImpl> files, // + @JsonProperty("storage") StoragePluginConfig storageConfig, // + @JsonProperty("format") FormatPluginConfig formatConfig, // + @JacksonInject StoragePluginRegistry engineRegistry, // + @JsonProperty("ref") FieldReference ref, // + @JsonProperty("columns") List<SchemaPath> columns // + ) throws IOException, ExecutionSetupException { + + this.formatPlugin = (EasyFormatPlugin<?>) engineRegistry.getFormatPlugin(storageConfig, formatConfig); + Preconditions.checkNotNull(this.formatPlugin); + this.files = files; + this.ref = ref; + this.columns = columns; + } + + public EasySubScan(List<FileWorkImpl> files, EasyFormatPlugin<?> plugin, FieldReference ref, List<SchemaPath> columns){ + this.formatPlugin = plugin; + this.files = files; + this.ref = ref; + this.columns = columns; + } + + @JsonIgnore + public EasyFormatPlugin<?> getFormatPlugin(){ + return formatPlugin; + } + + @JsonProperty("files") + public List<FileWorkImpl> getWorkUnits() { + return files; + } + + @JsonProperty("storage") + public StoragePluginConfig getStorageConfig(){ + return formatPlugin.getStorageConfig(); + } + + @JsonProperty("format") + public FormatPluginConfig getFormatConfig(){ + return formatPlugin.getConfig(); + } + + @JsonProperty("ref") + public FieldReference getRef() { + return ref; + } + + @JsonProperty("columns") + public List<SchemaPath> getColumns(){ + return columns; + } + + +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/FileWork.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/FileWork.java new file mode 100644 index 000000000..170d339c3 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/FileWork.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.dfs.easy; + + +public interface FileWork { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileWork.class); + + public String getPath(); + public long getStart(); + public long getLength(); +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillFileSystem.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillFileSystem.java new file mode 100644 index 000000000..8af6aaf5a --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillFileSystem.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.dfs.shim; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +/** + * Wraps the underlying filesystem to provide advanced file system features. Delegates to underlying file system if + * those features are exposed. + */ +public abstract class DrillFileSystem implements AutoCloseable{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillFileSystem.class); + + public abstract FileSystem getUnderlying(); + + public abstract BlockLocation[] getBlockLocations(FileStatus status, long start, long length) throws IOException; + public abstract List<FileStatus> list(boolean recursive, Path... paths) throws IOException; + public abstract FileStatus getFileStatus(Path p) throws IOException; + public abstract DrillOutputStream create(Path p) throws IOException; + public abstract DrillInputStream open(Path p) throws IOException; + +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillInputStream.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillInputStream.java new file mode 100644 index 000000000..82764a371 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillInputStream.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.dfs.shim; + +import org.apache.hadoop.fs.FSDataInputStream; + +public abstract class DrillInputStream implements AutoCloseable{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillInputStream.class); + +// public abstract AccountingByteBuf readNow(long start, long length) throws IOException; +// public abstract void readNow(AccountingByteBuf b, long start, long length) throws IOException; +// public abstract AccountingByteBuf readNow() throws IOException; + + public abstract FSDataInputStream getInputStream(); +// public abstract CheckedFuture<Long, IOException> readFuture(AccountingByteBuf b, long start, long length) throws IOException; + +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillOutputStream.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillOutputStream.java new file mode 100644 index 000000000..c2446e996 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/DrillOutputStream.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.dfs.shim; + +import java.io.OutputStream; + + +public abstract class DrillOutputStream implements AutoCloseable{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillOutputStream.class); + + public abstract OutputStream getOuputStream(); +// public abstract CheckedFuture<Long, IOException> writeFuture(AccountingByteBuf b); + +} diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/TestPlan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/FileSystemCreator.java index 71e6283e7..eaab5ef93 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/TestPlan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/FileSystemCreator.java @@ -15,34 +15,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec; - -import org.apache.drill.exec.client.QuerySubmitter; -import org.junit.Ignore; -import org.junit.Test; +package org.apache.drill.exec.store.dfs.shim; import java.io.IOException; +import java.net.URI; -/** - * Created with IntelliJ IDEA. - * User: sphillips - * Date: 1/24/14 - * Time: 3:46 PM - * To change this template use File | Settings | File Templates. - */ -public class TestPlan { - - String location = "/Users/sphillips/hive-lineitem-orderkey"; - String type = "physical"; - String zkQuorum = null; - boolean local = true; - int bits = 1; - +import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.exec.store.dfs.shim.fallback.FallbackFileSystem; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; - @Test - @Ignore - public void testSubmitPlan() throws Exception { - QuerySubmitter submitter = new QuerySubmitter(); - submitter.submitQuery(location, type, zkQuorum, local, bits); +public class FileSystemCreator { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSystemCreator.class); + + public static DrillFileSystem getFileSystem(DrillConfig config, Configuration fsConf) throws IOException{ + FileSystem fs = FileSystem.get(fsConf); + return new FallbackFileSystem(config, fs); } + } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/fallback/FallbackFileSystem.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/fallback/FallbackFileSystem.java new file mode 100644 index 000000000..5743ca1a4 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/shim/fallback/FallbackFileSystem.java @@ -0,0 +1,146 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.dfs.shim.fallback; + +import java.io.IOException; +import java.util.List; + +import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.exec.store.dfs.shim.DrillFileSystem; +import org.apache.drill.exec.store.dfs.shim.DrillInputStream; +import org.apache.drill.exec.store.dfs.shim.DrillOutputStream; +import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import com.google.common.collect.Lists; + +public class FallbackFileSystem extends DrillFileSystem { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FallbackFileSystem.class); + + final FileSystem fs; + + public FallbackFileSystem(DrillConfig config, FileSystem fs) { + this.fs = fs; + } + + @Override + public FileSystem getUnderlying() { + return fs; + } + + @Override + public List<FileStatus> list(boolean recursive, Path... paths) throws IOException { + if (recursive) { + List<FileStatus> statuses = Lists.newArrayList(); + for (Path p : paths) { + addRecursiveStatus(fs.getFileStatus(p), statuses); + } + return statuses; + + } else { + return Lists.newArrayList(fs.listStatus(paths)); + } + } + + + private void addRecursiveStatus(FileStatus parent, List<FileStatus> listToFill) throws IOException { + if (parent.isDir()) { + Path pattern = new Path(parent.getPath(), "/*"); + FileStatus[] sub = fs.globStatus(pattern); + for(FileStatus s : sub){ + listToFill.add(s); + } + } else { + listToFill.add(parent); + } + } + + @Override + public FileStatus getFileStatus(Path p) throws IOException { + return fs.getFileStatus(p); + } + + @Override + public DrillOutputStream create(Path p) throws IOException { + return new Out(fs.create(p)); + } + + @Override + public DrillInputStream open(Path p) throws IOException { + return new In(fs.open(p)); + } + + @Override + public void close() throws Exception { + fs.close(); + } + + @Override + public BlockLocation[] getBlockLocations(FileStatus status, long start, long len) throws IOException { + return fs.getFileBlockLocations(status, start, len); + } + + private class Out extends DrillOutputStream { + + private final FSDataOutputStream out; + + public Out(FSDataOutputStream out) { + super(); + this.out = out; + } + + @Override + public void close() throws Exception { + out.close(); + } + + @Override + public FSDataOutputStream getOuputStream() { + return out; + } + + } + + private class In extends DrillInputStream { + + private final FSDataInputStream in; + + public In(FSDataInputStream in) { + super(); + this.in = in; + } + + @Override + public FSDataInputStream getInputStream() { + return in; + } + + @Override + public void close() throws Exception { + in.close(); + } + + } + + + +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java new file mode 100644 index 000000000..82bf3bfc8 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.easy.json; + +import java.util.List; + +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.common.expression.FieldReference; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.logical.FormatPluginConfig; +import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.exec.ops.FragmentContext; +import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.store.RecordReader; +import org.apache.drill.exec.store.dfs.easy.EasyFormatPlugin; +import org.apache.drill.exec.store.dfs.easy.FileWork; +import org.apache.drill.exec.store.dfs.shim.DrillFileSystem; +import org.apache.drill.exec.store.easy.json.JSONFormatPlugin.JSONFormatConfig; + +import com.fasterxml.jackson.annotation.JsonTypeName; + +public class JSONFormatPlugin extends EasyFormatPlugin<JSONFormatConfig> { + + public JSONFormatPlugin(String name, DrillbitContext context, DrillFileSystem fs, StoragePluginConfig storageConfig) { + this(name, context, fs, storageConfig, new JSONFormatConfig()); + } + + public JSONFormatPlugin(String name, DrillbitContext context, DrillFileSystem fs, StoragePluginConfig config, JSONFormatConfig formatPluginConfig) { + super(name, context, fs, config, formatPluginConfig, true, false, false, "json", "json"); + } + + @Override + public RecordReader getRecordReader(FragmentContext context, FileWork fileWork, FieldReference ref, + List<SchemaPath> columns) throws ExecutionSetupException { + return new JSONRecordReader(context, fileWork.getPath(), this.getFileSystem().getUnderlying(), ref, columns); + } + + @JsonTypeName("json") + public static class JSONFormatConfig implements FormatPluginConfig { + + @Override + public int hashCode() { + return 31; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() == obj.getClass()) + return true; + return false; + } + + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java index 11b972c83..67e8b3fc5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONRecordReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store.json; +package org.apache.drill.exec.store.easy.json; import static com.fasterxml.jackson.core.JsonToken.END_ARRAY; import static com.fasterxml.jackson.core.JsonToken.END_OBJECT; @@ -215,7 +215,7 @@ public class JSONRecordReader implements RecordReader { private boolean fieldSelected(String field){ SchemaPath sp = new SchemaPath(field, ExpressionPosition.UNKNOWN); - if (this.columns != null && this.columns.size() > 1){ + if (this.columns != null && this.columns.size() > 0){ for (SchemaPath expr : this.columns){ if ( sp.equals(expr)){ return true; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java index 6211e2186..f330a1e05 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java @@ -17,19 +17,19 @@ */ package org.apache.drill.exec.store.hive; -import com.beust.jcommander.internal.Lists; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; + import org.apache.drill.exec.physical.OperatorCost; -import org.apache.drill.exec.physical.ReadEntry; import org.apache.drill.exec.physical.base.Size; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; -import java.util.List; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.Lists; -public class HiveReadEntry implements ReadEntry { +public class HiveReadEntry { @JsonProperty("table") public HiveTable table; @@ -60,18 +60,5 @@ public class HiveReadEntry implements ReadEntry { return partitionsUnwrapped; } - @Override - public OperatorCost getCost() { - // TODO: need to come up with way to calculate the cost for Hive tables - return new OperatorCost(1, 1, 2, 2); - } - - @Override - public Size getSize() { - // TODO: contact the metastore and find the size of the data in table - Size size = new Size(1, 1); - - return size; - } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScan.java index bc2a16bc6..a1e8f1da2 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScan.java @@ -34,7 +34,7 @@ import org.apache.drill.exec.physical.base.Size; import org.apache.drill.exec.physical.base.SubScan; import org.apache.drill.exec.proto.CoordinationProtos; import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; -import org.apache.drill.exec.store.StorageEngineRegistry; +import org.apache.drill.exec.store.StoragePluginRegistry; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.api.Partition; @@ -58,9 +58,9 @@ public class HiveScan extends AbstractGroupScan { @JsonIgnore private List<InputSplit> inputSplits = Lists.newArrayList(); @JsonIgnore - public HiveStorageEngine storageEngine; - @JsonProperty("storageengine") - public HiveStorageEngineConfig engineConfig; + public HiveStoragePlugin storagePlugin; + @JsonProperty("storage-plugin") + public String storagePluginName; @JsonIgnore public List<Partition> partitions; @@ -77,27 +77,27 @@ public class HiveScan extends AbstractGroupScan { Map<InputSplit, Partition> partitionMap = new HashMap(); @JsonCreator - public HiveScan(@JsonProperty("hive-table") HiveReadEntry hiveReadEntry, @JsonProperty("storageengine") HiveStorageEngineConfig config, + public HiveScan(@JsonProperty("hive-table") HiveReadEntry hiveReadEntry, @JsonProperty("storage-plugin") String storagePluginName, @JsonProperty("columns") List<FieldReference> columns, - @JacksonInject StorageEngineRegistry engineRegistry) throws ExecutionSetupException { + @JacksonInject StoragePluginRegistry engineRegistry) throws ExecutionSetupException { this.hiveReadEntry = hiveReadEntry; this.table = hiveReadEntry.getTable(); - this.engineConfig = config; - this.storageEngine = (HiveStorageEngine) engineRegistry.getEngine(config); + this.storagePluginName = storagePluginName; + this.storagePlugin = (HiveStoragePlugin) engineRegistry.getEngine(storagePluginName); this.columns = columns; this.partitions = hiveReadEntry.getPartitions(); getSplits(); - endpoints = storageEngine.getContext().getBits(); + endpoints = storagePlugin.getContext().getBits(); } - public HiveScan(HiveReadEntry hiveReadEntry, HiveStorageEngine storageEngine, List<FieldReference> columns) throws ExecutionSetupException { + public HiveScan(HiveReadEntry hiveReadEntry, HiveStoragePlugin storageEngine, List<FieldReference> columns) throws ExecutionSetupException { this.table = hiveReadEntry.getTable(); this.hiveReadEntry = hiveReadEntry; this.columns = columns; this.partitions = hiveReadEntry.getPartitions(); getSplits(); endpoints = storageEngine.getContext().getBits(); - this.engineConfig = storageEngine.getConfig(); + this.storagePluginName = storageEngine.getName(); } public List<FieldReference> getColumns() { @@ -112,7 +112,7 @@ public class HiveScan extends AbstractGroupScan { for (Object obj : properties.keySet()) { job.set((String) obj, (String) properties.get(obj)); } - InputFormat format = (InputFormat) Class.forName(table.getSd().getInputFormat()).getConstructor().newInstance(); + InputFormat<?, ?> format = (InputFormat<?, ?>) Class.forName(table.getSd().getInputFormat()).getConstructor().newInstance(); job.setInputFormat(format.getClass()); Path path = new Path(table.getSd().getLocation()); FileInputFormat.addInputPath(job, path); @@ -130,7 +130,7 @@ public class HiveScan extends AbstractGroupScan { for (Object obj : properties.keySet()) { job.set((String) obj, (String) properties.get(obj)); } - InputFormat format = (InputFormat) Class.forName(partition.getSd().getInputFormat()).getConstructor().newInstance(); + InputFormat<?, ?> format = (InputFormat<?, ?>) Class.forName(partition.getSd().getInputFormat()).getConstructor().newInstance(); job.setInputFormat(format.getClass()); FileInputFormat.addInputPath(job, new Path(partition.getSd().getLocation())); format = job.getInputFormat(); @@ -192,12 +192,12 @@ public class HiveScan extends AbstractGroupScan { @Override public List<EndpointAffinity> getOperatorAffinity() { - Map<String, DrillbitEndpoint> endpointMap = new HashMap(); + Map<String, DrillbitEndpoint> endpointMap = new HashMap<>(); for (DrillbitEndpoint endpoint : endpoints) { endpointMap.put(endpoint.getAddress(), endpoint); logger.debug("endpoing address: {}", endpoint.getAddress()); } - Map<DrillbitEndpoint, EndpointAffinity> affinityMap = new HashMap(); + Map<DrillbitEndpoint, EndpointAffinity> affinityMap = new HashMap<>(); try { long totalSize = 0; for (InputSplit split : inputSplits) { @@ -242,6 +242,6 @@ public class HiveScan extends AbstractGroupScan { @Override public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) throws ExecutionSetupException { - return new HiveScan(hiveReadEntry, storageEngine, columns); + return new HiveScan(hiveReadEntry, storagePlugin, columns); } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStorageEngine.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStorageEngine.java deleted file mode 100644 index 0f6f3bc4a..000000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStorageEngine.java +++ /dev/null @@ -1,194 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.hive; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.common.exceptions.DrillRuntimeException; -import org.apache.drill.common.exceptions.ExecutionSetupException; -import org.apache.drill.common.logical.data.Scan; -import org.apache.drill.exec.physical.OperatorCost; -import org.apache.drill.exec.physical.ReadEntry; -import org.apache.drill.exec.physical.base.Size; -import org.apache.drill.exec.server.DrillbitContext; -import org.apache.drill.exec.store.AbstractStorageEngine; -import org.apache.drill.exec.store.SchemaProvider; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; -import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.thrift.TException; - -import java.io.IOException; -import java.util.List; - -public class HiveStorageEngine extends AbstractStorageEngine { - - private HiveStorageEngineConfig config; - private HiveConf hiveConf; - private HiveSchemaProvider schemaProvider; - static private DrillbitContext context; - - public HiveStorageEngine(HiveStorageEngineConfig config, DrillbitContext context) throws ExecutionSetupException { - this.config = config; - this.context = context; - this.hiveConf = config.getHiveConf(); - } - - public HiveStorageEngineConfig getConfig() { - return config; - } - - public DrillbitContext getContext() { - return context; - } - - @Override - public HiveScan getPhysicalScan(Scan scan) throws IOException { - HiveReadEntry hiveReadEntry = scan.getSelection().getListWith(new ObjectMapper(), new TypeReference<HiveReadEntry>(){}); - try { - List<Partition> partitions = getSchemaProvider().getPartitions(hiveReadEntry.getTable().getDbName(), hiveReadEntry.getTable().getTableName()); - return new HiveScan(hiveReadEntry, this, null); - } catch (ExecutionSetupException | TException e) { - throw new DrillRuntimeException(e); - } - } - - @Override - public HiveSchemaProvider getSchemaProvider() { - try { - if (schemaProvider == null) { - schemaProvider = new HiveSchemaProvider(config, context.getConfig()); - } - return schemaProvider; - } catch (ExecutionSetupException e) { - throw new DrillRuntimeException(e); - } - } - - List<String> getPartitions(String dbName, String tableName) throws TException { - List<Partition> partitions = getSchemaProvider().getMetaClient().listPartitions(dbName, tableName, Short.MAX_VALUE); - List<String> partitionLocations = Lists.newArrayList(); - if (partitions == null) return null; - for (Partition part : partitions) { - partitionLocations.add(part.getSd().getLocation()); - } - return partitionLocations; - } - - public static class HiveEntry implements ReadEntry { - - private Table table; - - public HiveEntry(Table table) { - this.table = table; - } - - public Table getTable() { - return table; - } - - @Override - public OperatorCost getCost() { - throw new UnsupportedOperationException(this.getClass().getCanonicalName() + " is only for extracting path data from " + - "selections inside a scan node from a logical plan, it cannot be used in an executing plan and has no cost."); - } - - @Override - public Size getSize() { - throw new UnsupportedOperationException(this.getClass().getCanonicalName() + " is only for extracting path data from " + - "selections inside a scan node from a logical plan, it cannot be used in an executing plan and has no cost."); - } - } - - public static class HiveSchemaProvider implements SchemaProvider { - - private HiveConf hiveConf; - private HiveMetaStoreClient metaClient; - - public HiveSchemaProvider(HiveStorageEngineConfig config, DrillConfig dConfig) throws ExecutionSetupException { - hiveConf = config.getHiveConf(); - } - - public HiveMetaStoreClient getMetaClient() throws MetaException { - if (metaClient == null) { - metaClient = new HiveMetaStoreClient(hiveConf); - } - return metaClient; - } - - public Table getTable(String dbName, String tableName) throws TException { - HiveMetaStoreClient mClient = getMetaClient(); - try { - return mClient.getTable(dbName, tableName); - }catch (NoSuchObjectException e) { - logger.error("Database: {} table: {} not found", dbName, tableName); - throw new RuntimeException(e); - } catch (TException e) { - mClient.reconnect(); - return mClient.getTable(dbName, tableName); - } - } - - List<Partition> getPartitions(String dbName, String tableName) throws TException { - HiveMetaStoreClient mClient = getMetaClient(); - List<Partition> partitions; - try { - partitions = getMetaClient().listPartitions(dbName, tableName, Short.MAX_VALUE); - } catch (TException e) { - mClient.reconnect(); - partitions = getMetaClient().listPartitions(dbName, tableName, Short.MAX_VALUE); - } - return partitions; - } - - @Override - public HiveReadEntry getSelectionBaseOnName(String name) { - String[] dbNameTableName = name.split("\\."); - String dbName; - String t; - if (dbNameTableName.length > 1) { - dbName = dbNameTableName[0]; - t = dbNameTableName[1]; - } else { - dbName = "default"; - t = name; - } - - try { - Table table = getTable(dbName, t); - List<Partition> partitions = getPartitions(dbName, t); - List<HiveTable.HivePartition> hivePartitions = Lists.newArrayList(); - for(Partition part : partitions) { - hivePartitions.add(new HiveTable.HivePartition(part)); - } - if (hivePartitions.size() == 0) hivePartitions = null; - return new HiveReadEntry(new HiveTable(table), hivePartitions); - } catch (NoSuchObjectException e) { - throw new DrillRuntimeException(e); - } catch (TException e) { - throw new DrillRuntimeException(e); - } - } - } -} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java new file mode 100644 index 000000000..a1d575c9f --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.hive; + +import java.io.IOException; + +import net.hydromatic.optiq.Schema; +import net.hydromatic.optiq.SchemaPlus; + +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.common.logical.data.Scan; +import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.store.AbstractStoragePlugin; +import org.apache.drill.exec.store.hive.schema.HiveSchemaFactory; +import org.apache.hadoop.hive.conf.HiveConf; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class HiveStoragePlugin extends AbstractStoragePlugin { + + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HiveStoragePlugin.class); + + private final HiveStoragePluginConfig config; + private final HiveConf hiveConf; + private final HiveSchemaFactory schemaFactory; + private final DrillbitContext context; + private final String name; + + public HiveStoragePlugin(HiveStoragePluginConfig config, DrillbitContext context, String name) throws ExecutionSetupException { + this.config = config; + this.context = context; + this.schemaFactory = new HiveSchemaFactory(config, name, config.getHiveConf()); + this.hiveConf = config.getHiveConf(); + this.name = name; + } + + public HiveStoragePluginConfig getConfig() { + return config; + } + + public String getName(){ + return name; + } + + public DrillbitContext getContext() { + return context; + } + + @Override + public HiveScan getPhysicalScan(Scan scan) throws IOException { + HiveReadEntry hiveReadEntry = scan.getSelection().getListWith(new ObjectMapper(), new TypeReference<HiveReadEntry>(){}); + try { + return new HiveScan(hiveReadEntry, this, null); + } catch (ExecutionSetupException e) { + throw new IOException(e); + } + } + + @Override + public Schema createAndAddSchema(SchemaPlus parent) { + return schemaFactory.add(parent); + } + + +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStorageEngineConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePluginConfig.java index 91fec3b20..c9d76e54e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStorageEngineConfig.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePluginConfig.java @@ -21,13 +21,13 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; -import org.apache.drill.common.logical.StorageEngineConfigBase; +import org.apache.drill.common.logical.StoragePluginConfigBase; import org.apache.hadoop.hive.conf.HiveConf; import java.util.Map; @JsonTypeName("hive") -public class HiveStorageEngineConfig extends StorageEngineConfigBase { +public class HiveStoragePluginConfig extends StoragePluginConfigBase { @JsonProperty public Map<String, String> configProps; @JsonIgnore @@ -48,7 +48,7 @@ public class HiveStorageEngineConfig extends StorageEngineConfigBase { } @JsonCreator - public HiveStorageEngineConfig(@JsonProperty("config") Map<String, String> props) { + public HiveStoragePluginConfig(@JsonProperty("config") Map<String, String> props) { this.configProps = props; } @@ -62,7 +62,7 @@ public class HiveStorageEngineConfig extends StorageEngineConfigBase { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - HiveStorageEngineConfig that = (HiveStorageEngineConfig) o; + HiveStoragePluginConfig that = (HiveStoragePluginConfig) o; if (configProps != null ? !configProps.equals(that.configProps) : that.configProps != null) return false; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java new file mode 100644 index 000000000..abec2c572 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java @@ -0,0 +1,177 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.hive.schema; + +import java.nio.charset.Charset; +import java.util.ArrayList; + +import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.exec.planner.logical.DrillTable; +import org.apache.drill.exec.store.hive.HiveReadEntry; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.eigenbase.reltype.RelDataType; +import org.eigenbase.reltype.RelDataTypeFactory; +import org.eigenbase.sql.SqlCollation; +import org.eigenbase.sql.type.SqlTypeName; + +public class DrillHiveTable extends DrillTable{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillHiveTable.class); + + private final Table hiveTable; + + public DrillHiveTable(String storageEngineName, HiveReadEntry readEntry, StoragePluginConfig storageEngineConfig) { + super(storageEngineName, readEntry, storageEngineConfig); + this.hiveTable = new org.apache.hadoop.hive.ql.metadata.Table(readEntry.getTable()); + } + + @Override + public RelDataType getRowType(RelDataTypeFactory typeFactory) { + ArrayList<RelDataType> typeList = new ArrayList<>(); + ArrayList<String> fieldNameList = new ArrayList<>(); + + ArrayList<StructField> hiveFields = hiveTable.getFields(); + for(StructField hiveField : hiveFields) { + fieldNameList.add(hiveField.getFieldName()); + typeList.add(getRelDataTypeFromHiveType(typeFactory, hiveField.getFieldObjectInspector())); + } + + for (FieldSchema field : hiveTable.getPartitionKeys()) { + fieldNameList.add(field.getName()); + typeList.add(getRelDataTypeFromHiveTypeString(typeFactory, field.getType())); + } + + final RelDataType rowType = typeFactory.createStructType(typeList, fieldNameList); + return rowType; + } + + private RelDataType getRelDataTypeFromHiveTypeString(RelDataTypeFactory typeFactory, String type) { + switch(type) { + case "boolean": + return typeFactory.createSqlType(SqlTypeName.BOOLEAN); + + case "tinyint": + return typeFactory.createSqlType(SqlTypeName.TINYINT); + + case "smallint": + return typeFactory.createSqlType(SqlTypeName.SMALLINT); + + case "int": + return typeFactory.createSqlType(SqlTypeName.INTEGER); + + case "bigint": + return typeFactory.createSqlType(SqlTypeName.BIGINT); + + case "float": + return typeFactory.createSqlType(SqlTypeName.FLOAT); + + case "double": + return typeFactory.createSqlType(SqlTypeName.DOUBLE); + + case "date": + return typeFactory.createSqlType(SqlTypeName.DATE); + + case "timestamp": + return typeFactory.createSqlType(SqlTypeName.TIMESTAMP); + + case "binary": + return typeFactory.createSqlType(SqlTypeName.BINARY); + + case "decimal": + return typeFactory.createSqlType(SqlTypeName.DECIMAL); + + case "string": + case "varchar": { + return typeFactory.createTypeWithCharsetAndCollation( + typeFactory.createSqlType(SqlTypeName.VARCHAR), /*input type*/ + Charset.forName("ISO-8859-1"), /*unicode char set*/ + SqlCollation.IMPLICIT /* TODO: need to decide if implicit is the correct one */ + ); + } + + default: + throw new RuntimeException("Unknown or unsupported hive type: " + type); + } + } + + private RelDataType getRelDataTypeFromHivePrimitiveType(RelDataTypeFactory typeFactory, PrimitiveObjectInspector poi) { + switch(poi.getPrimitiveCategory()) { + case BOOLEAN: + return typeFactory.createSqlType(SqlTypeName.BOOLEAN); + case BYTE: + return typeFactory.createSqlType(SqlTypeName.TINYINT); + + case SHORT: + return typeFactory.createSqlType(SqlTypeName.SMALLINT); + + case INT: + return typeFactory.createSqlType(SqlTypeName.INTEGER); + + case LONG: + return typeFactory.createSqlType(SqlTypeName.BIGINT); + + case FLOAT: + return typeFactory.createSqlType(SqlTypeName.FLOAT); + + case DOUBLE: + return typeFactory.createSqlType(SqlTypeName.DOUBLE); + + case DATE: + return typeFactory.createSqlType(SqlTypeName.DATE); + + case TIMESTAMP: + return typeFactory.createSqlType(SqlTypeName.TIMESTAMP); + + case BINARY: + return typeFactory.createSqlType(SqlTypeName.BINARY); + + case DECIMAL: + return typeFactory.createSqlType(SqlTypeName.DECIMAL); + + case STRING: + case VARCHAR: { + return typeFactory.createTypeWithCharsetAndCollation( + typeFactory.createSqlType(SqlTypeName.VARCHAR), /*input type*/ + Charset.forName("ISO-8859-1"), /*unicode char set*/ + SqlCollation.IMPLICIT /* TODO: need to decide if implicit is the correct one */ + ); + } + + case UNKNOWN: + case VOID: + default: + throw new RuntimeException("Unknown or unsupported hive type"); + } + } + + private RelDataType getRelDataTypeFromHiveType(RelDataTypeFactory typeFactory, ObjectInspector oi) { + switch(oi.getCategory()) { + case PRIMITIVE: + return getRelDataTypeFromHivePrimitiveType(typeFactory, ((PrimitiveObjectInspector) oi)); + case LIST: + case MAP: + case STRUCT: + case UNION: + default: + throw new RuntimeException("Unknown or unsupported hive type"); + } + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveDatabaseSchema.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveDatabaseSchema.java new file mode 100644 index 000000000..e4ff37255 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveDatabaseSchema.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.hive.schema; + +import java.util.List; +import java.util.Set; + +import org.apache.drill.exec.planner.logical.DrillTable; +import org.apache.drill.exec.store.AbstractSchema; +import org.apache.drill.exec.store.SchemaHolder; +import org.apache.drill.exec.store.hive.schema.HiveSchemaFactory.HiveSchema; + +import com.google.common.collect.Sets; + +public class HiveDatabaseSchema extends AbstractSchema{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HiveDatabaseSchema.class); + + private final HiveSchema hiveSchema; + private final Set<String> tables; + + public HiveDatabaseSchema( // + List<String> tableList, // + HiveSchema hiveSchema, // + SchemaHolder parentSchema, // + String name) { + super(parentSchema, name); + this.hiveSchema = hiveSchema; + this.tables = Sets.newHashSet(tableList); + } + + @Override + public DrillTable getTable(String tableName) { + return hiveSchema.getDrillTable(this.name, tableName); + } + + @Override + public Set<String> getTableNames() { + return tables; + } + +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java new file mode 100644 index 000000000..091381fc6 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java @@ -0,0 +1,274 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.hive.schema; + +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +import net.hydromatic.optiq.Schema; +import net.hydromatic.optiq.SchemaPlus; + +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.exec.planner.logical.DrillTable; +import org.apache.drill.exec.store.AbstractSchema; +import org.apache.drill.exec.store.SchemaFactory; +import org.apache.drill.exec.store.SchemaHolder; +import org.apache.drill.exec.store.dfs.WorkspaceSchemaFactory.WorkspaceSchema; +import org.apache.drill.exec.store.hive.HiveReadEntry; +import org.apache.drill.exec.store.hive.HiveStoragePluginConfig; +import org.apache.drill.exec.store.hive.HiveTable; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.UnknownTableException; +import org.apache.thrift.TException; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; + +public class HiveSchemaFactory implements SchemaFactory { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HiveSchemaFactory.class); + + private static final String DATABASES = "databases"; + + private final HiveMetaStoreClient mClient; + private LoadingCache<String, List<String>> databases; + private LoadingCache<String, List<String>> tableNameLoader; + private LoadingCache<String, LoadingCache<String, HiveReadEntry>> tableLoaders; + private HiveStoragePluginConfig pluginConfig; + private final String schemaName; + + public HiveSchemaFactory(HiveStoragePluginConfig pluginConfig, String name, HiveConf hiveConf) throws ExecutionSetupException { + this.schemaName = name; + this.pluginConfig = pluginConfig; + + try { + this.mClient = new HiveMetaStoreClient(hiveConf); + } catch (MetaException e) { + throw new ExecutionSetupException("Failure setting up Hive metastore client.", e); + } + + databases = CacheBuilder // + .newBuilder() // + .expireAfterAccess(1, TimeUnit.MINUTES) // + .build(new DatabaseLoader()); + + tableNameLoader = CacheBuilder // + .newBuilder() // + .expireAfterAccess(1, TimeUnit.MINUTES) // + .build(new TableNameLoader()); + + tableLoaders = CacheBuilder // + .newBuilder() // + .expireAfterAccess(4, TimeUnit.HOURS) // + .maximumSize(20) // + .build(new TableLoaderLoader()); + } + + private class TableNameLoader extends CacheLoader<String, List<String>> { + + @Override + public List<String> load(String dbName) throws Exception { + try { + return mClient.getAllTables(dbName); + } catch (TException e) { + logger.warn("Failure while attempting to get hive tables", e); + mClient.reconnect(); + return mClient.getAllTables(dbName); + } + } + + } + + private class DatabaseLoader extends CacheLoader<String, List<String>> { + + @Override + public List<String> load(String key) throws Exception { + if (!DATABASES.equals(key)) + throw new UnsupportedOperationException(); + try { + return mClient.getAllDatabases(); + } catch (TException e) { + logger.warn("Failure while attempting to get hive tables", e); + mClient.reconnect(); + return mClient.getAllDatabases(); + } + } + } + + private class TableLoaderLoader extends CacheLoader<String, LoadingCache<String, HiveReadEntry>> { + + @Override + public LoadingCache<String, HiveReadEntry> load(String key) throws Exception { + return CacheBuilder.newBuilder().expireAfterAccess(1, TimeUnit.MINUTES).build(new TableLoader(key)); + } + + } + + private class TableLoader extends CacheLoader<String, HiveReadEntry> { + + private final String dbName; + + public TableLoader(String dbName) { + super(); + this.dbName = dbName; + } + + @Override + public HiveReadEntry load(String key) throws Exception { + Table t = null; + try { + t = mClient.getTable(dbName, key); + } catch (TException e) { + mClient.reconnect(); + t = mClient.getTable(dbName, key); + } + + if (t == null) + throw new UnknownTableException(String.format("Unable to find table '%s'.", key)); + + List<Partition> partitions = null; + try { + partitions = mClient.listPartitions(dbName, key, Short.MAX_VALUE); + } catch (TException e) { + mClient.reconnect(); + partitions = mClient.listPartitions(dbName, key, Short.MAX_VALUE); + } + + List<HiveTable.HivePartition> hivePartitions = Lists.newArrayList(); + for (Partition part : partitions) { + hivePartitions.add(new HiveTable.HivePartition(part)); + } + + if (hivePartitions.size() == 0) + hivePartitions = null; + return new HiveReadEntry(new HiveTable(t), hivePartitions); + + } + + } + + @Override + public Schema add(SchemaPlus parent) { + HiveSchema schema = new HiveSchema(new SchemaHolder(parent), schemaName); + SchemaPlus hPlus = parent.add(schema); + schema.setHolder(hPlus); + return schema; + } + + class HiveSchema extends AbstractSchema { + + private final SchemaHolder holder = new SchemaHolder(); + + private HiveDatabaseSchema defaultSchema; + + public HiveSchema(SchemaHolder parentSchema, String name) { + super(parentSchema, name); + getSubSchema("default"); + } + + @Override + public Schema getSubSchema(String name) { + List<String> tables; + try { + tables = tableNameLoader.get(name); + HiveDatabaseSchema schema = new HiveDatabaseSchema(tables, this, holder, name); + if(name.equals("default")){ + this.defaultSchema = schema; + } + return schema; + } catch (ExecutionException e) { + logger.warn("Failure while attempting to access HiveDatabase '{}'.", name, e.getCause()); + return null; + } + + } + + + void setHolder(SchemaPlus plusOfThis){ + holder.setSchema(plusOfThis); + for(String s : getSubSchemaNames()){ + plusOfThis.add(getSubSchema(s)); + } + } + + + @Override + public Set<String> getSubSchemaNames() { + try{ + List<String> dbs = databases.get(DATABASES); + return Sets.newHashSet(dbs); + }catch(ExecutionException e){ + logger.warn("Failure while getting Hive database list.", e); + } + return super.getSubSchemaNames(); + } + + @Override + public DrillTable getTable(String name) { + if(defaultSchema == null){ + return super.getTable(name); + } + return defaultSchema.getTable(name); + } + + @Override + public Set<String> getTableNames() { + if(defaultSchema == null){ + return super.getTableNames(); + } + return defaultSchema.getTableNames(); + } + + List<String> getTableNames(String dbName){ + try{ + return tableNameLoader.get(dbName); + }catch(ExecutionException e){ + logger.warn("Failure while loading table names for database '{}'.", dbName, e.getCause()); + return Collections.emptyList(); + } + } + + DrillTable getDrillTable(String dbName, String t){ + HiveReadEntry entry = getSelectionBaseOnName(dbName, t); + if(entry == null) return null; + return new DrillHiveTable(schemaName, entry, pluginConfig); + } + + HiveReadEntry getSelectionBaseOnName(String dbName, String t) { + if(dbName == null) dbName = "default"; + try{ + return tableLoaders.get(dbName).get(t); + }catch(ExecutionException e){ + logger.warn("Exception occurred while trying to read table. {}.{}", dbName, t, e.getCause()); + return null; + } + } + + } + + +}
\ No newline at end of file diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONGroupScan.java deleted file mode 100644 index 4782d82a1..000000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONGroupScan.java +++ /dev/null @@ -1,162 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.json; - -import static com.google.common.base.Preconditions.checkArgument; - -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; - -import org.apache.drill.common.exceptions.ExecutionSetupException; -import org.apache.drill.common.expression.FieldReference; -import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.exec.physical.EndpointAffinity; -import org.apache.drill.exec.physical.OperatorCost; -import org.apache.drill.exec.physical.ReadEntry; -import org.apache.drill.exec.physical.base.AbstractGroupScan; -import org.apache.drill.exec.physical.base.PhysicalOperator; -import org.apache.drill.exec.physical.base.Size; -import org.apache.drill.exec.physical.base.SubScan; -import org.apache.drill.exec.proto.CoordinationProtos; -import org.apache.drill.exec.store.StorageEngineRegistry; - -import com.fasterxml.jackson.annotation.JacksonInject; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; - -@JsonTypeName("json-scan") -public class JSONGroupScan extends AbstractGroupScan { - private static int ESTIMATED_RECORD_SIZE = 1024; // 1kb - private final JSONStorageEngine engine; - - private LinkedList<JSONGroupScan.ScanEntry>[] mappings; - private final List<JSONGroupScan.ScanEntry> readEntries; - private final OperatorCost cost; - private final Size size; - private final FieldReference ref; - private final List<SchemaPath> columns; - - @JsonCreator - public JSONGroupScan(@JsonProperty("entries") List<ScanEntry> entries, - @JsonProperty("storageengine") JSONStorageEngineConfig storageEngineConfig, - @JacksonInject StorageEngineRegistry engineRegistry, @JsonProperty("ref") FieldReference ref, - @JsonProperty("columns") List<SchemaPath> columns) throws ExecutionSetupException { - this(entries, (JSONStorageEngine) engineRegistry.getEngine(storageEngineConfig), ref, columns); - } - - public JSONGroupScan(List<ScanEntry> entries, JSONStorageEngine engine, FieldReference ref, - List<SchemaPath> columns) { - this.engine = engine; - this.readEntries = entries; - OperatorCost cost = new OperatorCost(0, 0, 0, 0); - Size size = new Size(0, 0); - for (JSONGroupScan.ScanEntry r : readEntries) { - cost = cost.add(r.getCost()); - size = size.add(r.getSize()); - } - this.cost = cost; - this.size = size; - this.ref = ref; - this.columns = columns; - } - - @SuppressWarnings("unchecked") - @Override - public void applyAssignments(List<CoordinationProtos.DrillbitEndpoint> endpoints) { - checkArgument(endpoints.size() <= readEntries.size()); - - mappings = new LinkedList[endpoints.size()]; - - int i = 0; - for (ScanEntry e : readEntries) { - if (i == endpoints.size()) i = 0; - LinkedList entries = mappings[i]; - if (entries == null) { - entries = new LinkedList<>(); - mappings[i] = entries; - } - entries.add(e); - i++; - } - } - - @SuppressWarnings("unchecked") - @Override - public SubScan getSpecificScan(int minorFragmentId) throws ExecutionSetupException{ - checkArgument(minorFragmentId < mappings.length, "Mappings length [%s] should be longer than minor fragment id [%s] but it isn't.", mappings.length, minorFragmentId); - return new JSONSubScan(mappings[minorFragmentId], engine, ref, columns); - } - - @Override - public List<EndpointAffinity> getOperatorAffinity() { - return Collections.emptyList(); - } - - @Override - @JsonIgnore - public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) { - return new JSONGroupScan(readEntries, engine, ref, columns); - } - - public static class ScanEntry implements ReadEntry { - private final String path; - private Size size; - - @JsonCreator - public ScanEntry(@JsonProperty("path") String path) { - this.path = path; - size = new Size(ESTIMATED_RECORD_SIZE, ESTIMATED_RECORD_SIZE); - } - - @Override - public OperatorCost getCost() { - return new OperatorCost(1, 1, 2, 2); - } - - @Override - public Size getSize() { - return size; - } - - public String getPath() { - return path; - } - } - - @Override - public int getMaxParallelizationWidth() { - return readEntries.size(); - } - - public List<SchemaPath> getColumns() { - return columns; - } - - @Override - public OperatorCost getCost() { - return cost; - } - - @Override - public Size getSize() { - return size; - } -} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONStorageEngine.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONStorageEngine.java deleted file mode 100644 index e4f20706f..000000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONStorageEngine.java +++ /dev/null @@ -1,78 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.json; - -import java.io.IOException; -import java.util.ArrayList; - -import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.common.logical.data.Scan; -import org.apache.drill.exec.server.DrillbitContext; -import org.apache.drill.exec.store.AbstractStorageEngine; -import org.apache.drill.exec.store.ClassPathFileSystem; -import org.apache.drill.exec.store.SchemaProvider; -import org.apache.drill.exec.store.json.JSONGroupScan.ScanEntry; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - -public class JSONStorageEngine extends AbstractStorageEngine { - private final JSONStorageEngineConfig config; - private final Configuration conf; - private FileSystem fileSystem; - public static final String HADOOP_DEFAULT_NAME = "fs.default.name"; - private final JsonSchemaProvider schemaProvider; - - public JSONStorageEngine(JSONStorageEngineConfig config, DrillbitContext context) { - this.config = config; - this.schemaProvider = new JsonSchemaProvider(config, context.getConfig()); - - try { - this.conf = new Configuration(); - this.conf.set("fs.classpath.impl", ClassPathFileSystem.class.getName()); - this.conf.set(HADOOP_DEFAULT_NAME, config.getDfsName()); - this.fileSystem = FileSystem.get(conf); - - } catch (IOException ie) { - throw new RuntimeException("Error setting up filesystem"); - } - } - - public FileSystem getFileSystem() { - return fileSystem; - } - - public JSONStorageEngineConfig getConfig(){ - return config; - } - - @Override - public JSONGroupScan getPhysicalScan(Scan scan) throws IOException { - ArrayList<ScanEntry> readEntries = scan.getSelection().getListWith(new ObjectMapper(), new TypeReference<ArrayList<ScanEntry>>() {}); - return new JSONGroupScan(readEntries, this, scan.getOutputReference(), null); - } - - @Override - public SchemaProvider getSchemaProvider() { - return schemaProvider; - } - - -} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONSubScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONSubScan.java deleted file mode 100644 index 92f6c0af1..000000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONSubScan.java +++ /dev/null @@ -1,122 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.json; - -import java.util.Iterator; -import java.util.List; - -import org.apache.drill.common.exceptions.ExecutionSetupException; -import org.apache.drill.common.expression.FieldReference; -import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.common.logical.StorageEngineConfig; -import org.apache.drill.exec.physical.OperatorCost; -import org.apache.drill.exec.physical.base.AbstractBase; -import org.apache.drill.exec.physical.base.PhysicalOperator; -import org.apache.drill.exec.physical.base.PhysicalVisitor; -import org.apache.drill.exec.physical.base.Size; -import org.apache.drill.exec.physical.base.SubScan; -import org.apache.drill.exec.store.StorageEngineRegistry; - -import com.fasterxml.jackson.annotation.JacksonInject; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import com.google.common.collect.Iterators; - -@JsonTypeName("json-sub-scan") -public class JSONSubScan extends AbstractBase implements SubScan { - - protected final List<JSONGroupScan.ScanEntry> readEntries; - private final OperatorCost cost; - private final Size size; - private final JSONStorageEngine storageEngine; - private final FieldReference ref; - private final List<SchemaPath> columns; - - @JsonCreator - public JSONSubScan(@JacksonInject StorageEngineRegistry registry, - @JsonProperty("engineConfig") StorageEngineConfig engineConfig, - @JsonProperty("readEntries") List<JSONGroupScan.ScanEntry> readEntries, - @JsonProperty("ref") FieldReference ref, - @JsonProperty("columns") List<SchemaPath> columns) throws ExecutionSetupException { - this(readEntries, (JSONStorageEngine) registry.getEngine(engineConfig), ref, columns); - } - - JSONSubScan(List<JSONGroupScan.ScanEntry> readEntries, JSONStorageEngine engine, FieldReference ref, - List<SchemaPath> columns){ - this.readEntries = readEntries; - this.storageEngine = engine; - OperatorCost cost = new OperatorCost(0, 0, 0, 0); - Size size = new Size(0, 0); - for (JSONGroupScan.ScanEntry r : readEntries) { - cost = cost.add(r.getCost()); - size = size.add(r.getSize()); - } - this.cost = cost; - this.size = size; - this.ref = ref; - this.columns = columns; - } - - public FieldReference getRef() { - return ref; - } - - public List<JSONGroupScan.ScanEntry> getReadEntries() { - return readEntries; - } - - public StorageEngineConfig getEngineConfig() { - return storageEngine.getConfig(); - } - - @JsonIgnore - public JSONStorageEngine getStorageEngine() { - return storageEngine; - } - - @Override - public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) throws ExecutionSetupException{ - throw new UnsupportedOperationException(); - } - - @Override - public OperatorCost getCost() { - return cost; - } - - @Override - public Size getSize() { - return size; - } - - @Override - public <T, X, E extends Throwable> T accept(PhysicalVisitor<T, X, E> physicalVisitor, X value) throws E { - return physicalVisitor.visitSubScan(this, value); - } - - @Override - public Iterator<PhysicalOperator> iterator() { - return Iterators.emptyIterator(); - } - - public List<SchemaPath> getColumns() { - return columns; - } -} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JsonSchemaProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JsonSchemaProvider.java deleted file mode 100644 index 66a45421b..000000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JsonSchemaProvider.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.json; - -import java.io.IOException; - -import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.exec.physical.ReadEntryWithPath; -import org.apache.drill.exec.store.ClassPathFileSystem; -import org.apache.drill.exec.store.SchemaProvider; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; - -import com.beust.jcommander.internal.Lists; - -public class JsonSchemaProvider implements SchemaProvider{ - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JsonSchemaProvider.class); - - public static final String HADOOP_DEFAULT_NAME = "fs.default.name"; - final JSONStorageEngineConfig configuration; - final FileSystem fs; - final Configuration conf; - - public JsonSchemaProvider(JSONStorageEngineConfig configuration, DrillConfig config){ - this.configuration = configuration; - try { - this.conf = new Configuration(); - this.conf.set(HADOOP_DEFAULT_NAME, "file:///"); - this.conf.set("fs.classpath.impl", ClassPathFileSystem.class.getName()); - this.fs = FileSystem.get(conf); - } catch (IOException ie) { - throw new RuntimeException("Error setting up filesystem"); - } - } - - @Override - public Object getSelectionBaseOnName(String tableName) { - try{ -// if(!fs.exists(new Path(tableName))) return null; - ReadEntryWithPath re = new ReadEntryWithPath(tableName); - return Lists.newArrayList(re); - }catch(Exception e){ - logger.warn(String.format("Failure while checking table name %s.", tableName), e); - return null; - } - } -} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockGroupScanPOP.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockGroupScanPOP.java index cb9cbf677..b71784ba0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockGroupScanPOP.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockGroupScanPOP.java @@ -27,7 +27,6 @@ import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.expr.TypeHelper; import org.apache.drill.exec.physical.EndpointAffinity; import org.apache.drill.exec.physical.OperatorCost; -import org.apache.drill.exec.physical.ReadEntry; import org.apache.drill.exec.physical.base.AbstractGroupScan; import org.apache.drill.exec.physical.base.PhysicalOperator; import org.apache.drill.exec.physical.base.Size; @@ -75,7 +74,7 @@ public class MockGroupScanPOP extends AbstractGroupScan { return readEntries; } - public static class MockScanEntry implements ReadEntry { + public static class MockScanEntry{ private final int records; private final MockColumn[] types; @@ -93,7 +92,7 @@ public class MockGroupScanPOP extends AbstractGroupScan { this.recordSize = size; } - @Override + @JsonIgnore public OperatorCost getCost() { return new OperatorCost(1, 2, 1, 1); } @@ -106,7 +105,7 @@ public class MockGroupScanPOP extends AbstractGroupScan { return types; } - @Override + @JsonIgnore public Size getSize() { return new Size(records, recordSize); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java index b58ab5013..0578b0698 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java @@ -20,20 +20,22 @@ package org.apache.drill.exec.store.mock; import java.io.IOException; import java.util.ArrayList; +import net.hydromatic.optiq.Schema; +import net.hydromatic.optiq.SchemaPlus; + import org.apache.drill.common.logical.data.Scan; import org.apache.drill.exec.physical.base.AbstractGroupScan; import org.apache.drill.exec.server.DrillbitContext; -import org.apache.drill.exec.store.AbstractStorageEngine; -import org.apache.drill.exec.store.SchemaProvider; +import org.apache.drill.exec.store.AbstractStoragePlugin; import org.apache.drill.exec.store.mock.MockGroupScanPOP.MockScanEntry; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -public class MockStorageEngine extends AbstractStorageEngine { +public class MockStorageEngine extends AbstractStoragePlugin { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MockStorageEngine.class); - public MockStorageEngine(MockStorageEngineConfig configuration, DrillbitContext context) { + public MockStorageEngine(MockStorageEngineConfig configuration, DrillbitContext context, String name) { } @@ -48,8 +50,9 @@ public class MockStorageEngine extends AbstractStorageEngine { } @Override - public SchemaProvider getSchemaProvider() { - throw new UnsupportedOperationException(); + public Schema createAndAddSchema(SchemaPlus parent) { + return null; } + } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngineConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngineConfig.java index 786c0ec51..826254420 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngineConfig.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngineConfig.java @@ -17,14 +17,14 @@ */ package org.apache.drill.exec.store.mock; -import org.apache.drill.common.logical.StorageEngineConfigBase; +import org.apache.drill.common.logical.StoragePluginConfigBase; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; @JsonTypeName("mock") -public class MockStorageEngineConfig extends StorageEngineConfigBase{ +public class MockStorageEngineConfig extends StoragePluginConfigBase{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MockStorageEngineConfig.class); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockSubScanPOP.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockSubScanPOP.java index 800c7c5a7..0753be50d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockSubScanPOP.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockSubScanPOP.java @@ -41,21 +41,21 @@ public class MockSubScanPOP extends AbstractBase implements SubScan { private final String url; protected final List<MockGroupScanPOP.MockScanEntry> readEntries; - private final OperatorCost cost; - private final Size size; +// private final OperatorCost cost; +// private final Size size; private LinkedList<MockGroupScanPOP.MockScanEntry>[] mappings; @JsonCreator public MockSubScanPOP(@JsonProperty("url") String url, @JsonProperty("entries") List<MockGroupScanPOP.MockScanEntry> readEntries) { this.readEntries = readEntries; - OperatorCost cost = new OperatorCost(0,0,0,0); - Size size = new Size(0,0); - for(MockGroupScanPOP.MockScanEntry r : readEntries){ - cost = cost.add(r.getCost()); - size = size.add(r.getSize()); - } - this.cost = cost; - this.size = size; +// OperatorCost cost = new OperatorCost(0,0,0,0); +// Size size = new Size(0,0); +// for(MockGroupScanPOP.MockScanEntry r : readEntries){ +// cost = cost.add(r.getCost()); +// size = size.add(r.getSize()); +// } +// this.cost = cost; +// this.size = size; this.url = url; } @@ -73,12 +73,12 @@ public class MockSubScanPOP extends AbstractBase implements SubScan { return Iterators.emptyIterator(); } - @Override + @Override @JsonIgnore public OperatorCost getCost() { throw new UnsupportedOperationException(); } - @Override + @Override @JsonIgnore public Size getSize() { throw new UnsupportedOperationException(); } @@ -86,7 +86,7 @@ public class MockSubScanPOP extends AbstractBase implements SubScan { // will want to replace these two methods with an interface above for AbstractSubScan @Override public boolean isExecutable() { - return true; //To change body of implemented methods use File | Settings | File Templates. + return true; } @Override diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/BitReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/BitReader.java index d0049c7f6..c489d5b27 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/BitReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/BitReader.java @@ -23,7 +23,7 @@ import org.apache.drill.exec.vector.ValueVector; import parquet.column.ColumnDescriptor; import parquet.hadoop.metadata.ColumnChunkMetaData; -public final class BitReader extends ColumnReader { +final class BitReader extends ColumnReader { private byte currentByte; private byte nextByte; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnDataReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnDataReader.java index ec447477c..a890f1c97 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnDataReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnDataReader.java @@ -28,7 +28,7 @@ import parquet.bytes.BytesInput; import parquet.format.PageHeader; import parquet.format.Util; -public class ColumnDataReader { +class ColumnDataReader { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ColumnDataReader.class); private final long endPosition; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnReader.java index 94ccbfcea..2cc126c7b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnReader.java @@ -29,7 +29,7 @@ import parquet.schema.PrimitiveType; import java.io.IOException; -public abstract class ColumnReader { +abstract class ColumnReader { final ParquetRecordReader parentReader; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/FixedByteAlignedReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/FixedByteAlignedReader.java index 3aae189e9..4f14f606c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/FixedByteAlignedReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/FixedByteAlignedReader.java @@ -23,7 +23,7 @@ import org.apache.drill.exec.vector.ValueVector; import parquet.column.ColumnDescriptor; import parquet.hadoop.metadata.ColumnChunkMetaData; -public class FixedByteAlignedReader extends ColumnReader { +class FixedByteAlignedReader extends ColumnReader { private byte[] bytes; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableBitReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableBitReader.java index 1dde7c767..4c060f2b7 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableBitReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableBitReader.java @@ -35,7 +35,7 @@ import java.io.IOException; * because page/batch boundaries that do not land on byte boundaries require shifting of all of the values * in the next batch. */ -public final class NullableBitReader extends ColumnReader { +final class NullableBitReader extends ColumnReader { NullableBitReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, ValueVector v) throws ExecutionSetupException { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java index 5ac9bb1aa..4c33aeba5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java @@ -26,7 +26,7 @@ import parquet.hadoop.metadata.ColumnChunkMetaData; import java.io.IOException; -public abstract class NullableColumnReader extends ColumnReader{ +abstract class NullableColumnReader extends ColumnReader{ int nullsFound; // used to skip nulls found diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableFixedByteAlignedReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableFixedByteAlignedReader.java index bb8102488..c2fc606d3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableFixedByteAlignedReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableFixedByteAlignedReader.java @@ -23,7 +23,7 @@ import org.apache.drill.exec.vector.ValueVector; import parquet.column.ColumnDescriptor; import parquet.hadoop.metadata.ColumnChunkMetaData; -public class NullableFixedByteAlignedReader extends NullableColumnReader { +class NullableFixedByteAlignedReader extends NullableColumnReader { private byte[] bytes; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/PageReadStatus.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/PageReadStatus.java index c08dcf351..1aef7b4fe 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/PageReadStatus.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/PageReadStatus.java @@ -31,7 +31,7 @@ import parquet.format.PageHeader; import parquet.hadoop.metadata.ColumnChunkMetaData; // class to keep track of the read position of variable length columns -public final class PageReadStatus { +final class PageReadStatus { private final ColumnReader parentColumnReader; private final ColumnDataReader dataReader; @@ -90,9 +90,9 @@ public final class PageReadStatus { bytesIn, pageHeader.data_page_header.num_values, pageHeader.uncompressed_page_size, - ParquetStorageEngine.parquetMetadataConverter.getEncoding(pageHeader.data_page_header.repetition_level_encoding), - ParquetStorageEngine.parquetMetadataConverter.getEncoding(pageHeader.data_page_header.definition_level_encoding), - ParquetStorageEngine.parquetMetadataConverter.getEncoding(pageHeader.data_page_header.encoding) + ParquetFormatPlugin.parquetMetadataConverter.getEncoding(pageHeader.data_page_header.repetition_level_encoding), + ParquetFormatPlugin.parquetMetadataConverter.getEncoding(pageHeader.data_page_header.definition_level_encoding), + ParquetFormatPlugin.parquetMetadataConverter.getEncoding(pageHeader.data_page_header.encoding) ); byteLength = pageHeader.uncompressed_page_size; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONStorageEngineConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatConfig.java index 359e7edde..33eb68a2f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONStorageEngineConfig.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatConfig.java @@ -15,40 +15,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store.json; +package org.apache.drill.exec.store.parquet; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import org.apache.drill.common.logical.StorageEngineConfigBase; - -@JsonTypeName("json") -public class JSONStorageEngineConfig extends StorageEngineConfigBase { - private String dfsName; +import org.apache.drill.common.logical.FormatPluginConfig; - public String getDfsName() { - return dfsName; - } +import com.fasterxml.jackson.annotation.JsonTypeName; - @JsonCreator - public JSONStorageEngineConfig(@JsonProperty("dfsName") String dfsName) { - this.dfsName = dfsName; - } +@JsonTypeName("parquet") +public class ParquetFormatConfig implements FormatPluginConfig{ @Override public int hashCode() { - return dfsName != null ? dfsName.hashCode() : 0; + return 7; } @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - JSONStorageEngineConfig that = (JSONStorageEngineConfig) o; - - if (dfsName != null ? !dfsName.equals(that.dfsName) : that.dfsName != null) return false; - - return true; + public boolean equals(Object obj) { + return obj instanceof ParquetFormatConfig; } + } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java new file mode 100644 index 000000000..6d02046b7 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java @@ -0,0 +1,178 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.parquet; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.regex.Pattern; + +import org.apache.drill.common.expression.FieldReference; +import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.store.QueryOptimizerRule; +import org.apache.drill.exec.store.dfs.BasicFormatMatcher; +import org.apache.drill.exec.store.dfs.FileSelection; +import org.apache.drill.exec.store.dfs.FormatMatcher; +import org.apache.drill.exec.store.dfs.FormatPlugin; +import org.apache.drill.exec.store.dfs.FormatSelection; +import org.apache.drill.exec.store.dfs.MagicString; +import org.apache.drill.exec.store.dfs.shim.DrillFileSystem; +import org.apache.drill.exec.store.mock.MockStorageEngine; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; + +import parquet.format.converter.ParquetMetadataConverter; +import parquet.hadoop.CodecFactoryExposer; +import parquet.hadoop.ParquetFileWriter; + +import com.google.common.collect.Lists; + +public class ParquetFormatPlugin implements FormatPlugin{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MockStorageEngine.class); + + private final DrillbitContext context; + static final ParquetMetadataConverter parquetMetadataConverter = new ParquetMetadataConverter(); + private CodecFactoryExposer codecFactoryExposer; + private final DrillFileSystem fs; + private final ParquetFormatMatcher formatMatcher; + private final ParquetFormatConfig config; + private final StoragePluginConfig storageConfig; + private final String name; + + public ParquetFormatPlugin(String name, DrillbitContext context, DrillFileSystem fs, StoragePluginConfig storageConfig){ + this(name, context, fs, storageConfig, new ParquetFormatConfig()); + } + + public ParquetFormatPlugin(String name, DrillbitContext context, DrillFileSystem fs, StoragePluginConfig storageConfig, ParquetFormatConfig formatConfig){ + this.context = context; + this.codecFactoryExposer = new CodecFactoryExposer(fs.getUnderlying().getConf()); + this.config = formatConfig; + this.formatMatcher = new ParquetFormatMatcher(this, fs); + this.storageConfig = storageConfig; + this.fs = fs; + this.name = name == null ? "parquet" : name; + } + + Configuration getHadoopConfig() { + return fs.getUnderlying().getConf(); + } + + public DrillFileSystem getFileSystem() { + return fs; + } + + @Override + public ParquetFormatConfig getConfig() { + return config; + } + + public DrillbitContext getContext() { + return this.context; + } + + @Override + public boolean supportsRead() { + return true; + } + + @Override + public List<QueryOptimizerRule> getOptimizerRules() { + return Collections.emptyList(); + } + + @Override + public ParquetGroupScan getGroupScan(FieldReference outputRef, FileSelection selection) throws IOException { + return new ParquetGroupScan( selection.getFileStatusList(fs), this, outputRef); + } + + @Override + public StoragePluginConfig getStorageConfig() { + return storageConfig; + } + + public CodecFactoryExposer getCodecFactoryExposer() { + return codecFactoryExposer; + } + + public String getName(){ + return name; + } + + @Override + public boolean supportsWrite() { + return false; + } + + + + @Override + public FormatMatcher getMatcher() { + return formatMatcher; + } + + private static class ParquetFormatMatcher extends BasicFormatMatcher{ + + private final DrillFileSystem fs; + + public ParquetFormatMatcher(ParquetFormatPlugin plugin, DrillFileSystem fs) { + super(plugin, fs, // + Lists.newArrayList( // + Pattern.compile(".*\\.parquet$"), // + Pattern.compile(".*/" + ParquetFileWriter.PARQUET_METADATA_FILE) // + // + ), + Lists.newArrayList(new MagicString(0, ParquetFileWriter.MAGIC)) + + ); + this.fs = fs; + + } + + @Override + public boolean supportDirectoryReads() { + return true; + } + + @Override + public FormatSelection isReadable(FileSelection file) throws IOException { + // TODO: we only check the first file for directory reading. This is because + if(file.containsDirectories(fs)){ + if(isDirReadable(file.getFirstPath(fs))){ + return new FormatSelection(plugin.getConfig(), file); + } + } + return super.isReadable(file); + } + + boolean isDirReadable(FileStatus dir) { + Path p = new Path(dir.getPath(), "/" + ParquetFileWriter.PARQUET_METADATA_FILE); + try { + return fs.getUnderlying().exists(p); + } catch (IOException e) { + logger.info("Failure while attempting to check for Parquet metadata file.", e); + return false; + } + } + + + + } + +}
\ No newline at end of file diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java index aa0111545..f76e59a6e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java @@ -18,37 +18,35 @@ package org.apache.drill.exec.store.parquet; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedList; import java.util.List; -import java.util.Map; import java.util.concurrent.TimeUnit; -import com.codahale.metrics.Histogram; -import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.Timer; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.common.exceptions.PhysicalOperatorSetupException; import org.apache.drill.common.expression.FieldReference; -import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.common.logical.data.NamedExpression; +import org.apache.drill.common.logical.FormatPluginConfig; +import org.apache.drill.common.logical.StoragePluginConfig; import org.apache.drill.exec.metrics.DrillMetrics; import org.apache.drill.exec.physical.EndpointAffinity; import org.apache.drill.exec.physical.OperatorCost; -import org.apache.drill.exec.physical.ReadEntryFromHDFS; -import org.apache.drill.exec.physical.ReadEntryWithPath; import org.apache.drill.exec.physical.base.AbstractGroupScan; import org.apache.drill.exec.physical.base.PhysicalOperator; import org.apache.drill.exec.physical.base.Size; import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; -import org.apache.drill.exec.store.AffinityCalculator; -import org.apache.drill.exec.store.StorageEngineRegistry; +import org.apache.drill.exec.store.StoragePluginRegistry; +import org.apache.drill.exec.store.dfs.FileSystemPlugin; +import org.apache.drill.exec.store.dfs.ReadEntryFromHDFS; +import org.apache.drill.exec.store.dfs.ReadEntryWithPath; +import org.apache.drill.exec.store.dfs.easy.FileWork; +import org.apache.drill.exec.store.schedule.AffinityCreator; +import org.apache.drill.exec.store.schedule.AssignmentCreator; +import org.apache.drill.exec.store.schedule.BlockMapBuilder; +import org.apache.drill.exec.store.schedule.CompleteWork; +import org.apache.drill.exec.store.schedule.EndpointByteMap; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -59,16 +57,17 @@ import parquet.hadoop.metadata.ColumnChunkMetaData; import parquet.hadoop.metadata.ParquetMetadata; import parquet.org.codehaus.jackson.annotate.JsonCreator; +import com.codahale.metrics.Histogram; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.Timer; import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Preconditions; import com.google.common.base.Stopwatch; -import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ListMultimap; import com.google.common.collect.Lists; -import com.google.common.collect.Multimap; - @JsonTypeName("parquet-scan") public class ParquetGroupScan extends AbstractGroupScan { @@ -78,11 +77,18 @@ public class ParquetGroupScan extends AbstractGroupScan { static final String ENDPOINT_BYTES_TIMER = MetricRegistry.name(ParquetGroupScan.class, "endpointBytes"); static final String ASSIGNMENT_TIMER = MetricRegistry.name(ParquetGroupScan.class, "applyAssignments"); static final String ASSIGNMENT_AFFINITY_HIST = MetricRegistry.name(ParquetGroupScan.class, "assignmentAffinity"); + final Histogram assignmentAffinityStats = metrics.histogram(ASSIGNMENT_AFFINITY_HIST); - private ArrayListMultimap<Integer, ParquetRowGroupScan.RowGroupReadEntry> mappings; + private ListMultimap<Integer, RowGroupInfo> mappings; private List<RowGroupInfo> rowGroupInfos; - private Stopwatch watch = new Stopwatch(); + private final List<ReadEntryWithPath> entries; + private final Stopwatch watch = new Stopwatch(); + private final ParquetFormatPlugin formatPlugin; + private final ParquetFormatConfig formatConfig; + private final FileSystem fs; + private final FieldReference ref; + private List<EndpointAffinity> endpointAffinities; private List<SchemaPath> columns; @@ -91,79 +97,87 @@ public class ParquetGroupScan extends AbstractGroupScan { } @JsonProperty("storageengine") - public ParquetStorageEngineConfig getEngineConfig() { - return this.engineConfig; + public ParquetFormatConfig getEngineConfig() { + return this.formatConfig; } - private List<ReadEntryWithPath> entries; - private long totalBytes; - private Collection<DrillbitEndpoint> availableEndpoints; - private ParquetStorageEngine storageEngine; - private ParquetStorageEngineConfig engineConfig; - private FileSystem fs; - private final FieldReference ref; - private List<EndpointAffinity> endpointAffinities; - @JsonCreator - public ParquetGroupScan(@JsonProperty("entries") List<ReadEntryWithPath> entries, - @JsonProperty("storageengine") ParquetStorageEngineConfig storageEngineConfig, - @JacksonInject StorageEngineRegistry engineRegistry, - @JsonProperty("ref") FieldReference ref, - @JsonProperty("columns") List<SchemaPath> columns - )throws IOException, ExecutionSetupException { + public ParquetGroupScan( // + @JsonProperty("entries") List<ReadEntryWithPath> entries, // + @JsonProperty("storage") StoragePluginConfig storageConfig, // + @JsonProperty("format") FormatPluginConfig formatConfig, // + @JacksonInject StoragePluginRegistry engineRegistry, // + @JsonProperty("ref") FieldReference ref, // + @JsonProperty("columns") List<SchemaPath> columns // + ) throws IOException, ExecutionSetupException { engineRegistry.init(DrillConfig.create()); this.columns = columns; - this.storageEngine = (ParquetStorageEngine) engineRegistry.getEngine(storageEngineConfig); - this.availableEndpoints = storageEngine.getContext().getBits(); - this.fs = storageEngine.getFileSystem(); - this.engineConfig = storageEngineConfig; + if(formatConfig == null) formatConfig = new ParquetFormatConfig(); + Preconditions.checkNotNull(storageConfig); + Preconditions.checkNotNull(formatConfig); + this.formatPlugin = (ParquetFormatPlugin) engineRegistry.getFormatPlugin(storageConfig, formatConfig); + Preconditions.checkNotNull(formatPlugin); + this.fs = formatPlugin.getFileSystem().getUnderlying(); + this.formatConfig = formatPlugin.getConfig(); this.entries = entries; this.ref = ref; - readFooter(); - calculateEndpointBytes(); + this.readFooterFromEntries(); + } - public ParquetGroupScan(List<ReadEntryWithPath> entries, // - ParquetStorageEngine storageEngine, // - FieldReference ref, // - List<SchemaPath> columns) throws IOException { - this.storageEngine = storageEngine; - this.columns = columns; - this.engineConfig = storageEngine.getEngineConfig(); - this.availableEndpoints = storageEngine.getContext().getBits(); - this.fs = storageEngine.getFileSystem(); - this.entries = entries; + public ParquetGroupScan(List<FileStatus> files, // + ParquetFormatPlugin formatPlugin, // + FieldReference ref) // + throws IOException { + this.formatPlugin = formatPlugin; + this.columns = null; + this.formatConfig = formatPlugin.getConfig(); + this.fs = formatPlugin.getFileSystem().getUnderlying(); + + this.entries = Lists.newArrayList(); + for(FileStatus file : files){ + entries.add(new ReadEntryWithPath(file.getPath().toString())); + } + this.ref = ref; - readFooter(); - calculateEndpointBytes(); + readFooter(files); } - private void readFooter() throws IOException { + private void readFooterFromEntries() throws IOException { + List<FileStatus> files = Lists.newArrayList(); + for(ReadEntryWithPath e : entries){ + files.add(fs.getFileStatus(new Path(e.getPath()))); + } + readFooter(files); + } + + private void readFooter(List<FileStatus> statuses) throws IOException { watch.reset(); watch.start(); Timer.Context tContext = metrics.timer(READ_FOOTER_TIMER).time(); - rowGroupInfos = new ArrayList(); + + + rowGroupInfos = Lists.newArrayList(); long start = 0, length = 0; ColumnChunkMetaData columnChunkMetaData; - for (ReadEntryWithPath readEntryWithPath : entries){ - Path path = new Path(readEntryWithPath.getPath()); - List<Footer> footers = ParquetFileReader.readFooters(this.storageEngine.getHadoopConfig(), path); + for (FileStatus status : statuses) { + List<Footer> footers = ParquetFileReader.readFooters(formatPlugin.getHadoopConfig(), status); if (footers.size() == 0) { - logger.warn("No footers found"); + throw new IOException(String.format("Unable to find footer for file %s", status.getPath().getName())); } -// readEntryWithPath.getPath(); for (Footer footer : footers) { int index = 0; ParquetMetadata metadata = footer.getParquetMetadata(); - for (BlockMetaData rowGroup : metadata.getBlocks()){ + for (BlockMetaData rowGroup : metadata.getBlocks()) { // need to grab block information from HDFS columnChunkMetaData = rowGroup.getColumns().iterator().next(); start = columnChunkMetaData.getFirstDataPageOffset(); - // this field is not being populated correctly, but the column chunks know their sizes, just summing them for now - //end = start + rowGroup.getTotalByteSize(); + // this field is not being populated correctly, but the column chunks know their sizes, just summing them for + // now + // end = start + rowGroup.getTotalByteSize(); length = 0; - for (ColumnChunkMetaData col : rowGroup.getColumns()){ + for (ColumnChunkMetaData col : rowGroup.getColumns()) { length += col.getTotalSize(); } String filePath = footer.getFile().toUri().getPath(); @@ -179,203 +193,109 @@ public class ParquetGroupScan extends AbstractGroupScan { logger.debug("Took {} ms to get row group infos", watch.elapsed(TimeUnit.MILLISECONDS)); } - private void calculateEndpointBytes() { - Timer.Context tContext = metrics.timer(ENDPOINT_BYTES_TIMER).time(); - watch.reset(); - watch.start(); - AffinityCalculator ac = new AffinityCalculator(fs, availableEndpoints); - for (RowGroupInfo e : rowGroupInfos) { - ac.setEndpointBytes(e); - totalBytes += e.getLength(); - } - watch.stop(); - tContext.stop(); - logger.debug("Took {} ms to calculate EndpointBytes", watch.elapsed(TimeUnit.MILLISECONDS)); - } - @JsonIgnore public FileSystem getFileSystem() { return this.fs; } - public static class RowGroupInfo extends ReadEntryFromHDFS { + public static class RowGroupInfo extends ReadEntryFromHDFS implements CompleteWork, FileWork { - private HashMap<DrillbitEndpoint,Long> endpointBytes; - private long maxBytes; + private EndpointByteMap byteMap; private int rowGroupIndex; @JsonCreator public RowGroupInfo(@JsonProperty("path") String path, @JsonProperty("start") long start, - @JsonProperty("length") long length, @JsonProperty("rowGroupIndex") int rowGroupIndex) { + @JsonProperty("length") long length, @JsonProperty("rowGroupIndex") int rowGroupIndex) { super(path, start, length); this.rowGroupIndex = rowGroupIndex; } - @Override - public OperatorCost getCost() { - return new OperatorCost(1, 2, 1, 1); + public RowGroupReadEntry getRowGroupReadEntry() { + return new RowGroupReadEntry(this.getPath(), this.getStart(), this.getLength(), this.rowGroupIndex); } - @Override - public Size getSize() { - // TODO - these values are wrong, I cannot know these until after I read a file - return new Size(10, 10); - } - - public HashMap<DrillbitEndpoint,Long> getEndpointBytes() { - return endpointBytes; - } - - public void setEndpointBytes(HashMap<DrillbitEndpoint,Long> endpointBytes) { - this.endpointBytes = endpointBytes; - } - - public void setMaxBytes(long bytes) { - this.maxBytes = bytes; + public int getRowGroupIndex() { + return this.rowGroupIndex; } - public long getMaxBytes() { - return maxBytes; + @Override + public int compareTo(CompleteWork o) { + return Long.compare(getTotalBytes(), o.getTotalBytes()); } - public ParquetRowGroupScan.RowGroupReadEntry getRowGroupReadEntry() { - return new ParquetRowGroupScan.RowGroupReadEntry(this.getPath(), this.getStart(), this.getLength(), this.rowGroupIndex); + @Override + public long getTotalBytes() { + return this.getLength(); } - public int getRowGroupIndex() { - return this.rowGroupIndex; + @Override + public EndpointByteMap getByteMap() { + return byteMap; } - } - private class ParquetReadEntryComparator implements Comparator<RowGroupInfo> { - public int compare(RowGroupInfo e1, RowGroupInfo e2) { - if (e1.getMaxBytes() == e2.getMaxBytes()) return 0; - return (e1.getMaxBytes() > e2.getMaxBytes()) ? 1 : -1; + public void setEndpointByteMap(EndpointByteMap byteMap) { + this.byteMap = byteMap; } } /** - *Calculates the affinity each endpoint has for this scan, by adding up the affinity each endpoint has for each + * Calculates the affinity each endpoint has for this scan, by adding up the affinity each endpoint has for each * rowGroup + * * @return a list of EndpointAffinity objects */ @Override public List<EndpointAffinity> getOperatorAffinity() { - watch.reset(); - watch.start(); + if (this.endpointAffinities == null) { - HashMap<DrillbitEndpoint, Float> affinities = new HashMap<>(); - for (RowGroupInfo entry : rowGroupInfos) { - for (DrillbitEndpoint d : entry.getEndpointBytes().keySet()) { - long bytes = entry.getEndpointBytes().get(d); - float affinity = (float)bytes / (float)totalBytes; - logger.debug("RowGroup: {} Endpoint: {} Bytes: {}", entry.getRowGroupIndex(), d.getAddress(), bytes); - if (affinities.keySet().contains(d)) { - affinities.put(d, affinities.get(d) + affinity); - } else { - affinities.put(d, affinity); - } + BlockMapBuilder bmb = new BlockMapBuilder(fs, formatPlugin.getContext().getBits()); + try{ + for (RowGroupInfo rgi : rowGroupInfos) { + EndpointByteMap ebm = bmb.getEndpointByteMap(rgi); + rgi.setEndpointByteMap(ebm); } + } catch (IOException e) { + logger.warn("Failure while determining operator affinity.", e); + return Collections.emptyList(); } - List<EndpointAffinity> affinityList = new LinkedList<>(); - for (DrillbitEndpoint d : affinities.keySet()) { - logger.debug("Endpoint {} has affinity {}", d.getAddress(), affinities.get(d).floatValue()); - affinityList.add(new EndpointAffinity(d,affinities.get(d).floatValue())); - } - this.endpointAffinities = affinityList; + + this.endpointAffinities = AffinityCreator.getAffinityMap(rowGroupInfos); } - watch.stop(); - logger.debug("Took {} ms to get operator affinity", watch.elapsed(TimeUnit.MILLISECONDS)); return this.endpointAffinities; } + @Override + public void applyAssignments(List<DrillbitEndpoint> incomingEndpoints) throws PhysicalOperatorSetupException { - static final double[] ASSIGNMENT_CUTOFFS = {0.99, 0.50, 0.25, 0.00}; + this.mappings = AssignmentCreator.getMappings(incomingEndpoints, rowGroupInfos); - /** - * - * @param incomingEndpoints - */ - @Override - public void applyAssignments(List<DrillbitEndpoint> incomingEndpoints) { - watch.reset(); - watch.start(); - final Timer.Context tcontext = metrics.timer(ASSIGNMENT_TIMER).time(); - Preconditions.checkArgument(incomingEndpoints.size() <= rowGroupInfos.size(), String.format("Incoming endpoints %d " + - "is greater than number of row groups %d", incomingEndpoints.size(), rowGroupInfos.size())); - mappings = ArrayListMultimap.create(); - ArrayList rowGroupList = new ArrayList(rowGroupInfos); - List<DrillbitEndpoint> endpointLinkedlist = Lists.newLinkedList(incomingEndpoints); - for(double cutoff : ASSIGNMENT_CUTOFFS ){ - scanAndAssign(mappings, endpointLinkedlist, rowGroupList, cutoff, false); - } - scanAndAssign(mappings, endpointLinkedlist, rowGroupList, 0.0, true); - tcontext.stop(); - watch.stop(); - logger.debug("Took {} ms to apply assignments", watch.elapsed(TimeUnit.MILLISECONDS)); - Preconditions.checkState(rowGroupList.isEmpty(), "All readEntries should be assigned by now, but some are still unassigned"); - Preconditions.checkState(!rowGroupInfos.isEmpty()); } - public int fragmentPointer = 0; + @Override + public ParquetRowGroupScan getSpecificScan(int minorFragmentId) { + assert minorFragmentId < mappings.size() : String.format( + "Mappings length [%d] should be longer than minor fragment id [%d] but it isn't.", mappings.size(), + minorFragmentId); - /** - * - * @param endpointAssignments the mapping between fragment/endpoint and rowGroup - * @param endpoints the list of drillbits, ordered by the corresponding fragment - * @param rowGroups the list of rowGroups to assign - * @param requiredPercentage the percentage of max bytes required to make an assignment - * @param assignAll if true, will assign even if no affinity - */ - private void scanAndAssign (Multimap<Integer, ParquetRowGroupScan.RowGroupReadEntry> endpointAssignments, List<DrillbitEndpoint> endpoints, - List<RowGroupInfo> rowGroups, double requiredPercentage, boolean assignAll) { - Collections.sort(rowGroups, new ParquetReadEntryComparator()); - final boolean requireAffinity = requiredPercentage > 0; - int maxAssignments = (int) (rowGroups.size() / endpoints.size()); - - if (maxAssignments < 1) maxAssignments = 1; - - for(Iterator<RowGroupInfo> iter = rowGroups.iterator(); iter.hasNext();){ - RowGroupInfo rowGroupInfo = iter.next(); - for (int i = 0; i < endpoints.size(); i++) { - int minorFragmentId = (fragmentPointer + i) % endpoints.size(); - DrillbitEndpoint currentEndpoint = endpoints.get(minorFragmentId); - Map<DrillbitEndpoint, Long> bytesPerEndpoint = rowGroupInfo.getEndpointBytes(); - boolean haveAffinity = bytesPerEndpoint.containsKey(currentEndpoint) ; - - if (assignAll || - (!bytesPerEndpoint.isEmpty() && - (!requireAffinity || haveAffinity) && - (!endpointAssignments.containsKey(minorFragmentId) || endpointAssignments.get(minorFragmentId).size() < maxAssignments) && - (!requireAffinity || bytesPerEndpoint.get(currentEndpoint) >= rowGroupInfo.getMaxBytes() * requiredPercentage))) { - - endpointAssignments.put(minorFragmentId, rowGroupInfo.getRowGroupReadEntry()); - logger.debug("Assigned rowGroup {} to minorFragmentId {} endpoint {}", rowGroupInfo.getRowGroupIndex(), minorFragmentId, endpoints.get(minorFragmentId).getAddress()); - if (bytesPerEndpoint.get(currentEndpoint) != null) { - assignmentAffinityStats.update(bytesPerEndpoint.get(currentEndpoint) / rowGroupInfo.getLength()); - } else { - assignmentAffinityStats.update(0); - } - iter.remove(); - fragmentPointer = (minorFragmentId + 1) % endpoints.size(); - break; - } - } + List<RowGroupInfo> rowGroupsForMinor = mappings.get(minorFragmentId); - } + Preconditions.checkArgument(!rowGroupsForMinor.isEmpty(), + String.format("MinorFragmentId %d has no read entries assigned", minorFragmentId)); + + return new ParquetRowGroupScan(formatPlugin, convertToReadEntries(rowGroupsForMinor), ref, columns); } - @Override - public ParquetRowGroupScan getSpecificScan(int minorFragmentId) { - assert minorFragmentId < mappings.size() : String.format("Mappings length [%d] should be longer than minor fragment id [%d] but it isn't.", mappings.size(), minorFragmentId); - for (ParquetRowGroupScan.RowGroupReadEntry rg : mappings.get(minorFragmentId)) { - logger.debug("minorFragmentId: {} Path: {} RowGroupIndex: {}",minorFragmentId, rg.getPath(),rg.getRowGroupIndex()); + + + private List<RowGroupReadEntry> convertToReadEntries(List<RowGroupInfo> rowGroups){ + List<RowGroupReadEntry> entries = Lists.newArrayList(); + for (RowGroupInfo rgi : rowGroups) { + RowGroupReadEntry rgre = new RowGroupReadEntry(rgi.getPath(), rgi.getStart(), rgi.getLength(), + rgi.getRowGroupIndex()); + entries.add(rgre); } - Preconditions.checkArgument(!mappings.get(minorFragmentId).isEmpty(), String.format("MinorFragmentId %d has no read entries assigned", minorFragmentId)); - return new ParquetRowGroupScan(storageEngine, engineConfig, mappings.get(minorFragmentId), ref, - columns); + return entries; } - public FieldReference getRef() { return ref; @@ -392,21 +312,21 @@ public class ParquetGroupScan extends AbstractGroupScan { @Override public OperatorCost getCost() { - //TODO Figure out how to properly calculate cost - return new OperatorCost(1,rowGroupInfos.size(),1,1); + // TODO Figure out how to properly calculate cost + return new OperatorCost(1, rowGroupInfos.size(), 1, 1); } @Override public Size getSize() { // TODO - this is wrong, need to populate correctly - return new Size(10,10); + return new Size(10, 10); } @Override @JsonIgnore public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) { Preconditions.checkArgument(children.isEmpty()); - //TODO return copy of self + // TODO return copy of self return this; } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java index 1e6c31a85..9e1cc6664 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java @@ -49,7 +49,7 @@ import parquet.schema.PrimitiveType; import com.google.common.base.Joiner; -public class ParquetRecordReader implements RecordReader { +class ParquetRecordReader implements RecordReader { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetRecordReader.class); // this value has been inflated to read in multiple value vectors at once, and then break them up into smaller vectors diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java index b3ce9b454..0e672d073 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java @@ -25,15 +25,16 @@ import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.common.expression.FieldReference; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.common.logical.StorageEngineConfig; +import org.apache.drill.common.logical.FormatPluginConfig; +import org.apache.drill.common.logical.StoragePluginConfig; import org.apache.drill.exec.physical.OperatorCost; -import org.apache.drill.exec.physical.ReadEntryFromHDFS; import org.apache.drill.exec.physical.base.AbstractBase; import org.apache.drill.exec.physical.base.PhysicalOperator; import org.apache.drill.exec.physical.base.PhysicalVisitor; import org.apache.drill.exec.physical.base.Size; import org.apache.drill.exec.physical.base.SubScan; -import org.apache.drill.exec.store.StorageEngineRegistry; +import org.apache.drill.exec.store.StoragePluginRegistry; +import org.apache.drill.exec.store.dfs.FileSystemPlugin; import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; @@ -48,43 +49,53 @@ import com.google.common.collect.Iterators; public class ParquetRowGroupScan extends AbstractBase implements SubScan { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetRowGroupScan.class); - public final StorageEngineConfig engineConfig; - private final ParquetStorageEngine parquetStorageEngine; + public final ParquetFormatConfig formatConfig; + private final ParquetFormatPlugin formatPlugin; private final List<RowGroupReadEntry> rowGroupReadEntries; private final FieldReference ref; private final List<SchemaPath> columns; @JsonCreator - public ParquetRowGroupScan(@JacksonInject StorageEngineRegistry registry, - @JsonProperty("engineConfig") StorageEngineConfig engineConfig, - @JsonProperty("rowGroupReadEntries") LinkedList<RowGroupReadEntry> rowGroupReadEntries, - @JsonProperty("ref") FieldReference ref, - @JsonProperty("columns") List<SchemaPath> columns - ) throws ExecutionSetupException { - parquetStorageEngine = (ParquetStorageEngine) registry.getEngine(engineConfig); + public ParquetRowGroupScan( // + @JacksonInject StoragePluginRegistry registry, // + @JsonProperty("storage") StoragePluginConfig storageConfig, // + @JsonProperty("format") FormatPluginConfig formatConfig, // + @JsonProperty("entries") LinkedList<RowGroupReadEntry> rowGroupReadEntries, // + @JsonProperty("ref") FieldReference ref, // + @JsonProperty("columns") List<SchemaPath> columns // + ) throws ExecutionSetupException { + + if(formatConfig == null) formatConfig = new ParquetFormatConfig(); + Preconditions.checkNotNull(storageConfig); + Preconditions.checkNotNull(formatConfig); + this.formatPlugin = (ParquetFormatPlugin) registry.getFormatPlugin(storageConfig, formatConfig); + Preconditions.checkNotNull(formatPlugin); this.rowGroupReadEntries = rowGroupReadEntries; - this.engineConfig = engineConfig; + this.formatConfig = formatPlugin.getConfig(); this.ref = ref; this.columns = columns; } - public ParquetRowGroupScan(ParquetStorageEngine engine, ParquetStorageEngineConfig config, - List<RowGroupReadEntry> rowGroupReadEntries, FieldReference ref, - List<SchemaPath> columns - ) { - parquetStorageEngine = engine; - engineConfig = config; + public ParquetRowGroupScan( // + ParquetFormatPlugin formatPlugin, // + List<RowGroupReadEntry> rowGroupReadEntries, // + FieldReference ref, // + List<SchemaPath> columns) { + this.formatPlugin = formatPlugin; + this.formatConfig = formatPlugin.getConfig(); this.rowGroupReadEntries = rowGroupReadEntries; this.ref = ref; this.columns = columns; } + @JsonProperty("entries") public List<RowGroupReadEntry> getRowGroupReadEntries() { return rowGroupReadEntries; } - public StorageEngineConfig getEngineConfig() { - return engineConfig; + @JsonProperty("storage") + public StoragePluginConfig getEngineConfig() { + return formatPlugin.getStorageConfig(); } @Override @@ -92,7 +103,6 @@ public class ParquetRowGroupScan extends AbstractBase implements SubScan { return null; } - public FieldReference getRef() { return ref; } @@ -108,8 +118,8 @@ public class ParquetRowGroupScan extends AbstractBase implements SubScan { } @JsonIgnore - public ParquetStorageEngine getStorageEngine(){ - return parquetStorageEngine; + public ParquetFormatPlugin getStorageEngine() { + return formatPlugin; } @Override @@ -120,8 +130,7 @@ public class ParquetRowGroupScan extends AbstractBase implements SubScan { @Override public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) throws ExecutionSetupException { Preconditions.checkArgument(children.isEmpty()); - return new ParquetRowGroupScan(parquetStorageEngine, (ParquetStorageEngineConfig) engineConfig, rowGroupReadEntries, - ref, columns); + return new ParquetRowGroupScan(formatPlugin, rowGroupReadEntries, ref, columns); } @Override @@ -133,36 +142,4 @@ public class ParquetRowGroupScan extends AbstractBase implements SubScan { return columns; } - public static class RowGroupReadEntry extends ReadEntryFromHDFS { - - private int rowGroupIndex; - - @parquet.org.codehaus.jackson.annotate.JsonCreator - public RowGroupReadEntry(@JsonProperty("path") String path, @JsonProperty("start") long start, - @JsonProperty("length") long length, @JsonProperty("rowGroupIndex") int rowGroupIndex) { - super(path, start, length); - this.rowGroupIndex = rowGroupIndex; - } - - @Override - public OperatorCost getCost() { - return new OperatorCost(1, 2, 1, 1); - } - - @Override - public Size getSize() { - // TODO - these values are wrong, I cannot know these until after I read a file - return new Size(10, 10); - } - - @JsonIgnore - public RowGroupReadEntry getRowGroupReadEntry() { - return new RowGroupReadEntry(this.getPath(), this.getStart(), this.getLength(), this.rowGroupIndex); - } - - public int getRowGroupIndex(){ - return rowGroupIndex; - } - } - } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java index 966a16ba5..17e7da266 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java @@ -26,7 +26,10 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import com.google.common.base.Stopwatch; + import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.common.expression.FieldReference; +import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.physical.impl.BatchCreator; import org.apache.drill.exec.physical.impl.ScanBatch; @@ -38,8 +41,10 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import parquet.hadoop.CodecFactoryExposer; import parquet.hadoop.ParquetFileReader; import parquet.hadoop.metadata.ParquetMetadata; @@ -50,9 +55,12 @@ public class ParquetScanBatchCreator implements BatchCreator<ParquetRowGroupScan public RecordBatch getBatch(FragmentContext context, ParquetRowGroupScan rowGroupScan, List<RecordBatch> children) throws ExecutionSetupException { Preconditions.checkArgument(children.isEmpty()); List<RecordReader> readers = Lists.newArrayList(); + + FileSystem fs = rowGroupScan.getStorageEngine().getFileSystem().getUnderlying(); + // keep footers in a map to avoid re-reading them Map<String, ParquetMetadata> footers = new HashMap<String, ParquetMetadata>(); - for(ParquetRowGroupScan.RowGroupReadEntry e : rowGroupScan.getRowGroupReadEntries()){ + for(RowGroupReadEntry e : rowGroupScan.getRowGroupReadEntries()){ /* Here we could store a map from file names to footers, to prevent re-reading the footer for each row group in a file TODO - to prevent reading the footer again in the parquet record reader (it is read earlier in the ParquetStorageEngine) @@ -63,11 +71,11 @@ public class ParquetScanBatchCreator implements BatchCreator<ParquetRowGroupScan try { if ( ! footers.containsKey(e.getPath())){ footers.put(e.getPath(), - ParquetFileReader.readFooter( rowGroupScan.getStorageEngine().getFileSystem().getConf(), new Path(e.getPath()))); + ParquetFileReader.readFooter( fs.getConf(), new Path(e.getPath()))); } readers.add( new ParquetRecordReader( - context, e.getPath(), e.getRowGroupIndex(), rowGroupScan.getStorageEngine().getFileSystem(), + context, e.getPath(), e.getRowGroupIndex(), fs, rowGroupScan.getStorageEngine().getCodecFactoryExposer(), footers.get(e.getPath()), rowGroupScan.getRef(), diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java deleted file mode 100644 index c17a9e356..000000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.parquet; - -import java.io.IOException; - -import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.exec.physical.ReadEntryWithPath; -import org.apache.drill.exec.store.ClassPathFileSystem; -import org.apache.drill.exec.store.SchemaProvider; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; - -import com.beust.jcommander.internal.Lists; - -public class ParquetSchemaProvider implements SchemaProvider{ - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetSchemaProvider.class); - - public static final String HADOOP_DEFAULT_NAME = "fs.default.name"; - final ParquetStorageEngineConfig configuration; - final FileSystem fs; - final Configuration conf; - - public ParquetSchemaProvider(ParquetStorageEngineConfig configuration, DrillConfig config){ - this.configuration = configuration; - try { - this.conf = new Configuration(); - this.conf.set("fs.classpath.impl", ClassPathFileSystem.class.getName()); - this.conf.set(HADOOP_DEFAULT_NAME, configuration.getDfsName()); - logger.debug("{}: {}",HADOOP_DEFAULT_NAME, configuration.getDfsName()); - this.fs = FileSystem.get(conf); - } catch (IOException ie) { - throw new RuntimeException("Error setting up filesystem", ie); - } - } - - @Override - public Object getSelectionBaseOnName(String tableName) { - try{ - if(!fs.exists(new Path(tableName))) return null; - ReadEntryWithPath re = new ReadEntryWithPath(tableName); - return Lists.newArrayList(re); - }catch(Exception e){ - logger.warn(String.format("Failure while checking table name %s.", tableName), e); - return null; - } - } -} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java deleted file mode 100644 index ad9756edd..000000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java +++ /dev/null @@ -1,123 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.parquet; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; - -import com.beust.jcommander.internal.Lists; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - -import org.apache.drill.common.exceptions.ExecutionSetupException; -import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.common.logical.StorageEngineConfig; -import org.apache.drill.common.logical.data.Scan; -import org.apache.drill.exec.ops.FragmentContext; -import org.apache.drill.exec.physical.ReadEntry; -import org.apache.drill.exec.physical.ReadEntryWithPath; -import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; -import org.apache.drill.exec.server.DrillbitContext; -import org.apache.drill.exec.store.AbstractStorageEngine; -import org.apache.drill.exec.store.RecordReader; -import org.apache.drill.exec.store.SchemaProvider; -import org.apache.drill.exec.store.mock.MockStorageEngine; - -import com.google.common.collect.ListMultimap; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.FileSystem; - -import parquet.format.converter.ParquetMetadataConverter; -import parquet.hadoop.CodecFactoryExposer; -import parquet.hadoop.ParquetFileReader; -import parquet.hadoop.metadata.BlockMetaData; -import parquet.hadoop.metadata.ColumnChunkMetaData; -import parquet.hadoop.metadata.ParquetMetadata; - -public class ParquetStorageEngine extends AbstractStorageEngine{ - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MockStorageEngine.class); - - private final DrillbitContext context; - static final ParquetMetadataConverter parquetMetadataConverter = new ParquetMetadataConverter(); - private CodecFactoryExposer codecFactoryExposer; - private final ParquetSchemaProvider schemaProvider; - private final ParquetStorageEngineConfig engineConfig; - - public ParquetStorageEngine(ParquetStorageEngineConfig configuration, DrillbitContext context){ - this.context = context; - this.schemaProvider = new ParquetSchemaProvider(configuration, context.getConfig()); - codecFactoryExposer = new CodecFactoryExposer(schemaProvider.conf); - this.engineConfig = configuration; - } - - public Configuration getHadoopConfig() { - double y = 5; - int x = (int) y; - return schemaProvider.conf; - } - - public FileSystem getFileSystem() { - return schemaProvider.fs; - } - - public ParquetStorageEngineConfig getEngineConfig() { - return engineConfig; - } - - public DrillbitContext getContext() { - return this.context; - } - - @Override - public boolean supportsRead() { - return true; - } - - @Override - public ParquetGroupScan getPhysicalScan(Scan scan) throws IOException { - - ArrayList<ReadEntryWithPath> readEntries = scan.getSelection().getListWith(new ObjectMapper(), - new TypeReference<ArrayList<ReadEntryWithPath>>() {}); - - return new ParquetGroupScan( readEntries, this, scan.getOutputReference(), null); - } - - @Override - public ListMultimap<ReadEntry, DrillbitEndpoint> getReadLocations(Collection<ReadEntry> entries) { - return null; - } - - @Override - public RecordReader getReader(FragmentContext context, ReadEntry readEntry) throws IOException { - return null; - } - - - public CodecFactoryExposer getCodecFactoryExposer() { - return codecFactoryExposer; - } - - @Override - public ParquetSchemaProvider getSchemaProvider() { - return schemaProvider; - } -}
\ No newline at end of file diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngineConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngineConfig.java deleted file mode 100644 index f2d612464..000000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngineConfig.java +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.parquet; - -import java.util.HashMap; - -import org.apache.drill.common.logical.StorageEngineConfigBase; -import org.apache.drill.exec.store.DistributedStorageEngine; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; - -@JsonTypeName("parquet") -public class ParquetStorageEngineConfig extends StorageEngineConfigBase implements DistributedStorageEngine{ - - public String getDfsName() { - return dfsName; - } - - // information needed to identify an HDFS instance - private String dfsName; - private HashMap<String,String> map; - - @JsonCreator - public ParquetStorageEngineConfig(@JsonProperty("dfsName") String dfsName) { - this.dfsName = dfsName; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - ParquetStorageEngineConfig that = (ParquetStorageEngineConfig) o; - - if (dfsName != null ? !dfsName.equals(that.dfsName) : that.dfsName != null) return false; - - return true; - } - - @Override - public int hashCode() { - return dfsName != null ? dfsName.hashCode() : 0; - } - public void set(String key, String value) { - map.put(key, value); - } - - public String get(String key) { - return map.get(key); - } -} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/RowGroupReadEntry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/RowGroupReadEntry.java new file mode 100644 index 000000000..986328eb7 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/RowGroupReadEntry.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.parquet; + +import org.apache.drill.exec.store.dfs.ReadEntryFromHDFS; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class RowGroupReadEntry extends ReadEntryFromHDFS { + + private int rowGroupIndex; + + @parquet.org.codehaus.jackson.annotate.JsonCreator + public RowGroupReadEntry(@JsonProperty("path") String path, @JsonProperty("start") long start, + @JsonProperty("length") long length, @JsonProperty("rowGroupIndex") int rowGroupIndex) { + super(path, start, length); + this.rowGroupIndex = rowGroupIndex; + } + + @JsonIgnore + public RowGroupReadEntry getRowGroupReadEntry() { + return new RowGroupReadEntry(this.getPath(), this.getStart(), this.getLength(), this.rowGroupIndex); + } + + public int getRowGroupIndex(){ + return rowGroupIndex; + } +}
\ No newline at end of file diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java index 0321838f3..d9e498efc 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java @@ -103,6 +103,10 @@ public class VarLenBinaryReader { do { lengthVarFieldsInCurrentRecord = 0; for (ColumnReader columnReader : columns) { + if (recordsReadInCurrentPass == columnReader.valueVecHolder.getValueVector().getValueCapacity()){ + rowGroupFinished = true; + break; + } if (columnReader.pageReadStatus.currentPage == null || columnReader.pageReadStatus.valuesRead == columnReader.pageReadStatus.currentPage.getValueCount()) { columnReader.totalValuesRead += columnReader.pageReadStatus.valuesRead; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AffinityCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AffinityCreator.java new file mode 100644 index 000000000..d25abad06 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AffinityCreator.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.schedule; + +import java.util.List; +import java.util.concurrent.TimeUnit; + +import org.apache.drill.exec.physical.EndpointAffinity; +import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; + +import com.beust.jcommander.internal.Lists; +import com.carrotsearch.hppc.ObjectFloatOpenHashMap; +import com.carrotsearch.hppc.cursors.ObjectFloatCursor; +import com.carrotsearch.hppc.cursors.ObjectLongCursor; +import com.google.common.base.Stopwatch; + +public class AffinityCreator { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AffinityCreator.class); + + public static <T extends CompleteWork> List<EndpointAffinity> getAffinityMap(List<T> work){ + Stopwatch watch = new Stopwatch(); + + long totalBytes = 0; + for (CompleteWork entry : work) { + totalBytes += entry.getTotalBytes(); + } + + ObjectFloatOpenHashMap<DrillbitEndpoint> affinities = new ObjectFloatOpenHashMap<DrillbitEndpoint>(); + for (CompleteWork entry : work) { + for (ObjectLongCursor<DrillbitEndpoint> cursor : entry.getByteMap()) { + long bytes = cursor.value; + float affinity = (float)bytes / (float)totalBytes; + logger.debug("Work: {} Endpoint: {} Bytes: {}", work, cursor.key.getAddress(), bytes); + affinities.putOrAdd(cursor.key, affinity, affinity); + } + } + + List<EndpointAffinity> affinityList = Lists.newLinkedList(); + for (ObjectFloatCursor<DrillbitEndpoint> d : affinities) { + logger.debug("Endpoint {} has affinity {}", d.key.getAddress(), d.value); + affinityList.add(new EndpointAffinity(d.key, d.value)); + } + + logger.debug("Took {} ms to get operator affinity", watch.elapsed(TimeUnit.MILLISECONDS)); + return affinityList; + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AssignmentCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AssignmentCreator.java new file mode 100644 index 000000000..eaa4f174e --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AssignmentCreator.java @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.schedule; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; + +import com.google.common.base.Preconditions; +import com.google.common.base.Stopwatch; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Lists; + +/** + * The AssignmentCreator is responsible for assigning a set of work units to the available slices. + */ +public class AssignmentCreator<T extends CompleteWork> { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AssignmentCreator.class); + + static final double[] ASSIGNMENT_CUTOFFS = { 0.99, 0.50, 0.25, 0.00 }; + private final ArrayListMultimap<Integer, T> mappings; + private final List<DrillbitEndpoint> endpoints; + + + + /** + * Given a set of endpoints to assign work to, attempt to evenly assign work based on affinity of work units to + * Drillbits. + * + * @param incomingEndpoints + * The set of nodes to assign work to. Note that nodes can be listed multiple times if we want to have + * multiple slices on a node working on the task simultaneously. + * @param units + * The work units to assign. + * @return ListMultimap of Integer > List<CompleteWork> (based on their incoming order) to with + */ + public static <T extends CompleteWork> ListMultimap<Integer, T> getMappings(List<DrillbitEndpoint> incomingEndpoints, + List<T> units) { + AssignmentCreator<T> creator = new AssignmentCreator<T>(incomingEndpoints, units); + return creator.mappings; + } + + private AssignmentCreator(List<DrillbitEndpoint> incomingEndpoints, List<T> units) { + Stopwatch watch = new Stopwatch(); + + Preconditions.checkArgument(incomingEndpoints.size() <= units.size(), String.format("Incoming endpoints %d " + + "is greater than number of row groups %d", incomingEndpoints.size(), units.size())); + this.mappings = ArrayListMultimap.create(); + this.endpoints = Lists.newLinkedList(incomingEndpoints); + + ArrayList<T> rowGroupList = new ArrayList<>(units); + for (double cutoff : ASSIGNMENT_CUTOFFS) { + scanAndAssign(rowGroupList, cutoff, false); + } + scanAndAssign(rowGroupList, 0.0, true); + + logger.debug("Took {} ms to apply assignments", watch.elapsed(TimeUnit.MILLISECONDS)); + Preconditions.checkState(rowGroupList.isEmpty(), "All readEntries should be assigned by now, but some are still unassigned"); + Preconditions.checkState(!units.isEmpty()); + + } + + /** + * + * @param mappings + * the mapping between fragment/endpoint and rowGroup + * @param endpoints + * the list of drillbits, ordered by the corresponding fragment + * @param workunits + * the list of rowGroups to assign + * @param requiredPercentage + * the percentage of max bytes required to make an assignment + * @param assignAll + * if true, will assign even if no affinity + */ + private void scanAndAssign(List<T> workunits, double requiredPercentage, boolean assignAll) { + Collections.sort(workunits); + int fragmentPointer = 0; + final boolean requireAffinity = requiredPercentage > 0; + int maxAssignments = (int) (workunits.size() / endpoints.size()); + + if (maxAssignments < 1) + maxAssignments = 1; + + for (Iterator<T> iter = workunits.iterator(); iter.hasNext();) { + T unit = iter.next(); + for (int i = 0; i < endpoints.size(); i++) { + int minorFragmentId = (fragmentPointer + i) % endpoints.size(); + DrillbitEndpoint currentEndpoint = endpoints.get(minorFragmentId); + EndpointByteMap endpointByteMap = unit.getByteMap(); + boolean haveAffinity = endpointByteMap.isSet(currentEndpoint); + + if (assignAll + || (!endpointByteMap.isEmpty() && (!requireAffinity || haveAffinity) + && (!mappings.containsKey(minorFragmentId) || mappings.get(minorFragmentId).size() < maxAssignments) && (!requireAffinity || endpointByteMap + .get(currentEndpoint) >= endpointByteMap.getMaxBytes() * requiredPercentage))) { + + mappings.put(minorFragmentId, unit); + // logger.debug("Assigned rowGroup {} to minorFragmentId {} endpoint {}", rowGroupInfo.getRowGroupIndex(), + // minorFragmentId, endpoints.get(minorFragmentId).getAddress()); + // if (bytesPerEndpoint.get(currentEndpoint) != null) { + // // assignmentAffinityStats.update(bytesPerEndpoint.get(currentEndpoint) / rowGroupInfo.getLength()); + // } else { + // // assignmentAffinityStats.update(0); + // } + iter.remove(); + fragmentPointer = (minorFragmentId + 1) % endpoints.size(); + break; + } + } + + } + } + +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/BlockMapBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/BlockMapBuilder.java new file mode 100644 index 000000000..432c1d764 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/BlockMapBuilder.java @@ -0,0 +1,215 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.schedule; + + +import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; + +import org.apache.drill.exec.metrics.DrillMetrics; +import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; +import org.apache.drill.exec.store.dfs.easy.FileWork; +import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import com.beust.jcommander.internal.Lists; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.Timer; +import com.google.common.base.Stopwatch; +import com.google.common.collect.ImmutableRangeMap; +import com.google.common.collect.Range; + +public class BlockMapBuilder { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BlockMapBuilder.class); + static final MetricRegistry metrics = DrillMetrics.getInstance(); + static final String BLOCK_MAP_BUILDER_TIMER = MetricRegistry.name(BlockMapBuilder.class, "blockMapBuilderTimer"); + + private HashMap<Path,ImmutableRangeMap<Long,BlockLocation>> blockMapMap = new HashMap<>(); + private Collection<DrillbitEndpoint> endpoints; + private FileSystem fs; + private HashMap<String,DrillbitEndpoint> endPointMap; + + public BlockMapBuilder(FileSystem fs, Collection<DrillbitEndpoint> endpoints) { + this.fs = fs; + this.endpoints = endpoints; + buildEndpointMap(); + } + + + public List<CompleteFileWork> generateFileWork(List<FileStatus> files, boolean blockify) throws IOException{ + List<CompleteFileWork> work = Lists.newArrayList(); + for(FileStatus f : files){ + ImmutableRangeMap<Long,BlockLocation> rangeMap = getBlockMap(f); + if(!blockify){ + work.add(new CompleteFileWork(this.getEndpointByteMap(new FileStatusWork(f)), 0, f.getLen(), f.getPath().toString())); + continue; + } + + for(Entry<Range<Long>, BlockLocation> l : rangeMap.asMapOfRanges().entrySet()){ + work.add(new CompleteFileWork(this.getEndpointByteMap(new FileStatusWork(f)), l.getValue().getOffset(), l.getValue().getLength(), f.getPath().toString())); + } + } + return work; + } + + private class FileStatusWork implements FileWork{ + private FileStatus status; + + public FileStatusWork(FileStatus status) { + if(status.isDir()) throw new IllegalStateException("FileStatus work only works with files, not directories."); + this.status = status; + } + + @Override + public String getPath() { + return status.getPath().toString(); + } + + @Override + public long getStart() { + return 0; + } + + @Override + public long getLength() { + return status.getLen(); + } + + + + } + + private ImmutableRangeMap<Long,BlockLocation> buildBlockMap(Path path) throws IOException { + FileStatus status = fs.getFileStatus(path); + return buildBlockMap(status); + } + + /** + * Builds a mapping of block locations to file byte range + */ + private ImmutableRangeMap<Long,BlockLocation> buildBlockMap(FileStatus status) throws IOException { + final Timer.Context context = metrics.timer(BLOCK_MAP_BUILDER_TIMER).time(); + BlockLocation[] blocks; + ImmutableRangeMap<Long,BlockLocation> blockMap; + blocks = fs.getFileBlockLocations(status, 0 , status.getLen()); + ImmutableRangeMap.Builder<Long, BlockLocation> blockMapBuilder = new ImmutableRangeMap.Builder<Long,BlockLocation>(); + for (BlockLocation block : blocks) { + long start = block.getOffset(); + long end = start + block.getLength(); + Range<Long> range = Range.closedOpen(start, end); + blockMapBuilder = blockMapBuilder.put(range, block); + } + blockMap = blockMapBuilder.build(); + blockMapMap.put(status.getPath(), blockMap); + context.stop(); + return blockMap; + } + + private ImmutableRangeMap<Long,BlockLocation> getBlockMap(Path path) throws IOException{ + ImmutableRangeMap<Long,BlockLocation> blockMap = blockMapMap.get(path); + if(blockMap == null){ + blockMap = buildBlockMap(path); + } + return blockMap; + } + + private ImmutableRangeMap<Long,BlockLocation> getBlockMap(FileStatus status) throws IOException{ + ImmutableRangeMap<Long,BlockLocation> blockMap = blockMapMap.get(status.getPath()); + if(blockMap == null){ + blockMap = buildBlockMap(status); + } + return blockMap; + } + + + /** + * For a given FileWork, calculate how many bytes are available on each on drillbit endpoint + * + * @param work the FileWork to calculate endpoint bytes for + * @throws IOException + */ + public EndpointByteMap getEndpointByteMap(FileWork work) throws IOException { + Stopwatch watch = new Stopwatch(); + watch.start(); + Path fileName = new Path(work.getPath()); + + + ImmutableRangeMap<Long,BlockLocation> blockMap = getBlockMap(fileName); + EndpointByteMapImpl endpointByteMap = new EndpointByteMapImpl(); + long start = work.getStart(); + long end = start + work.getLength(); + Range<Long> rowGroupRange = Range.closedOpen(start, end); + + // Find submap of ranges that intersect with the rowGroup + ImmutableRangeMap<Long,BlockLocation> subRangeMap = blockMap.subRangeMap(rowGroupRange); + + // Iterate through each block in this submap and get the host for the block location + for (Map.Entry<Range<Long>,BlockLocation> block : subRangeMap.asMapOfRanges().entrySet()) { + String[] hosts; + Range<Long> blockRange = block.getKey(); + try { + hosts = block.getValue().getHosts(); + } catch (IOException ioe) { + throw new RuntimeException("Failed to get hosts for block location", ioe); + } + Range<Long> intersection = rowGroupRange.intersection(blockRange); + long bytes = intersection.upperEndpoint() - intersection.lowerEndpoint(); + + // For each host in the current block location, add the intersecting bytes to the corresponding endpoint + for (String host : hosts) { + DrillbitEndpoint endpoint = getDrillBitEndpoint(host); + if(endpoint != null){ + endpointByteMap.add(endpoint, bytes); + }else{ + logger.debug("Failure finding Drillbit running on host {}. Skipping affinity to that host.", host); + } + } + } + + logger.debug("FileWork group ({},{}) max bytes {}", work.getPath(), work.getStart(), endpointByteMap.getMaxBytes()); + + logger.debug("Took {} ms to set endpoint bytes", watch.stop().elapsed(TimeUnit.MILLISECONDS)); + return endpointByteMap; + } + + private DrillbitEndpoint getDrillBitEndpoint(String hostName) { + return endPointMap.get(hostName); + } + + /** + * Builds a mapping of Drillbit endpoints to hostnames + */ + private void buildEndpointMap() { + Stopwatch watch = new Stopwatch(); + watch.start(); + endPointMap = new HashMap<String, DrillbitEndpoint>(); + for (DrillbitEndpoint d : endpoints) { + String hostName = d.getAddress(); + endPointMap.put(hostName, d); + } + watch.stop(); + logger.debug("Took {} ms to build endpoint map", watch.elapsed(TimeUnit.MILLISECONDS)); + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/CompleteFileWork.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/CompleteFileWork.java new file mode 100644 index 000000000..30b08f6fa --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/CompleteFileWork.java @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.schedule; + +import org.apache.drill.exec.store.dfs.easy.FileWork; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class CompleteFileWork implements FileWork, CompleteWork{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CompleteFileWork.class); + + private long start; + private long length; + private String path; + private EndpointByteMap byteMap; + + public CompleteFileWork(EndpointByteMap byteMap, long start, long length, String path) { + super(); + this.start = start; + this.length = length; + this.path = path; + this.byteMap = byteMap; + } + + @Override + public int compareTo(CompleteWork o) { + return Long.compare(getTotalBytes(), o.getTotalBytes()); + } + + @Override + public long getTotalBytes() { + return length; + } + + @Override + public EndpointByteMap getByteMap() { + return byteMap; + } + + @Override + public String getPath() { + return path; + } + + @Override + public long getStart() { + return start; + } + + @Override + public long getLength() { + return length; + } + + public FileWorkImpl getAsFileWork(){ + return new FileWorkImpl(start, length, path); + } + + public static class FileWorkImpl implements FileWork{ + + @JsonCreator + public FileWorkImpl(@JsonProperty("start") long start, @JsonProperty("length") long length, @JsonProperty("path") String path) { + super(); + this.start = start; + this.length = length; + this.path = path; + } + + public long start; + public long length; + public String path; + + @Override + public String getPath() { + return path; + } + + @Override + public long getStart() { + return start; + } + + @Override + public long getLength() { + return length; + } + + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/CompleteWork.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/CompleteWork.java new file mode 100644 index 000000000..44e27d45e --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/CompleteWork.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.schedule; + + +/** + * Container that holds a complete work unit. Can contain one or more partial units. + */ +public interface CompleteWork extends Comparable<CompleteWork>{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CompleteWork.class); + + public long getTotalBytes(); + public EndpointByteMap getByteMap(); +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/EndpointByteMap.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/EndpointByteMap.java new file mode 100644 index 000000000..f543d7523 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/EndpointByteMap.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.schedule; + +import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; + +import com.carrotsearch.hppc.cursors.ObjectLongCursor; + +/** + * Presents an interface that describes the number of bytes for a particular work unit associated with a particular DrillbitEndpoint. + */ +public interface EndpointByteMap extends Iterable<ObjectLongCursor<DrillbitEndpoint>>{ + + public boolean isSet(DrillbitEndpoint endpoint); + public long get(DrillbitEndpoint endpoint); + public boolean isEmpty(); + public long getMaxBytes(); + public void add(DrillbitEndpoint endpoint, long bytes); +}
\ No newline at end of file diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/EndpointByteMapImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/EndpointByteMapImpl.java new file mode 100644 index 000000000..64e52eb0c --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/EndpointByteMapImpl.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.schedule; + +import java.util.Iterator; + +import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; + +import com.carrotsearch.hppc.ObjectLongOpenHashMap; +import com.carrotsearch.hppc.cursors.ObjectLongCursor; + +public class EndpointByteMapImpl implements EndpointByteMap{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(EndpointByteMapImpl.class); + + private final ObjectLongOpenHashMap<DrillbitEndpoint> map = new ObjectLongOpenHashMap<>(); + + private long maxBytes; + + public boolean isSet(DrillbitEndpoint endpoint){ + return map.containsKey(endpoint); + } + + public long get(DrillbitEndpoint endpoint){ + return map.get(endpoint); + } + + public boolean isEmpty(){ + return map.isEmpty(); + } + + public void add(DrillbitEndpoint endpoint, long bytes){ + assert endpoint != null; + maxBytes = Math.max(maxBytes, map.putOrAdd(endpoint, bytes, bytes)+1); + } + + public long getMaxBytes() { + return maxBytes; + } + + @Override + public Iterator<ObjectLongCursor<DrillbitEndpoint>> iterator() { + return map.iterator(); + } + + +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/PartialWork.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/PartialWork.java new file mode 100644 index 000000000..bb8d9508b --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/PartialWork.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.schedule; + +import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; + +public class PartialWork { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PartialWork.class); + + private final long length; + private final DrillbitEndpoint[] locations; + + public PartialWork(long length, DrillbitEndpoint[] locations) { + super(); + this.length = length; + this.locations = locations; + } + + public long getLength() { + return length; + } + public DrillbitEndpoint[] getLocations() { + return locations; + } + + + +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java index 329815d20..baecc3fb8 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java @@ -346,7 +346,7 @@ public class Foreman implements Runnable, Closeable, Comparable<Object>{ private void runSQL(String sql) { try{ - DrillSqlWorker sqlWorker = new DrillSqlWorker(context.getSchemaFactory(), context.getFunctionRegistry()); + DrillSqlWorker sqlWorker = new DrillSqlWorker(context.getFactory(), context.getFunctionRegistry()); LogicalPlan plan = sqlWorker.getPlan(sql); diff --git a/exec/java-exec/src/main/resources/storage-engines.json b/exec/java-exec/src/main/resources/storage-engines.json deleted file mode 100644 index d1d0413a8..000000000 --- a/exec/java-exec/src/main/resources/storage-engines.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "storage":{ - "parquet-local" : - { - "type":"parquet", - "dfsName" : "file:///" - }, - "parquet-cp" : - { - "type":"parquet", - "dfsName" : "classpath:///" - }, - "jsonl" : - { - "type":"json", - "dfsName" : "file:///" - }, - "json-cp" : - { - "type":"json", - "dfsName" : "classpath:///" - }, - "parquet" : - { - "type":"parquet", - "dfsName" : "file:///" - } - } -}
\ No newline at end of file diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java index 2dd861131..62895095a 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java @@ -55,7 +55,7 @@ import org.apache.drill.exec.server.BootStrapContext; import org.apache.drill.exec.server.Drillbit; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.server.RemoteServiceSet; -import org.apache.drill.exec.store.StorageEngineRegistry; +import org.apache.drill.exec.store.StoragePluginRegistry; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.VarBinaryVector; import org.junit.AfterClass; @@ -304,7 +304,7 @@ public class TestOptiqPlans { } }; - StorageEngineRegistry reg = new StorageEngineRegistry(bitContext); + StoragePluginRegistry reg = new StoragePluginRegistry(bitContext); PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), reg); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java index 493fea6e2..27fae088d 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java @@ -43,7 +43,7 @@ import org.apache.drill.exec.proto.ExecProtos; import org.apache.drill.exec.proto.BitControl.PlanFragment; import org.apache.drill.exec.rpc.user.UserServer; import org.apache.drill.exec.server.DrillbitContext; -import org.apache.drill.exec.store.StorageEngineRegistry; +import org.apache.drill.exec.store.StoragePluginRegistry; import org.apache.drill.exec.vector.ValueVector; import org.junit.AfterClass; import org.junit.Test; @@ -119,7 +119,7 @@ public class TestMergeJoin { bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); }}; - PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(),CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), new StorageEngineRegistry(bitContext)); + PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(),CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), new StoragePluginRegistry(bitContext)); PhysicalPlan plan = reader.readPhysicalPlan( Files.toString( FileUtils.getResourceAsFile("/join/merge_single_batch.json"), Charsets.UTF_8) @@ -172,7 +172,7 @@ public class TestMergeJoin { bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); }}; - PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(),CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), new StorageEngineRegistry(bitContext)); + PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(),CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), new StoragePluginRegistry(bitContext)); PhysicalPlan plan = reader.readPhysicalPlan( Files.toString( FileUtils.getResourceAsFile("/join/merge_inner_single_batch.json"), Charsets.UTF_8) @@ -225,7 +225,7 @@ public class TestMergeJoin { bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); }}; - PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(),CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), new StorageEngineRegistry(bitContext)); + PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(),CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), new StoragePluginRegistry(bitContext)); PhysicalPlan plan = reader.readPhysicalPlan( Files.toString( FileUtils.getResourceAsFile("/join/merge_multi_batch.json"), Charsets.UTF_8) diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/CachedSingleFileSystem.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/CachedSingleFileSystem.java new file mode 100644 index 000000000..46a278583 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/CachedSingleFileSystem.java @@ -0,0 +1,171 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufInputStream; +import io.netty.buffer.UnpooledByteBufAllocator; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PositionedReadable; +import org.apache.hadoop.fs.Seekable; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.util.Progressable; + +public class CachedSingleFileSystem extends FileSystem{ + + private ByteBuf file; + private String path; + + public CachedSingleFileSystem(String path) throws IOException{ + this.path = path; + File f = new File(path); + long length = f.length(); + if(length > Integer.MAX_VALUE) throw new UnsupportedOperationException("Cached file system only supports files of less than 2GB."); + System.out.println(length); + try(InputStream is = new BufferedInputStream(new FileInputStream(path))){ + byte[] buffer = new byte[64*1024]; + this.file = UnpooledByteBufAllocator.DEFAULT.directBuffer((int) length); + int read; + while( (read = is.read(buffer)) > 0){ + file.writeBytes(buffer, 0, read); + } + } + } + + public void close() throws IOException{ + file.release(); + super.close(); + } + + @Override + public FSDataOutputStream append(Path arg0, int arg1, Progressable arg2) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public FSDataOutputStream create(Path arg0, FsPermission arg1, boolean arg2, int arg3, short arg4, long arg5, + Progressable arg6) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public boolean delete(Path arg0) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public boolean delete(Path arg0, boolean arg1) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public FileStatus getFileStatus(Path arg0) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public URI getUri() { + throw new UnsupportedOperationException(); + } + + @Override + public Path getWorkingDirectory() { + throw new UnsupportedOperationException(); + } + + @Override + public FileStatus[] listStatus(Path arg0) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public boolean mkdirs(Path path, FsPermission arg1) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public FSDataInputStream open(Path path, int arg1) throws IOException { + if(!path.toString().equals(this.path)) throw new IOException(String.format("You requested file %s but this cached single file system only has the file %s.", path.toString(), this.path)); + return new FSDataInputStream(new CachedFSDataInputStream(file.slice())); + } + + @Override + public boolean rename(Path arg0, Path arg1) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void setWorkingDirectory(Path arg0) { + throw new UnsupportedOperationException(); + } + + + private class CachedFSDataInputStream extends ByteBufInputStream implements Seekable, PositionedReadable{ + private ByteBuf buf; + public CachedFSDataInputStream(ByteBuf buffer) { + super(buffer); + this.buf = buffer; + + } + + @Override + public long getPos() throws IOException { + return buf.readerIndex(); + } + + @Override + public void seek(long arg0) throws IOException { + buf.readerIndex((int) arg0); + } + + @Override + public boolean seekToNewSource(long arg0) throws IOException { + return false; + } + + @Override + public int read(long pos, byte[] buffer, int offset, int length) throws IOException { + ByteBuf local = buf.slice( (int) pos, (int) Math.min( buf.capacity() - pos, length)); + local.readBytes(buffer, offset, buf.capacity()); + return buf.capacity(); + } + + @Override + public void readFully(long pos, byte[] buffer) throws IOException { + readFully(pos, buffer, 0, buffer.length); + } + + @Override + public void readFully(long pos, byte[] buffer, int offset, int length) throws IOException { + if(length + pos > buf.capacity()) throw new IOException("Read was too big."); + read(pos, buffer, offset, length); + } + } +}
\ No newline at end of file diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java index 574e847fd..67b11cbf9 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java @@ -19,15 +19,18 @@ package org.apache.drill.exec.store; import com.google.common.collect.ImmutableRangeMap; import com.google.common.collect.Range; + import junit.framework.Assert; import mockit.Injectable; import mockit.NonStrictExpectations; + import org.apache.drill.exec.physical.EndpointAffinity; import org.apache.drill.exec.proto.CoordinationProtos; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.store.parquet.ParquetGroupScan; import org.apache.drill.exec.store.parquet.ParquetRowGroupScan; -import org.apache.drill.exec.store.parquet.ParquetStorageEngine; +import org.apache.drill.exec.store.parquet.ParquetFormatPlugin; +import org.apache.drill.exec.store.schedule.BlockMapBuilder; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -100,60 +103,60 @@ public class TestAffinityCalculator { return endPoints; } - @Test - public void testSetEndpointBytes(@Injectable final FileSystem fs, @Injectable final FileStatus file) throws Throwable{ - final long blockSize = 256*1024*1024; - LinkedList<ParquetGroupScan.RowGroupInfo> rowGroups = new LinkedList<>(); - int numberOfHosts = 4; - int numberOfBlocks = 3; - String port = "1234"; - String[] hosts = new String[numberOfHosts]; - - final BlockLocation[] blockLocations = buildBlockLocations(hosts, blockSize); - final LinkedList<CoordinationProtos.DrillbitEndpoint> endPoints = buildEndpoints(numberOfHosts); - buildRowGroups(rowGroups, numberOfBlocks, blockSize, 3); - - new NonStrictExpectations() {{ - fs.getFileBlockLocations(file, 0, 3*blockSize); result = blockLocations; - fs.getFileStatus(new Path(path)); result = file; - file.getLen(); result = 3*blockSize; - }}; - - - AffinityCalculator ac = new AffinityCalculator(fs, endPoints); - for (ParquetGroupScan.RowGroupInfo rowGroup : rowGroups) { - ac.setEndpointBytes(rowGroup); - } - ParquetGroupScan.RowGroupInfo rg = rowGroups.get(0); - Long b = rg.getEndpointBytes().get(endPoints.get(0)); - assertEquals(blockSize,b.longValue()); - b = rg.getEndpointBytes().get(endPoints.get(3)); - assertNull(b); - - buildRowGroups(rowGroups, numberOfBlocks, blockSize, 2); - - ac = new AffinityCalculator(fs, endPoints); - for (ParquetGroupScan.RowGroupInfo rowGroup : rowGroups) { - ac.setEndpointBytes(rowGroup); - } - rg = rowGroups.get(0); - b = rg.getEndpointBytes().get(endPoints.get(0)); - assertEquals(blockSize*3/2,b.longValue()); - b = rg.getEndpointBytes().get(endPoints.get(3)); - assertEquals(blockSize / 2, b.longValue()); - - buildRowGroups(rowGroups, numberOfBlocks, blockSize, 6); - - ac = new AffinityCalculator(fs, endPoints); - for (ParquetGroupScan.RowGroupInfo rowGroup : rowGroups) { - ac.setEndpointBytes(rowGroup); - } - rg = rowGroups.get(0); - b = rg.getEndpointBytes().get(endPoints.get(0)); - assertEquals(blockSize/2,b.longValue()); - b = rg.getEndpointBytes().get(endPoints.get(3)); - assertNull(b); - } +// @Test +// public void testSetEndpointBytes(@Injectable final FileSystem fs, @Injectable final FileStatus file) throws Throwable{ +// final long blockSize = 256*1024*1024; +// LinkedList<ParquetGroupScan.RowGroupInfo> rowGroups = new LinkedList<>(); +// int numberOfHosts = 4; +// int numberOfBlocks = 3; +// String port = "1234"; +// String[] hosts = new String[numberOfHosts]; +// +// final BlockLocation[] blockLocations = buildBlockLocations(hosts, blockSize); +// final LinkedList<CoordinationProtos.DrillbitEndpoint> endPoints = buildEndpoints(numberOfHosts); +// buildRowGroups(rowGroups, numberOfBlocks, blockSize, 3); +// +// new NonStrictExpectations() {{ +// fs.getFileBlockLocations(file, 0, 3*blockSize); result = blockLocations; +// fs.getFileStatus(new Path(path)); result = file; +// file.getLen(); result = 3*blockSize; +// }}; +// +// +// BlockMapBuilder ac = new BlockMapBuilder(fs, endPoints); +// for (ParquetGroupScan.RowGroupInfo rowGroup : rowGroups) { +// ac.setEndpointBytes(rowGroup); +// } +// ParquetGroupScan.RowGroupInfo rg = rowGroups.get(0); +// Long b = rg.getEndpointBytes().get(endPoints.get(0)); +// assertEquals(blockSize,b.longValue()); +// b = rg.getEndpointBytes().get(endPoints.get(3)); +// assertNull(b); +// +// buildRowGroups(rowGroups, numberOfBlocks, blockSize, 2); +// +// ac = new BlockMapBuilder(fs, endPoints); +// for (ParquetGroupScan.RowGroupInfo rowGroup : rowGroups) { +// ac.setEndpointBytes(rowGroup); +// } +// rg = rowGroups.get(0); +// b = rg.getEndpointBytes().get(endPoints.get(0)); +// assertEquals(blockSize*3/2,b.longValue()); +// b = rg.getEndpointBytes().get(endPoints.get(3)); +// assertEquals(blockSize / 2, b.longValue()); +// +// buildRowGroups(rowGroups, numberOfBlocks, blockSize, 6); +// +// ac = new BlockMapBuilder(fs, endPoints); +// for (ParquetGroupScan.RowGroupInfo rowGroup : rowGroups) { +// ac.setEndpointBytes(rowGroup); +// } +// rg = rowGroups.get(0); +// b = rg.getEndpointBytes().get(endPoints.get(0)); +// assertEquals(blockSize/2,b.longValue()); +// b = rg.getEndpointBytes().get(endPoints.get(3)); +// assertNull(b); +// } @Test public void testBuildRangeMap() { diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java new file mode 100644 index 000000000..51a0b0b08 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store; + +import java.util.Iterator; +import java.util.Map; + +import org.apache.drill.exec.exception.SchemaChangeException; +import org.apache.drill.exec.physical.impl.OutputMutator; +import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; +import org.apache.drill.exec.record.MaterializedField; +import org.apache.drill.exec.record.VectorContainer; +import org.apache.drill.exec.record.VectorWrapper; +import org.apache.drill.exec.vector.ValueVector; + +import com.google.common.collect.Maps; + +public class TestOutputMutator implements OutputMutator, Iterable<VectorWrapper<?>> { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestOutputMutator.class); + + private final VectorContainer container = new VectorContainer(); + private final Map<MaterializedField, ValueVector> fieldVectorMap = Maps.newHashMap(); + + public void removeField(MaterializedField field) throws SchemaChangeException { + ValueVector vector = fieldVectorMap.remove(field); + if (vector == null) + throw new SchemaChangeException("Failure attempting to remove an unknown field."); + container.remove(vector); + vector.close(); + } + + public void addField(ValueVector vector) { + container.add(vector); + fieldVectorMap.put(vector.getField(), vector); + } + + @Override + public void removeAllFields() { + for (VectorWrapper<?> vw : container) { + vw.clear(); + } + container.clear(); + fieldVectorMap.clear(); + } + + @Override + public void setNewSchema() throws SchemaChangeException { + container.buildSchema(SelectionVectorMode.NONE); + } + + public Iterator<VectorWrapper<?>> iterator() { + return container.iterator(); + } + + public void clear(){ + removeAllFields(); + } + +} diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JSONRecordReaderTest.java index 62b9dfa22..41960b681 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JSONRecordReaderTest.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store; +package org.apache.drill.exec.store.json; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -45,7 +45,7 @@ import org.apache.drill.exec.physical.impl.OutputMutator; import org.apache.drill.exec.proto.SchemaDefProtos; import org.apache.drill.exec.proto.UserBitShared; import org.apache.drill.exec.record.MaterializedField; -import org.apache.drill.exec.store.json.JSONRecordReader; +import org.apache.drill.exec.store.easy.json.JSONRecordReader; import org.apache.drill.exec.vector.ValueVector; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -204,7 +204,7 @@ public class JSONRecordReaderTest { new Expectations() { { context.getAllocator(); - returns(new DirectBufferAllocator()); + returns(new TopLevelAllocator()); } }; diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/FieldInfo.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/FieldInfo.java new file mode 100644 index 000000000..34f60ba23 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/FieldInfo.java @@ -0,0 +1,40 @@ +/******************************************************************************* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.drill.exec.store.parquet; + +import org.apache.drill.common.types.TypeProtos; + +public class FieldInfo { + String parquetType; + String name; + int bitLength; + int numberOfPages; + Object[] values; + TypeProtos.MinorType type; + + FieldInfo(String parquetType, String name, int bitLength, Object[] values, TypeProtos.MinorType type, ParquetTestProperties props){ + this.parquetType = parquetType; + this.name = name; + this.bitLength = bitLength; + this.numberOfPages = Math.max(1, (int) Math.ceil( ((long) props.recordsPerRowGroup) * bitLength / 8.0 / props.bytesPerPage)); + this.values = values; + // generator is designed to use 3 values + assert values.length == 3; + this.type = type; + } +} diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java index 52430e132..9e999e344 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java @@ -17,38 +17,45 @@ */ package org.apache.drill.exec.store.parquet; +import static org.apache.drill.exec.store.parquet.TestFileGenerator.populateFieldInfoMap; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import java.io.File; import java.io.IOException; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import mockit.Injectable; + import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.expression.ExpressionPosition; +import org.apache.drill.common.expression.FieldReference; +import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.client.DrillClient; import org.apache.drill.exec.exception.SchemaChangeException; +import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; +import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.physical.impl.OutputMutator; -import org.apache.drill.exec.proto.UserBitShared.QueryId; +import org.apache.drill.exec.proto.BitControl.PlanFragment; import org.apache.drill.exec.proto.UserProtos; import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.record.RecordBatchLoader; -import org.apache.drill.exec.record.VectorWrapper; -import org.apache.drill.exec.rpc.RpcException; -import org.apache.drill.exec.rpc.user.ConnectionThrottle; -import org.apache.drill.exec.rpc.user.QueryResultBatch; -import org.apache.drill.exec.rpc.user.UserResultsListener; +import org.apache.drill.exec.rpc.user.UserServer; import org.apache.drill.exec.server.Drillbit; +import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.server.RemoteServiceSet; -import org.apache.drill.exec.store.parquet.TestFileGenerator.FieldInfo; -import org.apache.drill.exec.vector.BaseDataValueVector; +import org.apache.drill.exec.store.CachedSingleFileSystem; +import org.apache.drill.exec.store.TestOutputMutator; import org.apache.drill.exec.vector.ValueVector; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; @@ -57,7 +64,9 @@ import parquet.bytes.BytesInput; import parquet.column.page.Page; import parquet.column.page.PageReadStore; import parquet.column.page.PageReader; +import parquet.hadoop.CodecFactoryExposer; import parquet.hadoop.Footer; +import parquet.hadoop.ParquetFileReader; import parquet.hadoop.metadata.ParquetMetadata; import parquet.schema.MessageType; @@ -65,30 +74,33 @@ import com.google.common.base.Charsets; import com.google.common.base.Stopwatch; import com.google.common.collect.Lists; import com.google.common.io.Files; -import com.google.common.util.concurrent.SettableFuture; public class ParquetRecordReaderTest { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetRecordReaderTest.class); - private boolean VERBOSE_DEBUG = false; + static boolean VERBOSE_DEBUG = false; private boolean checkValues = true; static final int numberRowGroups = 1; static final int recordsPerRowGroup = 300; + static int DEFAULT_BYTES_PER_PAGE = 1024 * 1024 * 1; static final String fileName = "/tmp/parquet_test_file_many_types"; @BeforeClass public static void generateFile() throws Exception{ File f = new File(fileName); - if(!f.exists()) TestFileGenerator.generateParquetFile(fileName, numberRowGroups, recordsPerRowGroup); + ParquetTestProperties props = new ParquetTestProperties(numberRowGroups, recordsPerRowGroup, DEFAULT_BYTES_PER_PAGE, new HashMap<String, FieldInfo>()); + populateFieldInfoMap(props); + if(!f.exists()) TestFileGenerator.generateParquetFile(fileName, props); } - + + @Test - public void testMultipleRowGroupsAndReads() throws Exception { + public void testMultipleRowGroupsAndReads3() throws Exception { String planName = "/parquet/parquet_scan_screen.json"; testParquetFullEngineLocalPath(planName, fileName, 2, numberRowGroups, recordsPerRowGroup); } - + @Test public void testMultipleRowGroupsAndReads2() throws Exception { String readEntries; @@ -96,7 +108,7 @@ public class ParquetRecordReaderTest { // number of times to read the file int i = 3; for (int j = 0; j < i; j++){ - readEntries += "{path: \""+fileName+"\"}"; + readEntries += "\""+fileName+"\""; if (j < i - 1) readEntries += ","; } @@ -117,150 +129,6 @@ public class ParquetRecordReaderTest { testParquetFullEngineRemote(planName, fileName, 1, numberRowGroups, recordsPerRowGroup); } - - private class ParquetResultListener implements UserResultsListener { - private SettableFuture<Void> future = SettableFuture.create(); - RecordBatchLoader batchLoader; - - int batchCounter = 1; - private final HashMap<String, Long> valuesChecked = new HashMap<>(); - private final Map<String, FieldInfo> fields; - private final long totalRecords; - - ParquetResultListener(int recordsPerRowGroup, RecordBatchLoader batchLoader, int numberRowGroups, int numberOfTimesRead){ - this.batchLoader = batchLoader; - this.fields = TestFileGenerator.getFieldMap(recordsPerRowGroup); - this.totalRecords = recordsPerRowGroup * numberRowGroups * numberOfTimesRead; - } - - @Override - public void submissionFailed(RpcException ex) { - logger.debug("Submission failed.", ex); - future.setException(ex); - } - - @Override - public void resultArrived(QueryResultBatch result, ConnectionThrottle throttle) { - long columnValCounter = 0; - int i = 0; - FieldInfo currentField; - - boolean schemaChanged = false; - try { - schemaChanged = batchLoader.load(result.getHeader().getDef(), result.getData()); - } catch (SchemaChangeException e) { - logger.error("Failure while loading batch", e); - } - - // print headers. - if (schemaChanged) { - } // do not believe any change is needed for when the schema changes, with the current mock scan use case - - for (VectorWrapper<?> vw : batchLoader) { - ValueVector vv = vw.getValueVector(); - currentField = fields.get(vv.getField().getName()); - if (VERBOSE_DEBUG){ - System.out.println("\n" + (String) currentField.name); - } - if ( ! valuesChecked.containsKey(vv.getField().getName())){ - valuesChecked.put(vv.getField().getName(), (long) 0); - columnValCounter = 0; - } else { - columnValCounter = valuesChecked.get(vv.getField().getName()); - } - for (int j = 0; j < ((BaseDataValueVector)vv).getAccessor().getValueCount(); j++) { - if (VERBOSE_DEBUG){ - System.out.print(vv.getAccessor().getObject(j) + ", " + (j % 25 == 0 ? "\n batch:" + batchCounter + " v:" + j + " - " : "")); - } - if (checkValues) { - try { - assertField(vv, j, (TypeProtos.MinorType) currentField.type, - currentField.values[(int) (columnValCounter % 3)], (String) currentField.name + "/"); - } catch (AssertionError e) { submissionFailed(new RpcException(e)); } - } - columnValCounter++; - } - if (VERBOSE_DEBUG){ - System.out.println("\n" + ((BaseDataValueVector)vv).getAccessor().getValueCount()); - } - valuesChecked.remove(vv.getField().getName()); - valuesChecked.put(vv.getField().getName(), columnValCounter); - } - - - if (VERBOSE_DEBUG){ - for (i = 0; i < batchLoader.getRecordCount(); i++) { - if (i % 50 == 0){ - System.out.println(); - for (VectorWrapper<?> vw : batchLoader) { - ValueVector v = vw.getValueVector(); - System.out.print(pad(v.getField().getName(), 20) + " "); - } - System.out.println(); - System.out.println(); - } - - for (VectorWrapper<?> vw : batchLoader) { - ValueVector v = vw.getValueVector(); - System.out.print(pad(v.getAccessor().getObject(i).toString(), 20) + " "); - } - System.out.println( - - ); - } - } - - for(VectorWrapper<?> vw : batchLoader){ - vw.clear(); - } - result.release(); - - batchCounter++; - if(result.getHeader().getIsLastChunk()){ - for (String s : valuesChecked.keySet()) { - try { - assertEquals("Record count incorrect for column: " + s, totalRecords, (long) valuesChecked.get(s)); - } catch (AssertionError e) { submissionFailed(new RpcException(e)); } - } - - assert valuesChecked.keySet().size() > 0; - future.set(null); - } - } - - public void get() throws RpcException{ - try{ - future.get(); - return; - }catch(Throwable t){ - throw RpcException.mapException(t); - } - } - - @Override - public void queryIdArrived(QueryId queryId) { - } - } - - - - - public void testParquetFullEngineRemote(String plan, String filename, int numberOfTimesRead /* specified in json plan */, int numberOfRowGroups, int recordsPerRowGroup) throws Exception{ - - DrillConfig config = DrillConfig.create(); - - checkValues = false; - - try(DrillClient client = new DrillClient(config);){ - client.connect(); - RecordBatchLoader batchLoader = new RecordBatchLoader(client.getAllocator()); - ParquetResultListener resultListener = new ParquetResultListener(recordsPerRowGroup, batchLoader, numberOfRowGroups, numberOfTimesRead); - client.runQuery(UserProtos.QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile(plan), Charsets.UTF_8), resultListener); - resultListener.get(); - } - - } - public void testParquetFullEngineLocalPath(String planFileName, String filename, int numberOfTimesRead /* specified in json plan */, int numberOfRowGroups, int recordsPerRowGroup) throws Exception{ testParquetFullEngineLocalText(Files.toString(FileUtils.getResourceAsFile(planFileName), Charsets.UTF_8), filename, numberOfTimesRead, numberOfRowGroups, recordsPerRowGroup); @@ -277,14 +145,17 @@ public class ParquetRecordReaderTest { bit1.run(); client.connect(); RecordBatchLoader batchLoader = new RecordBatchLoader(client.getAllocator()); - ParquetResultListener resultListener = new ParquetResultListener(recordsPerRowGroup, batchLoader, numberOfRowGroups, numberOfTimesRead); + HashMap<String, FieldInfo> fields = new HashMap<>(); + ParquetTestProperties props = new ParquetTestProperties(numberRowGroups, recordsPerRowGroup, DEFAULT_BYTES_PER_PAGE, fields); + TestFileGenerator.populateFieldInfoMap(props); + ParquetResultListener resultListener = new ParquetResultListener(batchLoader, props, numberOfTimesRead, true); Stopwatch watch = new Stopwatch().start(); client.runQuery(UserProtos.QueryType.LOGICAL, planText, resultListener); - resultListener.get(); + resultListener.getResults(); System.out.println(String.format("Took %d ms to run query", watch.elapsed(TimeUnit.MILLISECONDS))); } - + } @@ -301,10 +172,13 @@ public class ParquetRecordReaderTest { bit1.run(); client.connect(); RecordBatchLoader batchLoader = new RecordBatchLoader(client.getAllocator()); - ParquetResultListener resultListener = new ParquetResultListener(recordsPerRowGroup, batchLoader, numberOfRowGroups, numberOfTimesRead); + HashMap<String, FieldInfo> fields = new HashMap<>(); + ParquetTestProperties props = new ParquetTestProperties(numberRowGroups, recordsPerRowGroup, DEFAULT_BYTES_PER_PAGE, fields); + TestFileGenerator.populateFieldInfoMap(props); + ParquetResultListener resultListener = new ParquetResultListener(batchLoader, props, numberOfTimesRead, true); Stopwatch watch = new Stopwatch().start(); client.runQuery(UserProtos.QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile(planName), Charsets.UTF_8), resultListener); - resultListener.get(); + resultListener.getResults(); System.out.println(String.format("Took %d ms to run query", watch.elapsed(TimeUnit.MILLISECONDS))); } @@ -326,6 +200,25 @@ public class ParquetRecordReaderTest { return result.toString(); } + public void testParquetFullEngineRemote(String plan, String filename, int numberOfTimesRead /* specified in json plan */, int numberOfRowGroups, int recordsPerRowGroup) throws Exception{ + + DrillConfig config = DrillConfig.create(); + + checkValues = false; + + try(DrillClient client = new DrillClient(config);){ + client.connect(); + RecordBatchLoader batchLoader = new RecordBatchLoader(client.getAllocator()); + HashMap<String, FieldInfo> fields = new HashMap<>(); + ParquetTestProperties props = new ParquetTestProperties(numberRowGroups, recordsPerRowGroup, DEFAULT_BYTES_PER_PAGE, fields); + TestFileGenerator.populateFieldInfoMap(props); + ParquetResultListener resultListener = new ParquetResultListener(batchLoader, props, numberOfTimesRead, true); + client.runQuery(UserProtos.QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile(plan), Charsets.UTF_8), resultListener); + resultListener.getResults(); + } + + } + class MockOutputMutator implements OutputMutator { List<MaterializedField> removedFields = Lists.newArrayList(); List<ValueVector> addFields = Lists.newArrayList(); @@ -358,24 +251,6 @@ public class ParquetRecordReaderTest { } } - private <T> void assertField(ValueVector valueVector, int index, TypeProtos.MinorType expectedMinorType, Object value, String name) { - assertField(valueVector, index, expectedMinorType, value, name, 0); - } - - @SuppressWarnings("unchecked") - private <T> void assertField(ValueVector valueVector, int index, TypeProtos.MinorType expectedMinorType, T value, String name, int parentFieldId) { - - if (expectedMinorType == TypeProtos.MinorType.MAP) { - return; - } - - T val = (T) valueVector.getAccessor().getObject(index); - if (val instanceof byte[]) { - assertTrue(Arrays.equals((byte[]) value, (byte[]) val)); - } else { - assertEquals(value, val); - } - } private void validateFooters(final List<Footer> metadata) { logger.debug(metadata.toString()); @@ -391,8 +266,8 @@ public class ParquetRecordReaderTest { assertEquals(footer.getFile().getName(), keyValueMetaData.get(footer.getFile().getName())); } } - - + + private void validateContains(MessageType schema, PageReadStore pages, String[] path, int values, BytesInput bytes) throws IOException { PageReader pageReader = pages.getPageReader(schema.getColumnDescription(path)); @@ -402,5 +277,208 @@ public class ParquetRecordReaderTest { } - + @Test + public void testMultipleRowGroups() throws Exception { + HashMap<String, FieldInfo> fields = new HashMap<>(); + ParquetTestProperties props = new ParquetTestProperties(3, 3000, DEFAULT_BYTES_PER_PAGE, fields); + populateFieldInfoMap(props); + testParquetFullEngineEventBased(true, "/parquet_scan_screen.json", "/tmp/test.parquet", 1, props); + } + + // TODO - Test currently marked ignore to prevent breaking of the build process, requires a binary file that was + // generated using pig. Will need to find a good place to keep files like this. + // For now I will upload it to the JIRA as an attachment. + @Ignore + @Test + public void testNullableColumns() throws Exception { + HashMap<String, FieldInfo> fields = new HashMap<>(); + ParquetTestProperties props = new ParquetTestProperties(1, 3000000, DEFAULT_BYTES_PER_PAGE, fields); + Object[] boolVals = {true, null, null}; + props.fields.put("a", new FieldInfo("boolean", "a", 1, boolVals, TypeProtos.MinorType.BIT, props)); + testParquetFullEngineEventBased(false, "/parquet_nullable.json", "/tmp/nullable_test.parquet", 1, props); + } + + @Ignore + @Test + public void testNullableColumnsVarLen() throws Exception { + HashMap<String, FieldInfo> fields = new HashMap<>(); + ParquetTestProperties props = new ParquetTestProperties(1, 300000, DEFAULT_BYTES_PER_PAGE, fields); + byte[] val = {'b'}; + byte[] val2 = {'b', '2'}; + byte[] val3 = { 'l','o','n','g','e','r',' ','s','t','r','i','n','g'}; + Object[] boolVals = { val, val2, val3}; + props.fields.put("a", new FieldInfo("boolean", "a", 1, boolVals, TypeProtos.MinorType.BIT, props)); + testParquetFullEngineEventBased(false, "/parquet_nullable_varlen.json", "/tmp/nullable_varlen.parquet", 1, props); + } + + @Test + public void testMultipleRowGroupsAndReads() throws Exception { + HashMap<String, FieldInfo> fields = new HashMap<>(); + ParquetTestProperties props = new ParquetTestProperties(4, 3000, DEFAULT_BYTES_PER_PAGE, fields); + populateFieldInfoMap(props); + String readEntries = ""; + // number of times to read the file + int i = 3; + for (int j = 0; j < i; j++){ + readEntries += "\"/tmp/test.parquet\""; + if (j < i - 1) + readEntries += ","; + } + testParquetFullEngineEventBased(true, "/parquet/parquet_scan_screen_read_entry_replace.json", readEntries, + "/tmp/test.parquet", i, props); + } + + // requires binary file generated by pig from TPCH data, also have to disable assert where data is coming in + @Ignore + @Test + public void testMultipleRowGroupsAndReadsPigError() throws Exception { + HashMap<String, FieldInfo> fields = new HashMap<>(); + ParquetTestProperties props = new ParquetTestProperties(5, 300000, DEFAULT_BYTES_PER_PAGE, fields); + TestFileGenerator.populatePigTPCHCustomerFields(props); + String readEntries = "{path: \"/tmp/tpc-h/customer\"}"; + testParquetFullEngineEventBased(false, false, "/parquet_scan_screen_read_entry_replace.json", readEntries, + "unused, no file is generated", 1, props, true); + + fields = new HashMap(); + props = new ParquetTestProperties(5, 300000, DEFAULT_BYTES_PER_PAGE, fields); + TestFileGenerator.populatePigTPCHSupplierFields(props); + readEntries = "{path: \"/tmp/tpc-h/supplier\"}"; + testParquetFullEngineEventBased(false, false, "/parquet_scan_screen_read_entry_replace.json", readEntries, + "unused, no file is generated", 1, props, true); + } + + @Test + public void testMultipleRowGroupsEvent() throws Exception { + HashMap<String, FieldInfo> fields = new HashMap<>(); + ParquetTestProperties props = new ParquetTestProperties(4, 3000, DEFAULT_BYTES_PER_PAGE, fields); + populateFieldInfoMap(props); + testParquetFullEngineEventBased(true, "/parquet_scan_screen.json", "/tmp/test.parquet", 1, props); + } + + + /** + * Tests the attribute in a scan node to limit the columns read by a scan. + * + * The functionality of selecting all columns is tested in all of the other tests that leave out the attribute. + * @throws Exception + */ + @Test + public void testSelectColumnRead() throws Exception { + HashMap<String, FieldInfo> fields = new HashMap<>(); + ParquetTestProperties props = new ParquetTestProperties(4, 3000, DEFAULT_BYTES_PER_PAGE, fields); + // generate metatdata for a series of test columns, these columns are all generated in the test file + populateFieldInfoMap(props); + TestFileGenerator.generateParquetFile("/tmp/test.parquet", props); + fields.clear(); + // create a new object to describe the dataset expected out of the scan operation + // the fields added below match those requested in the plan specified in parquet_selective_column_read.json + // that is used below in the test query + props = new ParquetTestProperties(4, 3000, DEFAULT_BYTES_PER_PAGE, fields); + props.fields.put("integer", new FieldInfo("int32", "integer", 32, TestFileGenerator.intVals, TypeProtos.MinorType.INT, props)); + props.fields.put("bigInt", new FieldInfo("int64", "bigInt", 64, TestFileGenerator.longVals, TypeProtos.MinorType.BIGINT, props)); + props.fields.put("bin", new FieldInfo("binary", "bin", -1, TestFileGenerator.binVals, TypeProtos.MinorType.VARBINARY, props)); + props.fields.put("bin2", new FieldInfo("binary", "bin2", -1, TestFileGenerator.bin2Vals, TypeProtos.MinorType.VARBINARY, props)); + testParquetFullEngineEventBased(true, false, "/parquet_selective_column_read.json", null, "/tmp/test.parquet", 1, props, false); + } + + public static void main(String[] args) throws Exception{ + // TODO - not sure why this has a main method, test below can be run directly + //new ParquetRecordReaderTest().testPerformance(); + } + + @Test + @Ignore + public void testPerformance(@Injectable final DrillbitContext bitContext, + @Injectable UserServer.UserClientConnection connection) throws Exception { + DrillConfig c = DrillConfig.create(); + FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); + FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); + +// new NonStrictExpectations() { +// { +// context.getAllocator(); result = BufferAllocator.getAllocator(DrillConfig.create()); +// } +// }; + + final String fileName = "/tmp/parquet_test_performance.parquet"; + HashMap<String, FieldInfo> fields = new HashMap<>(); + ParquetTestProperties props = new ParquetTestProperties(1, 20 * 1000 * 1000, DEFAULT_BYTES_PER_PAGE, fields); + populateFieldInfoMap(props); + //generateParquetFile(fileName, props); + + Configuration dfsConfig = new Configuration(); + List<Footer> footers = ParquetFileReader.readFooters(dfsConfig, new Path(fileName)); + Footer f = footers.iterator().next(); + + List<SchemaPath> columns = Lists.newArrayList(); + columns.add(new SchemaPath("_MAP.integer", ExpressionPosition.UNKNOWN)); + columns.add(new SchemaPath("_MAP.bigInt", ExpressionPosition.UNKNOWN)); + columns.add(new SchemaPath("_MAP.f", ExpressionPosition.UNKNOWN)); + columns.add(new SchemaPath("_MAP.d", ExpressionPosition.UNKNOWN)); + columns.add(new SchemaPath("_MAP.b", ExpressionPosition.UNKNOWN)); + columns.add(new SchemaPath("_MAP.bin", ExpressionPosition.UNKNOWN)); + columns.add(new SchemaPath("_MAP.bin2", ExpressionPosition.UNKNOWN)); + int totalRowCount = 0; + + FileSystem fs = new CachedSingleFileSystem(fileName); + for(int i = 0; i < 25; i++){ + ParquetRecordReader rr = new ParquetRecordReader(context, 256000, fileName, 0, fs, + new CodecFactoryExposer(dfsConfig), f.getParquetMetadata(), new FieldReference("_MAP", + ExpressionPosition.UNKNOWN), columns); + TestOutputMutator mutator = new TestOutputMutator(); + rr.setup(mutator); + Stopwatch watch = new Stopwatch(); + watch.start(); + + int rowCount = 0; + while ((rowCount = rr.next()) > 0) { + totalRowCount += rowCount; + } + System.out.println(String.format("Time completed: %s. ", watch.elapsed(TimeUnit.MILLISECONDS))); + rr.cleanup(); + } + System.out.println(String.format("Total row count %s", totalRowCount)); + } + + // specific tests should call this method, but it is not marked as a test itself intentionally + public void testParquetFullEngineEventBased(boolean generateNew, String plan, String readEntries, String filename, + int numberOfTimesRead /* specified in json plan */, ParquetTestProperties props) throws Exception{ + testParquetFullEngineEventBased(true, generateNew, plan, readEntries,filename, + numberOfTimesRead /* specified in json plan */, props, true); + } + + + // specific tests should call this method, but it is not marked as a test itself intentionally + public void testParquetFullEngineEventBased(boolean generateNew, String plan, String filename, int numberOfTimesRead /* specified in json plan */, ParquetTestProperties props) throws Exception{ + testParquetFullEngineEventBased(true, generateNew, plan, null, filename, numberOfTimesRead, props, true); + } + + // specific tests should call this method, but it is not marked as a test itself intentionally + public void testParquetFullEngineEventBased(boolean testValues, boolean generateNew, String plan, String readEntries, String filename, + int numberOfTimesRead /* specified in json plan */, ParquetTestProperties props, + boolean runAsLogicalPlan) throws Exception{ + RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet(); + if (generateNew) TestFileGenerator.generateParquetFile(filename, props); + DrillConfig config = DrillConfig.create(); + try(Drillbit bit1 = new Drillbit(config, serviceSet); DrillClient client = new DrillClient(config, serviceSet.getCoordinator());){ + bit1.run(); + client.connect(); + RecordBatchLoader batchLoader = new RecordBatchLoader(bit1.getContext().getAllocator()); + ParquetResultListener resultListener = new ParquetResultListener(batchLoader, props, numberOfTimesRead, testValues); + long C = System.nanoTime(); + String planText = Files.toString(FileUtils.getResourceAsFile(plan), Charsets.UTF_8); + // substitute in the string for the read entries, allows reuse of the plan file for several tests + if (readEntries != null) { + planText = planText.replaceFirst( "&REPLACED_IN_PARQUET_TEST&", readEntries); + } + if (runAsLogicalPlan) + client.runQuery(UserProtos.QueryType.LOGICAL, planText, resultListener); + else + client.runQuery(UserProtos.QueryType.PHYSICAL, planText, resultListener); + resultListener.getResults(); + long D = System.nanoTime(); + System.out.println(String.format("Took %f s to run query", (float)(D-C) / 1E9)); + } + } + } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/ParquetRecordReaderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest2.java index 6f568fb83..2ab12cb21 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/ParquetRecordReaderTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest2.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store; +package org.apache.drill.exec.store.parquet; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -42,7 +42,7 @@ import org.apache.drill.exec.rpc.user.QueryResultBatch; import org.apache.drill.exec.rpc.user.UserResultsListener; import org.apache.drill.exec.server.Drillbit; import org.apache.drill.exec.server.RemoteServiceSet; -import org.apache.drill.exec.store.json.JsonSchemaProvider; +import org.apache.drill.exec.store.ByteArrayUtil; import org.apache.drill.exec.vector.BaseDataValueVector; import org.apache.drill.exec.vector.ValueVector; import org.apache.hadoop.conf.Configuration; @@ -63,8 +63,8 @@ import com.google.common.base.Charsets; import com.google.common.io.Files; import com.google.common.util.concurrent.SettableFuture; -public class ParquetRecordReaderTest { - org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetRecordReaderTest.class); +public class ParquetRecordReaderTest2 { + org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetRecordReaderTest2.class); private static final boolean VERBOSE_DEBUG = false; @@ -270,7 +270,7 @@ public class ParquetRecordReaderTest { WrapAroundCounter booleanBitCounter = new WrapAroundCounter(7); Configuration configuration = new Configuration(); - configuration.set(JsonSchemaProvider.HADOOP_DEFAULT_NAME, "file:///"); + configuration.set("fs.default.name", "file:///"); //"message m { required int32 integer; required int64 integer64; required boolean b; required float f; required double d;}" FileSystem fs = FileSystem.get(configuration); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetResultListener.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetResultListener.java new file mode 100644 index 000000000..f99721a9b --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetResultListener.java @@ -0,0 +1,203 @@ +/******************************************************************************* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.drill.exec.store.parquet; + +import static junit.framework.Assert.assertEquals; + +import java.util.Arrays; +import java.util.HashMap; + +import org.apache.drill.common.types.TypeProtos; +import org.apache.drill.exec.exception.SchemaChangeException; +import org.apache.drill.exec.proto.UserBitShared; +import org.apache.drill.exec.record.RecordBatchLoader; +import org.apache.drill.exec.record.VectorWrapper; +import org.apache.drill.exec.rpc.RpcException; +import org.apache.drill.exec.rpc.user.ConnectionThrottle; +import org.apache.drill.exec.rpc.user.QueryResultBatch; +import org.apache.drill.exec.rpc.user.UserResultsListener; +import org.apache.drill.exec.vector.ValueVector; + +import com.google.common.base.Strings; +import com.google.common.util.concurrent.SettableFuture; + +public class ParquetResultListener implements UserResultsListener { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetResultListener.class); + + private SettableFuture<Void> future = SettableFuture.create(); + int count = 0; + int totalRecords; + RecordBatchLoader batchLoader; + boolean testValues; + + int batchCounter = 1; + HashMap<String, Integer> valuesChecked = new HashMap(); + ParquetTestProperties props; + + ParquetResultListener(RecordBatchLoader batchLoader, ParquetTestProperties props, int numberOfTimesRead, boolean testValues){ + this.batchLoader = batchLoader; + this.props = props; + this.totalRecords = props.recordsPerRowGroup * props.numberRowGroups * numberOfTimesRead; + this.testValues = testValues; + } + + @Override + public void submissionFailed(RpcException ex) { + logger.debug("Submission failed.", ex); + future.setException(ex); + } + + + private <T> void assertField(ValueVector valueVector, int index, TypeProtos.MinorType expectedMinorType, Object value, String name) { + assertField(valueVector, index, expectedMinorType, value, name, 0); + } + + @SuppressWarnings("unchecked") + private <T> void assertField(ValueVector valueVector, int index, TypeProtos.MinorType expectedMinorType, T value, String name, int parentFieldId) { + + if (expectedMinorType == TypeProtos.MinorType.MAP) { + return; + } + + T val = (T) valueVector.getAccessor().getObject(index); + if (val instanceof byte[]) { + assertEquals(true, Arrays.equals((byte[]) value, (byte[]) val)); + } else { + assertEquals(value, val); + } + } + + @Override + public void resultArrived(QueryResultBatch result, ConnectionThrottle throttle) { + logger.debug("result arrived in test batch listener."); + if(result.getHeader().getIsLastChunk()){ + future.set(null); + } + int columnValCounter = 0; + FieldInfo currentField; + count += result.getHeader().getRowCount(); + boolean schemaChanged = false; + try { + schemaChanged = batchLoader.load(result.getHeader().getDef(), result.getData()); + } catch (SchemaChangeException e) { + throw new RuntimeException(e); + } + + int recordCount = 0; + // print headers. + if (schemaChanged) { + } // do not believe any change is needed for when the schema changes, with the current mock scan use case + + for (VectorWrapper vw : batchLoader) { + ValueVector vv = vw.getValueVector(); + currentField = props.fields.get(vv.getField().getName()); + if (ParquetRecordReaderTest.VERBOSE_DEBUG){ + System.out.println("\n" + (String) currentField.name); + } + if ( ! valuesChecked.containsKey(vv.getField().getName())){ + valuesChecked.put(vv.getField().getName(), 0); + columnValCounter = 0; + } else { + columnValCounter = valuesChecked.get(vv.getField().getName()); + } + for (int j = 0; j < vv.getAccessor().getValueCount(); j++) { + if (ParquetRecordReaderTest.VERBOSE_DEBUG){ + if (vv.getAccessor().getObject(j) instanceof byte[]){ + System.out.print("[len:" + ((byte[]) vv.getAccessor().getObject(j)).length + " - ("); + for (int k = 0; k < ((byte[]) vv.getAccessor().getObject(j)).length; k++){ + System.out.print((char)((byte[])vv.getAccessor().getObject(j))[k] + ","); + } + System.out.print(") ]"); + } + else{ + System.out.print(Strings.padStart(vv.getAccessor().getObject(j) + "", 20, ' ') + " "); + } + System.out.print(", " + (j % 25 == 0 ? "\n batch:" + batchCounter + " v:" + j + " - " : "")); + } + if (testValues){ + assertField(vv, j, currentField.type, + currentField.values[columnValCounter % 3], currentField.name + "/"); + } + columnValCounter++; + } + if (ParquetRecordReaderTest.VERBOSE_DEBUG){ + System.out.println("\n" + vv.getAccessor().getValueCount()); + } + valuesChecked.remove(vv.getField().getName()); + valuesChecked.put(vv.getField().getName(), columnValCounter); + } + + if (ParquetRecordReaderTest.VERBOSE_DEBUG){ + for (int i = 0; i < batchLoader.getRecordCount(); i++) { + recordCount++; + if (i % 50 == 0){ + System.out.println(); + for (VectorWrapper vw : batchLoader) { + ValueVector v = vw.getValueVector(); + System.out.print(Strings.padStart(v.getField().getName(), 20, ' ') + " "); + + } + System.out.println(); + System.out.println(); + } + + for (VectorWrapper vw : batchLoader) { + ValueVector v = vw.getValueVector(); + if (v.getAccessor().getObject(i) instanceof byte[]){ + System.out.print("[len:" + ((byte[]) v.getAccessor().getObject(i)).length + " - ("); + for (int j = 0; j < ((byte[]) v.getAccessor().getObject(i)).length; j++){ + System.out.print(((byte[])v.getAccessor().getObject(i))[j] + ","); + } + System.out.print(") ]"); + } + else{ + System.out.print(Strings.padStart(v.getAccessor().getObject(i) + "", 20, ' ') + " "); + } + } + System.out.println( + + ); + } + } + batchCounter++; + if(result.getHeader().getIsLastChunk()){ + // ensure the right number of columns was returned, especially important to ensure selective column read is working + assert valuesChecked.keySet().size() == props.fields.keySet().size() : "Unexpected number of output columns from parquet scan,"; + for (String s : valuesChecked.keySet()) { + try { + assertEquals("Record count incorrect for column: " + s, totalRecords, (long) valuesChecked.get(s)); + } catch (AssertionError e) { submissionFailed(new RpcException(e)); } + } + + assert valuesChecked.keySet().size() > 0; + future.set(null); + } + } + + public void getResults() throws RpcException{ + try{ + future.get(); + }catch(Throwable t){ + throw RpcException.mapException(t); + } + } + + @Override + public void queryIdArrived(UserBitShared.QueryId queryId) { + } +} diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetTestProperties.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetTestProperties.java new file mode 100644 index 000000000..7c68a16fa --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetTestProperties.java @@ -0,0 +1,37 @@ +/******************************************************************************* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.drill.exec.store.parquet; + +import java.util.HashMap; + +public class ParquetTestProperties { + + int numberRowGroups; + int recordsPerRowGroup; + int bytesPerPage = 1024 * 1024 * 1; + HashMap<String, FieldInfo> fields = new HashMap<>(); + + public ParquetTestProperties(int numberRowGroups, int recordsPerRowGroup, int bytesPerPage, + HashMap<String, FieldInfo> fields){ + this.numberRowGroups = numberRowGroups; + this.recordsPerRowGroup = recordsPerRowGroup; + this.bytesPerPage = bytesPerPage; + this.fields = fields; + + } +} diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFileGenerator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFileGenerator.java index d2c95fe5e..689b168af 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFileGenerator.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFileGenerator.java @@ -20,7 +20,6 @@ package org.apache.drill.exec.store.parquet; import static parquet.column.Encoding.PLAIN; import java.util.HashMap; -import java.util.Map; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.exec.store.ByteArrayUtil; @@ -58,84 +57,60 @@ public class TestFileGenerator { static final Object[] binVals = { varLen1, varLen2, varLen3 }; static final Object[] bin2Vals = { varLen3, varLen2, varLen1 }; - static class FieldInfo { - - String parquetType; - String name; - int bitLength; - int numberOfPages; - Object[] values; - TypeProtos.MinorType type; - - FieldInfo(int recordsPerRowGroup, String parquetType, String name, int bitLength, Object[] values, TypeProtos.MinorType type) { - this.parquetType = parquetType; - this.name = name; - this.bitLength = bitLength; - this.numberOfPages = Math.max(1, (int) Math.ceil( ((long) recordsPerRowGroup) * bitLength / 8.0 / bytesPerPage)); - this.values = values; - // generator is designed to use 3 values - assert values.length == 3; - this.type = type; - } + static void populateFieldInfoMap(ParquetTestProperties props){ + props.fields.put("integer", new FieldInfo("int32", "integer", 32, intVals, TypeProtos.MinorType.INT, props)); + props.fields.put("bigInt", new FieldInfo("int64", "bigInt", 64, longVals, TypeProtos.MinorType.BIGINT, props)); + props.fields.put("f", new FieldInfo("float", "f", 32, floatVals, TypeProtos.MinorType.FLOAT4, props)); + props.fields.put("d", new FieldInfo("double", "d", 64, doubleVals, TypeProtos.MinorType.FLOAT8, props)); + props.fields.put("b", new FieldInfo("boolean", "b", 1, boolVals, TypeProtos.MinorType.BIT, props)); + props.fields.put("bin", new FieldInfo("binary", "bin", -1, binVals, TypeProtos.MinorType.VARBINARY, props)); + props.fields.put("bin2", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY, props)); } - private static class WrapAroundCounter { - - int maxVal; - int val; - - public WrapAroundCounter(int maxVal) { - this.maxVal = maxVal; - } - - public int increment() { - val++; - if (val > maxVal) { - val = 0; - } - return val; - } - - public void reset() { - val = 0; - } - + static void populatePigTPCHCustomerFields(ParquetTestProperties props){ + // all of the data in the fieldInfo constructors doesn't matter because the file is generated outside the test + props.fields.put("C_CUSTKEY", new FieldInfo("int32", "integer", 32, intVals, TypeProtos.MinorType.INT, props)); + props.fields.put("C_NATIONKEY", new FieldInfo("int64", "bigInt", 64, longVals, TypeProtos.MinorType.BIGINT, props)); + props.fields.put("C_ACCTBAL", new FieldInfo("float", "f", 32, floatVals, TypeProtos.MinorType.FLOAT4, props)); + props.fields.put("C_NAME", new FieldInfo("double", "d", 64, doubleVals, TypeProtos.MinorType.FLOAT8, props)); + props.fields.put("C_ADDRESS", new FieldInfo("boolean", "b", 1, boolVals, TypeProtos.MinorType.BIT, props)); + props.fields.put("C_PHONE", new FieldInfo("binary", "bin", -1, binVals, TypeProtos.MinorType.VARBINARY, props)); + props.fields.put("C_MKTSEGMENT", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY, props)); + props.fields.put("C_COMMENT", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY, props)); } - public static HashMap<String, FieldInfo> getFieldMap(int recordsPerRowGroup) { - HashMap<String, FieldInfo> fields = new HashMap<>(); - fields.put("integer", new FieldInfo(recordsPerRowGroup, "int32", "integer", 32, intVals, TypeProtos.MinorType.INT)); - fields.put("bigInt", new FieldInfo(recordsPerRowGroup, "int64", "bigInt", 64, longVals, TypeProtos.MinorType.BIGINT)); - fields.put("f", new FieldInfo(recordsPerRowGroup, "float", "f", 32, floatVals, TypeProtos.MinorType.FLOAT4)); - fields.put("d", new FieldInfo(recordsPerRowGroup, "double", "d", 64, doubleVals, TypeProtos.MinorType.FLOAT8)); - fields.put("b", new FieldInfo(recordsPerRowGroup, "boolean", "b", 1, boolVals, TypeProtos.MinorType.BIT)); - fields.put("bin", new FieldInfo(recordsPerRowGroup, "binary", "bin", -1, binVals, TypeProtos.MinorType.VARBINARY)); - fields.put("bin2", new FieldInfo(recordsPerRowGroup, "binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY)); - return fields; + static void populatePigTPCHSupplierFields(ParquetTestProperties props){ + // all of the data in the fieldInfo constructors doesn't matter because the file is generated outside the test + props.fields.put("S_SUPPKEY", new FieldInfo("int32", "integer", 32, intVals, TypeProtos.MinorType.INT, props)); + props.fields.put("S_NATIONKEY", new FieldInfo("int64", "bigInt", 64, longVals, TypeProtos.MinorType.BIGINT, props)); + props.fields.put("S_ACCTBAL", new FieldInfo("float", "f", 32, floatVals, TypeProtos.MinorType.FLOAT4, props)); + props.fields.put("S_NAME", new FieldInfo("double", "d", 64, doubleVals, TypeProtos.MinorType.FLOAT8, props)); + props.fields.put("S_ADDRESS", new FieldInfo("boolean", "b", 1, boolVals, TypeProtos.MinorType.BIT, props)); + props.fields.put("S_PHONE", new FieldInfo("binary", "bin", -1, binVals, TypeProtos.MinorType.VARBINARY, props)); + props.fields.put("S_COMMENT", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY, props)); } - public static void generateParquetFile(String filename, int numberRowGroups, int recordsPerRowGroup) throws Exception { - final Map<String, FieldInfo> fields = getFieldMap(recordsPerRowGroup); + public static void generateParquetFile(String filename, ParquetTestProperties props) throws Exception { int currentBooleanByte = 0; WrapAroundCounter booleanBitCounter = new WrapAroundCounter(7); - + Configuration configuration = new Configuration(); - configuration.set(ParquetSchemaProvider.HADOOP_DEFAULT_NAME, "file:///"); - // "message m { required int32 integer; required int64 integer64; required boolean b; required float f; required double d;}" + configuration.set("fs.default.name", "file:///"); + //"message m { required int32 integer; required int64 integer64; required boolean b; required float f; required double d;}" FileSystem fs = FileSystem.get(configuration); Path path = new Path(filename); - if (fs.exists(path)) - fs.delete(path, false); + if (fs.exists(path)) fs.delete(path, false); + String messageSchema = "message m {"; - for (FieldInfo fieldInfo : fields.values()) { + for (FieldInfo fieldInfo : props.fields.values()) { messageSchema += " required " + fieldInfo.parquetType + " " + fieldInfo.name + ";"; } // remove the last semicolon, java really needs a join method for strings... // TODO - nvm apparently it requires a semicolon after every field decl, might want to file a bug - // messageSchema = messageSchema.substring(schemaType, messageSchema.length() - 1); + //messageSchema = messageSchema.substring(schemaType, messageSchema.length() - 1); messageSchema += "}"; MessageType schema = MessageTypeParser.parseMessageType(messageSchema); @@ -145,25 +120,25 @@ public class TestFileGenerator { w.start(); HashMap<String, Integer> columnValuesWritten = new HashMap(); int valsWritten; - for (int k = 0; k < numberRowGroups; k++) { + for (int k = 0; k < props.numberRowGroups; k++){ w.startBlock(1); currentBooleanByte = 0; booleanBitCounter.reset(); - for (FieldInfo fieldInfo : fields.values()) { + for (FieldInfo fieldInfo : props.fields.values()) { - if (!columnValuesWritten.containsKey(fieldInfo.name)) { + if ( ! columnValuesWritten.containsKey(fieldInfo.name)){ columnValuesWritten.put((String) fieldInfo.name, 0); valsWritten = 0; } else { valsWritten = columnValuesWritten.get(fieldInfo.name); } - String[] path1 = { (String) fieldInfo.name }; + String[] path1 = {(String) fieldInfo.name}; ColumnDescriptor c1 = schema.getColumnDescription(path1); - w.startColumn(c1, recordsPerRowGroup, codec); - int valsPerPage = (int) Math.ceil(recordsPerRowGroup / (float) fieldInfo.numberOfPages); + w.startColumn(c1, props.recordsPerRowGroup, codec); + int valsPerPage = (int) Math.ceil(props.recordsPerRowGroup / (float) fieldInfo.numberOfPages); byte[] bytes; // for variable length binary fields int bytesNeededToEncodeLength = 4; @@ -171,21 +146,18 @@ public class TestFileGenerator { bytes = new byte[(int) Math.ceil(valsPerPage * (int) fieldInfo.bitLength / 8.0)]; } else { // the twelve at the end is to account for storing a 4 byte length with each value - int totalValLength = ((byte[]) fieldInfo.values[0]).length + ((byte[]) fieldInfo.values[1]).length - + ((byte[]) fieldInfo.values[2]).length + 3 * bytesNeededToEncodeLength; + int totalValLength = ((byte[]) fieldInfo.values[0]).length + ((byte[]) fieldInfo.values[1]).length + ((byte[]) fieldInfo.values[2]).length + 3 * bytesNeededToEncodeLength; // used for the case where there is a number of values in this row group that is not divisible by 3 int leftOverBytes = 0; - if (valsPerPage % 3 > 0) - leftOverBytes += ((byte[]) fieldInfo.values[1]).length + 4; - if (valsPerPage % 3 > 1) - leftOverBytes += ((byte[]) fieldInfo.values[2]).length + 4; + if ( valsPerPage % 3 > 0 ) leftOverBytes += ((byte[])fieldInfo.values[1]).length + bytesNeededToEncodeLength; + if ( valsPerPage % 3 > 1 ) leftOverBytes += ((byte[])fieldInfo.values[2]).length + bytesNeededToEncodeLength; bytes = new byte[valsPerPage / 3 * totalValLength + leftOverBytes]; } int bytesPerPage = (int) (valsPerPage * ((int) fieldInfo.bitLength / 8.0)); int bytesWritten = 0; for (int z = 0; z < (int) fieldInfo.numberOfPages; z++, bytesWritten = 0) { for (int i = 0; i < valsPerPage; i++) { - // System.out.print(i + ", " + (i % 25 == 0 ? "\n gen " + fieldInfo.name + ": " : "")); + //System.out.print(i + ", " + (i % 25 == 0 ? "\n gen " + fieldInfo.name + ": " : "")); if (fieldInfo.values[0] instanceof Boolean) { bytes[currentBooleanByte] |= bitFields[booleanBitCounter.val] @@ -195,25 +167,23 @@ public class TestFileGenerator { currentBooleanByte++; } valsWritten++; - if (currentBooleanByte > bytesPerPage) - break; + if (currentBooleanByte > bytesPerPage) break; } else { - if (fieldInfo.values[valsWritten % 3] instanceof byte[]) { - System.arraycopy(ByteArrayUtil.toByta(((byte[]) fieldInfo.values[valsWritten % 3]).length), 0, bytes, - bytesWritten, bytesNeededToEncodeLength); - System.arraycopy(fieldInfo.values[valsWritten % 3], 0, bytes, bytesWritten + bytesNeededToEncodeLength, - ((byte[]) fieldInfo.values[valsWritten % 3]).length); - bytesWritten += ((byte[]) fieldInfo.values[valsWritten % 3]).length + bytesNeededToEncodeLength; - } else { - System.arraycopy(ByteArrayUtil.toByta(fieldInfo.values[valsWritten % 3]), 0, bytes, i - * ((int) fieldInfo.bitLength / 8), (int) fieldInfo.bitLength / 8); + if (fieldInfo.values[valsWritten % 3] instanceof byte[]){ + System.arraycopy(ByteArrayUtil.toByta(((byte[])fieldInfo.values[valsWritten % 3]).length), + 0, bytes, bytesWritten, bytesNeededToEncodeLength); + System.arraycopy(fieldInfo.values[valsWritten % 3], + 0, bytes, bytesWritten + bytesNeededToEncodeLength, ((byte[])fieldInfo.values[valsWritten % 3]).length); + bytesWritten += ((byte[])fieldInfo.values[valsWritten % 3]).length + bytesNeededToEncodeLength; + } else{ + System.arraycopy( ByteArrayUtil.toByta(fieldInfo.values[valsWritten % 3]), + 0, bytes, i * ((int) fieldInfo.bitLength / 8), (int) fieldInfo.bitLength / 8); } valsWritten++; } } - w.writeDataPage((int) (recordsPerRowGroup / (int) fieldInfo.numberOfPages), bytes.length, - BytesInput.from(bytes), PLAIN, PLAIN, PLAIN); + w.writeDataPage((int) (props.recordsPerRowGroup / (int) fieldInfo.numberOfPages), bytes.length, BytesInput.from(bytes), PLAIN, PLAIN, PLAIN); currentBooleanByte = 0; } w.endColumn(); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetPhysicalPlan.java index 9c4aeea14..48c077343 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetPhysicalPlan.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.store; +package org.apache.drill.exec.store.parquet; import java.util.List; import java.util.concurrent.CountDownLatch; diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/WrapAroundCounter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/WrapAroundCounter.java new file mode 100644 index 000000000..eef856000 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/WrapAroundCounter.java @@ -0,0 +1,40 @@ +/******************************************************************************* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.drill.exec.store.parquet; + +public class WrapAroundCounter { + + int maxVal; + int val; + + public WrapAroundCounter(int maxVal) { + this.maxVal = maxVal; + } + + public int increment() { + val++; + if (val > maxVal) { + val = 0; + } + return val; + } + + public void reset() { + val = 0; + } +} diff --git a/exec/java-exec/src/test/resources/join/merge_inner_single_batch.json b/exec/java-exec/src/test/resources/join/merge_inner_single_batch.json index 5c7b8e362..5f00ce7c8 100644 --- a/exec/java-exec/src/test/resources/join/merge_inner_single_batch.json +++ b/exec/java-exec/src/test/resources/join/merge_inner_single_batch.json @@ -9,25 +9,21 @@ graph:[ { @id:1, - pop:"json-sub-scan", - readEntries:[ - {path: "#{LEFT_FILE}"} - ], - engineConfig:{ - "type":"json", - "dfsName" : "file:///" - } + pop:"fs-sub-scan", + format: {type: "json"}, + storage:{type: "file", connection: "file:///"}, + files:[ + { path: "#{LEFT_FILE}" } + ] }, { @id:2, - pop:"json-sub-scan", - readEntries:[ - {path: "#{RIGHT_FILE}"} - ], - engineConfig:{ - "type":"json", - "dfsName" : "file:///" - } + pop:"fs-sub-scan", + format: {type: "json"}, + storage:{type: "file", connection: "file:///"}, + files:[ + { path: "#{RIGHT_FILE}" } + ] }, { @id: 3, diff --git a/exec/java-exec/src/test/resources/join/merge_multi_batch.json b/exec/java-exec/src/test/resources/join/merge_multi_batch.json index 7abd19f54..285995113 100644 --- a/exec/java-exec/src/test/resources/join/merge_multi_batch.json +++ b/exec/java-exec/src/test/resources/join/merge_multi_batch.json @@ -9,27 +9,21 @@ graph:[ { @id:1, - pop:"json-sub-scan", - readEntries:[ - {path: "#{LEFT_FILE}"} - ], - engineConfig:{ - "type":"json", - "dfsName" : "file:///" - } - + pop:"fs-sub-scan", + format: {type: "json"}, + storage:{type: "file", connection: "file:///"}, + files:[ + { path: "#{LEFT_FILE}" } + ] }, { @id:2, - pop:"json-sub-scan", - readEntries:[ - {path: "#{RIGHT_FILE}"} - ], - engineConfig:{ - "type":"json", - "dfsName" : "file:///" - } - + pop:"fs-sub-scan", + format: {type: "json"}, + storage:{type: "file", connection: "file:///"}, + files:[ + { path: "#{RIGHT_FILE}" } + ] }, { @id: 3, diff --git a/exec/java-exec/src/test/resources/join/merge_single_batch.json b/exec/java-exec/src/test/resources/join/merge_single_batch.json index 1f7e0b5fa..285995113 100644 --- a/exec/java-exec/src/test/resources/join/merge_single_batch.json +++ b/exec/java-exec/src/test/resources/join/merge_single_batch.json @@ -9,25 +9,21 @@ graph:[ { @id:1, - pop:"json-sub-scan", - readEntries:[ - {path: "#{LEFT_FILE}"} - ], - engineConfig:{ - "type":"json", - "dfsName" : "file:///" - } + pop:"fs-sub-scan", + format: {type: "json"}, + storage:{type: "file", connection: "file:///"}, + files:[ + { path: "#{LEFT_FILE}" } + ] }, { @id:2, - pop:"json-sub-scan", - readEntries:[ - {path: "#{RIGHT_FILE}"} - ], - engineConfig:{ - "type":"json", - "dfsName" : "file:///" - } + pop:"fs-sub-scan", + format: {type: "json"}, + storage:{type: "file", connection: "file:///"}, + files:[ + { path: "#{RIGHT_FILE}" } + ] }, { @id: 3, diff --git a/exec/java-exec/src/test/resources/logical_json_scan.json b/exec/java-exec/src/test/resources/logical_json_scan.json index f9a8ffa21..b1cc4cecd 100644 --- a/exec/java-exec/src/test/resources/logical_json_scan.json +++ b/exec/java-exec/src/test/resources/logical_json_scan.json @@ -9,8 +9,8 @@ }, storage : { jsonl : { - type : "json", - dfsName : "file:///" + type : "file", + connection : "file:///" } }, query : [ { @@ -18,9 +18,7 @@ @id : 1, memo : "initial_scan", storageengine : "jsonl", - selection : [ { - path : "/src/drill/sandbox/prototype/exec/ref/src/test/resources/employees.json" - } ], + files : [ path : "/src/drill/sandbox/prototype/exec/ref/src/test/resources/employees.json" ], ref : "_MAP" }, { op : "store", diff --git a/exec/java-exec/src/test/resources/parquet/parquet_scan_screen.json b/exec/java-exec/src/test/resources/parquet/parquet_scan_screen.json index 29cab6848..50281f43b 100644 --- a/exec/java-exec/src/test/resources/parquet/parquet_scan_screen.json +++ b/exec/java-exec/src/test/resources/parquet/parquet_scan_screen.json @@ -8,10 +8,10 @@ } }, storage:{ - "parquet" : + "dfs" : { - "type":"parquet", - "dfsName" : "file:///" + "type":"named", + "name":"dfs" } }, query:[ @@ -19,15 +19,14 @@ @id:"1", op:"scan", memo:"initial_scan", - storageengine:"parquet", - selection: [ - { - path: "/tmp/parquet_test_file_many_types" - }, - { - path: "/tmp/parquet_test_file_many_types" - } - ] + storageengine:"dfs", + selection: { + format: {type: "parquet"}, + files: [ + "/tmp/parquet_test_file_many_types", + "/tmp/parquet_test_file_many_types" + ] + } }, { @id:"2", diff --git a/exec/java-exec/src/test/resources/parquet/parquet_scan_screen_read_entry_replace.json b/exec/java-exec/src/test/resources/parquet/parquet_scan_screen_read_entry_replace.json index af76e01c7..1bd52b46f 100644 --- a/exec/java-exec/src/test/resources/parquet/parquet_scan_screen_read_entry_replace.json +++ b/exec/java-exec/src/test/resources/parquet/parquet_scan_screen_read_entry_replace.json @@ -8,10 +8,10 @@ } }, storage:{ - "parquet" : + "fs" : { - "type":"parquet", - "dfsName" : "file:///" + "type":"file", + "connection" : "file:///" } }, query:[ @@ -19,10 +19,11 @@ @id:"1", op:"scan", memo:"initial_scan", - storageengine:"parquet", - selection: [ - &REPLACED_IN_PARQUET_TEST& - ] + storageengine:"fs", + selection: { + format: {type: "parquet"}, + files: [&REPLACED_IN_PARQUET_TEST&] + } }, { @id:"2", diff --git a/exec/java-exec/src/test/resources/parquet/parquet_scan_union_screen_physical.json b/exec/java-exec/src/test/resources/parquet/parquet_scan_union_screen_physical.json index 5efecafae..a49b9145c 100644 --- a/exec/java-exec/src/test/resources/parquet/parquet_scan_union_screen_physical.json +++ b/exec/java-exec/src/test/resources/parquet/parquet_scan_union_screen_physical.json @@ -14,9 +14,9 @@ path : "/tmp/parquet_test_file_many_types" } ], - storageengine:{ - "type":"parquet", - "dfsName" : "file:///" + storage:{ + "type":"file", + "connection" : "file:///" } }, { diff --git a/exec/java-exec/src/test/resources/parquet_scan_screen.json b/exec/java-exec/src/test/resources/parquet_scan_screen.json index f18c73813..3b3f86a8e 100644 --- a/exec/java-exec/src/test/resources/parquet_scan_screen.json +++ b/exec/java-exec/src/test/resources/parquet_scan_screen.json @@ -8,10 +8,10 @@ } }, storage:{ - "parquet" : + "dfs" : { - "type":"parquet", - "dfsName" : "file:///" + "type":"file", + "connection" : "file:///" } }, query:[ @@ -19,12 +19,11 @@ @id:"1", op:"scan", memo:"initial_scan", - storageengine:"parquet", - selection: [ - { - path: "/tmp/test.parquet" - } - ] + storageengine:"dfs", + selection: { + format: {type: "parquet"}, + files: ["/tmp/test.parquet"] + } }, { @id:"2", diff --git a/exec/java-exec/src/test/resources/parquet_scan_screen_read_entry_replace.json b/exec/java-exec/src/test/resources/parquet_scan_screen_read_entry_replace.json index af76e01c7..ab11a5fd0 100644 --- a/exec/java-exec/src/test/resources/parquet_scan_screen_read_entry_replace.json +++ b/exec/java-exec/src/test/resources/parquet_scan_screen_read_entry_replace.json @@ -8,10 +8,10 @@ } }, storage:{ - "parquet" : + "fs" : { - "type":"parquet", - "dfsName" : "file:///" + "type":"file", + "connection" : "file:///" } }, query:[ @@ -19,10 +19,11 @@ @id:"1", op:"scan", memo:"initial_scan", - storageengine:"parquet", - selection: [ - &REPLACED_IN_PARQUET_TEST& - ] + storageengine:"fs", + selection: { + format: {type: "parquet"} + files: [ &REPLACED_IN_PARQUET_TEST& ] + } }, { @id:"2", diff --git a/exec/java-exec/src/test/resources/parquet_selective_column_read.json b/exec/java-exec/src/test/resources/parquet_selective_column_read.json index 61e582d5e..515affcbd 100644 --- a/exec/java-exec/src/test/resources/parquet_selective_column_read.json +++ b/exec/java-exec/src/test/resources/parquet_selective_column_read.json @@ -13,12 +13,11 @@ entries : [ { path : "/tmp/test.parquet" } ], - storageengine : { - type : "parquet", - dfsName : "file:///" + storage : { + type : "file", + connection : "file:///" }, - columns: [ "integer", "bigInt", "bin", "bin2"], - fragmentPointer : 0 + columns: [ "integer", "bigInt", "bin", "bin2"] }, { pop : "screen", @id : 2, diff --git a/exec/java-exec/src/test/resources/physical_json_scan_test1.json b/exec/java-exec/src/test/resources/physical_json_scan_test1.json index 93bd9666d..5293d3bcb 100644 --- a/exec/java-exec/src/test/resources/physical_json_scan_test1.json +++ b/exec/java-exec/src/test/resources/physical_json_scan_test1.json @@ -9,16 +9,10 @@ graph:[ { @id:1, - pop:"json-scan", - entries: [ - { - path : "#{TEST_FILE}" - } - ], - storageengine: { - "type": "json", - "dfsName": "file:///" - } + pop:"fs-scan", + format: {type: "json"}, + storage:{type: "file", connection: "file:///"}, + files: [ "#{TEST_FILE}" ] }, { @id: 2, diff --git a/exec/java-exec/src/test/resources/sender/broadcast_exchange.json b/exec/java-exec/src/test/resources/sender/broadcast_exchange.json index 64f160644..408e35b2c 100644 --- a/exec/java-exec/src/test/resources/sender/broadcast_exchange.json +++ b/exec/java-exec/src/test/resources/sender/broadcast_exchange.json @@ -9,29 +9,17 @@ graph:[ { @id:1, - pop:"json-scan", - entries: [ - { - path : "#{LEFT_FILE}" - } - ], - storageengine: { - "type": "json", - "dfsName": "file:///" - } + pop:"fs-scan", + format: {type: "json"}, + storage:{type: "file", connection: "file:///"}, + files:["#{LEFT_FILE}"] }, { - @id:2, - pop:"json-scan", - entries: [ - { - path : "#{RIGHT_FILE}" - } - ], - storageengine: { - "type": "json", - "dfsName": "file:///" - } + @id:2, + pop:"fs-scan", + format: {type: "json"}, + storage:{type: "file", connection: "file:///"}, + files:["#{RIGHT_FILE}"] }, { @id: 3, diff --git a/exec/java-exec/src/test/resources/storage-engines.json b/exec/java-exec/src/test/resources/storage-engines.json new file mode 100644 index 000000000..7d3dc38cc --- /dev/null +++ b/exec/java-exec/src/test/resources/storage-engines.json @@ -0,0 +1,20 @@ +{ + "storage":{ + dfs: { + type: "file", + connection: "file:///" + }, + hive : + { + type:"hive", + config : + { + "hive.metastore.uris" : "", + "javax.jdo.option.ConnectionURL" : "jdbc:derby:;databaseName=../../sample-data/drill_hive_db;create=true", + "hive.metastore.warehouse.dir" : "/tmp/drill_hive_wh", + "fs.default.name" : "file:///", + "hive.metastore.sasl.enabled" : "false" + } + } + } +}
\ No newline at end of file |