diff options
Diffstat (limited to 'contrib/storage-hive/core/src/main/java/org/apache/drill/exec')
6 files changed, 23 insertions, 18 deletions
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/HivePartitionDescriptor.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/HivePartitionDescriptor.java index 25a0c080e..a52b48ded 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/HivePartitionDescriptor.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/HivePartitionDescriptor.java @@ -36,6 +36,7 @@ import org.apache.drill.exec.store.hive.HiveUtilities; import org.apache.drill.exec.store.hive.HiveReadEntry; import org.apache.drill.exec.store.hive.HiveScan; import org.apache.drill.exec.vector.ValueVector; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.api.Partition; import java.util.BitSet; @@ -87,9 +88,9 @@ public class HivePartitionDescriptor extends AbstractPartitionDescriptor { } @Override - public String getBaseTableLocation() { + public Path getBaseTableLocation() { HiveReadEntry origEntry = ((HiveScan) scanRel.getGroupScan()).getHiveReadEntry(); - return origEntry.table.getTable().getSd().getLocation(); + return new Path(origEntry.table.getTable().getSd().getLocation()); } @Override @@ -145,7 +146,7 @@ public class HivePartitionDescriptor extends AbstractPartitionDescriptor { List<PartitionLocation> locations = new LinkedList<>(); HiveReadEntry origEntry = ((HiveScan) scanRel.getGroupScan()).getHiveReadEntry(); for (Partition partition: origEntry.getPartitions()) { - locations.add(new HivePartitionLocation(partition.getValues(), partition.getSd().getLocation())); + locations.add(new HivePartitionLocation(partition.getValues(), new Path(partition.getSd().getLocation()))); } locationSuperList = Lists.partition(locations, PartitionDescriptor.PARTITION_BATCH_SIZE); sublistsCreated = true; @@ -170,7 +171,7 @@ public class HivePartitionDescriptor extends AbstractPartitionDescriptor { List<HiveTableWrapper.HivePartitionWrapper> newPartitions = Lists.newLinkedList(); for (HiveTableWrapper.HivePartitionWrapper part: oldPartitions) { - String partitionLocation = part.getPartition().getSd().getLocation(); + Path partitionLocation = new Path(part.getPartition().getSd().getLocation()); for (PartitionLocation newPartitionLocation: newPartitionLocations) { if (partitionLocation.equals(newPartitionLocation.getEntirePartitionLocation())) { newPartitions.add(part); diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/HivePartitionLocation.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/HivePartitionLocation.java index bb0efe841..25821f58a 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/HivePartitionLocation.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/HivePartitionLocation.java @@ -19,17 +19,19 @@ package org.apache.drill.exec.planner.sql; import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableList; import org.apache.drill.exec.planner.SimplePartitionLocation; +import org.apache.hadoop.fs.Path; import java.util.List; public class HivePartitionLocation extends SimplePartitionLocation { - private final String partitionLocation; + private final Path partitionLocation; private final List<String> partitionValues; - public HivePartitionLocation(final List<String> partitionValues, final String partitionLocation) { + public HivePartitionLocation(List<String> partitionValues, Path partitionLocation) { this.partitionValues = ImmutableList.copyOf(partitionValues); this.partitionLocation = partitionLocation; } + @Override public String getPartitionValue(int index) { assert index < partitionValues.size(); @@ -37,7 +39,7 @@ public class HivePartitionLocation extends SimplePartitionLocation { } @Override - public String getEntirePartitionLocation() { + public Path getEntirePartitionLocation() { return partitionLocation; } } diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveDrillNativeParquetRowGroupScan.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveDrillNativeParquetRowGroupScan.java index bea06e073..78e107a85 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveDrillNativeParquetRowGroupScan.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveDrillNativeParquetRowGroupScan.java @@ -124,7 +124,7 @@ public class HiveDrillNativeParquetRowGroupScan extends AbstractParquetRowGroupS @Override public Configuration getFsConf(RowGroupReadEntry rowGroupReadEntry) throws IOException { - Path path = new Path(rowGroupReadEntry.getPath()).getParent(); + Path path = rowGroupReadEntry.getPath().getParent(); return new ProjectionPusher().pushProjectionsAndFilters( new JobConf(HiveUtilities.generateHiveConf(hiveStoragePlugin.getHiveConf(), confProperties)), path.getParent()); diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveDrillNativeParquetScan.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveDrillNativeParquetScan.java index 617f6a59f..79a07f17b 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveDrillNativeParquetScan.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveDrillNativeParquetScan.java @@ -114,7 +114,7 @@ public class HiveDrillNativeParquetScan extends AbstractParquetGroupScan { assert split instanceof FileSplit; FileSplit fileSplit = (FileSplit) split; Path finalPath = fileSplit.getPath(); - String pathString = Path.getPathWithoutSchemeAndAuthority(finalPath).toString(); + Path pathString = Path.getPathWithoutSchemeAndAuthority(finalPath); entries.add(new ReadEntryWithPath(pathString)); // store partition values per path @@ -205,7 +205,7 @@ public class HiveDrillNativeParquetScan extends AbstractParquetGroupScan { protected void initInternal() throws IOException { Map<FileStatus, FileSystem> fileStatusConfMap = new LinkedHashMap<>(); for (ReadEntryWithPath entry : entries) { - Path path = new Path(entry.getPath()); + Path path = entry.getPath(); Configuration conf = new ProjectionPusher().pushProjectionsAndFilters( new JobConf(hiveStoragePlugin.getHiveConf()), path.getParent()); @@ -221,7 +221,7 @@ public class HiveDrillNativeParquetScan extends AbstractParquetGroupScan { } @Override - protected AbstractParquetGroupScan cloneWithFileSelection(Collection<String> filePaths) throws IOException { + protected AbstractParquetGroupScan cloneWithFileSelection(Collection<Path> filePaths) throws IOException { FileSelection newSelection = new FileSelection(null, new ArrayList<>(filePaths), null, null, false); return clone(newSelection); } diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveDrillNativeParquetScanBatchCreator.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveDrillNativeParquetScanBatchCreator.java index 339e1bda4..be906febe 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveDrillNativeParquetScanBatchCreator.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveDrillNativeParquetScanBatchCreator.java @@ -28,6 +28,7 @@ import org.apache.drill.exec.server.options.OptionManager; import org.apache.drill.exec.store.dfs.DrillFileSystem; import org.apache.drill.exec.store.parquet.AbstractParquetScanBatchCreator; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import java.io.IOException; import java.util.HashMap; @@ -54,7 +55,7 @@ public class HiveDrillNativeParquetScanBatchCreator extends AbstractParquetScanB */ private class HiveDrillNativeParquetDrillFileSystemManager extends AbstractDrillFileSystemManager { - private final Map<String, DrillFileSystem> fileSystems; + private final Map<Path, DrillFileSystem> fileSystems; HiveDrillNativeParquetDrillFileSystemManager(OperatorContext operatorContext) { super(operatorContext); @@ -62,7 +63,7 @@ public class HiveDrillNativeParquetScanBatchCreator extends AbstractParquetScanB } @Override - protected DrillFileSystem get(Configuration config, String path) throws ExecutionSetupException { + protected DrillFileSystem get(Configuration config, Path path) throws ExecutionSetupException { DrillFileSystem fs = fileSystems.get(path); if (fs == null) { try { diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HivePartitionHolder.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HivePartitionHolder.java index 803144e0b..3e16acba4 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HivePartitionHolder.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HivePartitionHolder.java @@ -19,6 +19,7 @@ package org.apache.drill.exec.store.hive; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.hadoop.fs.Path; import java.util.ArrayList; import java.util.Collections; @@ -34,11 +35,11 @@ import java.util.Map; */ public class HivePartitionHolder { - private final Map<String, Integer> keyToIndexMapper; + private final Map<Path, Integer> keyToIndexMapper; private final List<List<String>> partitionValues; @JsonCreator - public HivePartitionHolder(@JsonProperty("keyToIndexMapper") Map<String, Integer> keyToIndexMapper, + public HivePartitionHolder(@JsonProperty("keyToIndexMapper") Map<Path, Integer> keyToIndexMapper, @JsonProperty("partitionValues") List<List<String>> partitionValues) { this.keyToIndexMapper = keyToIndexMapper; this.partitionValues = partitionValues; @@ -50,7 +51,7 @@ public class HivePartitionHolder { } @JsonProperty - public Map<String, Integer> getKeyToIndexMapper() { + public Map<Path, Integer> getKeyToIndexMapper() { return keyToIndexMapper; } @@ -67,7 +68,7 @@ public class HivePartitionHolder { * @param key mapper key * @param values partition values */ - public void add(String key, List<String> values) { + public void add(Path key, List<String> values) { int index = partitionValues.indexOf(values); if (index == -1) { index = partitionValues.size(); @@ -84,7 +85,7 @@ public class HivePartitionHolder { * @param key mapper key * @return list of partition values */ - public List<String> get(String key) { + public List<String> get(Path key) { Integer index = keyToIndexMapper.get(key); if (index == null) { return Collections.emptyList(); |