aboutsummaryrefslogtreecommitdiff
path: root/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata
diff options
context:
space:
mode:
authorVitalii Diravka <vitalii.diravka@gmail.com>2019-02-18 22:30:36 +0200
committerVitalii Diravka <vitalii.diravka@gmail.com>2019-03-05 16:32:05 +0200
commit3d29faf81da593035f6bd38dd56d48e719afe7d4 (patch)
treed77b926b0de304677bf81a883e20e9776f986ea0 /exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata
parent7e3b45967dbb97da18ba49a2fa6a67a48e33b092 (diff)
DRILL-5603: Replace String file paths to Hadoop Path
- replaced all String path representation with org.apache.hadoop.fs.Path - added PathSerDe.Se JSON serializer - refactoring of DFSPartitionLocation code by leveraging existing listPartitionValues() functionality closes #1657
Diffstat (limited to 'exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata')
-rw-r--r--exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/Metadata.java37
-rw-r--r--exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/MetadataBase.java5
-rw-r--r--exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/MetadataPathUtils.java29
-rw-r--r--exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/Metadata_V1.java13
-rw-r--r--exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/Metadata_V2.java15
-rw-r--r--exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/Metadata_V3.java15
-rw-r--r--exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/ParquetTableMetadataDirs.java7
7 files changed, 68 insertions, 53 deletions
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/Metadata.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/Metadata.java
index 0db007ab6..d0e2734d2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/Metadata.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/Metadata.java
@@ -24,6 +24,7 @@ import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.fasterxml.jackson.module.afterburner.AfterburnerModule;
+import org.apache.drill.exec.serialization.PathSerDe;
import org.apache.drill.exec.store.parquet.ParquetReaderConfig;
import org.apache.drill.shaded.guava.com.google.common.base.Stopwatch;
import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
@@ -82,6 +83,9 @@ import static org.apache.drill.exec.store.parquet.metadata.Metadata_V3.ParquetFi
import static org.apache.drill.exec.store.parquet.metadata.Metadata_V3.ParquetTableMetadata_v3;
import static org.apache.drill.exec.store.parquet.metadata.Metadata_V3.RowGroupMetadata_v3;
+/**
+ * This is an utility class, holder for Parquet Table Metadata and {@link ParquetReaderConfig}
+ */
public class Metadata {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Metadata.class);
@@ -106,7 +110,7 @@ public class Metadata {
* @param path path
* @param readerConfig parquet reader configuration
*/
- public static void createMeta(FileSystem fs, String path, ParquetReaderConfig readerConfig) throws IOException {
+ public static void createMeta(FileSystem fs, Path path, ParquetReaderConfig readerConfig) throws IOException {
Metadata metadata = new Metadata(readerConfig);
metadata.createMetaFilesRecursively(path, fs);
}
@@ -208,13 +212,13 @@ public class Metadata {
* {@code path} directory).
* @throws IOException if parquet metadata can't be serialized and written to the json file
*/
- private Pair<ParquetTableMetadata_v3, ParquetTableMetadataDirs> createMetaFilesRecursively(final String path, FileSystem fs) throws IOException {
+ private Pair<ParquetTableMetadata_v3, ParquetTableMetadataDirs> createMetaFilesRecursively(final Path path, FileSystem fs) throws IOException {
Stopwatch timer = logger.isDebugEnabled() ? Stopwatch.createStarted() : null;
List<ParquetFileMetadata_v3> metaDataList = Lists.newArrayList();
- List<String> directoryList = Lists.newArrayList();
+ List<Path> directoryList = Lists.newArrayList();
ConcurrentHashMap<ColumnTypeMetadata_v3.Key, ColumnTypeMetadata_v3> columnTypeInfoSet =
new ConcurrentHashMap<>();
- Path p = new Path(path);
+ Path p = path;
FileStatus fileStatus = fs.getFileStatus(p);
assert fileStatus.isDirectory() : "Expected directory";
@@ -222,10 +226,10 @@ public class Metadata {
for (final FileStatus file : DrillFileSystemUtil.listAll(fs, p, false)) {
if (file.isDirectory()) {
- ParquetTableMetadata_v3 subTableMetadata = (createMetaFilesRecursively(file.getPath().toString(), fs)).getLeft();
+ ParquetTableMetadata_v3 subTableMetadata = (createMetaFilesRecursively(file.getPath(), fs)).getLeft();
metaDataList.addAll(subTableMetadata.files);
directoryList.addAll(subTableMetadata.directories);
- directoryList.add(file.getPath().toString());
+ directoryList.add(file.getPath());
// Merge the schema from the child level into the current level
//TODO: We need a merge method that merges two columns with the same name but different types
columnTypeInfoSet.putAll(subTableMetadata.columnTypeInfo);
@@ -268,7 +272,7 @@ public class Metadata {
ParquetTableMetadataDirs parquetTableMetadataDirs = new ParquetTableMetadataDirs(directoryList);
return Pair.of(parquetTableMetadata, parquetTableMetadataDirs);
}
- List<String> emptyDirList = Lists.newArrayList();
+ List<Path> emptyDirList = new ArrayList<>();
if (timer != null) {
logger.debug("Creating metadata files recursively took {} ms", timer.elapsed(TimeUnit.MILLISECONDS));
timer.stop();
@@ -495,7 +499,7 @@ public class Metadata {
rowGroupMetadataList.add(rowGroupMeta);
}
- String path = Path.getPathWithoutSchemeAndAuthority(file.getPath()).toString();
+ Path path = Path.getPathWithoutSchemeAndAuthority(file.getPath());
return new ParquetFileMetadata_v3(path, file.getLen(), rowGroupMetadataList);
}
@@ -535,6 +539,8 @@ public class Metadata {
*
* @param parquetTableMetadata parquet table metadata
* @param p file path
+ * @param fs Drill file system
+ * @throws IOException if metadata can't be serialized
*/
private void writeFile(ParquetTableMetadata_v3 parquetTableMetadata, Path p, FileSystem fs) throws IOException {
JsonFactory jsonFactory = new JsonFactory();
@@ -542,6 +548,7 @@ public class Metadata {
jsonFactory.configure(JsonParser.Feature.AUTO_CLOSE_SOURCE, false);
ObjectMapper mapper = new ObjectMapper(jsonFactory);
SimpleModule module = new SimpleModule();
+ module.addSerializer(Path.class, new PathSerDe.Se());
module.addSerializer(ColumnMetadata_v3.class, new ColumnMetadata_v3.Serializer());
mapper.registerModule(module);
OutputStream os = fs.create(p);
@@ -556,6 +563,7 @@ public class Metadata {
jsonFactory.configure(JsonParser.Feature.AUTO_CLOSE_SOURCE, false);
ObjectMapper mapper = new ObjectMapper(jsonFactory);
SimpleModule module = new SimpleModule();
+ module.addSerializer(Path.class, new PathSerDe.Se());
mapper.registerModule(module);
OutputStream os = fs.create(p);
mapper.writerWithDefaultPrettyPrinter().writeValue(os, parquetTableMetadataDirs);
@@ -602,7 +610,7 @@ public class Metadata {
parquetTableMetadataDirs.updateRelativePaths(metadataParentDirPath);
if (!alreadyCheckedModification && tableModified(parquetTableMetadataDirs.getDirectories(), path, metadataParentDir, metaContext, fs)) {
parquetTableMetadataDirs =
- (createMetaFilesRecursively(Path.getPathWithoutSchemeAndAuthority(path.getParent()).toString(), fs)).getRight();
+ (createMetaFilesRecursively(Path.getPathWithoutSchemeAndAuthority(path.getParent()), fs)).getRight();
newMetadata = true;
}
} else {
@@ -616,7 +624,7 @@ public class Metadata {
}
if (!alreadyCheckedModification && tableModified(parquetTableMetadata.getDirectories(), path, metadataParentDir, metaContext, fs)) {
parquetTableMetadata =
- (createMetaFilesRecursively(Path.getPathWithoutSchemeAndAuthority(path.getParent()).toString(), fs)).getLeft();
+ (createMetaFilesRecursively(Path.getPathWithoutSchemeAndAuthority(path.getParent()), fs)).getLeft();
newMetadata = true;
}
@@ -647,9 +655,10 @@ public class Metadata {
* @return true if metadata needs to be updated, false otherwise
* @throws IOException if some resources are not accessible
*/
- private boolean tableModified(List<String> directories, Path metaFilePath, Path parentDir, MetadataContext metaContext, FileSystem fs) throws IOException {
+ private boolean tableModified(List<Path> directories, Path metaFilePath, Path parentDir,
+ MetadataContext metaContext, FileSystem fs) throws IOException {
Stopwatch timer = logger.isDebugEnabled() ? Stopwatch.createStarted() : null;
- metaContext.setStatus(parentDir.toUri().getPath());
+ metaContext.setStatus(parentDir);
long metaFileModifyTime = fs.getFileStatus(metaFilePath).getModificationTime();
FileStatus directoryStatus = fs.getFileStatus(parentDir);
int numDirs = 1;
@@ -661,10 +670,10 @@ public class Metadata {
}
return true;
}
- for (String directory : directories) {
+ for (Path directory : directories) {
numDirs++;
metaContext.setStatus(directory);
- directoryStatus = fs.getFileStatus(new Path(directory));
+ directoryStatus = fs.getFileStatus(directory);
if (directoryStatus.getModificationTime() > metaFileModifyTime) {
if (timer != null) {
logger.debug("Directory {} was modified. Took {} ms to check modification time of {} directories",
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/MetadataBase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/MetadataBase.java
index bed8be67a..ee07470d2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/MetadataBase.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/MetadataBase.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.store.parquet.metadata;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import org.apache.hadoop.fs.Path;
import org.apache.parquet.schema.OriginalType;
import org.apache.parquet.schema.PrimitiveType;
@@ -57,7 +58,7 @@ public class MetadataBase {
public static abstract class ParquetTableMetadataBase {
@JsonIgnore
- public abstract List<String> getDirectories();
+ public abstract List<Path> getDirectories();
@JsonIgnore public abstract List<? extends ParquetFileMetadata> getFiles();
@@ -83,7 +84,7 @@ public class MetadataBase {
}
public static abstract class ParquetFileMetadata {
- @JsonIgnore public abstract String getPath();
+ @JsonIgnore public abstract Path getPath();
@JsonIgnore public abstract Long getLength();
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/MetadataPathUtils.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/MetadataPathUtils.java
index 3e7c2ffcc..2794e2b14 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/MetadataPathUtils.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/MetadataPathUtils.java
@@ -21,6 +21,7 @@ import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
import org.apache.drill.common.util.DrillVersionInfo;
import org.apache.hadoop.fs.Path;
+import java.util.ArrayList;
import java.util.List;
import static org.apache.drill.exec.store.parquet.metadata.MetadataVersion.Constants.SUPPORTED_VERSIONS;
@@ -39,12 +40,12 @@ public class MetadataPathUtils {
* @param baseDir base parent directory
* @return list of absolute paths
*/
- public static List<String> convertToAbsolutePaths(List<String> paths, String baseDir) {
+ public static List<Path> convertToAbsolutePaths(List<Path> paths, String baseDir) {
if (!paths.isEmpty()) {
- List<String> absolutePaths = Lists.newArrayList();
- for (String relativePath : paths) {
- String absolutePath = (new Path(relativePath).isAbsolute()) ? relativePath
- : new Path(baseDir, relativePath).toUri().getPath();
+ List<Path> absolutePaths = Lists.newArrayList();
+ for (Path relativePath : paths) {
+ Path absolutePath = (relativePath.isAbsolute()) ? relativePath
+ : new Path(baseDir, relativePath);
absolutePaths.add(absolutePath);
}
return absolutePaths;
@@ -64,10 +65,10 @@ public class MetadataPathUtils {
if (!files.isEmpty()) {
List<ParquetFileMetadata_v3> filesWithAbsolutePaths = Lists.newArrayList();
for (ParquetFileMetadata_v3 file : files) {
- Path relativePath = new Path(file.getPath());
+ Path relativePath = file.getPath();
// create a new file if old one contains a relative path, otherwise use an old file
ParquetFileMetadata_v3 fileWithAbsolutePath = (relativePath.isAbsolute()) ? file
- : new ParquetFileMetadata_v3(new Path(baseDir, relativePath).toUri().getPath(), file.length, file.rowGroups);
+ : new ParquetFileMetadata_v3(new Path(baseDir, relativePath), file.length, file.rowGroups);
filesWithAbsolutePaths.add(fileWithAbsolutePath);
}
return filesWithAbsolutePaths;
@@ -84,9 +85,9 @@ public class MetadataPathUtils {
* @return parquet table metadata with relative paths for the files and directories
*/
public static ParquetTableMetadata_v3 createMetadataWithRelativePaths(
- ParquetTableMetadata_v3 tableMetadataWithAbsolutePaths, String baseDir) {
- List<String> directoriesWithRelativePaths = Lists.newArrayList();
- for (String directory : tableMetadataWithAbsolutePaths.getDirectories()) {
+ ParquetTableMetadata_v3 tableMetadataWithAbsolutePaths, Path baseDir) {
+ List<Path> directoriesWithRelativePaths = new ArrayList<>();
+ for (Path directory : tableMetadataWithAbsolutePaths.getDirectories()) {
directoriesWithRelativePaths.add(relativize(baseDir, directory));
}
List<ParquetFileMetadata_v3> filesWithRelativePaths = Lists.newArrayList();
@@ -105,9 +106,9 @@ public class MetadataPathUtils {
* @param baseDir base path (the part of the Path, which should be cut off from child path)
* @return relative path
*/
- public static String relativize(String baseDir, String childPath) {
- Path fullPathWithoutSchemeAndAuthority = Path.getPathWithoutSchemeAndAuthority(new Path(childPath));
- Path basePathWithoutSchemeAndAuthority = Path.getPathWithoutSchemeAndAuthority(new Path(baseDir));
+ public static Path relativize(Path baseDir, Path childPath) {
+ Path fullPathWithoutSchemeAndAuthority = Path.getPathWithoutSchemeAndAuthority(childPath);
+ Path basePathWithoutSchemeAndAuthority = Path.getPathWithoutSchemeAndAuthority(baseDir);
// Since hadoop Path hasn't relativize() we use uri.relativize() to get relative path
Path relativeFilePath = new Path(basePathWithoutSchemeAndAuthority.toUri()
@@ -116,7 +117,7 @@ public class MetadataPathUtils {
throw new IllegalStateException(String.format("Path %s is not a subpath of %s.",
basePathWithoutSchemeAndAuthority.toUri().getPath(), fullPathWithoutSchemeAndAuthority.toUri().getPath()));
}
- return relativeFilePath.toUri().getPath();
+ return relativeFilePath;
}
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/Metadata_V1.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/Metadata_V1.java
index 92feb5f8a..4b0dca803 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/Metadata_V1.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/Metadata_V1.java
@@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
import org.apache.drill.common.expression.SchemaPath;
+import org.apache.hadoop.fs.Path;
import org.apache.parquet.io.api.Binary;
import org.apache.parquet.schema.OriginalType;
import org.apache.parquet.schema.PrimitiveType;
@@ -43,19 +44,19 @@ public class Metadata_V1 {
@JsonProperty(value = "metadata_version", access = JsonProperty.Access.WRITE_ONLY) private String metadataVersion;
@JsonProperty
List<ParquetFileMetadata_v1> files;
- @JsonProperty List<String> directories;
+ @JsonProperty List<Path> directories;
public ParquetTableMetadata_v1() {
}
- public ParquetTableMetadata_v1(String metadataVersion, List<ParquetFileMetadata_v1> files, List<String> directories) {
+ public ParquetTableMetadata_v1(String metadataVersion, List<ParquetFileMetadata_v1> files, List<Path> directories) {
this.metadataVersion = metadataVersion;
this.files = files;
this.directories = directories;
}
@JsonIgnore
- @Override public List<String> getDirectories() {
+ @Override public List<Path> getDirectories() {
return directories;
}
@@ -114,7 +115,7 @@ public class Metadata_V1 {
*/
public static class ParquetFileMetadata_v1 extends ParquetFileMetadata {
@JsonProperty
- public String path;
+ public Path path;
@JsonProperty
public Long length;
@JsonProperty
@@ -123,7 +124,7 @@ public class Metadata_V1 {
public ParquetFileMetadata_v1() {
}
- public ParquetFileMetadata_v1(String path, Long length, List<RowGroupMetadata_v1> rowGroups) {
+ public ParquetFileMetadata_v1(Path path, Long length, List<RowGroupMetadata_v1> rowGroups) {
this.path = path;
this.length = length;
this.rowGroups = rowGroups;
@@ -134,7 +135,7 @@ public class Metadata_V1 {
return String.format("path: %s rowGroups: %s", path, rowGroups);
}
- @JsonIgnore @Override public String getPath() {
+ @JsonIgnore @Override public Path getPath() {
return path;
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/Metadata_V2.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/Metadata_V2.java
index 7eddc1279..a78fca4bf 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/Metadata_V2.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/Metadata_V2.java
@@ -28,6 +28,7 @@ import com.fasterxml.jackson.databind.JsonDeserializer;
import com.fasterxml.jackson.databind.JsonSerializer;
import com.fasterxml.jackson.databind.KeyDeserializer;
import com.fasterxml.jackson.databind.SerializerProvider;
+import org.apache.hadoop.fs.Path;
import org.apache.parquet.io.api.Binary;
import org.apache.parquet.schema.OriginalType;
import org.apache.parquet.schema.PrimitiveType;
@@ -59,7 +60,7 @@ public class Metadata_V2 {
@JsonProperty public ConcurrentHashMap<ColumnTypeMetadata_v2.Key, ColumnTypeMetadata_v2> columnTypeInfo;
@JsonProperty
List<ParquetFileMetadata_v2> files;
- @JsonProperty List<String> directories;
+ @JsonProperty List<Path> directories;
@JsonProperty String drillVersion;
public ParquetTableMetadata_v2() {
@@ -71,7 +72,7 @@ public class Metadata_V2 {
}
public ParquetTableMetadata_v2(String metadataVersion, ParquetTableMetadataBase parquetTable,
- List<ParquetFileMetadata_v2> files, List<String> directories, String drillVersion) {
+ List<ParquetFileMetadata_v2> files, List<Path> directories, String drillVersion) {
this.metadataVersion = metadataVersion;
this.files = files;
this.directories = directories;
@@ -79,7 +80,7 @@ public class Metadata_V2 {
this.drillVersion = drillVersion;
}
- public ParquetTableMetadata_v2(String metadataVersion, List<ParquetFileMetadata_v2> files, List<String> directories,
+ public ParquetTableMetadata_v2(String metadataVersion, List<ParquetFileMetadata_v2> files, List<Path> directories,
ConcurrentHashMap<ColumnTypeMetadata_v2.Key, ColumnTypeMetadata_v2> columnTypeInfo, String drillVersion) {
this.metadataVersion = metadataVersion;
this.files = files;
@@ -93,7 +94,7 @@ public class Metadata_V2 {
}
@JsonIgnore
- @Override public List<String> getDirectories() {
+ @Override public List<Path> getDirectories() {
return directories;
}
@@ -152,14 +153,14 @@ public class Metadata_V2 {
* Struct which contains the metadata for a single parquet file
*/
public static class ParquetFileMetadata_v2 extends ParquetFileMetadata {
- @JsonProperty public String path;
+ @JsonProperty public Path path;
@JsonProperty public Long length;
@JsonProperty public List<RowGroupMetadata_v2> rowGroups;
public ParquetFileMetadata_v2() {
}
- public ParquetFileMetadata_v2(String path, Long length, List<RowGroupMetadata_v2> rowGroups) {
+ public ParquetFileMetadata_v2(Path path, Long length, List<RowGroupMetadata_v2> rowGroups) {
this.path = path;
this.length = length;
this.rowGroups = rowGroups;
@@ -169,7 +170,7 @@ public class Metadata_V2 {
return String.format("path: %s rowGroups: %s", path, rowGroups);
}
- @JsonIgnore @Override public String getPath() {
+ @JsonIgnore @Override public Path getPath() {
return path;
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/Metadata_V3.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/Metadata_V3.java
index 4bb07f78f..a5ff89795 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/Metadata_V3.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/Metadata_V3.java
@@ -25,6 +25,7 @@ import com.fasterxml.jackson.databind.JsonSerializer;
import com.fasterxml.jackson.databind.KeyDeserializer;
import com.fasterxml.jackson.databind.SerializerProvider;
import org.apache.drill.common.expression.SchemaPath;
+import org.apache.hadoop.fs.Path;
import org.apache.parquet.io.api.Binary;
import org.apache.parquet.schema.OriginalType;
import org.apache.parquet.schema.PrimitiveType;
@@ -54,7 +55,7 @@ public class Metadata_V3 {
@JsonProperty public ConcurrentHashMap<ColumnTypeMetadata_v3.Key, ColumnTypeMetadata_v3> columnTypeInfo;
@JsonProperty
List<ParquetFileMetadata_v3> files;
- @JsonProperty List<String> directories;
+ @JsonProperty List<Path> directories;
@JsonProperty String drillVersion;
/**
@@ -74,7 +75,7 @@ public class Metadata_V3 {
}
public ParquetTableMetadata_v3(String metadataVersion, ParquetTableMetadataBase parquetTable,
- List<ParquetFileMetadata_v3> files, List<String> directories, String drillVersion) {
+ List<ParquetFileMetadata_v3> files, List<Path> directories, String drillVersion) {
this.metadataVersion = metadataVersion;
this.files = files;
this.directories = directories;
@@ -82,7 +83,7 @@ public class Metadata_V3 {
this.drillVersion = drillVersion;
}
- public ParquetTableMetadata_v3(String metadataVersion, List<ParquetFileMetadata_v3> files, List<String> directories,
+ public ParquetTableMetadata_v3(String metadataVersion, List<ParquetFileMetadata_v3> files, List<Path> directories,
ConcurrentHashMap<ColumnTypeMetadata_v3.Key, ColumnTypeMetadata_v3> columnTypeInfo,
String drillVersion) {
this.metadataVersion = metadataVersion;
@@ -97,7 +98,7 @@ public class Metadata_V3 {
}
@JsonIgnore
- @Override public List<String> getDirectories() {
+ @Override public List<Path> getDirectories() {
return directories;
}
@@ -168,14 +169,14 @@ public class Metadata_V3 {
* Struct which contains the metadata for a single parquet file
*/
public static class ParquetFileMetadata_v3 extends ParquetFileMetadata {
- @JsonProperty public String path;
+ @JsonProperty public Path path;
@JsonProperty public Long length;
@JsonProperty public List<RowGroupMetadata_v3> rowGroups;
public ParquetFileMetadata_v3() {
}
- public ParquetFileMetadata_v3(String path, Long length, List<RowGroupMetadata_v3> rowGroups) {
+ public ParquetFileMetadata_v3(Path path, Long length, List<RowGroupMetadata_v3> rowGroups) {
this.path = path;
this.length = length;
this.rowGroups = rowGroups;
@@ -185,7 +186,7 @@ public class Metadata_V3 {
return String.format("path: %s rowGroups: %s", path, rowGroups);
}
- @JsonIgnore @Override public String getPath() {
+ @JsonIgnore @Override public Path getPath() {
return path;
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/ParquetTableMetadataDirs.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/ParquetTableMetadataDirs.java
index 186f53415..b1fd7f23e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/ParquetTableMetadataDirs.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/ParquetTableMetadataDirs.java
@@ -19,24 +19,25 @@ package org.apache.drill.exec.store.parquet.metadata;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.fs.Path;
import java.util.List;
public class ParquetTableMetadataDirs {
@JsonProperty
- List<String> directories;
+ List<Path> directories;
public ParquetTableMetadataDirs() {
// default constructor needed for deserialization
}
- public ParquetTableMetadataDirs(List<String> directories) {
+ public ParquetTableMetadataDirs(List<Path> directories) {
this.directories = directories;
}
@JsonIgnore
- public List<String> getDirectories() {
+ public List<Path> getDirectories() {
return directories;
}