diff options
| author | Parth Chandra <pchandra@maprtech.com> | 2014-08-28 22:23:56 -0700 |
|---|---|---|
| committer | Jacques Nadeau <jacques@apache.org> | 2014-08-29 08:29:43 -0700 |
| commit | 8bf3c4e0d8f3b2d7dc5b798f31272afa3d7eaf15 (patch) | |
| tree | 3d2a80d7b540f02703f0ace74f042f3fe01de838 /exec/java-exec/src/main | |
| parent | 4dcac1909eb687d226136c5794b2cf0c07e5733a (diff) | |
DRILL-1359 Fix reading and writing of repeated scalar types in Parquet
Diffstat (limited to 'exec/java-exec/src/main')
| -rw-r--r-- | exec/java-exec/src/main/codegen/templates/ParquetOutputRecordWriter.java | 4 | ||||
| -rw-r--r-- | exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java | 8 |
2 files changed, 10 insertions, 2 deletions
diff --git a/exec/java-exec/src/main/codegen/templates/ParquetOutputRecordWriter.java b/exec/java-exec/src/main/codegen/templates/ParquetOutputRecordWriter.java index d4b5af517..cb37a1b52 100644 --- a/exec/java-exec/src/main/codegen/templates/ParquetOutputRecordWriter.java +++ b/exec/java-exec/src/main/codegen/templates/ParquetOutputRecordWriter.java @@ -215,9 +215,9 @@ public abstract class ParquetOutputRecordWriter extends AbstractRecordWriter imp <#elseif minor.class == "VarChar" || minor.class == "Var16Char" || minor.class == "VarBinary"> <#if mode.prefix == "Repeated"> reader.read(i, holder); - consumer.startField(fieldName, fieldId); + //consumer.startField(fieldName, fieldId); consumer.addBinary(Binary.fromByteBuffer(holder.buffer.nioBuffer(holder.start, holder.end - holder.start))); - consumer.endField(fieldName, fieldId); + //consumer.endField(fieldName, fieldId); <#else> reader.read(holder); ByteBuf buf = holder.buffer; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java index f9b6d91b5..608c6898b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java @@ -44,6 +44,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import parquet.column.ColumnDescriptor; import parquet.hadoop.ParquetFileReader; import parquet.hadoop.metadata.ParquetMetadata; import parquet.schema.MessageType; @@ -153,11 +154,18 @@ public class ParquetScanBatchCreator implements BatchCreator<ParquetRowGroupScan private static boolean isComplex(ParquetMetadata footer) { MessageType schema = footer.getFileMetaData().getSchema(); + for (Type type : schema.getFields()) { if (!type.isPrimitive()) { return true; } } + for (ColumnDescriptor col : schema.getColumns()) { + if (col.getMaxRepetitionLevel() > 0) { + return true; + } + } return false; } + } |