diff options
author | Jason Altekruse <altekrusejason@gmail.com> | 2014-08-29 19:45:22 -0700 |
---|---|---|
committer | Jacques Nadeau <jacques@apache.org> | 2014-08-29 23:05:56 -0700 |
commit | 749dbe40ae0fcec3a4387c3b7454ba3ad0fe7902 (patch) | |
tree | 2bc47ae8dc81b5d5327a8b191fa4842b64f77219 /exec/java-exec/src | |
parent | ffbf3ae15a2752acc60693972b9a37eb948ef872 (diff) |
Fix for Parquet reader in larger files brought about by more aggressive memory management.
Diffstat (limited to 'exec/java-exec/src')
3 files changed, 23 insertions, 4 deletions
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableVarLengthValuesColumn.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableVarLengthValuesColumn.java index dc29fbd1f..8bff0b0be 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableVarLengthValuesColumn.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableVarLengthValuesColumn.java @@ -88,12 +88,10 @@ public abstract class NullableVarLengthValuesColumn<V extends ValueVector> exten dataTypeLengthInBits = pageReader.pageDataByteArray.getInt((int) pageReader.readyToReadPosInBytes); } // I think this also needs to happen if it is null for the random access - if (! variableWidthVector.getMutator().setValueLengthSafe((int) valuesReadInCurrentPass + pageReader.valuesReadyToRead, dataTypeLengthInBits)) { - return true; - } boolean success = setSafe(valuesReadInCurrentPass + pageReader.valuesReadyToRead, pageReader.pageDataByteArray, (int) pageReader.readyToReadPosInBytes + 4, dataTypeLengthInBits); - assert success; + if ( ! success ) + return true; return false; } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java index a61b40f4d..268d03d99 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java @@ -236,6 +236,22 @@ public class TestParquetWriter extends BaseTestQuery { } } + @Ignore + @Test + public void testParquetRead2() throws Exception { + test("alter system set `store.parquet.use_new_reader` = true"); + List<QueryResultBatch> expected = testSqlWithResults("select s_comment,s_suppkey from dfs.`/tmp/sf100_supplier.parquet`"); + test("alter system set `store.parquet.use_new_reader` = false"); + List<QueryResultBatch> results = testSqlWithResults("select s_comment,s_suppkey from dfs.`/tmp/sf100_supplier.parquet`"); + compareResults(expected, results); + for (QueryResultBatch result : results) { + result.release(); + } + for (QueryResultBatch result : expected) { + result.release(); + } + } + public void runTestAndValidate(String selection, String validationSelection, String inputTable, String outputFile) throws Exception { Path path = new Path("/tmp/" + outputFile); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java index def6dac80..ecdb990c2 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java @@ -152,6 +152,11 @@ public class ParquetRecordReaderTest extends BaseTestQuery{ } @Test + public void testNullableVarCharMemory() throws Exception { + testFull(QueryType.SQL, "select s_comment,s_suppkey from dfs.`/tmp/sf100_supplier.parquet`", "", 1, 1, 1000, false); + } + + @Test public void testReadVoter() throws Exception { testFull(QueryType.SQL, "select * from dfs.`/tmp/voter.parquet`", "", 1, 1, 1000, false); } |