diff options
| author | Paul Rogers <progers@cloudera.com> | 2019-01-28 22:04:31 -0800 |
|---|---|---|
| committer | Arina Ielchiieva <arina.yelchiyeva@gmail.com> | 2019-02-08 11:13:22 +0200 |
| commit | 304293a46e66ba27b6b38bbc2fef63743f78d598 (patch) | |
| tree | 978c1cfc24b0753094cf91f85ff77c439e9a0c42 | |
| parent | 1fa12e71c45efd9d1dd8ff383588d30360753bac (diff) | |
DRILL-7024: Refactor ColumnWriter to simplify type-conversion shim
DRILL-7006 added a type conversion "shim" within the row set framework. Basically, we insert a "shim" column writer that takes data in one form (String, say), and does reader-specific conversions to a target format (INT, say).
The code works fine, but the shim class ends up needing to override a bunch of methods which it then passes along to the base writer. This PR refactors the code so that the conversion shim is simpler.
closes #1633
65 files changed, 587 insertions, 470 deletions
diff --git a/common/src/test/java/org/apache/drill/categories/RowSetTests.java b/common/src/test/java/org/apache/drill/categories/RowSetTests.java new file mode 100644 index 000000000..98ad675f0 --- /dev/null +++ b/common/src/test/java/org/apache/drill/categories/RowSetTests.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.categories; + +/** + * A category for tests that test the RowSet, ResultSetLoader + * and related mechanisms. + */ +public interface RowSetTests { + // Junit category marker +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/ColumnBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/ColumnBuilder.java index 33a8eeb8c..19d86453d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/ColumnBuilder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/ColumnBuilder.java @@ -162,7 +162,7 @@ public class ColumnBuilder { (NullableVector) vector); } else { vectorState = SimpleVectorState.vectorState(columnSchema, - colWriter.scalar(), vector); + colWriter.events(), vector); } // Create the column state which binds the vector and writer together. @@ -261,7 +261,7 @@ public class ColumnBuilder { offsetVectorState = new OffsetVectorState( (((AbstractArrayWriter) writer.array()).offsetWriter()), offsetVector, - writer.array().entry()); + writer.array().entry().events()); } else { offsetVectorState = new NullVectorState(); } @@ -397,7 +397,8 @@ public class ColumnBuilder { // Create the list vector state that tracks the list vector lifecycle. - final ListVectorState vectorState = new ListVectorState(listWriter, memberState.writer(), listVector); + final ListVectorState vectorState = new ListVectorState(listWriter, + memberState.writer().events(), listVector); // Assemble it all into a union column state. @@ -504,7 +505,7 @@ public class ColumnBuilder { // For a repeated list, we only care about final RepeatedListVectorState vectorState = new RepeatedListVectorState( - arrayWriter.array(), vector); + arrayWriter, vector); // Build the container that tracks the array contents diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/ColumnState.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/ColumnState.java index 95f011a23..36b9db810 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/ColumnState.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/ColumnState.java @@ -22,9 +22,10 @@ import org.apache.drill.exec.record.metadata.ColumnMetadata; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.accessor.ObjectType; import org.apache.drill.exec.vector.accessor.ScalarWriter; -import org.apache.drill.exec.vector.accessor.ScalarWriter.ColumnWriterListener; import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter; import org.apache.drill.exec.vector.accessor.writer.AbstractObjectWriter; +import org.apache.drill.exec.vector.accessor.writer.WriterEvents; +import org.apache.drill.exec.vector.accessor.writer.WriterEvents.ColumnWriterListener; /** * Represents the write-time state for a column including the writer and the (optional) @@ -53,13 +54,13 @@ public abstract class ColumnState { AbstractObjectWriter colWriter, VectorState vectorState) { super(loader, colWriter, vectorState); - ScalarWriter scalarWriter; + WriterEvents scalarEvents; if (colWriter.type() == ObjectType.ARRAY) { - scalarWriter = writer.array().scalar(); + scalarEvents = writer.array().entry().events(); } else { - scalarWriter = writer.scalar(); + scalarEvents = writer.events(); } - scalarWriter.bindListener(this); + scalarEvents.bindListener(this); } @Override diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/ListState.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/ListState.java index 3488e7bb7..37104f437 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/ListState.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/ListState.java @@ -32,12 +32,12 @@ import org.apache.drill.exec.record.metadata.VariantMetadata; import org.apache.drill.exec.record.metadata.VariantSchema; import org.apache.drill.exec.vector.accessor.ObjectWriter; import org.apache.drill.exec.vector.accessor.VariantWriter; -import org.apache.drill.exec.vector.accessor.WriterPosition; import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter; import org.apache.drill.exec.vector.accessor.writer.ListWriterImpl; import org.apache.drill.exec.vector.accessor.writer.SimpleListShim; import org.apache.drill.exec.vector.accessor.writer.UnionVectorShim; import org.apache.drill.exec.vector.accessor.writer.UnionWriterImpl; +import org.apache.drill.exec.vector.accessor.writer.WriterEvents; import org.apache.drill.exec.vector.complex.ListVector; import org.apache.drill.exec.vector.complex.UnionVector; @@ -123,7 +123,7 @@ public class ListState extends ContainerState memberVectorState = new NullVectorState(); } - public ListVectorState(ListWriterImpl writer, WriterPosition elementWriter, ListVector vector) { + public ListVectorState(ListWriterImpl writer, WriterEvents elementWriter, ListVector vector) { this.schema = writer.schema(); this.vector = vector; bitsVectorState = new IsSetVectorState(writer, vector.getBitsVector()); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/NullableVectorState.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/NullableVectorState.java index b01292074..90dc3fe58 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/NullableVectorState.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/NullableVectorState.java @@ -38,7 +38,7 @@ public class NullableVectorState implements VectorState { this.schema = writer.schema(); this.vector = vector; - this.writer = (NullableScalarWriter) writer.scalar(); + this.writer = (NullableScalarWriter) writer.events(); bitsState = new IsSetVectorState(this.writer.bitsWriter(), vector.getBitsVector()); valuesState = SimpleVectorState.vectorState(this.writer.schema(), this.writer.baseWriter(), vector.getValuesVector()); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/RepeatedListState.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/RepeatedListState.java index ab64dded3..39d7d4423 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/RepeatedListState.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/RepeatedListState.java @@ -87,12 +87,12 @@ public class RepeatedListState extends ContainerState implements RepeatedListWri private final RepeatedListVector vector; private final OffsetVectorState offsetsState; - public RepeatedListVectorState(ArrayWriter arrayWriter, RepeatedListVector vector) { + public RepeatedListVectorState(AbstractObjectWriter arrayWriter, RepeatedListVector vector) { this.vector = vector; - this.arrayWriter = arrayWriter; + this.arrayWriter = arrayWriter.array(); offsetsState = new OffsetVectorState( - arrayWriter, vector.getOffsetVector(), - arrayWriter.entryType() == null ? null : arrayWriter.array()); + arrayWriter.events(), vector.getOffsetVector(), + this.arrayWriter.entryType() == null ? null : arrayWriter.events()); } /** @@ -105,7 +105,7 @@ public class RepeatedListState extends ContainerState implements RepeatedListWri */ public void updateChildWriter(AbstractObjectWriter childWriter) { - offsetsState.setChildWriter(childWriter.array()); + offsetsState.setChildWriter(childWriter.events()); } @SuppressWarnings("unchecked") diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/RepeatedVectorState.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/RepeatedVectorState.java index 8dd4839ce..1d01b9d08 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/RepeatedVectorState.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/RepeatedVectorState.java @@ -23,7 +23,7 @@ import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.accessor.ArrayWriter; import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter; import org.apache.drill.exec.vector.accessor.writer.AbstractArrayWriter; -import org.apache.drill.exec.vector.accessor.writer.AbstractScalarWriter; +import org.apache.drill.exec.vector.accessor.writer.AbstractScalarWriterImpl; import org.apache.drill.exec.vector.complex.RepeatedValueVector; /** @@ -47,7 +47,7 @@ public class RepeatedVectorState implements VectorState { // vector, and the scalar (value) portion of the array writer. arrayWriter = (AbstractArrayWriter) writer; - AbstractScalarWriter colWriter = (AbstractScalarWriter) writer.scalar(); + AbstractScalarWriterImpl colWriter = (AbstractScalarWriterImpl) writer.entry().events(); valuesState = SimpleVectorState.vectorState(writer.schema(), colWriter, vector.getDataVector()); // Create the offsets state with the offset vector portion of the repeated diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/SingleVectorState.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/SingleVectorState.java index e57373caa..538435375 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/SingleVectorState.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/SingleVectorState.java @@ -29,6 +29,7 @@ import org.apache.drill.exec.vector.VariableWidthVector; import org.apache.drill.exec.vector.accessor.WriterPosition; import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter; import org.apache.drill.exec.vector.accessor.writer.OffsetVectorWriter; +import org.apache.drill.exec.vector.accessor.writer.WriterEvents; /** * Base class for a single vector. Handles the bulk of work for that vector. @@ -41,7 +42,7 @@ public abstract class SingleVectorState implements VectorState { public abstract static class SimpleVectorState extends SingleVectorState { - public SimpleVectorState(WriterPosition writer, + public SimpleVectorState(WriterEvents writer, ValueVector mainVector) { super(writer, mainVector); } @@ -74,7 +75,7 @@ public abstract class SingleVectorState implements VectorState { public static class FixedWidthVectorState extends SimpleVectorState { - public FixedWidthVectorState(WriterPosition writer, ValueVector mainVector) { + public FixedWidthVectorState(WriterEvents writer, ValueVector mainVector) { super(writer, mainVector); } @@ -87,7 +88,7 @@ public abstract class SingleVectorState implements VectorState { public static class IsSetVectorState extends FixedWidthVectorState { - public IsSetVectorState(WriterPosition writer, ValueVector mainVector) { + public IsSetVectorState(WriterEvents writer, ValueVector mainVector) { super(writer, mainVector); } @@ -112,7 +113,7 @@ public abstract class SingleVectorState implements VectorState { private final ColumnMetadata schema; - public VariableWidthVectorState(ColumnMetadata schema, WriterPosition writer, ValueVector mainVector) { + public VariableWidthVectorState(ColumnMetadata schema, WriterEvents writer, ValueVector mainVector) { super(writer, mainVector); this.schema = schema; } @@ -147,13 +148,13 @@ public abstract class SingleVectorState implements VectorState { private WriterPosition childWriter; - public OffsetVectorState(WriterPosition writer, ValueVector mainVector, + public OffsetVectorState(WriterEvents writer, ValueVector mainVector, WriterPosition childWriter) { super(writer, mainVector); this.childWriter = childWriter; } - public void setChildWriter(WriterPosition childWriter) { + public void setChildWriter(WriterEvents childWriter) { this.childWriter = childWriter; } @@ -224,11 +225,11 @@ public abstract class SingleVectorState implements VectorState { } } - protected final WriterPosition writer; + protected final WriterEvents writer; protected final ValueVector mainVector; protected ValueVector backupVector; - public SingleVectorState(WriterPosition writer, ValueVector mainVector) { + public SingleVectorState(WriterEvents writer, ValueVector mainVector) { this.writer = writer; this.mainVector = mainVector; } @@ -359,7 +360,7 @@ public abstract class SingleVectorState implements VectorState { @Override public boolean isProjected() { return true; } - public static SimpleVectorState vectorState(ColumnMetadata schema, WriterPosition writer, ValueVector mainVector) { + public static SimpleVectorState vectorState(ColumnMetadata schema, WriterEvents writer, ValueVector mainVector) { if (schema.isVariableWidth()) { return new VariableWidthVectorState(schema, writer, mainVector); } else { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/TupleState.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/TupleState.java index ff531f717..eace69ebb 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/TupleState.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/impl/TupleState.java @@ -35,7 +35,6 @@ import org.apache.drill.exec.record.metadata.TupleSchema; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.accessor.ObjectWriter; import org.apache.drill.exec.vector.accessor.TupleWriter; -import org.apache.drill.exec.vector.accessor.TupleWriter.TupleWriterListener; import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter; import org.apache.drill.exec.vector.accessor.writer.AbstractObjectWriter; import org.apache.drill.exec.vector.accessor.writer.AbstractTupleWriter; @@ -92,7 +91,7 @@ import org.apache.drill.exec.vector.complex.AbstractMapVector; */ public abstract class TupleState extends ContainerState - implements TupleWriterListener { + implements AbstractTupleWriter.TupleWriterListener { /** * Represents a map column (either single or repeated). Includes maps that diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/filter/RuntimeFilterRouter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/filter/RuntimeFilterRouter.java index a4946a96c..b59c17eeb 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/filter/RuntimeFilterRouter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/filter/RuntimeFilterRouter.java @@ -18,6 +18,7 @@ package org.apache.drill.exec.work.filter; import org.apache.commons.collections.CollectionUtils; +import org.apache.drill.common.exceptions.UserException; import org.apache.drill.exec.ops.SendingAccountor; import org.apache.drill.exec.physical.base.AbstractPhysicalVisitor; import org.apache.drill.exec.physical.base.Exchange; @@ -172,9 +173,14 @@ public class RuntimeFilterRouter { } } + @SuppressWarnings("unchecked") private Wrapper findTargetWrapper(Wrapper wrapper, TargetPhysicalOperatorVisitor targetOpVisitor) { targetOpVisitor.setCurrentFragment(wrapper.getNode()); - wrapper.getNode().getRoot().accept(targetOpVisitor, null); + try { + wrapper.getNode().getRoot().accept(targetOpVisitor, null); + } catch (Throwable e) { + throw UserException.systemError(e).build(); + } boolean contain = targetOpVisitor.isContain(); if (contain) { return wrapper; @@ -233,6 +239,7 @@ public class RuntimeFilterRouter { return null; } + @Override public boolean isContain() { return contain; } @@ -284,6 +291,7 @@ public class RuntimeFilterRouter { return null; } + @Override public boolean isContain() { return contain; } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/project/TestConstantColumnLoader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/project/TestConstantColumnLoader.java index ffdb06ecf..b45374be2 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/project/TestConstantColumnLoader.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/project/TestConstantColumnLoader.java @@ -20,6 +20,7 @@ package org.apache.drill.exec.physical.impl.scan.project; import java.util.ArrayList; import java.util.List; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; @@ -32,6 +33,7 @@ import org.apache.drill.test.SubOperatorTest; import org.apache.drill.test.rowSet.RowSet.SingleRowSet; import org.apache.drill.test.rowSet.RowSetComparison; import org.junit.Test; +import org.junit.experimental.categories.Category; /** * Drill allows file metadata columns (also called "implicit" columns.) @@ -39,6 +41,7 @@ import org.junit.Test; * values. The ConstantColumnLoader builds and populates these columns. */ +@Category(RowSetTests.class) public class TestConstantColumnLoader extends SubOperatorTest { private static class DummyColumn implements ConstantColumnSpec { diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/project/TestNullColumnLoader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/project/TestNullColumnLoader.java index 90b9f7182..a825fad36 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/project/TestNullColumnLoader.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/project/TestNullColumnLoader.java @@ -25,6 +25,7 @@ import java.util.List; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.types.Types; import org.apache.drill.exec.physical.rowSet.ResultVectorCache; import org.apache.drill.exec.physical.rowSet.impl.NullResultVectorCacheImpl; @@ -37,6 +38,7 @@ import org.apache.drill.test.SubOperatorTest; import org.apache.drill.test.rowSet.RowSet.SingleRowSet; import org.apache.drill.test.rowSet.RowSetComparison; import org.junit.Test; +import org.junit.experimental.categories.Category; /** * Test the mechanism that handles all-null columns during projection. @@ -49,6 +51,7 @@ import org.junit.Test; * any other type and mode. */ +@Category(RowSetTests.class) public class TestNullColumnLoader extends SubOperatorTest { private ResolvedNullColumn makeNullCol(String name, MajorType nullType) { diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/project/TestRowBatchMerger.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/project/TestRowBatchMerger.java index 289b418e5..ef7e01fe4 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/project/TestRowBatchMerger.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/project/TestRowBatchMerger.java @@ -33,11 +33,15 @@ import org.apache.drill.test.rowSet.RowSet.SingleRowSet; import org.apache.drill.test.rowSet.RowSetComparison; import org.junit.BeforeClass; import org.junit.Test; +import org.junit.experimental.categories.Category; import io.netty.buffer.DrillBuf; import static org.apache.drill.test.rowSet.RowSetUtilities.mapValue; import static org.apache.drill.test.rowSet.RowSetUtilities.singleMap; + +import org.apache.drill.categories.RowSetTests; + import static org.apache.drill.test.rowSet.RowSetUtilities.mapArray; @@ -50,6 +54,7 @@ import static org.apache.drill.test.rowSet.RowSetUtilities.mapArray; * vector. */ +@Category(RowSetTests.class) public class TestRowBatchMerger extends SubOperatorTest { public static class RowSetSource implements VectorSource { diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/project/TestScanLevelProjection.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/project/TestScanLevelProjection.java index c58991aa8..cc58b6cd7 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/project/TestScanLevelProjection.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/project/TestScanLevelProjection.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.exec.physical.impl.scan.ScanTestUtils; @@ -30,12 +31,14 @@ import org.apache.drill.exec.physical.rowSet.project.RequestedTuple.RequestedCol import org.apache.drill.exec.record.metadata.ProjectionType; import org.apache.drill.test.SubOperatorTest; import org.junit.Test; +import org.junit.experimental.categories.Category; /** * Test the level of projection done at the level of the scan as a whole; * before knowledge of table "implicit" columns or the specific table schema. */ +@Category(RowSetTests.class) public class TestScanLevelProjection extends SubOperatorTest { /** diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/project/TestSchemaLevelProjection.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/project/TestSchemaLevelProjection.java index c10e5fc7a..800d4dc29 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/project/TestSchemaLevelProjection.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/project/TestSchemaLevelProjection.java @@ -26,6 +26,7 @@ import static org.junit.Assert.fail; import java.util.List; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.physical.impl.scan.project.ResolvedTuple.ResolvedRow; @@ -35,6 +36,7 @@ import org.apache.drill.exec.record.metadata.SchemaBuilder; import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.test.SubOperatorTest; import org.junit.Test; +import org.junit.experimental.categories.Category; /** * "Schema level projection" describes one side of the projection @@ -44,6 +46,7 @@ import org.junit.Test; * combines these to map out the actual projection. */ +@Category(RowSetTests.class) public class TestSchemaLevelProjection extends SubOperatorTest { /** diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/project/TestSchemaSmoothing.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/project/TestSchemaSmoothing.java index d008a9d8c..a21b1e472 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/project/TestSchemaSmoothing.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/project/TestSchemaSmoothing.java @@ -24,6 +24,7 @@ import static org.junit.Assert.fail; import java.util.List; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.physical.impl.protocol.SchemaTracker; import org.apache.drill.exec.physical.impl.scan.ScanTestUtils; @@ -38,6 +39,7 @@ import org.apache.drill.test.SubOperatorTest; import org.apache.drill.test.rowSet.RowSet.SingleRowSet; import org.apache.drill.test.rowSet.RowSetComparison; import org.junit.Test; +import org.junit.experimental.categories.Category; /** * Tests schema smoothing at the schema projection level. @@ -79,6 +81,7 @@ import org.junit.Test; * the future to know what data will be scanned.) */ +@Category(RowSetTests.class) public class TestSchemaSmoothing extends SubOperatorTest { /** diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestProjectedTuple.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestProjectedTuple.java index 69d56a1bc..2966bd561 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestProjectedTuple.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestProjectedTuple.java @@ -27,6 +27,7 @@ import static org.junit.Assert.fail; import java.util.ArrayList; import java.util.List; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.exec.physical.rowSet.project.ImpliedTupleRequest; @@ -36,7 +37,9 @@ import org.apache.drill.exec.physical.rowSet.project.RequestedTupleImpl; import org.apache.drill.exec.record.metadata.ProjectionType; import org.junit.Ignore; import org.junit.Test; +import org.junit.experimental.categories.Category; +@Category(RowSetTests.class) public class TestProjectedTuple { @Test diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderEmptyProject.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderEmptyProject.java index 15fd04948..9150bc127 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderEmptyProject.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderEmptyProject.java @@ -24,6 +24,7 @@ import static org.junit.Assert.fail; import java.util.List; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.physical.rowSet.ResultSetLoader; @@ -36,9 +37,10 @@ import org.apache.drill.test.SubOperatorTest; import org.apache.drill.test.rowSet.RowSetBuilder; import org.apache.drill.test.rowSet.RowSetUtilities; import org.junit.Test; - +import org.junit.experimental.categories.Category; import org.apache.drill.shaded.guava.com.google.common.collect.Lists; +@Category(RowSetTests.class) public class TestResultSetLoaderEmptyProject extends SubOperatorTest { /** diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderLimits.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderLimits.java index fff86d754..059fc016f 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderLimits.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderLimits.java @@ -23,6 +23,7 @@ import static org.junit.Assert.fail; import java.util.Arrays; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.physical.rowSet.ResultSetLoader; @@ -32,6 +33,7 @@ import org.apache.drill.exec.record.metadata.SchemaBuilder; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.test.SubOperatorTest; import org.junit.Test; +import org.junit.experimental.categories.Category; /** * Tests of the row limit functionality of the result set loader. The @@ -45,6 +47,7 @@ import org.junit.Test; * the row limit turns out to be too large.) */ +@Category(RowSetTests.class) public class TestResultSetLoaderLimits extends SubOperatorTest { /** diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderMapArray.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderMapArray.java index 9b75e9bb3..4a203faa6 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderMapArray.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderMapArray.java @@ -28,6 +28,7 @@ import static org.junit.Assert.assertTrue; import java.util.Arrays; import java.util.Iterator; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.physical.rowSet.ResultSetLoader; @@ -49,6 +50,7 @@ import org.apache.drill.test.rowSet.RowSet.SingleRowSet; import org.apache.drill.test.rowSet.RowSetReader; import org.apache.drill.test.rowSet.RowSetUtilities; import org.junit.Test; +import org.junit.experimental.categories.Category; /** * Test map array support in the result set loader. @@ -59,6 +61,7 @@ import org.junit.Test; * constructs not to be tackled lightly. */ +@Category(RowSetTests.class) public class TestResultSetLoaderMapArray extends SubOperatorTest { @Test diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderMaps.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderMaps.java index ac4ab8f53..48614de55 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderMaps.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderMaps.java @@ -28,6 +28,7 @@ import static org.junit.Assert.fail; import java.util.Arrays; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MinorType; @@ -49,12 +50,14 @@ import org.apache.drill.test.rowSet.RowSet.SingleRowSet; import org.apache.drill.test.rowSet.RowSetReader; import org.apache.drill.test.rowSet.RowSetUtilities; import org.junit.Test; +import org.junit.experimental.categories.Category; /** * Test (non-array) map support in the result set loader and related classes. */ +@Category(RowSetTests.class) public class TestResultSetLoaderMaps extends SubOperatorTest { @Test diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderOmittedValues.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderOmittedValues.java index a888fce0b..ec37a6d6f 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderOmittedValues.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderOmittedValues.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue; import java.util.Arrays; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.physical.rowSet.ResultSetLoader; import org.apache.drill.exec.physical.rowSet.RowSetLoader; @@ -37,8 +38,10 @@ import org.apache.drill.test.rowSet.RowSet.SingleRowSet; import org.apache.drill.test.rowSet.RowSetReader; import org.apache.drill.test.rowSet.RowSetUtilities; import org.junit.Test; +import org.junit.experimental.categories.Category; +@Category(RowSetTests.class) public class TestResultSetLoaderOmittedValues extends SubOperatorTest { /** diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderOverflow.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderOverflow.java index 9b11380af..a82e3c3bf 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderOverflow.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderOverflow.java @@ -23,6 +23,7 @@ import static org.junit.Assert.fail; import java.util.Arrays; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MinorType; @@ -40,13 +41,14 @@ import org.apache.drill.test.SubOperatorTest; import org.apache.drill.test.rowSet.RowSet; import org.apache.drill.test.rowSet.RowSetReader; import org.junit.Test; - +import org.junit.experimental.categories.Category; import org.apache.drill.shaded.guava.com.google.common.base.Charsets; /** * Exercise the vector overflow functionality for the result set loader. */ +@Category(RowSetTests.class) public class TestResultSetLoaderOverflow extends SubOperatorTest { /** diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProjection.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProjection.java index 3ab0c3c2e..66295123f 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProjection.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProjection.java @@ -26,6 +26,7 @@ import static org.junit.Assert.assertTrue; import java.util.Arrays; import java.util.List; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MinorType; @@ -42,11 +43,13 @@ import org.apache.drill.test.rowSet.RowSet; import org.apache.drill.test.rowSet.RowSet.SingleRowSet; import org.apache.drill.test.rowSet.RowSetUtilities; import org.junit.Test; +import org.junit.experimental.categories.Category; /** * Test of the basics of the projection mechanism. */ +@Category(RowSetTests.class) public class TestResultSetLoaderProjection extends SubOperatorTest { /** diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java index 8fb600d34..dc0236d09 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java @@ -29,6 +29,7 @@ import static org.junit.Assert.fail; import java.util.Arrays; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MinorType; @@ -44,11 +45,11 @@ import org.apache.drill.exec.vector.accessor.TupleWriter.UndefinedColumnExceptio import org.apache.drill.test.SubOperatorTest; import org.apache.drill.test.rowSet.RowSet; import org.apache.drill.test.rowSet.RowSet.SingleRowSet; -import org.apache.drill.test.rowSet.test.TestColumnConvertor.TestConvertor; +import org.apache.drill.test.rowSet.test.TestColumnConverter.TestConverter; import org.apache.drill.test.rowSet.RowSetReader; import org.apache.drill.test.rowSet.RowSetUtilities; -import org.junit.Ignore; import org.junit.Test; +import org.junit.experimental.categories.Category; /** * Tests of the overall result set loader protocol focusing on which operations @@ -72,6 +73,7 @@ import org.junit.Test; * current state. */ +@Category(RowSetTests.class) public class TestResultSetLoaderProtocol extends SubOperatorTest { @Test @@ -611,7 +613,6 @@ public class TestResultSetLoaderProtocol extends SubOperatorTest { * required, nullable and repeated columns. */ - @Ignore("Not yet") @Test public void testTypeConversion() { TupleMetadata schema = new SchemaBuilder() @@ -620,18 +621,19 @@ public class TestResultSetLoaderProtocol extends SubOperatorTest { .addArray("n3", MinorType.INT) .buildSchema(); - // Add a type convertor. Passed in as a factory + // Add a type converter. Passed in as a factory // since we must create a new one for each row set writer. - schema.metadata("n1").setTypeConverter(TestConvertor.factory()); - schema.metadata("n2").setTypeConverter(TestConvertor.factory()); - schema.metadata("n3").setTypeConverter(TestConvertor.factory()); + schema.metadata("n1").setTypeConverter(TestConverter.factory()); + schema.metadata("n2").setTypeConverter(TestConverter.factory()); + schema.metadata("n3").setTypeConverter(TestConverter.factory()); ResultSetLoaderImpl.ResultSetOptions options = new OptionBuilder() .setSchema(schema) .setRowCountLimit(ValueVector.MAX_ROW_COUNT) .build(); ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator(), options); + rsLoader.startBatch(); // Write data as both a string as an integer @@ -640,7 +642,7 @@ public class TestResultSetLoaderProtocol extends SubOperatorTest { rootWriter.addRow(234, 23, intArray(234, 235)); RowSet actual = fixture.wrap(rsLoader.harvest()); - // Build the expected vector without a type convertor. + // Build the expected vector without a type converter. TupleMetadata expectedSchema = new SchemaBuilder() .add("n1", MinorType.INT) diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderRepeatedList.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderRepeatedList.java index 32940da95..b73440746 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderRepeatedList.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderRepeatedList.java @@ -29,6 +29,7 @@ import java.util.Arrays; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.types.Types; import org.apache.drill.exec.physical.rowSet.ResultSetLoader; import org.apache.drill.exec.physical.rowSet.RowSetLoader; @@ -53,7 +54,7 @@ import org.apache.drill.test.rowSet.RowSetReader; import org.apache.drill.test.rowSet.RowSetUtilities; import org.apache.drill.test.rowSet.RowSet.SingleRowSet; import org.junit.Test; - +import org.junit.experimental.categories.Category; import org.apache.drill.shaded.guava.com.google.common.base.Charsets; /** @@ -64,6 +65,7 @@ import org.apache.drill.shaded.guava.com.google.common.base.Charsets; * Repeated lists appear to be used only by JSON. */ +@Category(RowSetTests.class) public class TestResultSetLoaderRepeatedList extends SubOperatorTest { @Test diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderTorture.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderTorture.java index 6e7427548..31ea3c633 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderTorture.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderTorture.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertTrue; import java.util.Arrays; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.physical.rowSet.ResultSetLoader; import org.apache.drill.exec.physical.rowSet.RowSetLoader; @@ -40,7 +41,7 @@ import org.apache.drill.test.SubOperatorTest; import org.apache.drill.test.rowSet.RowSet; import org.apache.drill.test.rowSet.RowSetReader; import org.junit.Test; - +import org.junit.experimental.categories.Category; import org.apache.drill.shaded.guava.com.google.common.base.Charsets; /** @@ -64,6 +65,7 @@ import org.apache.drill.shaded.guava.com.google.common.base.Charsets; * things in a single query. */ +@Category(RowSetTests.class) public class TestResultSetLoaderTorture extends SubOperatorTest { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestResultSetLoaderTorture.class); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderUnions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderUnions.java index 09c7eb096..8329306d5 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderUnions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderUnions.java @@ -32,6 +32,7 @@ import java.util.Arrays; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.types.Types; import org.apache.drill.exec.physical.rowSet.ResultSetLoader; import org.apache.drill.exec.physical.rowSet.RowSetLoader; @@ -64,7 +65,7 @@ import org.apache.drill.test.rowSet.RowSetBuilder; import org.apache.drill.test.rowSet.RowSetReader; import org.apache.drill.test.rowSet.RowSetUtilities; import org.junit.Test; - +import org.junit.experimental.categories.Category; import org.apache.drill.shaded.guava.com.google.common.base.Charsets; /** @@ -74,6 +75,8 @@ import org.apache.drill.shaded.guava.com.google.common.base.Charsets; * Most operators do not support them. But, JSON uses them, so they must * be made to work in the result set loader layer. */ + +@Category(RowSetTests.class) public class TestResultSetLoaderUnions extends SubOperatorTest { @Test diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetSchemaChange.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetSchemaChange.java index 0055376bd..2dcd001dc 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetSchemaChange.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetSchemaChange.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue; import java.util.Arrays; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.physical.rowSet.ResultSetLoader; @@ -38,7 +39,9 @@ import org.apache.drill.test.rowSet.RowSet.SingleRowSet; import org.apache.drill.test.rowSet.RowSetReader; import org.apache.drill.test.rowSet.RowSetUtilities; import org.junit.Test; +import org.junit.experimental.categories.Category; +@Category(RowSetTests.class) public class TestResultSetSchemaChange extends SubOperatorTest { /** diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultVectorCache.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultVectorCache.java index 6d0cce708..03853f08c 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultVectorCache.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultVectorCache.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; @@ -32,7 +33,9 @@ import org.apache.drill.exec.vector.IntVector; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.test.SubOperatorTest; import org.junit.Test; +import org.junit.experimental.categories.Category; +@Category(RowSetTests.class) public class TestResultVectorCache extends SubOperatorTest { @Test diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestColumnConvertor.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestColumnConverter.java index b7d865de1..f989b0c95 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestColumnConvertor.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestColumnConverter.java @@ -18,6 +18,9 @@ package org.apache.drill.test.rowSet.test; import static org.apache.drill.test.rowSet.RowSetUtilities.strArray; + +import org.apache.drill.categories.RowSetTests; + import static org.apache.drill.test.rowSet.RowSetUtilities.intArray; import org.apache.drill.common.types.TypeProtos.MinorType; @@ -26,29 +29,31 @@ import org.apache.drill.exec.record.metadata.SchemaBuilder; import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.vector.accessor.ColumnConversionFactory; import org.apache.drill.exec.vector.accessor.ScalarWriter; -import org.apache.drill.exec.vector.accessor.writer.AbstractWriteConvertor; -import org.apache.drill.exec.vector.accessor.writer.ConcreteWriter; +import org.apache.drill.exec.vector.accessor.writer.AbstractWriteConverter; +import org.apache.drill.exec.vector.accessor.writer.AbstractScalarWriter; import org.apache.drill.test.SubOperatorTest; import org.apache.drill.test.rowSet.RowSet; import org.apache.drill.test.rowSet.RowSetBuilder; import org.apache.drill.test.rowSet.RowSetUtilities; import org.apache.drill.test.rowSet.RowSet.SingleRowSet; import org.junit.Test; +import org.junit.experimental.categories.Category; /** - * Tests the column type convertor feature of the column metadata + * Tests the column type converter feature of the column metadata * and of the RowSetWriter. */ -public class TestColumnConvertor extends SubOperatorTest { +@Category(RowSetTests.class) +public class TestColumnConverter extends SubOperatorTest { /** * Simple type converter that allows string-to-int conversions. * Inherits usual int value support from the base writer. */ - public static class TestConvertor extends AbstractWriteConvertor { + public static class TestConverter extends AbstractWriteConverter { - public TestConvertor(ScalarWriter baseWriter) { + public TestConverter(ScalarWriter baseWriter) { super(baseWriter); } @@ -60,16 +65,16 @@ public class TestColumnConvertor extends SubOperatorTest { public static ColumnConversionFactory factory() { return new ColumnConversionFactory() { @Override - public ConcreteWriter newWriter(ColumnMetadata colDefn, - ConcreteWriter baseWriter) { - return new TestConvertor(baseWriter); + public AbstractScalarWriter newWriter(ColumnMetadata colDefn, + ScalarWriter baseWriter) { + return new TestConverter(baseWriter); } }; } } @Test - public void testScalarConvertor() { + public void testScalarConverter() { // Create the schema @@ -78,11 +83,11 @@ public class TestColumnConvertor extends SubOperatorTest { .addNullable("n2", MinorType.INT) .buildSchema(); - // Add a type convertor. Passed in as a factory + // Add a type converter. Passed in as a factory // since we must create a new one for each row set writer. - schema.metadata("n1").setTypeConverter(TestConvertor.factory()); - schema.metadata("n2").setTypeConverter(TestConvertor.factory()); + schema.metadata("n1").setTypeConverter(TestConverter.factory()); + schema.metadata("n2").setTypeConverter(TestConverter.factory()); // Write data as both a string as an integer @@ -91,7 +96,7 @@ public class TestColumnConvertor extends SubOperatorTest { .addRow(234, 23) .build(); - // Build the expected vector without a type convertor. + // Build the expected vector without a type converter. TupleMetadata expectedSchema = new SchemaBuilder() .add("n1", MinorType.INT) @@ -108,7 +113,7 @@ public class TestColumnConvertor extends SubOperatorTest { } @Test - public void testArrayConvertor() { + public void testArrayConverter() { // Create the schema @@ -116,10 +121,10 @@ public class TestColumnConvertor extends SubOperatorTest { .addArray("n", MinorType.INT) .buildSchema(); - // Add a type convertor. Passed in as a factory + // Add a type converter. Passed in as a factory // since we must create a new one for each row set writer. - schema.metadata("n").setTypeConverter(TestConvertor.factory()); + schema.metadata("n").setTypeConverter(TestConverter.factory()); // Write data as both a string as an integer @@ -128,7 +133,7 @@ public class TestColumnConvertor extends SubOperatorTest { .addSingleCol(intArray(234, 235)) .build(); - // Build the expected vector without a type convertor. + // Build the expected vector without a type converter. TupleMetadata expectedSchema = new SchemaBuilder() .addArray("n", MinorType.INT) diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestFixedWidthWriter.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestFixedWidthWriter.java index 09d9d9ef5..3eba578ae 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestFixedWidthWriter.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestFixedWidthWriter.java @@ -21,6 +21,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.record.MaterializedField; @@ -29,10 +30,11 @@ import org.apache.drill.exec.vector.IntVector; import org.apache.drill.exec.vector.accessor.ColumnAccessors.IntColumnWriter; import org.apache.drill.exec.vector.accessor.ColumnWriterIndex; import org.apache.drill.exec.vector.accessor.ScalarWriter; -import org.apache.drill.exec.vector.accessor.ScalarWriter.ColumnWriterListener; import org.apache.drill.exec.vector.accessor.ValueType; +import org.apache.drill.exec.vector.accessor.writer.WriterEvents.ColumnWriterListener; import org.apache.drill.test.SubOperatorTest; import org.junit.Test; +import org.junit.experimental.categories.Category; /** * Test the int writer as a typical example of a fixed-width @@ -40,6 +42,7 @@ import org.junit.Test; * overflow, and filling in empty values. */ +@Category(RowSetTests.class) public class TestFixedWidthWriter extends SubOperatorTest { public static class TestIndex implements ColumnWriterIndex { diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestHyperVectorReaders.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestHyperVectorReaders.java index ea1490216..e07318edd 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestHyperVectorReaders.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestHyperVectorReaders.java @@ -25,6 +25,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.record.metadata.SchemaBuilder; import org.apache.drill.exec.record.metadata.TupleMetadata; @@ -39,6 +40,7 @@ import org.apache.drill.test.rowSet.RowSetReader; import org.apache.drill.test.rowSet.RowSetUtilities; import org.apache.drill.test.rowSet.RowSetWriter; import org.junit.Test; +import org.junit.experimental.categories.Category; /** * Test the reader mechanism that reads rows indexed via an SV4. @@ -51,6 +53,7 @@ import org.junit.Test; * This test does not cover repeated vectors; those tests should be added. */ +@Category(RowSetTests.class) public class TestHyperVectorReaders extends SubOperatorTest { /** diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestIndirectReaders.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestIndirectReaders.java index db1c882b4..727c6d010 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestIndirectReaders.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestIndirectReaders.java @@ -17,6 +17,7 @@ */ package org.apache.drill.test.rowSet.test; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.record.metadata.SchemaBuilder; import org.apache.drill.exec.record.metadata.TupleMetadata; @@ -32,6 +33,7 @@ import org.apache.drill.test.rowSet.RowSet.SingleRowSet; import org.apache.drill.test.rowSet.RowSetReader; import org.apache.drill.test.rowSet.RowSetUtilities; import org.junit.Test; +import org.junit.experimental.categories.Category; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -44,6 +46,7 @@ import static org.junit.Assert.assertTrue; * so if the index works for one reader, it will for for all. */ +@Category(RowSetTests.class) public class TestIndirectReaders extends SubOperatorTest { /** diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestMapAccessors.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestMapAccessors.java index 7c67fc43c..4e18546b3 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestMapAccessors.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestMapAccessors.java @@ -26,6 +26,7 @@ import static org.junit.Assert.assertTrue; import java.util.Iterator; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.record.metadata.SchemaBuilder; @@ -48,6 +49,7 @@ import org.apache.drill.test.rowSet.RowSetBuilder; import org.apache.drill.test.rowSet.RowSetComparison; import org.apache.drill.test.rowSet.RowSetReader; import org.junit.Test; +import org.junit.experimental.categories.Category; /** * Test map support in the column readers and writers. @@ -57,6 +59,7 @@ import org.junit.Test; * schema, which makes this mechanism far simpler. */ +@Category(RowSetTests.class) public class TestMapAccessors extends SubOperatorTest { @Test diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestOffsetVectorWriter.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestOffsetVectorWriter.java index 0801d84a8..91396e67e 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestOffsetVectorWriter.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestOffsetVectorWriter.java @@ -21,6 +21,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.record.MaterializedField; @@ -28,13 +29,14 @@ import org.apache.drill.exec.record.metadata.SchemaBuilder; import org.apache.drill.exec.vector.UInt4Vector; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.accessor.ScalarWriter; -import org.apache.drill.exec.vector.accessor.ScalarWriter.ColumnWriterListener; import org.apache.drill.exec.vector.accessor.ValueType; import org.apache.drill.exec.vector.accessor.writer.OffsetVectorWriterImpl; +import org.apache.drill.exec.vector.accessor.writer.WriterEvents.ColumnWriterListener; import org.apache.drill.test.SubOperatorTest; import org.apache.drill.test.rowSet.test.TestFixedWidthWriter.TestIndex; import org.junit.BeforeClass; import org.junit.Test; +import org.junit.experimental.categories.Category; import io.netty.buffer.DrillBuf; @@ -49,6 +51,7 @@ import io.netty.buffer.DrillBuf; * counts.) */ +@Category(RowSetTests.class) public class TestOffsetVectorWriter extends SubOperatorTest { /** diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestRepeatedListAccessors.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestRepeatedListAccessors.java index 3fb0f5686..72c145e56 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestRepeatedListAccessors.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestRepeatedListAccessors.java @@ -27,6 +27,7 @@ import static org.junit.Assert.assertTrue; import static org.apache.drill.test.rowSet.RowSetUtilities.objArray; import static org.apache.drill.test.rowSet.RowSetUtilities.singleObjArray; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.record.BatchSchema; @@ -56,6 +57,7 @@ import org.apache.drill.test.rowSet.RowSetReader; import org.apache.drill.test.rowSet.RowSetUtilities; import org.apache.drill.test.rowSet.RowSetWriter; import org.junit.Test; +import org.junit.experimental.categories.Category; /** * Test the basics of repeated list support in the schema builder, @@ -64,6 +66,7 @@ import org.junit.Test; * on to the result set loader tests. */ +@Category(RowSetTests.class) public class TestRepeatedListAccessors extends SubOperatorTest { /** diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestRowSet.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestRowSet.java index b660672e8..db871567d 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestRowSet.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestRowSet.java @@ -28,6 +28,7 @@ import static org.junit.Assert.fail; import java.io.UnsupportedEncodingException; import java.util.Arrays; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.record.metadata.SchemaBuilder; import org.apache.drill.exec.record.metadata.TupleMetadata; @@ -50,6 +51,7 @@ import org.apache.drill.test.rowSet.RowSetReader; import org.apache.drill.test.rowSet.RowSetUtilities; import org.apache.drill.test.rowSet.RowSetWriter; import org.junit.Test; +import org.junit.experimental.categories.Category; /** * Test row sets. Since row sets are a thin wrapper around vectors, @@ -71,6 +73,7 @@ import org.junit.Test; * A list is an array of variants. Variants are tested elsewhere. */ +@Category(RowSetTests.class) public class TestRowSet extends SubOperatorTest { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestRowSet.class); diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestRowSetComparison.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestRowSetComparison.java index 867f61f5b..41eec8016 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestRowSetComparison.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestRowSetComparison.java @@ -17,6 +17,7 @@ */ package org.apache.drill.test.rowSet.test; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.memory.RootAllocator; @@ -28,7 +29,9 @@ import org.apache.drill.test.rowSet.RowSetComparison; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.experimental.categories.Category; +@Category(RowSetTests.class) public class TestRowSetComparison { private BufferAllocator allocator; diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestScalarAccessors.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestScalarAccessors.java index 2f2d96472..e6635f081 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestScalarAccessors.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestScalarAccessors.java @@ -25,6 +25,7 @@ import static org.junit.Assert.assertTrue; import java.math.BigDecimal; import java.util.Arrays; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; @@ -39,7 +40,7 @@ import org.apache.drill.test.rowSet.RowSetReader; import org.joda.time.Period; import org.apache.drill.test.rowSet.RowSet.SingleRowSet; import org.junit.Test; - +import org.junit.experimental.categories.Category; import org.apache.drill.shaded.guava.com.google.common.collect.Lists; /** @@ -54,6 +55,7 @@ import org.apache.drill.shaded.guava.com.google.common.collect.Lists; // TODO: Decimal28Sparse // TODO: Decimal38Sparse +@Category(RowSetTests.class) public class TestScalarAccessors extends SubOperatorTest { @Test diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestSchemaBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestSchemaBuilder.java index 358330d2f..995019708 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestSchemaBuilder.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestSchemaBuilder.java @@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.types.Types; import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.record.metadata.ColumnMetadata; @@ -37,12 +38,15 @@ import org.apache.drill.exec.record.metadata.UnionBuilder; import org.apache.drill.exec.record.metadata.VariantMetadata; import org.apache.drill.test.DrillTest; import org.junit.Test; +import org.junit.experimental.categories.Category; /** * The schema builder for tests has grown complex to handle maps, unions, * lists and repeated lists. This test verifies that it assembles the various * pieces correctly for the various nesting combinations. */ + +@Category(RowSetTests.class) public class TestSchemaBuilder extends DrillTest { @Test diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestVariableWidthWriter.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestVariableWidthWriter.java index 1c430fd87..c40bbd772 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestVariableWidthWriter.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestVariableWidthWriter.java @@ -21,6 +21,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.record.MaterializedField; @@ -28,15 +29,16 @@ import org.apache.drill.exec.record.metadata.SchemaBuilder; import org.apache.drill.exec.vector.VarCharVector; import org.apache.drill.exec.vector.accessor.ColumnAccessors.VarCharColumnWriter; import org.apache.drill.exec.vector.accessor.ScalarWriter; -import org.apache.drill.exec.vector.accessor.ScalarWriter.ColumnWriterListener; import org.apache.drill.exec.vector.accessor.ValueType; +import org.apache.drill.exec.vector.accessor.writer.WriterEvents.ColumnWriterListener; import org.apache.drill.test.SubOperatorTest; import org.apache.drill.test.rowSet.test.TestFixedWidthWriter.TestIndex; import org.bouncycastle.util.Arrays; import org.junit.Test; - +import org.junit.experimental.categories.Category; import org.apache.drill.shaded.guava.com.google.common.base.Charsets; +@Category(RowSetTests.class) public class TestVariableWidthWriter extends SubOperatorTest { /** diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestVariantAccessors.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestVariantAccessors.java index 3d017c31f..b468c31f5 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestVariantAccessors.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestVariantAccessors.java @@ -26,6 +26,7 @@ import static org.junit.Assert.assertTrue; import java.util.List; +import org.apache.drill.categories.RowSetTests; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.record.BatchSchema; @@ -57,6 +58,7 @@ import org.apache.drill.test.rowSet.RowSetWriter; import org.apache.drill.test.rowSet.RowSet.ExtendableRowSet; import org.apache.drill.test.rowSet.RowSet.SingleRowSet; import org.junit.Test; +import org.junit.experimental.categories.Category; /** * Tests for readers and writers for union and list types. @@ -67,6 +69,8 @@ import org.junit.Test; * result set builder. It does not, however, work in the Project * and other operators. Some assembly required for future use.) */ + +@Category(RowSetTests.class) public class TestVariantAccessors extends SubOperatorTest { @Test diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ColumnConversionFactory.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ColumnConversionFactory.java index 02efd6def..691dcf168 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ColumnConversionFactory.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ColumnConversionFactory.java @@ -18,7 +18,6 @@ package org.apache.drill.exec.vector.accessor; import org.apache.drill.exec.record.metadata.ColumnMetadata; -import org.apache.drill.exec.vector.accessor.writer.ConcreteWriter; /** * Create a column type converter for the given column and base writer. @@ -36,5 +35,5 @@ public interface ColumnConversionFactory { * @return a new scalar writer to insert between the client and * the base vector */ - ConcreteWriter newWriter(ColumnMetadata colDefn, ConcreteWriter baseWriter); + ScalarWriter newWriter(ColumnMetadata colDefn, ScalarWriter baseWriter); } diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ColumnWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ColumnWriter.java index 5c2fde1b6..5117782dd 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ColumnWriter.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ColumnWriter.java @@ -18,9 +18,6 @@ package org.apache.drill.exec.vector.accessor; import org.apache.drill.exec.record.metadata.ColumnMetadata; -import org.apache.drill.exec.vector.accessor.ScalarWriter.ColumnWriterListener; -import org.apache.drill.exec.vector.accessor.TupleWriter.TupleWriterListener; -import org.apache.drill.exec.vector.accessor.VariantWriter.VariantWriterListener; /** * Generic information about a column writer including: @@ -33,36 +30,7 @@ import org.apache.drill.exec.vector.accessor.VariantWriter.VariantWriterListener * testing.</li> */ -public interface ColumnWriter extends WriterPosition { - - interface TupleListenable { - - /** - * Bind a listener to the underlying map or map array column. Not valid if the - * underlying writer is a scalar or scalar array. - * - * @param listener - * the tuple listener to bind - */ - - void bindListener(TupleWriterListener listener); - } - - interface ScalarListenable { - /** - * Bind a listener to the underlying scalar column, or array of scalar - * columns. Not valid if the underlying writer is a map or array of maps. - * - * @param listener - * the column listener to bind - */ - - void bindListener(ColumnWriterListener listener); - } - - interface VariantListenable { - void bindListener(VariantWriterListener listener); - } +public interface ColumnWriter { /** * Return the object (structure) type of this writer. diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ObjectWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ObjectWriter.java index af7df6e88..fffd4e588 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ObjectWriter.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ObjectWriter.java @@ -17,6 +17,8 @@ */ package org.apache.drill.exec.vector.accessor; +import org.apache.drill.exec.vector.accessor.writer.WriterEvents; + /** * Represents a column within a tuple. A column can be an array, a scalar or a * tuple. Each has an associated column metadata (schema) and a writer. The @@ -49,4 +51,11 @@ public interface ObjectWriter extends ColumnWriter { ArrayWriter array(); VariantWriter variant(); + + /** + * The internal state behind this writer. To be used only by the + * implementation, not by the client. + */ + + WriterEvents events(); } diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ScalarWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ScalarWriter.java index 87c798890..db86570d1 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ScalarWriter.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ScalarWriter.java @@ -21,8 +21,6 @@ import java.math.BigDecimal; import org.joda.time.Period; -import org.apache.drill.exec.vector.accessor.ColumnWriter.ScalarListenable; - /** * Represents a scalar value: a required column, a nullable column, * or one element within an array of scalars. @@ -44,41 +42,7 @@ import org.apache.drill.exec.vector.accessor.ColumnWriter.ScalarListenable; * {@see ScalarElementReader} */ -public interface ScalarWriter extends ColumnWriter, ScalarListenable { - - /** - * Listener (callback) for vector overflow events. To be optionally - * implemented and bound by the client code of the writer. If no - * listener is bound, and a vector overflows, then an exception is - * thrown. - */ - - public interface ColumnWriterListener { - - /** - * Alert the listener that a vector has overflowed. Upon return, - * all writers must have a new set of buffers available, ready - * to accept the in-flight value that triggered the overflow. - * - * @param writer the writer that triggered the overflow - */ - - void overflowed(ScalarWriter writer); - - /** - * A writer wants to expand its vector. Allows the listener to - * either allow the growth, or trigger and overflow to limit - * batch size. - * - * @param writer the writer that wishes to grow its vector - * @param delta the amount by which the vector is to grow - * @return true if the vector can be grown, false if the writer - * should instead trigger an overflow by calling - * <tt>overflowed()</tt> - */ - - boolean canExpand(ScalarWriter writer, int delta); - } +public interface ScalarWriter extends ColumnWriter { /** * Describe the type of the value. This is a compression of the diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/TupleWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/TupleWriter.java index 6f5d17931..05ff2fc5a 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/TupleWriter.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/TupleWriter.java @@ -21,7 +21,6 @@ import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.record.metadata.ColumnMetadata; import org.apache.drill.exec.record.metadata.ProjectionType; import org.apache.drill.exec.record.metadata.TupleMetadata; -import org.apache.drill.exec.vector.accessor.ColumnWriter.TupleListenable; /** * Writer for a tuple. A tuple is composed of columns with a fixed order and @@ -52,23 +51,7 @@ import org.apache.drill.exec.vector.accessor.ColumnWriter.TupleListenable; * @see {@link SingleMapWriter}, the class which this class replaces */ -public interface TupleWriter extends ColumnWriter, TupleListenable { - - /** - * Listener (callback) to handle requests to add a new column to a tuple (row - * or map). Implemented and bound by the client code that creates or uses the - * tuple writer. If no listener is bound, then an attempt to add a column - * throws an exception. - */ - - interface TupleWriterListener { - - ObjectWriter addColumn(TupleWriter tuple, ColumnMetadata column); - - ObjectWriter addColumn(TupleWriter tuple, MaterializedField field); - - ProjectionType projectionType(String columnName); - } +public interface TupleWriter extends ColumnWriter { /** * Unchecked exception thrown when attempting to access a column writer by diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractArrayWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractArrayWriter.java index d7b97e110..585696f52 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractArrayWriter.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractArrayWriter.java @@ -20,6 +20,7 @@ package org.apache.drill.exec.vector.accessor.writer; import org.apache.drill.exec.record.metadata.ColumnMetadata; import org.apache.drill.exec.vector.UInt4Vector; import org.apache.drill.exec.vector.accessor.ArrayWriter; +import org.apache.drill.exec.vector.accessor.ColumnWriter; import org.apache.drill.exec.vector.accessor.ColumnWriterIndex; import org.apache.drill.exec.vector.accessor.ObjectType; import org.apache.drill.exec.vector.accessor.ObjectWriter; @@ -95,7 +96,7 @@ public abstract class AbstractArrayWriter implements ArrayWriter, WriterEvents { public static class ArrayObjectWriter extends AbstractObjectWriter { - private AbstractArrayWriter arrayWriter; + private final AbstractArrayWriter arrayWriter; public ArrayObjectWriter(AbstractArrayWriter arrayWriter) { this.arrayWriter = arrayWriter; @@ -105,6 +106,9 @@ public abstract class AbstractArrayWriter implements ArrayWriter, WriterEvents { public ArrayWriter array() { return arrayWriter; } @Override + public ColumnWriter writer() { return arrayWriter; } + + @Override public WriterEvents events() { return arrayWriter; } @Override @@ -266,6 +270,12 @@ public abstract class AbstractArrayWriter implements ArrayWriter, WriterEvents { } @Override + public void bindListener(ColumnWriterListener listener) { + elementObjWriter.events().bindListener(listener); + offsetsWriter.bindListener(listener); + } + + @Override public ObjectType type() { return ObjectType.ARRAY; } @Override @@ -342,6 +352,7 @@ public abstract class AbstractArrayWriter implements ArrayWriter, WriterEvents { public OffsetVectorWriter offsetWriter() { return offsetsWriter; } + @Override public void dump(HierarchicalFormatter format) { format .startObject(this) diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractObjectWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractObjectWriter.java index 87a181b26..527a69cdc 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractObjectWriter.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractObjectWriter.java @@ -38,9 +38,6 @@ import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter; public abstract class AbstractObjectWriter implements ObjectWriter { @Override - public ColumnMetadata schema() { return baseWriter().schema(); } - - @Override public ScalarWriter scalar() { throw new UnsupportedOperationException(); } @@ -60,42 +57,24 @@ public abstract class AbstractObjectWriter implements ObjectWriter { throw new UnsupportedOperationException(); } - public abstract WriterEvents events(); - - public ColumnWriter baseWriter() { - return (ColumnWriter) events(); - } - + public abstract ColumnWriter writer(); @Override - public ObjectType type() { return baseWriter().type(); } + public abstract WriterEvents events(); @Override - public boolean nullable() { return baseWriter().nullable(); } + public ColumnMetadata schema() { return writer().schema(); } @Override - public void setNull() { - baseWriter().setNull(); - } + public ObjectType type() { return writer().type(); } @Override - public void setObject(Object value) { - baseWriter().setObject(value); - } - - public abstract void dump(HierarchicalFormatter format); + public boolean nullable() { return writer().nullable(); } @Override - public int rowStartIndex() { - return baseWriter().rowStartIndex(); - } + public void setNull() { writer().setNull(); } @Override - public int lastWriteIndex() { - return baseWriter().lastWriteIndex(); - } + public void setObject(Object value) { writer().setObject(value); } - @Override - public int writeIndex() { - return baseWriter().writeIndex(); - } + public abstract void dump(HierarchicalFormatter format); } diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractScalarWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractScalarWriter.java index 9c2e9861d..a596e14af 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractScalarWriter.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractScalarWriter.java @@ -17,109 +17,53 @@ */ package org.apache.drill.exec.vector.accessor.writer; -import org.apache.drill.exec.record.metadata.ColumnMetadata; -import org.apache.drill.exec.vector.BaseDataValueVector; -import org.apache.drill.exec.vector.accessor.ColumnConversionFactory; -import org.apache.drill.exec.vector.accessor.ColumnWriterIndex; -import org.apache.drill.exec.vector.accessor.ObjectType; +import java.math.BigDecimal; + import org.apache.drill.exec.vector.accessor.ScalarWriter; -import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter; +import org.apache.drill.exec.vector.accessor.UnsupportedConversionError; +import org.apache.drill.exec.vector.accessor.writer.WriterEvents.ColumnWriterListener; +import org.joda.time.Period; /** - * Column writer implementation that acts as the basis for the - * generated, vector-specific implementations. All set methods - * throw an exception; subclasses simply override the supported - * method(s). + * Base class for concrete scalar column writers including actual vector + * writers, wrappers for nullable types, and shims used to convert types. */ -public abstract class AbstractScalarWriter extends ConcreteWriter { - - public static class ScalarObjectWriter extends AbstractObjectWriter { - - private ConcreteWriter scalarWriter; - - public ScalarObjectWriter(ConcreteWriter scalarWriter) { - final ColumnMetadata metadata = scalarWriter.schema(); - final ColumnConversionFactory factory = metadata.typeConverter(); - if (factory == null) { - this.scalarWriter = scalarWriter; - } else { - this.scalarWriter = factory.newWriter(metadata, scalarWriter); - } - } - - @Override - public ScalarWriter scalar() { return scalarWriter; } - - @Override - public WriterEvents events() { return scalarWriter; } - - @Override - public void dump(HierarchicalFormatter format) { - format - .startObject(this) - .attribute("scalarWriter"); - scalarWriter.dump(format); - format.endObject(); - } - } - - protected ColumnMetadata schema; - - /** - * Indicates the position in the vector to write. Set via an object so that - * all writers (within the same subtree) can agree on the write position. - * For example, all top-level, simple columns see the same row index. - * All columns within a repeated map see the same (inner) index, etc. - */ - - protected ColumnWriterIndex vectorIndex; +public abstract class AbstractScalarWriter implements ScalarWriter { @Override - public ObjectType type() { return ObjectType.SCALAR; } - - public void bindSchema(ColumnMetadata schema) { - this.schema = schema; - } - - @Override - public void bindIndex(ColumnWriterIndex vectorIndex) { - this.vectorIndex = vectorIndex; - } - - @Override - public int rowStartIndex() { - return vectorIndex.rowStartIndex(); + public void setObject(Object value) { + if (value == null) { + setNull(); + } else if (value instanceof Integer) { + setInt((Integer) value); + } else if (value instanceof Long) { + setLong((Long) value); + } else if (value instanceof String) { + setString((String) value); + } else if (value instanceof BigDecimal) { + setDecimal((BigDecimal) value); + } else if (value instanceof Period) { + setPeriod((Period) value); + } else if (value instanceof byte[]) { + final byte[] bytes = (byte[]) value; + setBytes(bytes, bytes.length); + } else if (value instanceof Byte) { + setInt((Byte) value); + } else if (value instanceof Short) { + setInt((Short) value); + } else if (value instanceof Double) { + setDouble((Double) value); + } else if (value instanceof Float) { + setDouble((Float) value); + } else { + throw conversionError(value.getClass().getSimpleName()); + } } - @Override - public int writeIndex() { - return vectorIndex.vectorIndex(); + protected UnsupportedConversionError conversionError(String javaType) { + return UnsupportedConversionError.writeError(schema(), javaType); } - @Override - public ColumnMetadata schema() { return schema; } - - public abstract BaseDataValueVector vector(); - - @Override - public void startWrite() { } - - @Override - public void startRow() { } - - @Override - public void endArrayValue() { } - - @Override - public void saveRow() { } - - @Override - public void dump(HierarchicalFormatter format) { - format - .startObject(this) - .attributeIdentity("vector", vector()) - .attribute("schema", vector().getField()) - .endObject(); - } + public void bindListener(ColumnWriterListener listener) { } } diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractScalarWriterImpl.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractScalarWriterImpl.java new file mode 100644 index 000000000..c1306b618 --- /dev/null +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractScalarWriterImpl.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.vector.accessor.writer; + +import org.apache.drill.exec.record.metadata.ColumnMetadata; +import org.apache.drill.exec.vector.BaseDataValueVector; +import org.apache.drill.exec.vector.accessor.ColumnConversionFactory; +import org.apache.drill.exec.vector.accessor.ColumnWriter; +import org.apache.drill.exec.vector.accessor.ColumnWriterIndex; +import org.apache.drill.exec.vector.accessor.ObjectType; +import org.apache.drill.exec.vector.accessor.ScalarWriter; +import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter; + +/** + * Column writer implementation that acts as the basis for the + * generated, vector-specific implementations. All set methods + * throw an exception; subclasses simply override the supported + * method(s). + */ + +public abstract class AbstractScalarWriterImpl extends AbstractScalarWriter implements WriterEvents { + + /** + * Wraps a scalar writer and its event handler to provide a uniform + * JSON-like interface for all writer types. + * <p> + * The client sees only the scalar writer API. But, internals need + * visibility into a rather complex set of events to orchestrate + * vector events: mostly sent to the writer, but some times sent + * from the writer, such as vector overflow. Separating the two + * concepts makes it easier to add type-conversion shims on top of + * the actual vector writer. + */ + public static class ScalarObjectWriter extends AbstractObjectWriter { + + private final WriterEvents writerEvents; + private ScalarWriter scalarWriter; + + public ScalarObjectWriter(AbstractScalarWriterImpl scalarWriter) { + final ColumnMetadata metadata = scalarWriter.schema(); + final ColumnConversionFactory factory = metadata.typeConverter(); + writerEvents = scalarWriter; + if (factory == null) { + this.scalarWriter = scalarWriter; + } else { + this.scalarWriter = factory.newWriter(metadata, scalarWriter); + } + } + + @Override + public ScalarWriter scalar() { return scalarWriter; } + + @Override + public ColumnWriter writer() { return scalarWriter; } + + @Override + public WriterEvents events() { return writerEvents; } + + @Override + public void dump(HierarchicalFormatter format) { + format + .startObject(this) + .attribute("scalarWriter"); + writerEvents.dump(format); + format.endObject(); + } + } + + protected ColumnMetadata schema; + + /** + * Indicates the position in the vector to write. Set via an object so that + * all writers (within the same subtree) can agree on the write position. + * For example, all top-level, simple columns see the same row index. + * All columns within a repeated map see the same (inner) index, etc. + */ + + protected ColumnWriterIndex vectorIndex; + + @Override + public ObjectType type() { return ObjectType.SCALAR; } + + public void bindSchema(ColumnMetadata schema) { + this.schema = schema; + } + + @Override + public void bindIndex(ColumnWriterIndex vectorIndex) { + this.vectorIndex = vectorIndex; + } + + @Override + public int rowStartIndex() { + return vectorIndex.rowStartIndex(); + } + + @Override + public int writeIndex() { + return vectorIndex.vectorIndex(); + } + + @Override + public ColumnMetadata schema() { return schema; } + + public abstract BaseDataValueVector vector(); + + @Override + public void startWrite() { } + + @Override + public void startRow() { } + + @Override + public void endArrayValue() { } + + @Override + public void saveRow() { } + + @Override + public void dump(HierarchicalFormatter format) { + format + .startObject(this) + .attributeIdentity("vector", vector()) + .attribute("schema", vector().getField()) + .endObject(); + } +} diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractTupleWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractTupleWriter.java index aee806df8..006608125 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractTupleWriter.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractTupleWriter.java @@ -25,6 +25,7 @@ import org.apache.drill.exec.record.metadata.ColumnMetadata; import org.apache.drill.exec.record.metadata.ProjectionType; import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.vector.accessor.ArrayWriter; +import org.apache.drill.exec.vector.accessor.ColumnWriter; import org.apache.drill.exec.vector.accessor.ColumnWriterIndex; import org.apache.drill.exec.vector.accessor.ObjectType; import org.apache.drill.exec.vector.accessor.ObjectWriter; @@ -104,13 +105,16 @@ public abstract class AbstractTupleWriter implements TupleWriter, WriterEvents { public static class TupleObjectWriter extends AbstractObjectWriter { - private AbstractTupleWriter tupleWriter; + private final AbstractTupleWriter tupleWriter; public TupleObjectWriter(AbstractTupleWriter tupleWriter) { this.tupleWriter = tupleWriter; } @Override + public ColumnWriter writer() { return tupleWriter; } + + @Override public TupleWriter tuple() { return tupleWriter; } @Override @@ -126,11 +130,27 @@ public abstract class AbstractTupleWriter implements TupleWriter, WriterEvents { } } + /** + * Listener (callback) to handle requests to add a new column to a tuple (row + * or map). Implemented and bound by the client code that creates or uses the + * tuple writer. If no listener is bound, then an attempt to add a column + * throws an exception. + */ + + public static interface TupleWriterListener { + + ObjectWriter addColumn(TupleWriter tuple, ColumnMetadata column); + + ObjectWriter addColumn(TupleWriter tuple, MaterializedField field); + + ProjectionType projectionType(String columnName); + } + protected final TupleMetadata tupleSchema; protected final List<AbstractObjectWriter> writers; protected ColumnWriterIndex vectorIndex; protected ColumnWriterIndex childIndex; - protected TupleWriterListener listener; + protected AbstractTupleWriter.TupleWriterListener listener; protected State state = State.IDLE; protected AbstractTupleWriter(TupleMetadata schema, List<AbstractObjectWriter> writers) { @@ -174,7 +194,7 @@ public abstract class AbstractTupleWriter implements TupleWriter, WriterEvents { public int addColumnWriter(AbstractObjectWriter colWriter) { assert writers.size() == tupleSchema.size(); - int colIndex = tupleSchema.addColumn(colWriter.schema()); + final int colIndex = tupleSchema.addColumn(colWriter.schema()); writers.add(colWriter); colWriter.events().bindIndex(childIndex); if (state != State.IDLE) { @@ -197,7 +217,7 @@ public abstract class AbstractTupleWriter implements TupleWriter, WriterEvents { if (listener == null) { throw new UnsupportedOperationException("addColumn"); } - AbstractObjectWriter colWriter = (AbstractObjectWriter) listener.addColumn(this, column); + final AbstractObjectWriter colWriter = (AbstractObjectWriter) listener.addColumn(this, column); return addColumnWriter(colWriter); } @@ -206,7 +226,7 @@ public abstract class AbstractTupleWriter implements TupleWriter, WriterEvents { if (listener == null) { throw new UnsupportedOperationException("addColumn"); } - AbstractObjectWriter colWriter = (AbstractObjectWriter) listener.addColumn(this, field); + final AbstractObjectWriter colWriter = (AbstractObjectWriter) listener.addColumn(this, field); return addColumnWriter(colWriter); } @@ -317,7 +337,7 @@ public abstract class AbstractTupleWriter implements TupleWriter, WriterEvents { @Override public ObjectWriter column(String colName) { - int index = tupleSchema.index(colName); + final int index = tupleSchema.index(colName); if (index == -1) { throw new UndefinedColumnException(colName); } @@ -331,7 +351,7 @@ public abstract class AbstractTupleWriter implements TupleWriter, WriterEvents { @Override public void setObject(Object value) { - Object values[] = (Object[]) value; + final Object values[] = (Object[]) value; if (values.length != tupleSchema.size()) { throw new IllegalArgumentException( String.format("Map %s has %d columns, but value array has " + @@ -404,11 +424,14 @@ public abstract class AbstractTupleWriter implements TupleWriter, WriterEvents { return vectorIndex.vectorIndex(); } - @Override public void bindListener(TupleWriterListener listener) { this.listener = listener; } + @Override + public void bindListener(ColumnWriterListener listener) { } + + @Override public void dump(HierarchicalFormatter format) { format .startObject(this) diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractWriteConvertor.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractWriteConverter.java index 0d0bc8803..55cd991a7 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractWriteConvertor.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractWriteConverter.java @@ -20,11 +20,9 @@ package org.apache.drill.exec.vector.accessor.writer; import java.math.BigDecimal; import org.apache.drill.exec.record.metadata.ColumnMetadata; -import org.apache.drill.exec.vector.accessor.ColumnWriterIndex; import org.apache.drill.exec.vector.accessor.ObjectType; import org.apache.drill.exec.vector.accessor.ScalarWriter; import org.apache.drill.exec.vector.accessor.ValueType; -import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter; import org.joda.time.Period; /** @@ -38,20 +36,12 @@ import org.joda.time.Period; * for an int column in the case above. */ -// TODO: This organization works fine, but is a bit heavy-weight. -// It may be time to think about separating the pure writer aspect of -// a column writer from its plumbing aspects. That is, the base -// ConcreteWriter class combines the public API (ScalarWriter) with -// the internal implementation (WriterEvents) into a single class. -// Might be worth using composition rather than inheritance to keep -// these aspects distinct. +public class AbstractWriteConverter extends AbstractScalarWriter { -public class AbstractWriteConvertor extends ConcreteWriter { + private final ScalarWriter baseWriter; - private final ConcreteWriter baseWriter; - - public AbstractWriteConvertor(ScalarWriter baseWriter) { - this.baseWriter = (ConcreteWriter) baseWriter; + public AbstractWriteConverter(ScalarWriter baseWriter) { + this.baseWriter = baseWriter; } @Override @@ -60,31 +50,6 @@ public class AbstractWriteConvertor extends ConcreteWriter { } @Override - public int lastWriteIndex() { - return baseWriter.lastWriteIndex(); - } - - @Override - public void restartRow() { - baseWriter.restartRow(); - } - - @Override - public void endWrite() { - baseWriter.endWrite(); - } - - @Override - public void preRollover() { - baseWriter.preRollover(); - } - - @Override - public void postRollover() { - baseWriter.postRollover(); - } - - @Override public ObjectType type() { return baseWriter.type(); } @@ -105,46 +70,6 @@ public class AbstractWriteConvertor extends ConcreteWriter { } @Override - public int rowStartIndex() { - return baseWriter.rowStartIndex(); - } - - @Override - public int writeIndex() { - return baseWriter.writeIndex(); - } - - @Override - public void bindListener(ColumnWriterListener listener) { - baseWriter.bindListener(listener); - } - - @Override - public void bindIndex(ColumnWriterIndex index) { - baseWriter.bindIndex(index); - } - - @Override - public void startWrite() { - baseWriter.startWrite(); - } - - @Override - public void startRow() { - baseWriter.startRow(); - } - - @Override - public void endArrayValue() { - baseWriter.endArrayValue(); - } - - @Override - public void saveRow() { - baseWriter.saveRow(); - } - - @Override public void setInt(int value) { baseWriter.setInt(value); } @@ -178,9 +103,4 @@ public class AbstractWriteConvertor extends ConcreteWriter { public void setPeriod(Period value) { baseWriter.setPeriod(value); } - - @Override - public void dump(HierarchicalFormatter format) { - baseWriter.dump(format); - } } diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/BaseScalarWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/BaseScalarWriter.java index 7e121495f..e2e63d2ee 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/BaseScalarWriter.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/BaseScalarWriter.java @@ -129,7 +129,7 @@ import io.netty.buffer.DrillBuf; * a roll-over when a vector overflows. */ -public abstract class BaseScalarWriter extends AbstractScalarWriter { +public abstract class BaseScalarWriter extends AbstractScalarWriterImpl { public static final int MIN_BUFFER_SIZE = 256; diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/ColumnWriterFactory.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/ColumnWriterFactory.java index 6ee5bbcf7..206a0a66a 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/ColumnWriterFactory.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/ColumnWriterFactory.java @@ -26,7 +26,7 @@ import org.apache.drill.exec.vector.NullableVector; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.accessor.ColumnAccessorUtils; import org.apache.drill.exec.vector.accessor.writer.AbstractArrayWriter.ArrayObjectWriter; -import org.apache.drill.exec.vector.accessor.writer.AbstractScalarWriter.ScalarObjectWriter; +import org.apache.drill.exec.vector.accessor.writer.AbstractScalarWriterImpl.ScalarObjectWriter; import org.apache.drill.exec.vector.accessor.writer.dummy.DummyArrayWriter; import org.apache.drill.exec.vector.accessor.writer.dummy.DummyScalarWriter; import org.apache.drill.exec.vector.complex.RepeatedValueVector; diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/ConcreteWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/ConcreteWriter.java deleted file mode 100644 index 549431f76..000000000 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/ConcreteWriter.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.vector.accessor.writer; - -import java.math.BigDecimal; - -import org.apache.drill.exec.vector.accessor.ScalarWriter; -import org.apache.drill.exec.vector.accessor.UnsupportedConversionError; -import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter; -import org.joda.time.Period; - -/** - * Base class for concrete scalar column writers including actual vector - * writers, wrappers for nullable types, and shims used to convert types. - */ - -public abstract class ConcreteWriter implements ScalarWriter, WriterEvents { - - @Override - public void setObject(Object value) { - if (value == null) { - setNull(); - } else if (value instanceof Integer) { - setInt((Integer) value); - } else if (value instanceof Long) { - setLong((Long) value); - } else if (value instanceof String) { - setString((String) value); - } else if (value instanceof BigDecimal) { - setDecimal((BigDecimal) value); - } else if (value instanceof Period) { - setPeriod((Period) value); - } else if (value instanceof byte[]) { - final byte[] bytes = (byte[]) value; - setBytes(bytes, bytes.length); - } else if (value instanceof Byte) { - setInt((Byte) value); - } else if (value instanceof Short) { - setInt((Short) value); - } else if (value instanceof Double) { - setDouble((Double) value); - } else if (value instanceof Float) { - setDouble((Float) value); - } else { - throw conversionError(value.getClass().getSimpleName()); - } - } - - protected UnsupportedConversionError conversionError(String javaType) { - return UnsupportedConversionError.writeError(schema(), javaType); - } - - abstract void dump(HierarchicalFormatter format); -} diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/EmptyListShim.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/EmptyListShim.java index 1167d3385..d8c8947ca 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/EmptyListShim.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/EmptyListShim.java @@ -21,6 +21,7 @@ import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.record.metadata.ColumnMetadata; import org.apache.drill.exec.vector.accessor.ColumnWriterIndex; import org.apache.drill.exec.vector.accessor.ObjectWriter; +import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter; import org.apache.drill.exec.vector.accessor.writer.UnionWriterImpl.UnionShim; import org.apache.drill.shaded.guava.com.google.common.base.Preconditions; @@ -45,6 +46,9 @@ public class EmptyListShim implements UnionShim { public void bindIndex(ColumnWriterIndex index) { } @Override + public void bindListener(ColumnWriterListener listener) { } + + @Override public void startWrite() { } @Override @@ -127,6 +131,9 @@ public class EmptyListShim implements UnionShim { public int rowStartIndex() { return 0; } @Override + public int writeIndex() { return 0; } + + @Override public void addMember(AbstractObjectWriter colWriter) { // This shim has no types. If this is called, then the shim replacement @@ -135,4 +142,8 @@ public class EmptyListShim implements UnionShim { throw new UnsupportedOperationException(); } + @Override + public void dump(HierarchicalFormatter format) { + format.startObject(this).endObject(); + } } diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/NullableScalarWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/NullableScalarWriter.java index 75ef57fc4..e9fe11ae4 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/NullableScalarWriter.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/NullableScalarWriter.java @@ -23,12 +23,12 @@ import org.apache.drill.exec.record.metadata.ColumnMetadata; import org.apache.drill.exec.vector.BaseDataValueVector; import org.apache.drill.exec.vector.NullableVector; import org.apache.drill.exec.vector.accessor.ColumnAccessors.UInt1ColumnWriter; -import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter; import org.apache.drill.exec.vector.accessor.ColumnWriterIndex; import org.apache.drill.exec.vector.accessor.ValueType; +import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter; import org.joda.time.Period; -public class NullableScalarWriter extends AbstractScalarWriter { +public class NullableScalarWriter extends AbstractScalarWriterImpl { public static final class ChildIndex implements ColumnWriterIndex { @@ -93,7 +93,7 @@ public class NullableScalarWriter extends AbstractScalarWriter { @Override public void bindIndex(ColumnWriterIndex index) { writerIndex = index; - ColumnWriterIndex childIndex = new ChildIndex(index); + final ColumnWriterIndex childIndex = new ChildIndex(index); isSetWriter.bindIndex(childIndex); baseWriter.bindIndex(childIndex); } diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/ScalarArrayWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/ScalarArrayWriter.java index f271bfab0..c761a95de 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/ScalarArrayWriter.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/ScalarArrayWriter.java @@ -22,8 +22,9 @@ import java.math.BigDecimal; import org.apache.drill.exec.record.metadata.ColumnMetadata; import org.apache.drill.exec.vector.accessor.ColumnWriterIndex; +import org.apache.drill.exec.vector.accessor.ScalarWriter; import org.apache.drill.exec.vector.accessor.writer.AbstractArrayWriter.BaseArrayWriter; -import org.apache.drill.exec.vector.accessor.writer.AbstractScalarWriter.ScalarObjectWriter; +import org.apache.drill.exec.vector.accessor.writer.AbstractScalarWriterImpl.ScalarObjectWriter; import org.apache.drill.exec.vector.complex.RepeatedValueVector; import org.joda.time.Period; @@ -60,7 +61,7 @@ public class ScalarArrayWriter extends BaseArrayWriter { public final void nextElement() { next(); } } - private final ConcreteWriter elementWriter; + private final ScalarWriter elementWriter; public ScalarArrayWriter(ColumnMetadata schema, RepeatedValueVector vector, BaseScalarWriter elementWriter) { @@ -70,7 +71,7 @@ public class ScalarArrayWriter extends BaseArrayWriter { // Save the writer from the scalar object writer created above // which may have wrapped the element writer in a type convertor. - this.elementWriter = (ConcreteWriter) elementObjWriter.scalar(); + this.elementWriter = elementObjWriter.scalar(); } public static ArrayObjectWriter build(ColumnMetadata schema, @@ -83,7 +84,7 @@ public class ScalarArrayWriter extends BaseArrayWriter { public void bindIndex(ColumnWriterIndex index) { elementIndex = new ScalarElementWriterIndex(); super.bindIndex(index); - elementWriter.bindIndex(elementIndex); + elementObjWriter.events().bindIndex(elementIndex); } @Override diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/SimpleListShim.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/SimpleListShim.java index 6dd947503..66b81f976 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/SimpleListShim.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/SimpleListShim.java @@ -21,8 +21,8 @@ import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.record.metadata.ColumnMetadata; import org.apache.drill.exec.vector.accessor.ColumnWriterIndex; import org.apache.drill.exec.vector.accessor.ObjectWriter; +import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter; import org.apache.drill.exec.vector.accessor.writer.UnionWriterImpl.UnionShim; - import org.apache.drill.shaded.guava.com.google.common.base.Preconditions; /** @@ -56,6 +56,11 @@ public class SimpleListShim implements UnionShim { } @Override + public void bindListener(ColumnWriterListener listener) { + colWriter.events().bindListener(listener); + } + + @Override public boolean hasType(MinorType type) { return type == colWriter.schema().type(); } @@ -185,11 +190,23 @@ public class SimpleListShim implements UnionShim { @Override public int lastWriteIndex() { - return colWriter.lastWriteIndex(); + return events().lastWriteIndex(); } @Override public int rowStartIndex() { - return colWriter.rowStartIndex(); + return events().rowStartIndex(); + } + + @Override + public int writeIndex() { + return events().writeIndex(); + } + + @Override + public void dump(HierarchicalFormatter format) { + format.startObject(this).attribute("colWriter"); + colWriter.dump(format); + format.endObject(); } } diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/UnionVectorShim.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/UnionVectorShim.java index e39176f89..d18b382fb 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/UnionVectorShim.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/UnionVectorShim.java @@ -20,10 +20,10 @@ package org.apache.drill.exec.vector.accessor.writer; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.record.metadata.ColumnMetadata; import org.apache.drill.exec.vector.ValueVector; -import org.apache.drill.exec.vector.accessor.ColumnWriter; import org.apache.drill.exec.vector.accessor.ColumnWriterIndex; import org.apache.drill.exec.vector.accessor.ObjectWriter; import org.apache.drill.exec.vector.accessor.VariantWriter.VariantWriterListener; +import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter; import org.apache.drill.exec.vector.accessor.writer.AbstractFixedWidthWriter.BaseFixedWidthWriter; import org.apache.drill.exec.vector.accessor.writer.UnionWriterImpl.UnionShim; import org.apache.drill.exec.vector.complex.UnionVector; @@ -42,7 +42,7 @@ public class UnionVectorShim implements UnionShim { static class DefaultListener implements VariantWriterListener { - private UnionVectorShim shim; + private final UnionVectorShim shim; private DefaultListener(UnionVectorShim shim) { this.shim = shim; @@ -57,8 +57,8 @@ public class UnionVectorShim implements UnionShim { // will already be in the variant schema by the time we add the // writer to the variant writer in a few steps from now. - ValueVector memberVector = shim.vector.getMember(type); - ColumnMetadata memberSchema = shim.writer.variantSchema().addType(type); + final ValueVector memberVector = shim.vector.getMember(type); + final ColumnMetadata memberSchema = shim.writer.variantSchema().addType(type); return ColumnWriterFactory.buildColumnWriter(memberSchema, memberVector); } @@ -100,7 +100,7 @@ public class UnionVectorShim implements UnionShim { @Override public void bindWriter(UnionWriterImpl writer) { this.writer = writer; - ColumnWriterIndex index = writer.index(); + final ColumnWriterIndex index = writer.index(); if (index != null) { bindIndex(index); } @@ -116,6 +116,12 @@ public class UnionVectorShim implements UnionShim { } } + // Unions are complex: the listener should bind to the individual components + // as they are created. + + @Override + public void bindListener(ColumnWriterListener listener) { } + @Override public void setNull() { @@ -133,7 +139,7 @@ public class UnionVectorShim implements UnionShim { @Override public ObjectWriter member(MinorType type) { - AbstractObjectWriter colWriter = variants[type.ordinal()]; + final AbstractObjectWriter colWriter = variants[type.ordinal()]; if (colWriter != null) { return colWriter; } @@ -150,14 +156,14 @@ public class UnionVectorShim implements UnionShim { @Override public AbstractObjectWriter addMember(ColumnMetadata schema) { - AbstractObjectWriter colWriter = (AbstractObjectWriter) writer.listener().addMember(schema); + final AbstractObjectWriter colWriter = (AbstractObjectWriter) writer.listener().addMember(schema); addMember(colWriter); return colWriter; } @Override public AbstractObjectWriter addMember(MinorType type) { - AbstractObjectWriter colWriter = (AbstractObjectWriter) writer.listener().addType(type); + final AbstractObjectWriter colWriter = (AbstractObjectWriter) writer.listener().addType(type); addMember(colWriter); return colWriter; } @@ -189,7 +195,7 @@ public class UnionVectorShim implements UnionShim { */ public void addMemberWriter(AbstractObjectWriter colWriter) { - MinorType type = colWriter.schema().type(); + final MinorType type = colWriter.schema().type(); assert variants[type.ordinal()] == null; variants[type.ordinal()] = colWriter; } @@ -283,7 +289,10 @@ public class UnionVectorShim implements UnionShim { * @return the writer for the types vector */ - public ColumnWriter typeWriter() { return typeWriter; } + public AbstractScalarWriterImpl typeWriter() { return typeWriter; } + + @Override + public int writeIndex() { return typeWriter.writeIndex(); } @Override public int lastWriteIndex() { return typeWriter.lastWriteIndex(); } @@ -301,4 +310,11 @@ public class UnionVectorShim implements UnionShim { public void initTypeIndex(int typeFillCount) { ((BaseFixedWidthWriter) typeWriter).setLastWriteIndex(typeFillCount); } + + @Override + public void dump(HierarchicalFormatter format) { + format.startObject(this).attribute("typeWriter"); + typeWriter.dump(format); + format.endObject(); + } }
\ No newline at end of file diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/UnionWriterImpl.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/UnionWriterImpl.java index 928317f09..a308ceaf4 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/UnionWriterImpl.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/UnionWriterImpl.java @@ -23,6 +23,7 @@ import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.record.metadata.ColumnMetadata; import org.apache.drill.exec.record.metadata.VariantMetadata; import org.apache.drill.exec.vector.accessor.ArrayWriter; +import org.apache.drill.exec.vector.accessor.ColumnWriter; import org.apache.drill.exec.vector.accessor.ColumnWriterIndex; import org.apache.drill.exec.vector.accessor.ObjectType; import org.apache.drill.exec.vector.accessor.ObjectWriter; @@ -55,7 +56,9 @@ public class UnionWriterImpl implements VariantWriter, WriterEvents { ObjectWriter member(MinorType type); void setType(MinorType type); + @Override int lastWriteIndex(); + @Override int rowStartIndex(); AbstractObjectWriter addMember(ColumnMetadata colSchema); AbstractObjectWriter addMember(MinorType type); @@ -71,6 +74,9 @@ public class UnionWriterImpl implements VariantWriter, WriterEvents { } @Override + public ColumnWriter writer() { return writer; } + + @Override public VariantWriter variant() { return writer; } @Override @@ -136,6 +142,12 @@ public class UnionWriterImpl implements VariantWriter, WriterEvents { this.listener = listener; } + // Unions are complex: listeners should bind to the components as they + // are created. + + @Override + public void bindListener(ColumnWriterListener listener) { } + // The following are for coordinating with the shim. public State state() { return state; } @@ -192,7 +204,7 @@ public class UnionWriterImpl implements VariantWriter, WriterEvents { // conversion. Then set the type because, if the conversion is // done, the type vector exists only after creating the member. - ObjectWriter writer = shim.member(type); + final ObjectWriter writer = shim.member(type); setType(type); return writer; } @@ -222,7 +234,7 @@ public class UnionWriterImpl implements VariantWriter, WriterEvents { */ protected void addMember(AbstractObjectWriter writer) { - MinorType type = writer.schema().type(); + final MinorType type = writer.schema().type(); // If the metadata has not yet been added to the variant // schema, do so now. (Unfortunately, the default listener @@ -334,7 +346,7 @@ public class UnionWriterImpl implements VariantWriter, WriterEvents { // Can look for exactly one period type as is done for Object[] below throw new IllegalArgumentException("Period is ambiguous, please use scalar(type)"); } else if (value instanceof byte[]) { - byte[] bytes = (byte[]) value; + final byte[] bytes = (byte[]) value; scalar(MinorType.VARBINARY).setBytes(bytes, bytes.length); } else if (value instanceof Byte) { scalar(MinorType.TINYINT).setInt((Byte) value); @@ -361,6 +373,7 @@ public class UnionWriterImpl implements VariantWriter, WriterEvents { } } + @Override public void dump(HierarchicalFormatter format) { // TODO Auto-generated method stub diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/WriterEvents.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/WriterEvents.java index c8f9f4896..1983d1799 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/WriterEvents.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/WriterEvents.java @@ -18,6 +18,9 @@ package org.apache.drill.exec.vector.accessor.writer; import org.apache.drill.exec.vector.accessor.ColumnWriterIndex; +import org.apache.drill.exec.vector.accessor.ScalarWriter; +import org.apache.drill.exec.vector.accessor.WriterPosition; +import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter; /** * Internal interface used to control the behavior @@ -37,7 +40,7 @@ import org.apache.drill.exec.vector.accessor.ColumnWriterIndex; * calls, can change. */ -public interface WriterEvents { +public interface WriterEvents extends WriterPosition { /** * Tracks the write state of a tuple or variant to allow applying the correct @@ -72,6 +75,40 @@ public interface WriterEvents { } /** + * Listener (callback) for vector overflow events. To be optionally + * implemented and bound by the client code of the writer. If no + * listener is bound, and a vector overflows, then an exception is + * thrown. + */ + + interface ColumnWriterListener { + + /** + * Alert the listener that a vector has overflowed. Upon return, + * all writers must have a new set of buffers available, ready + * to accept the in-flight value that triggered the overflow. + * + * @param writer the writer that triggered the overflow + */ + + void overflowed(ScalarWriter writer); + + /** + * A writer wants to expand its vector. Allows the listener to + * either allow the growth, or trigger and overflow to limit + * batch size. + * + * @param writer the writer that wishes to grow its vector + * @param delta the amount by which the vector is to grow + * @return true if the vector can be grown, false if the writer + * should instead trigger an overflow by calling + * <tt>overflowed()</tt> + */ + + boolean canExpand(ScalarWriter writer, int delta); + } + + /** * Bind the writer to a writer index. * * @param index the writer index (top level or nested for @@ -81,6 +118,19 @@ public interface WriterEvents { void bindIndex(ColumnWriterIndex index); /** + * Bind a listener to the underlying vector writer. This listener reports on vector + * events (overflow, growth), and so is called only when the writer is backed by + * a vector. The listener is ignored (and never called) for dummy (non-projected) + * columns. If the column is compound (such as for a nullable or repeated column, + * or for a map), then the writer is bound to the individual components. + * + * @param listener + * the vector event listener to bind + */ + + void bindListener(ColumnWriterListener listener); + + /** * Start a write (batch) operation. Performs any vector initialization * required at the start of a batch (especially for offset vectors.) */ @@ -143,4 +193,6 @@ public interface WriterEvents { */ void postRollover(); + + abstract void dump(HierarchicalFormatter format); } diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/dummy/DummyScalarWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/dummy/DummyScalarWriter.java index 9cfe56e22..919f68956 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/dummy/DummyScalarWriter.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/dummy/DummyScalarWriter.java @@ -23,7 +23,7 @@ import org.apache.drill.exec.record.metadata.ColumnMetadata; import org.apache.drill.exec.vector.BaseDataValueVector; import org.apache.drill.exec.vector.accessor.ColumnWriterIndex; import org.apache.drill.exec.vector.accessor.ValueType; -import org.apache.drill.exec.vector.accessor.writer.AbstractScalarWriter; +import org.apache.drill.exec.vector.accessor.writer.AbstractScalarWriterImpl; import org.joda.time.Period; /** @@ -32,7 +32,7 @@ import org.joda.time.Period; * nor is it backed by a real vector, index or type. */ -public class DummyScalarWriter extends AbstractScalarWriter { +public class DummyScalarWriter extends AbstractScalarWriterImpl { public DummyScalarWriter(ColumnMetadata schema) { this.schema = schema; |