From cdc5daed5218ed70445d178f5fc25df09c573001 Mon Sep 17 00:00:00 2001 From: Jacques Nadeau Date: Sun, 11 May 2014 15:12:19 -0700 Subject: Add support for RepeatedMapVector, MapVector and RepeatedListVector. --- .../apache/drill/exec/expr/EvaluationVisitor.java | 246 +++++++++++++++------ 1 file changed, 173 insertions(+), 73 deletions(-) (limited to 'exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java') diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java index dce070f46..d700bf3f0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java @@ -20,8 +20,6 @@ package org.apache.drill.exec.expr; import java.util.List; import java.util.Set; -import io.netty.buffer.ByteBuf; -import com.google.common.collect.Lists; import org.apache.drill.common.expression.CastExpression; import org.apache.drill.common.expression.ConvertExpression; import org.apache.drill.common.expression.FunctionCall; @@ -29,25 +27,24 @@ import org.apache.drill.common.expression.FunctionHolderExpression; import org.apache.drill.common.expression.IfExpression; import org.apache.drill.common.expression.IfExpression.IfCondition; import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.common.expression.PathSegment; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.expression.TypedNullConstant; -import org.apache.drill.common.expression.ValueExpressions; import org.apache.drill.common.expression.ValueExpressions.BooleanExpression; -import org.apache.drill.common.expression.ValueExpressions.DoubleExpression; -import org.apache.drill.common.expression.ValueExpressions.LongExpression; -import org.apache.drill.common.expression.ValueExpressions.IntExpression; import org.apache.drill.common.expression.ValueExpressions.DateExpression; -import org.apache.drill.common.expression.ValueExpressions.IntervalYearExpression; -import org.apache.drill.common.expression.ValueExpressions.IntervalDayExpression; -import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression; -import org.apache.drill.common.expression.ValueExpressions.TimeExpression; -import org.apache.drill.common.expression.ValueExpressions.Decimal9Expression; import org.apache.drill.common.expression.ValueExpressions.Decimal18Expression; import org.apache.drill.common.expression.ValueExpressions.Decimal28Expression; import org.apache.drill.common.expression.ValueExpressions.Decimal38Expression; +import org.apache.drill.common.expression.ValueExpressions.Decimal9Expression; +import org.apache.drill.common.expression.ValueExpressions.DoubleExpression; +import org.apache.drill.common.expression.ValueExpressions.IntExpression; +import org.apache.drill.common.expression.ValueExpressions.IntervalDayExpression; +import org.apache.drill.common.expression.ValueExpressions.IntervalYearExpression; +import org.apache.drill.common.expression.ValueExpressions.LongExpression; import org.apache.drill.common.expression.ValueExpressions.QuotedString; +import org.apache.drill.common.expression.ValueExpressions.TimeExpression; +import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression; import org.apache.drill.common.expression.visitors.AbstractExprVisitor; -import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.common.types.Types; @@ -60,6 +57,8 @@ import org.apache.drill.exec.expr.fn.HiveFuncHolder; import org.apache.drill.exec.physical.impl.filter.ReturnValueExpression; import org.apache.drill.exec.record.NullExpression; import org.apache.drill.exec.vector.ValueHolderHelper; +import org.apache.drill.exec.vector.complex.reader.FieldReader; +import org.apache.drill.exec.vector.complex.writer.FieldWriter; import com.google.common.collect.Lists; import com.sun.codemodel.JBlock; @@ -68,6 +67,7 @@ import com.sun.codemodel.JConditional; import com.sun.codemodel.JExpr; import com.sun.codemodel.JExpression; import com.sun.codemodel.JInvocation; +import com.sun.codemodel.JLabel; import com.sun.codemodel.JType; import com.sun.codemodel.JVar; @@ -88,7 +88,7 @@ public class EvaluationVisitor { private class EvalVisitor extends AbstractExprVisitor, RuntimeException> { - + @Override public HoldingContainer visitFunctionCall(FunctionCall call, ClassGenerator generator) throws RuntimeException { throw new UnsupportedOperationException("FunctionCall is not expected here. "+ @@ -143,18 +143,18 @@ public class EvaluationVisitor { JConditional jc = null; JBlock conditionalBlock = new JBlock(false, false); for (IfCondition c : ifExpr.conditions) { - HoldingContainer HoldingContainer = c.condition.accept(this, generator); + HoldingContainer holdingContainer = c.condition.accept(this, generator); if (jc == null) { - if (HoldingContainer.isOptional()) { - jc = conditionalBlock._if(HoldingContainer.getIsSet().cand(HoldingContainer.getValue())); + if (holdingContainer.isOptional()) { + jc = conditionalBlock._if(holdingContainer.getIsSet().cand(holdingContainer.getValue())); } else { - jc = conditionalBlock._if(HoldingContainer.getValue().eq(JExpr.lit(1))); + jc = conditionalBlock._if(holdingContainer.getValue().eq(JExpr.lit(1))); } } else { - if (HoldingContainer.isOptional()) { - jc = jc._else()._if(HoldingContainer.getIsSet().cand(HoldingContainer.getValue())); + if (holdingContainer.isOptional()) { + jc = jc._else()._if(holdingContainer.getIsSet().cand(holdingContainer.getValue())); } else { - jc = jc._else()._if(HoldingContainer.getValue()); + jc = jc._else()._if(holdingContainer.getValue()); } } @@ -184,14 +184,14 @@ public class EvaluationVisitor { return output; } - + @Override public HoldingContainer visitSchemaPath(SchemaPath path, ClassGenerator generator) throws RuntimeException { throw new UnsupportedOperationException("All schema paths should have been replaced with ValueVectorExpressions."); } @Override - public HoldingContainer visitLongConstant(LongExpression e, ClassGenerator generator) throws RuntimeException { + public HoldingContainer visitLongConstant(LongExpression e, ClassGenerator generator) throws RuntimeException { HoldingContainer out = generator.declare(e.getMajorType()); generator.getEvalBlock().assign(out.getValue(), JExpr.lit(e.getLong())); return out; @@ -269,81 +269,180 @@ public class EvaluationVisitor { private HoldingContainer visitValueVectorWriteExpression(ValueVectorWriteExpression e, ClassGenerator generator) { - LogicalExpression child = e.getChild(); - HoldingContainer inputContainer = child.accept(this, generator); + final LogicalExpression child = e.getChild(); + final HoldingContainer inputContainer = child.accept(this, generator); + final boolean complex = Types.isComplex(inputContainer.getMajorType()); + JBlock block = generator.getEvalBlock(); JExpression outIndex = generator.getMappingSet().getValueWriteIndex(); JVar vv = generator.declareVectorValueSetupAndMember(generator.getMappingSet().getOutgoing(), e.getFieldId()); - String setMethod = e.isSafe() ? "setSafe" : "set"; - - JInvocation setMeth; - if (Types.usesHolderForGet(inputContainer.getMajorType())) { - setMeth = vv.invoke("getMutator").invoke(setMethod).arg(outIndex).arg(inputContainer.getHolder()); - }else{ - setMeth = vv.invoke("getMutator").invoke(setMethod).arg(outIndex).arg(inputContainer.getValue()); - } - - if(e.isSafe()){ - HoldingContainer outputContainer = generator.declare(Types.REQUIRED_BIT); - block.assign(outputContainer.getValue(), JExpr.lit(1)); - if(inputContainer.isOptional()){ -// block._if(vv.invoke("getMutator").invoke(setMethod).arg(outIndex).not())._then().assign(outputContainer.getValue(), JExpr.lit(0)); - JConditional jc = block._if(inputContainer.getIsSet().eq(JExpr.lit(0)).not()); - block = jc._then(); + + if(complex){ + JType writerImpl = generator.getModel()._ref(TypeHelper.getWriterImpl(inputContainer.getMinorType(), inputContainer.getMajorType().getMode())); + JType writerIFace = generator.getModel()._ref(TypeHelper.getWriterInterface(inputContainer.getMinorType(), inputContainer.getMajorType().getMode())); + JVar writer = generator.declareClassField("writer", writerIFace); + generator.getSetupBlock().assign(writer, JExpr._new(writerImpl).arg(vv).arg(JExpr._null())); + generator.getEvalBlock().add(writer.invoke("setPosition").arg(outIndex)); + String copyMethod = inputContainer.isSingularRepeated() ? "copyAsValueSingle" : "copyAsValue"; + generator.getEvalBlock().add(inputContainer.getHolder().invoke(copyMethod).arg(writer)); + if(e.isSafe()){ + HoldingContainer outputContainer = generator.declare(Types.REQUIRED_BIT); + JConditional ifOut = generator.getEvalBlock()._if(writer.invoke("ok")); + ifOut._then().assign(outputContainer.getValue(), JExpr.lit(1)); + ifOut._else().assign(outputContainer.getValue(), JExpr.lit(0)); + return outputContainer; } - block._if(setMeth.not())._then().assign(outputContainer.getValue(), JExpr.lit(0)); - return outputContainer; }else{ - if (inputContainer.isOptional()) { -// block.add(vv.invoke("getMutator").invoke(setMethod).arg(outIndex)); - JConditional jc = block._if(inputContainer.getIsSet().eq(JExpr.lit(0)).not()); - block = jc._then(); + String setMethod = e.isSafe() ? "setSafe" : "set"; + + JInvocation setMeth; + if (Types.usesHolderForGet(inputContainer.getMajorType())) { + setMeth = vv.invoke("getMutator").invoke(setMethod).arg(outIndex).arg(inputContainer.getHolder()); + }else{ + setMeth = vv.invoke("getMutator").invoke(setMethod).arg(outIndex).arg(inputContainer.getValue()); + } + + if(e.isSafe()){ + HoldingContainer outputContainer = generator.declare(Types.REQUIRED_BIT); + block.assign(outputContainer.getValue(), JExpr.lit(1)); + if(inputContainer.isOptional()){ +// block._if(vv.invoke("getMutator").invoke(setMethod).arg(outIndex).not())._then().assign(outputContainer.getValue(), JExpr.lit(0)); + JConditional jc = block._if(inputContainer.getIsSet().eq(JExpr.lit(0)).not()); + block = jc._then(); + } + block._if(setMeth.not())._then().assign(outputContainer.getValue(), JExpr.lit(0)); + return outputContainer; + }else{ + if (inputContainer.isOptional()) { +// block.add(vv.invoke("getMutator").invoke(setMethod).arg(outIndex)); + JConditional jc = block._if(inputContainer.getIsSet().eq(JExpr.lit(0)).not()); + block = jc._then(); + } + block.add(setMeth); } - block.add(setMeth); + } - + + return null; } private HoldingContainer visitValueVectorReadExpression(ValueVectorReadExpression e, ClassGenerator generator) throws RuntimeException { // declare value vector - - JVar vv1 = generator.declareVectorValueSetupAndMember(generator.getMappingSet().getIncoming(), e.getFieldId()); + + JExpression vv1 = generator.declareVectorValueSetupAndMember(generator.getMappingSet().getIncoming(), e.getFieldId()); JExpression indexVariable = generator.getMappingSet().getValueReadIndex(); - JInvocation getValueAccessor = vv1.invoke("getAccessor").invoke("get"); - JInvocation getValueAccessor2 = vv1.invoke("getAccessor"); + JExpression componentVariable = indexVariable.shrz(JExpr.lit(16)); if (e.isSuperReader()) { - - getValueAccessor = ((JExpression) vv1.component(indexVariable.shrz(JExpr.lit(16)))).invoke("getAccessor").invoke("get"); - getValueAccessor2 = ((JExpression) vv1.component(indexVariable.shrz(JExpr.lit(16)))).invoke("getAccessor"); + vv1 = ((JExpression) vv1.component(componentVariable)); indexVariable = indexVariable.band(JExpr.lit((int) Character.MAX_VALUE)); } // evaluation work. HoldingContainer out = generator.declare(e.getMajorType()); + final boolean primitive = !Types.usesHolderForGet(e.getMajorType()); + final boolean hasReadPath = e.hasReadPath(); + final boolean complex = Types.isComplex(e.getMajorType()); - if (out.isOptional()) { - JBlock blk = generator.getEvalBlock(); - blk.assign(out.getIsSet(), getValueAccessor2.invoke("isSet").arg(indexVariable)); - JConditional jc = blk._if(out.getIsSet().eq(JExpr.lit(1))); - if (Types.usesHolderForGet(e.getMajorType())) { - jc._then().add(getValueAccessor.arg(indexVariable).arg(out.getHolder())); - } else { - jc._then().assign(out.getValue(), getValueAccessor.arg(indexVariable)); + int[] fieldIds = e.getFieldId().getFieldIds(); + for(int i = 1; i < fieldIds.length; i++){ + + } + + if(!hasReadPath && !complex){ + + JInvocation getValueAccessor = vv1.invoke("getAccessor").invoke("get"); + JInvocation getValueAccessor2 = vv1.invoke("getAccessor"); + JBlock eval = new JBlock(); + + if(primitive){ + eval.assign(out.getValue(), getValueAccessor.arg(indexVariable)); + }else{ + eval.add(getValueAccessor.arg(indexVariable).arg(out.getHolder())); } - } else { - if (Types.usesHolderForGet(e.getMajorType())) { - if (e.isArrayElement()) { - generator.getEvalBlock().add(getValueAccessor.arg(indexVariable).arg(JExpr.lit(e.getIndex())).arg(out.getHolder())); - } else { - generator.getEvalBlock().add(getValueAccessor.arg(indexVariable).arg(out.getHolder())); + + if (out.isOptional()) { + JBlock blk = generator.getEvalBlock(); + blk.assign(out.getIsSet(), getValueAccessor2.invoke("isSet").arg(indexVariable)); + JConditional jc = blk._if(out.getIsSet().eq(JExpr.lit(1))); + jc._then().add(eval); + }else{ + generator.getEvalBlock().add(eval); + } + + }else{ + JExpression vector = e.isSuperReader() ? vv1.component(componentVariable) : vv1; + JExpression expr = vector.invoke("getAccessor").invoke("getReader"); + + JLabel label = generator.getEvalBlock().label("complex"); + JBlock eval = generator.getEvalBlock().block(); + + // position to the correct value. + eval.add(expr.invoke("setPosition").arg(indexVariable)); + PathSegment seg = e.getReadPath(); + int listNum = 0; + boolean lastWasArray = false; + while(true){ + if(seg.isArray()){ + lastWasArray = true; + + if(seg.isLastPath() && !complex) break; + + JVar list = generator.declareClassField("list", generator.getModel()._ref(FieldReader.class)); + generator.getSetupBlock().assign(list, expr); + expr = list; + + // if this is an array, set a single position for the expression to allow us to read the right data lower down. + JVar desiredIndex = eval.decl(generator.getModel().INT, "desiredIndex" + listNum, JExpr.lit(seg.getArraySegment().getIndex())); + // start with negative one so that we are at zero after first call to next. + JVar currentIndex = eval.decl(generator.getModel().INT, "currentIndex" + listNum, JExpr.lit(-1)); + + eval._while( // + currentIndex.lt(desiredIndex) // + .cand(expr.invoke("next")) ).body().assign(currentIndex, currentIndex.plus(JExpr.lit(1))); + + JBlock ifNoVal = eval._if(desiredIndex.ne(currentIndex))._then().block(); + if(!complex) ifNoVal.assign(out.getIsSet(), JExpr.lit(0)); + ifNoVal._break(label); + + listNum++; + + }else{ + lastWasArray = false; + JExpression fieldName = JExpr.lit(seg.getNameSegment().getPath()); + expr = expr.invoke("reader").arg(fieldName); + } + seg = seg.getChild(); + + // stop once we get to last column or when the segment is an array at the end of the reference. + if(seg == null || seg.isLastPath() && seg.isArray()) break; + } + MajorType secondaryType = e.getFieldId().getSecondaryFinal(); + JType readerImpl = generator.getModel()._ref(TypeHelper.getReaderClassName(secondaryType.getMinorType(), secondaryType.getMode())); + JVar complexReader = generator.declareClassField("reader", readerImpl); + generator.getSetupBlock().assign(complexReader, JExpr.cast(readerImpl, expr)); + expr = complexReader; + + if(complex){ + HoldingContainer hc = new HoldingContainer(e.getMajorType(), (JVar) expr, null, null, lastWasArray); + return hc; + //eval.assign(out.getHolder().ref("reader"), expr); + }else{ + if(seg != null){ + eval.add(expr.invoke("read").arg(JExpr.lit(seg.getArraySegment().getIndex())).arg(out.getHolder())); + }else{ + + eval.add(expr.invoke("read").arg(out.getHolder())); } - } else { - generator.getEvalBlock().assign(out.getValue(), getValueAccessor.arg(indexVariable)); } + } + + + + return out; } @@ -352,11 +451,11 @@ public class EvaluationVisitor { // Preconditions.checkArgument(child.getMajorType().equals(Types.REQUIRED_BOOLEAN)); HoldingContainer hc = child.accept(this, generator); if(e.isReturnTrueOnOne()){ - generator.getEvalBlock()._return(hc.getValue().eq(JExpr.lit(1))); + generator.getEvalBlock()._return(hc.getValue().eq(JExpr.lit(1))); }else{ generator.getEvalBlock()._return(hc.getValue()); } - + return null; } @@ -453,6 +552,7 @@ public class EvaluationVisitor { } } + private class ConstantFilter extends EvalVisitor { private Set constantBoundaries; -- cgit v1.2.3