diff options
author | Volodymyr Vysotskyi <vvovyk@gmail.com> | 2019-01-09 23:52:34 +0200 |
---|---|---|
committer | Sorabh Hamirwasia <sorabh@apache.org> | 2019-01-11 16:47:39 -0800 |
commit | 06a210c2d681c3972ca3885840da0559341c1c02 (patch) | |
tree | 6ff57991f5eec0fc6231083c7b4b5c09373b7a9d /exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical | |
parent | 7be4c53b31c543d2d43651664d4d85b265fbfec7 (diff) |
DRILL-6959: Fix loss of precision when casting time and timestamp literals in filter condition
closes #1607
Diffstat (limited to 'exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical')
-rw-r--r-- | exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java | 353 |
1 files changed, 176 insertions, 177 deletions
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java index 8991a7d50..dd5b62e89 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java @@ -18,7 +18,6 @@ package org.apache.drill.exec.planner.logical; import org.apache.calcite.rel.type.RelDataType; -import org.apache.drill.shaded.guava.com.google.common.base.Function; import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableList; import io.netty.buffer.DrillBuf; import org.apache.calcite.rex.RexExecutor; @@ -84,6 +83,7 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.util.Calendar; import java.util.List; +import java.util.function.Function; public class DrillConstExecutor implements RexExecutor { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillConstExecutor.class); @@ -115,8 +115,8 @@ public class DrillConstExecutor implements RexExecutor { TypeProtos.MinorType.UINT8) .build(); - FunctionImplementationRegistry funcImplReg; - UdfUtilities udfUtilities; + private final FunctionImplementationRegistry funcImplReg; + private final UdfUtilities udfUtilities; public DrillConstExecutor(FunctionImplementationRegistry funcImplReg, UdfUtilities udfUtilities, PlannerSettings plannerSettings) { this.funcImplReg = funcImplReg; @@ -125,12 +125,13 @@ public class DrillConstExecutor implements RexExecutor { } @Override - public void reduce(final RexBuilder rexBuilder, List<RexNode> constExps, final List<RexNode> reducedValues) { - for (final RexNode newCall : constExps) { + @SuppressWarnings("deprecation") + public void reduce(RexBuilder rexBuilder, List<RexNode> constExps, List<RexNode> reducedValues) { + for (RexNode newCall : constExps) { LogicalExpression logEx = DrillOptiq.toDrill(new DrillParseContext(plannerSettings), (RelNode) null /* input rel */, newCall); ErrorCollectorImpl errors = new ErrorCollectorImpl(); - final LogicalExpression materializedExpr = ExpressionTreeMaterializer.materialize(logEx, null, errors, funcImplReg); + LogicalExpression materializedExpr = ExpressionTreeMaterializer.materialize(logEx, null, errors, funcImplReg); if (errors.getErrorCount() != 0) { String message = String.format( "Failure while materializing expression in constant expression evaluator [%s]. Errors: %s", @@ -149,7 +150,7 @@ public class DrillConstExecutor implements RexExecutor { } ValueHolder output = InterpreterEvaluator.evaluateConstantExpr(udfUtilities, materializedExpr); - final RelDataTypeFactory typeFactory = rexBuilder.getTypeFactory(); + RelDataTypeFactory typeFactory = rexBuilder.getTypeFactory(); if (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL && TypeHelper.isNull(output)) { SqlTypeName sqlTypeName = TypeInferenceUtils.getCalciteTypeFromDrillType(materializedExpr.getMajorType().getMinorType()); @@ -166,199 +167,199 @@ public class DrillConstExecutor implements RexExecutor { continue; } - Function<ValueHolder, RexNode> literator = new Function<ValueHolder, RexNode>() { - @Override - public RexNode apply(ValueHolder output) { - switch(materializedExpr.getMajorType().getMinorType()) { - case INT: { - int value = (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) ? - ((NullableIntHolder) output).value : ((IntHolder) output).value; - return rexBuilder.makeLiteral(new BigDecimal(value), + Function<ValueHolder, RexNode> literator = valueHolder -> { + switch (materializedExpr.getMajorType().getMinorType()) { + case INT: { + int value = (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) ? + ((NullableIntHolder) valueHolder).value : ((IntHolder) valueHolder).value; + return rexBuilder.makeLiteral(new BigDecimal(value), TypeInferenceUtils.createCalciteTypeWithNullability(typeFactory, SqlTypeName.INTEGER, newCall.getType().isNullable()), false); - } - case BIGINT: { - long value = (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) ? - ((NullableBigIntHolder) output).value : ((BigIntHolder) output).value; - return rexBuilder.makeLiteral(new BigDecimal(value), + } + case BIGINT: { + long value = (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) ? + ((NullableBigIntHolder) valueHolder).value : ((BigIntHolder) valueHolder).value; + return rexBuilder.makeLiteral(new BigDecimal(value), TypeInferenceUtils.createCalciteTypeWithNullability(typeFactory, SqlTypeName.BIGINT, newCall.getType().isNullable()), false); - } - case FLOAT4: { - float value = (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) ? - ((NullableFloat4Holder) output).value : ((Float4Holder) output).value; - return rexBuilder.makeLiteral(new BigDecimal(value), + } + case FLOAT4: { + float value = (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) ? + ((NullableFloat4Holder) valueHolder).value : ((Float4Holder) valueHolder).value; + return rexBuilder.makeLiteral(new BigDecimal(value), TypeInferenceUtils.createCalciteTypeWithNullability(typeFactory, SqlTypeName.FLOAT, newCall.getType().isNullable()), false); - } - case FLOAT8: { - double value = (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) ? - ((NullableFloat8Holder) output).value : ((Float8Holder) output).value; - return rexBuilder.makeLiteral(new BigDecimal(value), + } + case FLOAT8: { + double value = (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) ? + ((NullableFloat8Holder) valueHolder).value : ((Float8Holder) valueHolder).value; + return rexBuilder.makeLiteral(new BigDecimal(value), TypeInferenceUtils.createCalciteTypeWithNullability(typeFactory, SqlTypeName.DOUBLE, newCall.getType().isNullable()), false); - } - case VARCHAR: { - String value = (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) ? - StringFunctionHelpers.getStringFromVarCharHolder((NullableVarCharHolder)output) : - StringFunctionHelpers.getStringFromVarCharHolder((VarCharHolder)output); - return rexBuilder.makeLiteral(value, - TypeInferenceUtils.createCalciteTypeWithNullability(typeFactory, SqlTypeName.VARCHAR, newCall.getType().isNullable()), false); - } - case BIT: { - boolean value = (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) ? - ((NullableBitHolder) output).value == 1 : ((BitHolder) output).value == 1; - return rexBuilder.makeLiteral(value, + } + case VARCHAR: { + String value = (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) ? + StringFunctionHelpers.getStringFromVarCharHolder((NullableVarCharHolder) valueHolder) : + StringFunctionHelpers.getStringFromVarCharHolder((VarCharHolder) valueHolder); + RelDataType type = typeFactory.createSqlType(SqlTypeName.VARCHAR, newCall.getType().getPrecision()); + RelDataType typeWithNullability = typeFactory.createTypeWithNullability(type, newCall.getType().isNullable()); + return rexBuilder.makeLiteral(value, typeWithNullability, false); + } + case BIT: { + boolean value = (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) ? + ((NullableBitHolder) valueHolder).value == 1 : ((BitHolder) valueHolder).value == 1; + return rexBuilder.makeLiteral(value, TypeInferenceUtils.createCalciteTypeWithNullability(typeFactory, SqlTypeName.BOOLEAN, newCall.getType().isNullable()), false); - } - case DATE: { - Calendar value = (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) ? - new DateTime(((NullableDateHolder) output).value, DateTimeZone.UTC).toCalendar(null) : - new DateTime(((DateHolder) output).value, DateTimeZone.UTC).toCalendar(null); - return rexBuilder.makeLiteral(DateString.fromCalendarFields(value), + } + case DATE: { + Calendar value = (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) ? + new DateTime(((NullableDateHolder) valueHolder).value, DateTimeZone.UTC).toCalendar(null) : + new DateTime(((DateHolder) valueHolder).value, DateTimeZone.UTC).toCalendar(null); + return rexBuilder.makeLiteral(DateString.fromCalendarFields(value), TypeInferenceUtils.createCalciteTypeWithNullability(typeFactory, SqlTypeName.DATE, newCall.getType().isNullable()), false); + } + case DECIMAL9: { + long value; + int scale; + if (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) { + NullableDecimal9Holder decimal9Out = (NullableDecimal9Holder) valueHolder; + value = decimal9Out.value; + scale = decimal9Out.scale; + } else { + Decimal9Holder decimal9Out = (Decimal9Holder) valueHolder; + value = decimal9Out.value; + scale = decimal9Out.scale; } - case DECIMAL9: { - long value; - int scale; - if (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) { - NullableDecimal9Holder decimal9Out = (NullableDecimal9Holder)output; - value = decimal9Out.value; - scale = decimal9Out.scale; - } else { - Decimal9Holder decimal9Out = (Decimal9Holder)output; - value = decimal9Out.value; - scale = decimal9Out.scale; - } - return rexBuilder.makeLiteral( + return rexBuilder.makeLiteral( new BigDecimal(BigInteger.valueOf(value), scale), TypeInferenceUtils.createCalciteTypeWithNullability(typeFactory, SqlTypeName.DECIMAL, newCall.getType().isNullable()), false); + } + case DECIMAL18: { + long value; + int scale; + if (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) { + NullableDecimal18Holder decimal18Out = (NullableDecimal18Holder) valueHolder; + value = decimal18Out.value; + scale = decimal18Out.scale; + } else { + Decimal18Holder decimal18Out = (Decimal18Holder) valueHolder; + value = decimal18Out.value; + scale = decimal18Out.scale; } - case DECIMAL18: { - long value; - int scale; - if (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) { - NullableDecimal18Holder decimal18Out = (NullableDecimal18Holder)output; - value = decimal18Out.value; - scale = decimal18Out.scale; - } else { - Decimal18Holder decimal18Out = (Decimal18Holder)output; - value = decimal18Out.value; - scale = decimal18Out.scale; - } - return rexBuilder.makeLiteral( + return rexBuilder.makeLiteral( new BigDecimal(BigInteger.valueOf(value), scale), TypeInferenceUtils.createCalciteTypeWithNullability(typeFactory, SqlTypeName.DECIMAL, newCall.getType().isNullable()), false); + } + case VARDECIMAL: { + DrillBuf buffer; + int start; + int end; + int scale; + int precision; + if (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) { + NullableVarDecimalHolder varDecimalHolder = (NullableVarDecimalHolder) valueHolder; + buffer = varDecimalHolder.buffer; + start = varDecimalHolder.start; + end = varDecimalHolder.end; + scale = varDecimalHolder.scale; + precision = varDecimalHolder.precision; + } else { + VarDecimalHolder varDecimalHolder = (VarDecimalHolder) valueHolder; + buffer = varDecimalHolder.buffer; + start = varDecimalHolder.start; + end = varDecimalHolder.end; + scale = varDecimalHolder.scale; + precision = varDecimalHolder.precision; } - case VARDECIMAL: { - DrillBuf buffer; - int start; - int end; - int scale; - int precision; - if (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) { - NullableVarDecimalHolder varDecimalHolder = (NullableVarDecimalHolder) output; - buffer = varDecimalHolder.buffer; - start = varDecimalHolder.start; - end = varDecimalHolder.end; - scale = varDecimalHolder.scale; - precision = varDecimalHolder.precision; - } else { - VarDecimalHolder varDecimalHolder = (VarDecimalHolder) output; - buffer = varDecimalHolder.buffer; - start = varDecimalHolder.start; - end = varDecimalHolder.end; - scale = varDecimalHolder.scale; - precision = varDecimalHolder.precision; - } - return rexBuilder.makeLiteral( - org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromDrillBuf(buffer, start, end - start, scale), - typeFactory.createSqlType(SqlTypeName.DECIMAL, precision, scale), - false); + return rexBuilder.makeLiteral( + org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromDrillBuf(buffer, start, end - start, scale), + typeFactory.createSqlType(SqlTypeName.DECIMAL, precision, scale), + false); + } + case DECIMAL28SPARSE: { + DrillBuf buffer; + int start; + int scale; + if (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) { + NullableDecimal28SparseHolder decimal28Out = (NullableDecimal28SparseHolder) valueHolder; + buffer = decimal28Out.buffer; + start = decimal28Out.start; + scale = decimal28Out.scale; + } else { + Decimal28SparseHolder decimal28Out = (Decimal28SparseHolder) valueHolder; + buffer = decimal28Out.buffer; + start = decimal28Out.start; + scale = decimal28Out.scale; } - case DECIMAL28SPARSE: { - DrillBuf buffer; - int start; - int scale; - if (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) { - NullableDecimal28SparseHolder decimal28Out = (NullableDecimal28SparseHolder)output; - buffer = decimal28Out.buffer; - start = decimal28Out.start; - scale = decimal28Out.scale; - } else { - Decimal28SparseHolder decimal28Out = (Decimal28SparseHolder)output; - buffer = decimal28Out.buffer; - start = decimal28Out.start; - scale = decimal28Out.scale; - } - return rexBuilder.makeLiteral( + return rexBuilder.makeLiteral( org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromSparse(buffer, start * 20, 5, scale), TypeInferenceUtils.createCalciteTypeWithNullability(typeFactory, SqlTypeName.DECIMAL, newCall.getType().isNullable()), false); + } + case DECIMAL38SPARSE: { + DrillBuf buffer; + int start; + int scale; + if (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) { + NullableDecimal38SparseHolder decimal38Out = (NullableDecimal38SparseHolder) valueHolder; + buffer = decimal38Out.buffer; + start = decimal38Out.start; + scale = decimal38Out.scale; + } else { + Decimal38SparseHolder decimal38Out = (Decimal38SparseHolder) valueHolder; + buffer = decimal38Out.buffer; + start = decimal38Out.start; + scale = decimal38Out.scale; } - case DECIMAL38SPARSE: { - DrillBuf buffer; - int start; - int scale; - if (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) { - NullableDecimal38SparseHolder decimal38Out = (NullableDecimal38SparseHolder)output; - buffer = decimal38Out.buffer; - start = decimal38Out.start; - scale = decimal38Out.scale; - } else { - Decimal38SparseHolder decimal38Out = (Decimal38SparseHolder)output; - buffer = decimal38Out.buffer; - start = decimal38Out.start; - scale = decimal38Out.scale; - } - return rexBuilder.makeLiteral(org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromSparse(buffer, start * 24, 6, scale), + return rexBuilder.makeLiteral(org.apache.drill.exec.util.DecimalUtility.getBigDecimalFromSparse(buffer, start * 24, 6, scale), TypeInferenceUtils.createCalciteTypeWithNullability(typeFactory, SqlTypeName.DECIMAL, newCall.getType().isNullable()), false); - } - case TIME: { - Calendar value = (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) ? - new DateTime(((NullableTimeHolder) output).value, DateTimeZone.UTC).toCalendar(null) : - new DateTime(((TimeHolder) output).value, DateTimeZone.UTC).toCalendar(null); - return rexBuilder.makeLiteral(TimeString.fromCalendarFields(value), - TypeInferenceUtils.createCalciteTypeWithNullability(typeFactory, SqlTypeName.TIME, newCall.getType().isNullable()), false); - } - case TIMESTAMP: { - Calendar value = (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) ? - new DateTime(((NullableTimeStampHolder) output).value, DateTimeZone.UTC).toCalendar(null) : - new DateTime(((TimeStampHolder) output).value, DateTimeZone.UTC).toCalendar(null); - return rexBuilder.makeLiteral(TimestampString.fromCalendarFields(value), - TypeInferenceUtils.createCalciteTypeWithNullability(typeFactory, SqlTypeName.TIMESTAMP, newCall.getType().isNullable()), false); - } - case INTERVALYEAR: { - BigDecimal value = (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) ? - new BigDecimal(((NullableIntervalYearHolder) output).value) : - new BigDecimal(((IntervalYearHolder) output).value); - return rexBuilder.makeLiteral(value, + } + case TIME: { + Calendar value = (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) ? + new DateTime(((NullableTimeHolder) valueHolder).value, DateTimeZone.UTC).toCalendar(null) : + new DateTime(((TimeHolder) valueHolder).value, DateTimeZone.UTC).toCalendar(null); + RelDataType type = typeFactory.createSqlType(SqlTypeName.TIME, newCall.getType().getPrecision()); + RelDataType typeWithNullability = typeFactory.createTypeWithNullability(type, newCall.getType().isNullable()); + return rexBuilder.makeLiteral(TimeString.fromCalendarFields(value), typeWithNullability, false); + } + case TIMESTAMP: { + Calendar value = (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) ? + new DateTime(((NullableTimeStampHolder) valueHolder).value, DateTimeZone.UTC).toCalendar(null) : + new DateTime(((TimeStampHolder) valueHolder).value, DateTimeZone.UTC).toCalendar(null); + RelDataType type = typeFactory.createSqlType(SqlTypeName.TIMESTAMP, newCall.getType().getPrecision()); + RelDataType typeWithNullability = typeFactory.createTypeWithNullability(type, newCall.getType().isNullable()); + return rexBuilder.makeLiteral(TimestampString.fromCalendarFields(value), typeWithNullability, false); + } + case INTERVALYEAR: { + BigDecimal value = (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) ? + new BigDecimal(((NullableIntervalYearHolder) valueHolder).value) : + new BigDecimal(((IntervalYearHolder) valueHolder).value); + return rexBuilder.makeLiteral(value, TypeInferenceUtils.createCalciteTypeWithNullability(typeFactory, SqlTypeName.INTERVAL_YEAR_MONTH, newCall.getType().isNullable()), false); + } + case INTERVALDAY: { + int days; + int milliseconds; + if (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) { + NullableIntervalDayHolder intervalDayOut = (NullableIntervalDayHolder) valueHolder; + days = intervalDayOut.days; + milliseconds = intervalDayOut.milliseconds; + } else { + IntervalDayHolder intervalDayOut = (IntervalDayHolder) valueHolder; + days = intervalDayOut.days; + milliseconds = intervalDayOut.milliseconds; } - case INTERVALDAY: { - int days; - int milliseconds; - if (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) { - NullableIntervalDayHolder intervalDayOut = (NullableIntervalDayHolder) output; - days = intervalDayOut.days; - milliseconds = intervalDayOut.milliseconds; - } else { - IntervalDayHolder intervalDayOut = (IntervalDayHolder) output; - days = intervalDayOut.days; - milliseconds = intervalDayOut.milliseconds; - } - return rexBuilder.makeLiteral( - new BigDecimal(days * (long) DateUtilities.daysToStandardMillis + milliseconds), - TypeInferenceUtils.createCalciteTypeWithNullability(typeFactory, SqlTypeName.INTERVAL_DAY, - newCall.getType().isNullable()), false); - } - // The list of known unsupported types is used to trigger this behavior of re-using the input expression - // before the expression is even attempted to be evaluated, this is just here as a last precaution a - // as new types may be added in the future. - default: - logger.debug("Constant expression not folded due to return type {}, complete expression: {}", + return rexBuilder.makeLiteral( + new BigDecimal(days * (long) DateUtilities.daysToStandardMillis + milliseconds), + TypeInferenceUtils.createCalciteTypeWithNullability(typeFactory, SqlTypeName.INTERVAL_DAY, + newCall.getType().isNullable()), false); + } + // The list of known unsupported types is used to trigger this behavior of re-using the input expression + // before the expression is even attempted to be evaluated, this is just here as a last precaution a + // as new types may be added in the future. + default: + logger.debug("Constant expression not folded due to return type {}, complete expression: {}", materializedExpr.getMajorType(), ExpressionStringBuilder.toString(materializedExpr)); - return newCall; - } + return newCall; } }; @@ -366,5 +367,3 @@ public class DrillConstExecutor implements RexExecutor { } } } - - |