aboutsummaryrefslogtreecommitdiff
path: root/exec/java-exec/src/main/java
diff options
context:
space:
mode:
authorVitalii Diravka <vitalii.diravka@gmail.com>2017-12-01 22:48:05 +0200
committerBen-Zvi <bben-zvi@mapr.com>2018-02-01 18:05:34 -0800
commitf30200812eb27c76b8d4d246008cbd9bd59fb0a5 (patch)
treebb3809b6b4cb7ae560d4a0eb2a7b3ea2c73ae386 /exec/java-exec/src/main/java
parent07dae3c34001f15b28f0332ddc1be23adb539b41 (diff)
DRILL-4185: UNION ALL involving empty directory on any side of union all results in Failed query
closes #1083
Diffstat (limited to 'exec/java-exec/src/main/java')
-rw-r--r--exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SchemalessBatchCreator.java40
-rw-r--r--exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SchemalessScan.java94
-rw-r--r--exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java86
-rw-r--r--exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java8
-rw-r--r--exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrule.java3
-rw-r--r--exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/RefreshMetadataHandler.java14
-rw-r--r--exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemalessBatch.java108
-rw-r--r--exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java21
-rw-r--r--exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java5
-rw-r--r--exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java12
-rw-r--r--exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java23
11 files changed, 352 insertions, 62 deletions
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SchemalessBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SchemalessBatchCreator.java
new file mode 100644
index 000000000..0fb8f45a2
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SchemalessBatchCreator.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.base;
+
+import java.util.List;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.exec.ops.ExecutorFragmentContext;
+import org.apache.drill.exec.physical.impl.BatchCreator;
+import org.apache.drill.exec.record.CloseableRecordBatch;
+import org.apache.drill.exec.record.SchemalessBatch;
+import org.apache.drill.exec.record.RecordBatch;
+
+/**
+ * The operator for creating {@link SchemalessBatch} instances
+ */
+public class SchemalessBatchCreator implements BatchCreator<SchemalessScan> {
+
+ @Override
+ public CloseableRecordBatch getBatch(ExecutorFragmentContext context, SchemalessScan subScan, List<RecordBatch> children)
+ throws ExecutionSetupException {
+ return new SchemalessBatch();
+ }
+
+}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SchemalessScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SchemalessScan.java
new file mode 100644
index 000000000..e0db1aeb6
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SchemalessScan.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.base;
+
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.base.Preconditions;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.exec.physical.PhysicalOperatorSetupException;
+import org.apache.drill.exec.planner.logical.DynamicDrillTable;
+import org.apache.drill.exec.proto.CoordinationProtos;
+
+import java.util.List;
+
+/**
+ * The type of scan operator, which allows to scan schemaless tables ({@link DynamicDrillTable} with null selection)
+ */
+@JsonTypeName("schemaless-scan")
+public class SchemalessScan extends AbstractFileGroupScan implements SubScan {
+
+ private final String selectionRoot;
+
+ public SchemalessScan(String userName, String selectionRoot) {
+ super(userName);
+ this.selectionRoot = selectionRoot;
+ }
+
+ public SchemalessScan(final SchemalessScan that) {
+ super(that);
+ this.selectionRoot = that.selectionRoot;
+ }
+
+ @Override
+ public void applyAssignments(List<CoordinationProtos.DrillbitEndpoint> endpoints) throws PhysicalOperatorSetupException {
+ }
+
+ @Override
+ public SubScan getSpecificScan(int minorFragmentId) throws ExecutionSetupException {
+ return this;
+ }
+
+ @Override
+ public int getMaxParallelizationWidth() {
+ return 1;
+ }
+
+ @Override
+ public String getDigest() {
+ return toString();
+ }
+
+ @Override
+ public String toString() {
+ final String pattern = "SchemalessScan [selectionRoot = %s]";
+ return String.format(pattern, selectionRoot);
+ }
+
+ @Override
+ public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) throws ExecutionSetupException {
+ Preconditions.checkArgument(children.isEmpty());
+ assert children == null || children.isEmpty();
+ return new SchemalessScan(this);
+ }
+
+ @Override
+ public GroupScan clone(List<SchemaPath> columns) {
+ return this;
+ }
+
+ @Override
+ public ScanStats getScanStats() {
+ return ScanStats.ZERO_RECORD_TABLE;
+ }
+
+ @Override
+ public boolean supportsPartitionFilterPushdown() {
+ return false;
+ }
+
+}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java
index fa8c13a92..9fc734dbb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java
@@ -146,7 +146,7 @@ public class NestedLoopJoinBatch extends AbstractBinaryRecordBatch<NestedLoopJoi
return IterOutcome.NONE;
}
- boolean drainRight = true;
+ boolean drainRight = rightUpstream != IterOutcome.NONE ? true : false;
while (drainRight) {
rightUpstream = next(RIGHT_INPUT, right);
switch (rightUpstream) {
@@ -267,22 +267,24 @@ public class NestedLoopJoinBatch extends AbstractBinaryRecordBatch<NestedLoopJoi
int fieldId = 0;
int outputFieldId = 0;
- // Set the input and output value vector references corresponding to the left batch
- for (MaterializedField field : leftSchema) {
- final TypeProtos.MajorType fieldType = field.getType();
-
- // Add the vector to the output container
- container.addOrGet(field);
-
- JVar inVV = nLJClassGenerator.declareVectorValueSetupAndMember("leftBatch",
- new TypedFieldId(fieldType, false, fieldId));
- JVar outVV = nLJClassGenerator.declareVectorValueSetupAndMember("outgoing",
- new TypedFieldId(fieldType, false, outputFieldId));
-
- nLJClassGenerator.getEvalBlock().add(outVV.invoke("copyFromSafe").arg(leftIndex).arg(outIndex).arg(inVV));
- nLJClassGenerator.rotateBlock();
- fieldId++;
- outputFieldId++;
+ if (leftSchema != null) {
+ // Set the input and output value vector references corresponding to the left batch
+ for (MaterializedField field : leftSchema) {
+ final TypeProtos.MajorType fieldType = field.getType();
+
+ // Add the vector to the output container
+ container.addOrGet(field);
+
+ JVar inVV = nLJClassGenerator.declareVectorValueSetupAndMember("leftBatch",
+ new TypedFieldId(fieldType, false, fieldId));
+ JVar outVV = nLJClassGenerator.declareVectorValueSetupAndMember("outgoing",
+ new TypedFieldId(fieldType, false, outputFieldId));
+
+ nLJClassGenerator.getEvalBlock().add(outVV.invoke("copyFromSafe").arg(leftIndex).arg(outIndex).arg(inVV));
+ nLJClassGenerator.rotateBlock();
+ fieldId++;
+ outputFieldId++;
+ }
}
// generate emitRight
@@ -291,32 +293,34 @@ public class NestedLoopJoinBatch extends AbstractBinaryRecordBatch<NestedLoopJoi
JExpression batchIndex = JExpr.direct("batchIndex");
JExpression recordIndexWithinBatch = JExpr.direct("recordIndexWithinBatch");
- // Set the input and output value vector references corresponding to the right batch
- for (MaterializedField field : rightSchema) {
+ if (rightSchema != null) {
+ // Set the input and output value vector references corresponding to the right batch
+ for (MaterializedField field : rightSchema) {
+
+ final TypeProtos.MajorType inputType = field.getType();
+ TypeProtos.MajorType outputType;
+ // if join type is LEFT, make sure right batch output fields data mode is optional
+ if (popConfig.getJoinType() == JoinRelType.LEFT && inputType.getMode() == TypeProtos.DataMode.REQUIRED) {
+ outputType = Types.overrideMode(inputType, TypeProtos.DataMode.OPTIONAL);
+ } else {
+ outputType = inputType;
+ }
- final TypeProtos.MajorType inputType = field.getType();
- TypeProtos.MajorType outputType;
- // if join type is LEFT, make sure right batch output fields data mode is optional
- if (popConfig.getJoinType() == JoinRelType.LEFT && inputType.getMode() == TypeProtos.DataMode.REQUIRED) {
- outputType = Types.overrideMode(inputType, TypeProtos.DataMode.OPTIONAL);
- } else {
- outputType = inputType;
+ MaterializedField newField = MaterializedField.create(field.getName(), outputType);
+ container.addOrGet(newField);
+
+ JVar inVV = nLJClassGenerator.declareVectorValueSetupAndMember("rightContainer",
+ new TypedFieldId(inputType, true, fieldId));
+ JVar outVV = nLJClassGenerator.declareVectorValueSetupAndMember("outgoing",
+ new TypedFieldId(outputType, false, outputFieldId));
+ nLJClassGenerator.getEvalBlock().add(outVV.invoke("copyFromSafe")
+ .arg(recordIndexWithinBatch)
+ .arg(outIndex)
+ .arg(inVV.component(batchIndex)));
+ nLJClassGenerator.rotateBlock();
+ fieldId++;
+ outputFieldId++;
}
-
- MaterializedField newField = MaterializedField.create(field.getName(), outputType);
- container.addOrGet(newField);
-
- JVar inVV = nLJClassGenerator.declareVectorValueSetupAndMember("rightContainer",
- new TypedFieldId(inputType, true, fieldId));
- JVar outVV = nLJClassGenerator.declareVectorValueSetupAndMember("outgoing",
- new TypedFieldId(outputType, false, outputFieldId));
- nLJClassGenerator.getEvalBlock().add(outVV.invoke("copyFromSafe")
- .arg(recordIndexWithinBatch)
- .arg(outIndex)
- .arg(inVV.component(batchIndex)));
- nLJClassGenerator.rotateBlock();
- fieldId++;
- outputFieldId++;
}
return context.getImplementationClass(nLJCodeGenerator);
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java
index 9a0d36940..4ee767146 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java
@@ -30,8 +30,10 @@ import org.apache.calcite.sql.SqlCall;
import org.apache.calcite.sql.SqlNode;
import org.apache.drill.common.JSONOptions;
import org.apache.drill.common.logical.StoragePluginConfig;
+import org.apache.drill.exec.physical.base.SchemalessScan;
import org.apache.drill.exec.physical.base.GroupScan;
import org.apache.drill.exec.store.StoragePlugin;
+import org.apache.drill.exec.store.dfs.FileSelection;
import org.apache.drill.exec.util.ImpersonationUtil;
public abstract class DrillTable implements Table {
@@ -85,7 +87,11 @@ public abstract class DrillTable implements Table {
public GroupScan getGroupScan() throws IOException{
if (scan == null) {
- this.scan = plugin.getPhysicalScan(userName, new JSONOptions(selection));
+ if (selection instanceof FileSelection && ((FileSelection) selection).isEmptyDirectory()) {
+ this.scan = new SchemalessScan(userName, ((FileSelection) selection).getSelectionRoot());
+ } else {
+ this.scan = plugin.getPhysicalScan(userName, new JSONOptions(selection));
+ }
}
return scan;
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrule.java
index 336ab3a96..7ea020299 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrule.java
@@ -40,8 +40,7 @@ public class UnionAllPrule extends Prule {
protected static final Logger tracer = CalciteTrace.getPlannerTracer();
private UnionAllPrule() {
- super(
- RelOptHelper.any(DrillUnionRel.class), "Prel.UnionAllPrule");
+ super(RelOptHelper.any(DrillUnionRel.class), "Prel.UnionAllPrule");
}
@Override
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/RefreshMetadataHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/RefreshMetadataHandler.java
index b36356ab7..6bfceb42a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/RefreshMetadataHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/RefreshMetadataHandler.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -33,6 +33,7 @@ import org.apache.drill.exec.planner.sql.DirectPlan;
import org.apache.drill.exec.planner.sql.SchemaUtilites;
import org.apache.drill.exec.planner.sql.parser.SqlRefreshMetadata;
import org.apache.drill.exec.store.dfs.DrillFileSystem;
+import org.apache.drill.exec.store.dfs.FileSelection;
import org.apache.drill.exec.store.dfs.FileSystemPlugin;
import org.apache.drill.exec.store.dfs.FormatSelection;
import org.apache.drill.exec.store.dfs.NamedFormatPluginConfig;
@@ -78,11 +79,11 @@ public class RefreshMetadataHandler extends DefaultSqlHandler {
final Table table = schema.getTable(tableName);
- if(table == null){
+ if (table == null) {
return direct(false, "Table %s does not exist.", tableName);
}
- if(! (table instanceof DrillTable) ){
+ if (!(table instanceof DrillTable)) {
return notSupported(tableName);
}
@@ -90,7 +91,12 @@ public class RefreshMetadataHandler extends DefaultSqlHandler {
final DrillTable drillTable = (DrillTable) table;
final Object selection = drillTable.getSelection();
- if( !(selection instanceof FormatSelection) ){
+
+ if (selection instanceof FileSelection && ((FileSelection) selection).isEmptyDirectory()) {
+ return direct(false, "Table %s is empty and doesn't contain any parquet files.", tableName);
+ }
+
+ if (!(selection instanceof FormatSelection)) {
return notSupported(tableName);
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemalessBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemalessBatch.java
new file mode 100644
index 000000000..ff0cfafcd
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemalessBatch.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.exec.record;
+
+
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.record.selection.SelectionVector2;
+import org.apache.drill.exec.record.selection.SelectionVector4;
+
+import java.util.Iterator;
+
+/**
+ * Empty batch without schema and data.
+ */
+public class SchemalessBatch implements CloseableRecordBatch {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SchemalessBatch.class);
+
+ public SchemalessBatch() {
+ logger.debug("Empty schemaless batch is created");
+ }
+
+ @Override
+ public FragmentContext getContext() {
+ return null;
+ }
+
+ @Override
+ public BatchSchema getSchema() {
+ return null;
+ }
+
+ @Override
+ public int getRecordCount() {
+ return 0;
+ }
+
+ @Override
+ public SelectionVector2 getSelectionVector2() {
+ throw new UnsupportedOperationException(String.format("You should not call getSelectionVector2() for class %s",
+ this.getClass().getCanonicalName()));
+ }
+
+ @Override
+ public SelectionVector4 getSelectionVector4() {
+ throw new UnsupportedOperationException(String.format("You should not call getSelectionVector4() for class %s",
+ this.getClass().getCanonicalName()));
+ }
+
+ @Override
+ public void kill(boolean sendUpstream) {
+ }
+
+ @Override
+ public VectorContainer getOutgoingContainer() {
+ throw new UnsupportedOperationException(String.format("You should not call getOutgoingContainer() for class %s",
+ this.getClass().getCanonicalName()));
+ }
+
+ @Override
+ public TypedFieldId getValueVectorId(SchemaPath path) {
+ throw new UnsupportedOperationException(String.format("You should not call getValueVectorId() for class %s",
+ this.getClass().getCanonicalName()));
+ }
+
+ @Override
+ public VectorWrapper<?> getValueAccessorById(Class<?> clazz, int... ids) {
+ throw new UnsupportedOperationException(String.format("You should not call getValueAccessorById() for class %s",
+ this.getClass().getCanonicalName()));
+ }
+
+ @Override
+ public IterOutcome next() {
+ return IterOutcome.NONE;
+ }
+
+ @Override
+ public WritableBatch getWritableBatch() {
+ throw new UnsupportedOperationException(String.format("You should not call getWritableBatch() for class %s",
+ this.getClass().getCanonicalName()));
+ }
+
+ @Override
+ public Iterator<VectorWrapper<?>> iterator() {
+ return null;
+ }
+
+ @Override
+ public void close() throws Exception {
+ // This is present to match BatchCreator#getBatch() returning type.
+ }
+}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
index 7fa981b00..7edb327b1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
@@ -56,6 +56,11 @@ public class FileSelection {
*/
private MetadataContext metaContext = null;
+ /**
+ * Indicates whether this selectionRoot is an empty directory
+ */
+ private boolean emptyDirectory;
+
private enum StatusType {
NOT_CHECKED, // initial state
NO_DIRS, // no directories in this selection
@@ -424,6 +429,22 @@ public class FileSelection {
return metaContext;
}
+ /**
+ * @return true if this {@link FileSelection#selectionRoot} points to an empty directory, false otherwise
+ */
+ public boolean isEmptyDirectory() {
+ return emptyDirectory;
+ }
+
+ /**
+ * Setting {@link FileSelection#emptyDirectory} as true allows to identify this {@link FileSelection#selectionRoot}
+ * as an empty directory
+ */
+ public void setEmptyDirectoryStatus() {
+ this.emptyDirectory = true;
+ }
+
+
@Override
public String toString() {
final StringBuilder sb = new StringBuilder();
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java
index a3886bb74..7640c1366 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java
@@ -69,7 +69,6 @@ import org.apache.drill.exec.util.DrillFileSystemUtil;
import org.apache.drill.exec.util.ImpersonationUtil;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -632,7 +631,9 @@ public class WorkspaceSchemaFactory {
final FileSelection newSelection = hasDirectories ? fileSelection.minusDirectories(getFS()) : fileSelection;
if (newSelection == null) {
- return null;
+ // empty directory / selection means that this is the empty and schemaless table
+ fileSelection.setEmptyDirectoryStatus();
+ return new DynamicDrillTable(plugin, storageEngineName, schemaConfig.getUserName(), fileSelection);
}
for (final FormatMatcher matcher : fileMatchers) {
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
index e457e1839..e48239b96 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
@@ -29,8 +29,10 @@ import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.base.AbstractFileGroupScan;
import org.apache.drill.exec.physical.base.AbstractWriter;
import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.base.SchemalessScan;
import org.apache.drill.exec.physical.impl.WriterRecordBatch;
import org.apache.drill.exec.planner.logical.DrillTable;
import org.apache.drill.exec.planner.logical.DynamicDrillTable;
@@ -165,9 +167,15 @@ public class ParquetFormatPlugin implements FormatPlugin{
}
@Override
- public ParquetGroupScan getGroupScan(String userName, FileSelection selection, List<SchemaPath> columns)
+ public AbstractFileGroupScan getGroupScan(String userName, FileSelection selection, List<SchemaPath> columns)
throws IOException {
- return new ParquetGroupScan(userName, selection, this, columns);
+ ParquetGroupScan parquetGroupScan = new ParquetGroupScan(userName, selection, this, columns);
+ if (parquetGroupScan.getEntries().isEmpty()) {
+ // If ParquetGroupScan does not contain any entries, it means selection directories are empty and
+ // metadata cache files are invalid, return schemaless scan
+ return new SchemalessScan(userName, parquetGroupScan.getSelectionRoot());
+ }
+ return parquetGroupScan;
}
@Override
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
index 75b18df07..aa001f97c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
@@ -210,18 +210,20 @@ public class ParquetGroupScan extends AbstractFileGroupScan {
this.entries = Lists.newArrayList();
- if (checkForInitializingEntriesWithSelectionRoot()) {
- // The fully expanded list is already stored as part of the fileSet
- entries.add(new ReadEntryWithPath(fileSelection.getSelectionRoot()));
- } else {
- for (String fileName : fileSelection.getFiles()) {
- entries.add(new ReadEntryWithPath(fileName));
+ if (fileSelection != null) {
+ if (checkForInitializingEntriesWithSelectionRoot()) {
+ // The fully expanded list is already stored as part of the fileSet
+ entries.add(new ReadEntryWithPath(fileSelection.getSelectionRoot()));
+ } else {
+ for (String fileName : fileSelection.getFiles()) {
+ entries.add(new ReadEntryWithPath(fileName));
+ }
}
- }
- this.filter = filter;
+ this.filter = filter;
- init();
+ init();
+ }
}
/*
@@ -875,7 +877,8 @@ public class ParquetGroupScan extends AbstractFileGroupScan {
if (fileSet.isEmpty()) {
// no files were found, most likely we tried to query some empty sub folders
- throw UserException.validationError().message("The table you tried to query is empty").build(logger);
+ logger.warn("The table is empty but with outdated invalid metadata cache files. Please, delete them.");
+ return null;
}
List<String> fileNames = Lists.newArrayList(fileSet);