DRILL-6810: Disable NULL_IF_NULL NullHandling for functions with ComplexWriter

closes #1509

4 files changed, 173 insertions, 0 deletions

diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/impl/TestStringFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/impl/TestStringFunctions.java
index 36e7ead50..8158c3950 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/impl/TestStringFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/impl/TestStringFunctions.java
@@ -1393,6 +1393,32 @@ public class TestStringFunctions extends BaseTestQuery {
   }
 
   @Test
+  public void testSplitWithNullInput() throws Exception {
+    // Contents of the generated file:
+    /*
+      {"a": "aaaaaa.bbb.cc.ddddd"}
+      {"a": null}
+      {"a": "aa"}
+     */
+    try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(dirTestWatcher.getRootDir(), "nullable_strings.json")))) {
+      String[] fieldValue = {"\"aaaaaa.bbb.cc.ddddd\"", null, "\"aa\""};
+      for (String value : fieldValue) {
+        String entry = String.format("{ \"a\": %s}\n", value);
+        writer.write(entry);
+      }
+    }
+
+    testBuilder()
+        .sqlQuery("select split(a, '.') wordsCount from dfs.`nullable_strings.json` t")
+        .unOrdered()
+        .baselineColumns("wordsCount")
+        .baselineValues(ImmutableList.of(new Text("aaaaaa"), new Text("bbb"), new Text("cc"), new Text("ddddd")))
+        .baselineValues(ImmutableList.of())
+        .baselineValues(ImmutableList.of(new Text("aa")))
+        .go();
+  }
+
+  @Test
   public void testReverse() throws Exception {
     testBuilder()
       .sqlQuery("select reverse('qwerty') words from (values(1))")
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestParseFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestParseFunctions.java
new file mode 100644
index 000000000..2e4c4b7fc
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestParseFunctions.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.fn.impl;
+
+import org.apache.drill.categories.SqlFunctionTest;
+import org.apache.drill.test.ClusterFixture;
+import org.apache.drill.test.ClusterFixtureBuilder;
+import org.apache.drill.test.ClusterTest;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+
+import static org.apache.drill.test.TestBuilder.mapOf;
+
+@Category(SqlFunctionTest.class)
+public class TestParseFunctions extends ClusterTest {
+
+  @BeforeClass
+  public static void setup() throws Exception {
+    generateDataSource();
+    ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher);
+    startCluster(builder);
+  }
+
+  private static void generateDataSource() throws Exception {
+    // Contents of the generated file:
+    /*
+      {"url": "ftp://somewhere.com:3190/someFile?a=12&b=someValue"}
+      {"url": null}
+      {"url": "http://someUrl?p1=v1&p2=v=2&"}
+     */
+    try (BufferedWriter writer = new BufferedWriter(new FileWriter(
+        new File(dirTestWatcher.getRootDir(), "nullable_urls.json")))) {
+      String[] urls = {"\"ftp://somewhere.com:3190/someFile?a=12&b=someValue\"", null, "\"http://someUrl?p1=v1&p2=v=2&\""};
+      for (String url : urls) {
+        String entry = String.format("{ \"url\": %s}\n", url);
+        writer.write(entry);
+      }
+    }
+  }
+
+  @Test
+  public void testParseQueryFunction() throws Exception {
+    testBuilder()
+        .sqlQuery("select parse_query(url) parameters from dfs.`nullable_urls.json`")
+        .unOrdered()
+        .baselineColumns("parameters")
+        .baselineValues(mapOf("a", "12", "b", "someValue"))
+        .baselineValues(mapOf())
+        .baselineValues(mapOf("p1", "v1", "p2", "v=2"))
+        .go();
+  }
+
+  @Test
+  public void testParseUrlFunction() throws Exception {
+    testBuilder()
+        .sqlQuery("select parse_url(url) data from dfs.`nullable_urls.json`")
+        .unOrdered()
+        .baselineColumns("data")
+        .baselineValues(mapOf("protocol", "ftp", "authority", "somewhere.com:3190", "host", "somewhere.com",
+            "path", "/someFile", "query", "a=12&b=someValue", "filename", "/someFile?a=12&b=someValue", "port", 3190))
+        .baselineValues(mapOf())
+        .baselineValues(mapOf("protocol", "http", "authority", "someUrl", "host", "someUrl", "path", "",
+            "query", "p1=v1&p2=v=2&", "filename", "?p1=v1&p2=v=2&"))
+        .go();
+  }
+}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestConvertFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestConvertFunctions.java
index 9ab09db99..d2e0020c3 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestConvertFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestConvertFunctions.java
@@ -25,6 +25,9 @@ import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
 import java.time.LocalDate;
 import java.time.LocalTime;
 import java.util.ArrayList;
@@ -242,6 +245,33 @@ public class TestConvertFunctions extends BaseTestQuery {
   }
 
   @Test
+  public void testConvertFromJsonNullableInput() throws Exception {
+    // Contents of the generated file:
+    /*
+      {"k": "{a: 1, b: 2}"}
+      {"k": null}
+      {"k": "{c: 3}"}
+     */
+    try (BufferedWriter writer = new BufferedWriter(new FileWriter(
+        new File(dirTestWatcher.getRootDir(), "nullable_json_strings.json")))) {
+      String[] fieldValue = {"\"{a: 1, b: 2}\"", null, "\"{c: 3}\""};
+      for (String value : fieldValue) {
+        String entry = String.format("{\"k\": %s}\n", value);
+        writer.write(entry);
+      }
+    }
+
+    testBuilder()
+        .sqlQuery("select convert_from(k, 'json') as col from dfs.`nullable_json_strings.json`")
+        .unOrdered()
+        .baselineColumns("col")
+        .baselineValues(mapOf("a", 1L, "b", 2L))
+        .baselineValues(mapOf())
+        .baselineValues(mapOf("c", 3L))
+        .go();
+  }
+
+  @Test
   public void testConvertToComplexJSON() throws Exception {
 
     String result1 =
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
index 3bc35bf6c..07f214699 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
@@ -24,6 +24,7 @@ import org.junit.Ignore;
 import org.junit.Test;
 
 import java.io.BufferedWriter;
+import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.PrintWriter;
@@ -31,6 +32,7 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 
 import static org.apache.drill.test.TestBuilder.listOf;
+import static org.apache.drill.test.TestBuilder.mapOf;
 
 public class TestComplexTypeReader extends BaseTestQuery {
   @BeforeClass
@@ -232,6 +234,35 @@ public class TestComplexTypeReader extends BaseTestQuery {
   }
 
   @Test
+  public void testKVGenWithNullableInput() throws Exception {
+    // Contents of the generated file:
+    /*
+      {"foo": {"obj":1, "bar":10}}
+      {"foo": {"obj":2, "bar":20}}
+      {"foo": null}
+      {"foo": {"obj": null, "bar": 30}}
+     */
+    try (BufferedWriter writer = new BufferedWriter(new FileWriter(
+        new File(dirTestWatcher.getRootDir(), "input_nested.json")))) {
+      String[] fieldValue = {"{\"obj\":1, \"bar\":10}", "{\"obj\":2, \"bar\":20}", null, "{\"obj\": null, \"bar\": 30}"};
+      for (String value : fieldValue) {
+        String entry = String.format("{\"foo\": %s}\n", value);
+        writer.write(entry);
+      }
+    }
+
+    testBuilder()
+        .sqlQuery("select kvgen(foo) kv from dfs.`input_nested.json`")
+        .unOrdered()
+        .baselineColumns("kv")
+        .baselineValues(listOf(mapOf("key", "obj", "value", 1L), mapOf("key", "bar", "value", 10L)))
+        .baselineValues(listOf(mapOf("key", "obj", "value", 2L), mapOf("key", "bar", "value", 20L)))
+        .baselineValues(listOf())
+        .baselineValues(listOf(mapOf("key", "bar", "value", 30L)))
+        .go();
+  }
+
+  @Test
   @Ignore( "until flattening code creates correct ListVector (DRILL-4045)" )
   public void testNestedFlatten() throws Exception {
     test("select flatten(rl) from cp.`jsoninput/input2.json`");