ODPI-66, ODPI-67. Add HADOOP_LIBJARSCONTENT and HADOOP_BINCONTENT tests

11 files changed, 288 insertions, 0 deletions

diff --git a/bigtop-tests/spec-tests/runtime/src/test/groovy/org/odpi/specs/runtime/TestSpecsRuntime.groovy b/bigtop-tests/spec-tests/runtime/src/test/groovy/org/odpi/specs/runtime/TestSpecsRuntime.groovy
index cf745a0c..138ed8c5 100644
--- a/bigtop-tests/spec-tests/runtime/src/test/groovy/org/odpi/specs/runtime/TestSpecsRuntime.groovy
+++ b/bigtop-tests/spec-tests/runtime/src/test/groovy/org/odpi/specs/runtime/TestSpecsRuntime.groovy
@@ -143,6 +143,38 @@ public class TestSpecsRuntime {
           missingFiles.size() == 0)
         break
 
+      case 'dircontent':
+        def expectedFiles = []
+        new File("${testsList}", "${arguments['referenceList']}").eachLine { line ->
+          expectedFiles << line
+        }
+
+        def baseDir = getEnv(arguments['baseDirEnv'], arguments['envcmd'])
+        def subDir = arguments['subDir']
+        if (!subDir && arguments['subDirEnv']) {
+          subDir = getEnv(arguments['subDirEnv'], arguments['envcmd'])
+        }
+
+        def dir = null
+        if (subDir) {
+          dir = new File(baseDir, subDir)
+        } else {
+          dir = new File(baseDir)
+        }
+        Assert.assertNotNull("Directory has to be set for the test to continue", dir)
+
+        def actualFiles = []
+        if (dir.exists()) {
+          dir.eachFile FileType.FILES, { file ->
+            def relPath = new File( dir.toURI().relativize( file.toURI() ).toString() ).path
+            actualFiles << relPath
+          }
+        }
+
+        def commonFiles = actualFiles.intersect(expectedFiles)
+        Assert.assertTrue("${testName} fail: Directory content for ${dir.path} does not match reference. ",
+           commonFiles.size()==actualFiles.size() && commonFiles.size()==expectedFiles.size())
+        break
       case 'hadoop_tools':
         def toolsPathStr = getEnv("HADOOP_TOOLS_PATH", "hadoop envvars")
         Assert.assertNotNull("${testName} fail: HADOOP_TOOLS_PATH environment variable should be set", toolsPathStr)
diff --git a/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-common-bin.list b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-common-bin.list
new file mode 100644
index 00000000..b3e508d5
--- /dev/null
+++ b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-common-bin.list
@@ -0,0 +1,3 @@
+rcc
+hadoop
+fuse_dfs
diff --git a/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-common-jar.list b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-common-jar.list
new file mode 100644
index 00000000..2a543932
--- /dev/null
+++ b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-common-jar.list
@@ -0,0 +1,61 @@
+api-util-1.0.0-M20.jar
+curator-recipes-2.7.1.jar
+curator-framework-2.7.1.jar
+netty-3.6.2.Final.jar
+gson-2.2.4.jar
+paranamer-2.3.jar
+jackson-core-asl-1.9.13.jar
+jackson-xc-1.9.13.jar
+jersey-server-1.9.jar
+stax-api-1.0-2.jar
+zookeeper-3.4.6.jar
+htrace-core-3.1.0-incubating.jar
+slf4j-api-1.7.10.jar
+avro-1.7.4.jar
+slf4j-log4j12-1.7.10.jar
+curator-client-2.7.1.jar
+jets3t-0.9.0.jar
+commons-net-3.1.jar
+jaxb-impl-2.2.3-1.jar
+httpclient-4.2.5.jar
+apacheds-kerberos-codec-2.0.0-M15.jar
+commons-cli-1.2.jar
+log4j-1.2.17.jar
+jackson-mapper-asl-1.9.13.jar
+java-xmlbuilder-0.4.jar
+jsp-api-2.1.jar
+guava-11.0.2.jar
+jetty-6.1.26.jar
+commons-logging-1.1.3.jar
+snappy-java-1.0.4.1.jar
+commons-httpclient-3.1.jar
+jsch-0.1.42.jar
+jersey-core-1.9.jar
+commons-compress-1.4.1.jar
+jettison-1.1.jar
+junit-4.11.jar
+commons-collections-3.2.1.jar
+xz-1.0.jar
+asm-3.2.jar
+commons-codec-1.4.jar
+commons-digester-1.8.jar
+api-asn1-api-1.0.0-M20.jar
+xmlenc-0.52.jar
+commons-configuration-1.6.jar
+mockito-all-1.8.5.jar
+commons-lang-2.6.jar
+jetty-util-6.1.26.jar
+jsr305-3.0.0.jar
+protobuf-java-2.5.0.jar
+httpcore-4.2.5.jar
+commons-io-2.4.jar
+activation-1.1.jar
+jersey-json-1.9.jar
+jaxb-api-2.2.2.jar
+commons-math3-3.1.1.jar
+hamcrest-core-1.3.jar
+commons-beanutils-core-1.8.0.jar
+apacheds-i18n-2.0.0-M15.jar
+servlet-api-2.5.jar
+commons-beanutils-1.7.0.jar
+jackson-jaxrs-1.9.13.jar
diff --git a/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-hdfs-bin.list b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-hdfs-bin.list
new file mode 100644
index 00000000..88879870
--- /dev/null
+++ b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-hdfs-bin.list
@@ -0,0 +1 @@
+hdfs
diff --git a/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-hdfs-jar.li b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-hdfs-jar.li
new file mode 100644
index 00000000..7677cd16
--- /dev/null
+++ b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-hdfs-jar.li
@@ -0,0 +1,22 @@
+netty-3.6.2.Final.jar
+leveldbjni-all-1.8.jar
+paranamer-2.3.jar
+jackson-core-asl-1.9.13.jar
+jersey-server-1.9.jar
+guice-3.0.jar
+avro-1.7.4.jar
+log4j-1.2.17.jar
+jackson-mapper-asl-1.9.13.jar
+snappy-java-1.0.4.1.jar
+jersey-core-1.9.jar
+jersey-guice-1.9.jar
+commons-compress-1.4.1.jar
+junit-4.11.jar
+xz-1.0.jar
+asm-3.2.jar
+aopalliance-1.0.jar
+javax.inject-1.jar
+protobuf-java-2.5.0.jar
+commons-io-2.4.jar
+hamcrest-core-1.3.jar
+guice-servlet-3.0.jar
diff --git a/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-hdfs-jar.list b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-hdfs-jar.list
new file mode 100644
index 00000000..d24d7640
--- /dev/null
+++ b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-hdfs-jar.list
@@ -0,0 +1,25 @@
+netty-3.6.2.Final.jar
+leveldbjni-all-1.8.jar
+jackson-core-asl-1.9.13.jar
+jersey-server-1.9.jar
+htrace-core-3.1.0-incubating.jar
+commons-daemon-1.0.13.jar
+commons-cli-1.2.jar
+log4j-1.2.17.jar
+jackson-mapper-asl-1.9.13.jar
+guava-11.0.2.jar
+jetty-6.1.26.jar
+commons-logging-1.1.3.jar
+jersey-core-1.9.jar
+asm-3.2.jar
+commons-codec-1.4.jar
+xml-apis-1.3.04.jar
+xercesImpl-2.9.1.jar
+xmlenc-0.52.jar
+commons-lang-2.6.jar
+netty-all-4.0.23.Final.jar
+jetty-util-6.1.26.jar
+jsr305-3.0.0.jar
+protobuf-java-2.5.0.jar
+commons-io-2.4.jar
+servlet-api-2.5.jar
diff --git a/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-mapreduce-bin.list b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-mapreduce-bin.list
new file mode 100644
index 00000000..0a7a9c57
--- /dev/null
+++ b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-mapreduce-bin.list
@@ -0,0 +1 @@
+mapred
diff --git a/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-mapreduce-jar.list b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-mapreduce-jar.list
new file mode 100644
index 00000000..7677cd16
--- /dev/null
+++ b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-mapreduce-jar.list
@@ -0,0 +1,22 @@
+netty-3.6.2.Final.jar
+leveldbjni-all-1.8.jar
+paranamer-2.3.jar
+jackson-core-asl-1.9.13.jar
+jersey-server-1.9.jar
+guice-3.0.jar
+avro-1.7.4.jar
+log4j-1.2.17.jar
+jackson-mapper-asl-1.9.13.jar
+snappy-java-1.0.4.1.jar
+jersey-core-1.9.jar
+jersey-guice-1.9.jar
+commons-compress-1.4.1.jar
+junit-4.11.jar
+xz-1.0.jar
+asm-3.2.jar
+aopalliance-1.0.jar
+javax.inject-1.jar
+protobuf-java-2.5.0.jar
+commons-io-2.4.jar
+hamcrest-core-1.3.jar
+guice-servlet-3.0.jar
diff --git a/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-yarn-bin.list b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-yarn-bin.list
new file mode 100644
index 00000000..d3861b97
--- /dev/null
+++ b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-yarn-bin.list
@@ -0,0 +1,3 @@
+mapred
+yarn
+container-executor
diff --git a/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-yarn-jar.list b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-yarn-jar.list
new file mode 100644
index 00000000..68b7120a
--- /dev/null
+++ b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-yarn-jar.list
@@ -0,0 +1,38 @@
+netty-3.6.2.Final.jar
+leveldbjni-all-1.8.jar
+jackson-core-asl-1.9.13.jar
+jackson-xc-1.9.13.jar
+jersey-server-1.9.jar
+stax-api-1.0-2.jar
+zookeeper-3.4.6.jar
+guice-3.0.jar
+jaxb-impl-2.2.3-1.jar
+zookeeper-3.4.6-tests.jar
+jersey-client-1.9.jar
+commons-cli-1.2.jar
+log4j-1.2.17.jar
+jackson-mapper-asl-1.9.13.jar
+guava-11.0.2.jar
+jetty-6.1.26.jar
+commons-logging-1.1.3.jar
+jersey-core-1.9.jar
+jersey-guice-1.9.jar
+commons-compress-1.4.1.jar
+jettison-1.1.jar
+commons-collections-3.2.1.jar
+xz-1.0.jar
+asm-3.2.jar
+commons-codec-1.4.jar
+aopalliance-1.0.jar
+javax.inject-1.jar
+commons-lang-2.6.jar
+jetty-util-6.1.26.jar
+jsr305-3.0.0.jar
+protobuf-java-2.5.0.jar
+commons-io-2.4.jar
+activation-1.1.jar
+jersey-json-1.9.jar
+jaxb-api-2.2.2.jar
+guice-servlet-3.0.jar
+servlet-api-2.5.jar
+jackson-jaxrs-1.9.13.jar
diff --git a/bigtop-tests/spec-tests/runtime/src/test/resources/testRuntimeSpecConf.groovy b/bigtop-tests/spec-tests/runtime/src/test/resources/testRuntimeSpecConf.groovy
index 9616e7ed..6bf60c7f 100644
--- a/bigtop-tests/spec-tests/runtime/src/test/resources/testRuntimeSpecConf.groovy
+++ b/bigtop-tests/spec-tests/runtime/src/test/resources/testRuntimeSpecConf.groovy
@@ -213,6 +213,86 @@ specs {
         referenceList = 'hadoop-subprojs.list'
       }
     }
+    'HADOOP_BINCONTENT_COMMON' {
+      name = 'HADOOP_BINCONTENT_COMMON'
+      type = 'dircontent'
+      arguments {
+        envcmd = 'hadoop envvars'
+        baseDirEnv = 'HADOOP_COMMON_HOME'
+        subDir = 'bin'
+        referenceList = 'hadoop-common-bin.list'
+      }
+    }
+    'HADOOP_BINCONTENT_HDFS' {
+      name = 'HADOOP_BINCONTENT_HDFS'
+      type = 'dircontent'
+      arguments {
+        envcmd = 'hdfs envvars'
+        baseDirEnv = 'HADOOP_HDFS_HOME'
+        subDir = 'bin'
+        referenceList = 'hadoop-hdfs-bin.list'
+      }
+    }
+    'HADOOP_BINCONTENT_MAPRED' {
+      name = 'HADOOP_BINCONTENT_MAPRED'
+      type = 'dircontent'
+      arguments {
+        envcmd = 'mapred envvars'
+        baseDirEnv = 'HADOOP_MAPRED_HOME'
+        subDir = 'bin'
+        referenceList = 'hadoop-mapreduce-bin.list'
+      }
+    }
+    'HADOOP_BINCONTENT_YARN' {
+      name = 'HADOOP_BINCONTENT_YARN'
+      type = 'dircontent'
+      arguments {
+        envcmd = 'yarn envvars'
+        baseDirEnv = 'HADOOP_YARN_HOME'
+        subDir = 'bin'
+        referenceList = 'hadoop-yarn-bin.list'
+      }
+    }
+    'HADOOP_JARCONTENT_COMMON' {
+      name = 'HADOOP_JARCONTENT_COMMON'
+      type = 'dircontent'
+      arguments {
+        envcmd = 'hadoop envvars'
+        baseDirEnv = 'HADOOP_COMMON_HOME'
+        subDirEnv = 'HADOOP_COMMON_LIB_JARS_DIR'
+        referenceList = 'hadoop-common-jar.list'
+      }
+    }
+    'HADOOP_JARCONTENT_HDFS' {
+      name = 'HADOOP_JARCONTENT_HDFS'
+      type = 'dircontent'
+      arguments {
+        envcmd = 'hdfs envvars'
+        baseDirEnv = 'HADOOP_HDFS_HOME'
+        subDirEnv = 'HDFS_LIB_JARS_DIR'
+        referenceList = 'hadoop-hdfs-jar.list'
+      }
+    }
+    'HADOOP_JARCONTENT_MAPRED' {
+      name = 'HADOOP_JARCONTENT_MAPRED'
+      type = 'dircontent'
+      arguments {
+        envcmd = 'mapred envvars'
+        baseDirEnv = 'HADOOP_MAPRED_HOME'
+        subDirEnv = 'MAPRED_LIB_JARS_DIR'
+        referenceList = 'hadoop-mapreduce-jar.list'
+      }
+    }
+    'HADOOP_JARCONTENT_YARN' {
+      name = 'HADOOP_JARCONTENT_YARN'
+      type = 'dircontent'
+      arguments {
+        envcmd = 'yarn envvars'
+        baseDirEnv = 'HADOOP_YARN_HOME'
+        subDirEnv = 'YARN_LIB_JARS_DIR'
+        referenceList = 'hadoop-yarn-jar.list'
+      }
+    }
     'HADOOP_GETCONF' {
       name = 'HADOOP_GETCONF'
       type = 'shell'