From f13840f12c83b90553da9bf950447a711838d32a Mon Sep 17 00:00:00 2001 From: Konstantin Boudnik Date: Fri, 4 Mar 2016 15:13:49 +0300 Subject: ODPI-66, ODPI-67. Add HADOOP_LIBJARSCONTENT and HADOOP_BINCONTENT tests --- .../org/odpi/specs/runtime/TestSpecsRuntime.groovy | 32 +++++++++ .../src/test/resources/hadoop-common-bin.list | 3 + .../src/test/resources/hadoop-common-jar.list | 61 +++++++++++++++++ .../src/test/resources/hadoop-hdfs-bin.list | 1 + .../runtime/src/test/resources/hadoop-hdfs-jar.li | 22 ++++++ .../src/test/resources/hadoop-hdfs-jar.list | 25 +++++++ .../src/test/resources/hadoop-mapreduce-bin.list | 1 + .../src/test/resources/hadoop-mapreduce-jar.list | 22 ++++++ .../src/test/resources/hadoop-yarn-bin.list | 3 + .../src/test/resources/hadoop-yarn-jar.list | 38 ++++++++++ .../src/test/resources/testRuntimeSpecConf.groovy | 80 ++++++++++++++++++++++ 11 files changed, 288 insertions(+) create mode 100644 bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-common-bin.list create mode 100644 bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-common-jar.list create mode 100644 bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-hdfs-bin.list create mode 100644 bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-hdfs-jar.li create mode 100644 bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-hdfs-jar.list create mode 100644 bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-mapreduce-bin.list create mode 100644 bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-mapreduce-jar.list create mode 100644 bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-yarn-bin.list create mode 100644 bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-yarn-jar.list (limited to 'bigtop-tests') diff --git a/bigtop-tests/spec-tests/runtime/src/test/groovy/org/odpi/specs/runtime/TestSpecsRuntime.groovy b/bigtop-tests/spec-tests/runtime/src/test/groovy/org/odpi/specs/runtime/TestSpecsRuntime.groovy index cf745a0c..138ed8c5 100644 --- a/bigtop-tests/spec-tests/runtime/src/test/groovy/org/odpi/specs/runtime/TestSpecsRuntime.groovy +++ b/bigtop-tests/spec-tests/runtime/src/test/groovy/org/odpi/specs/runtime/TestSpecsRuntime.groovy @@ -143,6 +143,38 @@ public class TestSpecsRuntime { missingFiles.size() == 0) break + case 'dircontent': + def expectedFiles = [] + new File("${testsList}", "${arguments['referenceList']}").eachLine { line -> + expectedFiles << line + } + + def baseDir = getEnv(arguments['baseDirEnv'], arguments['envcmd']) + def subDir = arguments['subDir'] + if (!subDir && arguments['subDirEnv']) { + subDir = getEnv(arguments['subDirEnv'], arguments['envcmd']) + } + + def dir = null + if (subDir) { + dir = new File(baseDir, subDir) + } else { + dir = new File(baseDir) + } + Assert.assertNotNull("Directory has to be set for the test to continue", dir) + + def actualFiles = [] + if (dir.exists()) { + dir.eachFile FileType.FILES, { file -> + def relPath = new File( dir.toURI().relativize( file.toURI() ).toString() ).path + actualFiles << relPath + } + } + + def commonFiles = actualFiles.intersect(expectedFiles) + Assert.assertTrue("${testName} fail: Directory content for ${dir.path} does not match reference. ", + commonFiles.size()==actualFiles.size() && commonFiles.size()==expectedFiles.size()) + break case 'hadoop_tools': def toolsPathStr = getEnv("HADOOP_TOOLS_PATH", "hadoop envvars") Assert.assertNotNull("${testName} fail: HADOOP_TOOLS_PATH environment variable should be set", toolsPathStr) diff --git a/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-common-bin.list b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-common-bin.list new file mode 100644 index 00000000..b3e508d5 --- /dev/null +++ b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-common-bin.list @@ -0,0 +1,3 @@ +rcc +hadoop +fuse_dfs diff --git a/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-common-jar.list b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-common-jar.list new file mode 100644 index 00000000..2a543932 --- /dev/null +++ b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-common-jar.list @@ -0,0 +1,61 @@ +api-util-1.0.0-M20.jar +curator-recipes-2.7.1.jar +curator-framework-2.7.1.jar +netty-3.6.2.Final.jar +gson-2.2.4.jar +paranamer-2.3.jar +jackson-core-asl-1.9.13.jar +jackson-xc-1.9.13.jar +jersey-server-1.9.jar +stax-api-1.0-2.jar +zookeeper-3.4.6.jar +htrace-core-3.1.0-incubating.jar +slf4j-api-1.7.10.jar +avro-1.7.4.jar +slf4j-log4j12-1.7.10.jar +curator-client-2.7.1.jar +jets3t-0.9.0.jar +commons-net-3.1.jar +jaxb-impl-2.2.3-1.jar +httpclient-4.2.5.jar +apacheds-kerberos-codec-2.0.0-M15.jar +commons-cli-1.2.jar +log4j-1.2.17.jar +jackson-mapper-asl-1.9.13.jar +java-xmlbuilder-0.4.jar +jsp-api-2.1.jar +guava-11.0.2.jar +jetty-6.1.26.jar +commons-logging-1.1.3.jar +snappy-java-1.0.4.1.jar +commons-httpclient-3.1.jar +jsch-0.1.42.jar +jersey-core-1.9.jar +commons-compress-1.4.1.jar +jettison-1.1.jar +junit-4.11.jar +commons-collections-3.2.1.jar +xz-1.0.jar +asm-3.2.jar +commons-codec-1.4.jar +commons-digester-1.8.jar +api-asn1-api-1.0.0-M20.jar +xmlenc-0.52.jar +commons-configuration-1.6.jar +mockito-all-1.8.5.jar +commons-lang-2.6.jar +jetty-util-6.1.26.jar +jsr305-3.0.0.jar +protobuf-java-2.5.0.jar +httpcore-4.2.5.jar +commons-io-2.4.jar +activation-1.1.jar +jersey-json-1.9.jar +jaxb-api-2.2.2.jar +commons-math3-3.1.1.jar +hamcrest-core-1.3.jar +commons-beanutils-core-1.8.0.jar +apacheds-i18n-2.0.0-M15.jar +servlet-api-2.5.jar +commons-beanutils-1.7.0.jar +jackson-jaxrs-1.9.13.jar diff --git a/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-hdfs-bin.list b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-hdfs-bin.list new file mode 100644 index 00000000..88879870 --- /dev/null +++ b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-hdfs-bin.list @@ -0,0 +1 @@ +hdfs diff --git a/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-hdfs-jar.li b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-hdfs-jar.li new file mode 100644 index 00000000..7677cd16 --- /dev/null +++ b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-hdfs-jar.li @@ -0,0 +1,22 @@ +netty-3.6.2.Final.jar +leveldbjni-all-1.8.jar +paranamer-2.3.jar +jackson-core-asl-1.9.13.jar +jersey-server-1.9.jar +guice-3.0.jar +avro-1.7.4.jar +log4j-1.2.17.jar +jackson-mapper-asl-1.9.13.jar +snappy-java-1.0.4.1.jar +jersey-core-1.9.jar +jersey-guice-1.9.jar +commons-compress-1.4.1.jar +junit-4.11.jar +xz-1.0.jar +asm-3.2.jar +aopalliance-1.0.jar +javax.inject-1.jar +protobuf-java-2.5.0.jar +commons-io-2.4.jar +hamcrest-core-1.3.jar +guice-servlet-3.0.jar diff --git a/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-hdfs-jar.list b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-hdfs-jar.list new file mode 100644 index 00000000..d24d7640 --- /dev/null +++ b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-hdfs-jar.list @@ -0,0 +1,25 @@ +netty-3.6.2.Final.jar +leveldbjni-all-1.8.jar +jackson-core-asl-1.9.13.jar +jersey-server-1.9.jar +htrace-core-3.1.0-incubating.jar +commons-daemon-1.0.13.jar +commons-cli-1.2.jar +log4j-1.2.17.jar +jackson-mapper-asl-1.9.13.jar +guava-11.0.2.jar +jetty-6.1.26.jar +commons-logging-1.1.3.jar +jersey-core-1.9.jar +asm-3.2.jar +commons-codec-1.4.jar +xml-apis-1.3.04.jar +xercesImpl-2.9.1.jar +xmlenc-0.52.jar +commons-lang-2.6.jar +netty-all-4.0.23.Final.jar +jetty-util-6.1.26.jar +jsr305-3.0.0.jar +protobuf-java-2.5.0.jar +commons-io-2.4.jar +servlet-api-2.5.jar diff --git a/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-mapreduce-bin.list b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-mapreduce-bin.list new file mode 100644 index 00000000..0a7a9c57 --- /dev/null +++ b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-mapreduce-bin.list @@ -0,0 +1 @@ +mapred diff --git a/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-mapreduce-jar.list b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-mapreduce-jar.list new file mode 100644 index 00000000..7677cd16 --- /dev/null +++ b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-mapreduce-jar.list @@ -0,0 +1,22 @@ +netty-3.6.2.Final.jar +leveldbjni-all-1.8.jar +paranamer-2.3.jar +jackson-core-asl-1.9.13.jar +jersey-server-1.9.jar +guice-3.0.jar +avro-1.7.4.jar +log4j-1.2.17.jar +jackson-mapper-asl-1.9.13.jar +snappy-java-1.0.4.1.jar +jersey-core-1.9.jar +jersey-guice-1.9.jar +commons-compress-1.4.1.jar +junit-4.11.jar +xz-1.0.jar +asm-3.2.jar +aopalliance-1.0.jar +javax.inject-1.jar +protobuf-java-2.5.0.jar +commons-io-2.4.jar +hamcrest-core-1.3.jar +guice-servlet-3.0.jar diff --git a/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-yarn-bin.list b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-yarn-bin.list new file mode 100644 index 00000000..d3861b97 --- /dev/null +++ b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-yarn-bin.list @@ -0,0 +1,3 @@ +mapred +yarn +container-executor diff --git a/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-yarn-jar.list b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-yarn-jar.list new file mode 100644 index 00000000..68b7120a --- /dev/null +++ b/bigtop-tests/spec-tests/runtime/src/test/resources/hadoop-yarn-jar.list @@ -0,0 +1,38 @@ +netty-3.6.2.Final.jar +leveldbjni-all-1.8.jar +jackson-core-asl-1.9.13.jar +jackson-xc-1.9.13.jar +jersey-server-1.9.jar +stax-api-1.0-2.jar +zookeeper-3.4.6.jar +guice-3.0.jar +jaxb-impl-2.2.3-1.jar +zookeeper-3.4.6-tests.jar +jersey-client-1.9.jar +commons-cli-1.2.jar +log4j-1.2.17.jar +jackson-mapper-asl-1.9.13.jar +guava-11.0.2.jar +jetty-6.1.26.jar +commons-logging-1.1.3.jar +jersey-core-1.9.jar +jersey-guice-1.9.jar +commons-compress-1.4.1.jar +jettison-1.1.jar +commons-collections-3.2.1.jar +xz-1.0.jar +asm-3.2.jar +commons-codec-1.4.jar +aopalliance-1.0.jar +javax.inject-1.jar +commons-lang-2.6.jar +jetty-util-6.1.26.jar +jsr305-3.0.0.jar +protobuf-java-2.5.0.jar +commons-io-2.4.jar +activation-1.1.jar +jersey-json-1.9.jar +jaxb-api-2.2.2.jar +guice-servlet-3.0.jar +servlet-api-2.5.jar +jackson-jaxrs-1.9.13.jar diff --git a/bigtop-tests/spec-tests/runtime/src/test/resources/testRuntimeSpecConf.groovy b/bigtop-tests/spec-tests/runtime/src/test/resources/testRuntimeSpecConf.groovy index 9616e7ed..6bf60c7f 100644 --- a/bigtop-tests/spec-tests/runtime/src/test/resources/testRuntimeSpecConf.groovy +++ b/bigtop-tests/spec-tests/runtime/src/test/resources/testRuntimeSpecConf.groovy @@ -213,6 +213,86 @@ specs { referenceList = 'hadoop-subprojs.list' } } + 'HADOOP_BINCONTENT_COMMON' { + name = 'HADOOP_BINCONTENT_COMMON' + type = 'dircontent' + arguments { + envcmd = 'hadoop envvars' + baseDirEnv = 'HADOOP_COMMON_HOME' + subDir = 'bin' + referenceList = 'hadoop-common-bin.list' + } + } + 'HADOOP_BINCONTENT_HDFS' { + name = 'HADOOP_BINCONTENT_HDFS' + type = 'dircontent' + arguments { + envcmd = 'hdfs envvars' + baseDirEnv = 'HADOOP_HDFS_HOME' + subDir = 'bin' + referenceList = 'hadoop-hdfs-bin.list' + } + } + 'HADOOP_BINCONTENT_MAPRED' { + name = 'HADOOP_BINCONTENT_MAPRED' + type = 'dircontent' + arguments { + envcmd = 'mapred envvars' + baseDirEnv = 'HADOOP_MAPRED_HOME' + subDir = 'bin' + referenceList = 'hadoop-mapreduce-bin.list' + } + } + 'HADOOP_BINCONTENT_YARN' { + name = 'HADOOP_BINCONTENT_YARN' + type = 'dircontent' + arguments { + envcmd = 'yarn envvars' + baseDirEnv = 'HADOOP_YARN_HOME' + subDir = 'bin' + referenceList = 'hadoop-yarn-bin.list' + } + } + 'HADOOP_JARCONTENT_COMMON' { + name = 'HADOOP_JARCONTENT_COMMON' + type = 'dircontent' + arguments { + envcmd = 'hadoop envvars' + baseDirEnv = 'HADOOP_COMMON_HOME' + subDirEnv = 'HADOOP_COMMON_LIB_JARS_DIR' + referenceList = 'hadoop-common-jar.list' + } + } + 'HADOOP_JARCONTENT_HDFS' { + name = 'HADOOP_JARCONTENT_HDFS' + type = 'dircontent' + arguments { + envcmd = 'hdfs envvars' + baseDirEnv = 'HADOOP_HDFS_HOME' + subDirEnv = 'HDFS_LIB_JARS_DIR' + referenceList = 'hadoop-hdfs-jar.list' + } + } + 'HADOOP_JARCONTENT_MAPRED' { + name = 'HADOOP_JARCONTENT_MAPRED' + type = 'dircontent' + arguments { + envcmd = 'mapred envvars' + baseDirEnv = 'HADOOP_MAPRED_HOME' + subDirEnv = 'MAPRED_LIB_JARS_DIR' + referenceList = 'hadoop-mapreduce-jar.list' + } + } + 'HADOOP_JARCONTENT_YARN' { + name = 'HADOOP_JARCONTENT_YARN' + type = 'dircontent' + arguments { + envcmd = 'yarn envvars' + baseDirEnv = 'HADOOP_YARN_HOME' + subDirEnv = 'YARN_LIB_JARS_DIR' + referenceList = 'hadoop-yarn-jar.list' + } + } 'HADOOP_GETCONF' { name = 'HADOOP_GETCONF' type = 'shell' -- cgit v1.2.3