mauve: tests to run are driven by files

Signed-off-by: Andrew McDermott <andrew.mcdermott@linaro.org>

5 files changed, 96 insertions, 66 deletions

diff --git a/openembedded/mauve-sanity-test.yaml b/openembedded/mauve-sanity-test.yaml
new file mode 100644
index 0000000..ecf05c8
--- /dev/null
+++ b/openembedded/mauve-sanity-test.yaml
@@ -0,0 +1,33 @@
+metadata:
+  name: mauve
+  format: "Lava-Test-Shell Test Definition 1.0"
+  description: "Mauve Sanity Tests for Open Embedded"
+  version: 1.0
+
+install:
+  git-repos:
+    - git://git.linaro.org/qa/test-definitions.git
+    - git://git.linaro.org/people/andrew.mcdermott/mauve.git
+
+run:
+  steps:
+    - 'PATH=$PATH:/lava/tests/$TESTRUN_ID/test-definitions/openembedded/scripts'
+    - 'PATH=/usr/lib/jvm/java-8-openjdk/bin:$PATH'
+    - 'which java'
+    - 'java -version'
+    - 'rm -rf /tmp/mauve'
+    - 'ln -s /lava/tests/$TESTRUN_ID/mauve /tmp/mauve'
+    - 'cd /tmp/mauve'
+    - 'mauve-lava-test -compile no -showpasses < /lava/tests/$TESTRUN_ID/test-definitions/openembedded/mauve/sanity-test.txt'
+
+# The following "parser" will correctly parse the output from the
+# tests but it is deliberately commented out because you end up with
+# one line per test case in the LAVA dashboard and, given the number
+# of tests that are run, there will be ~250 pages to wade through.
+
+# parse:
+#   pattern: "(?P<result>(PASS|FAIL|SKIP)):\\s+(?P<test_case_id>(.*))"
+#   fixupdict:
+#     PASS: pass
+#     FAIL: fail
+#     SKIP: skip
diff --git a/openembedded/mauve.yaml b/openembedded/mauve.yaml
index 3128a47..fdd8b4d 100644
--- a/openembedded/mauve.yaml
+++ b/openembedded/mauve.yaml
@@ -22,72 +22,7 @@ run:
     - 'rm -rf /tmp/mauve'
     - 'ln -s /lava/tests/$TESTRUN_ID/mauve /tmp/mauve'
     - 'cd /tmp/mauve'
-    - 'lava-test-case gnu.testlet.javax.security --shell mauve-test-pkg gnu/testlet/javax/security -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.javax.security.txt'
-    - 'lava-test-case-attach gnu.testlet.javax.security /tmp/mauve-gnu.testlet.javax.security.txt text/plain'
-    - 'lava-test-case gnu.testlet.java.awt --shell mauve-test-pkg gnu/testlet/java/awt -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.java.awt.txt'
-    - 'lava-test-case-attach gnu.testlet.java.awt /tmp/mauve-gnu.testlet.java.awt.txt text/plain'
-    - 'lava-test-case gnu.testlet.org.omg --shell mauve-test-pkg gnu/testlet/org/omg -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.org.omg.txt'
-    - 'lava-test-case-attach gnu.testlet.org.omg /tmp/mauve-gnu.testlet.org.omg.txt text/plain'
-    - 'lava-test-case gnu.testlet.javax.swing --shell mauve-test-pkg gnu/testlet/javax/swing -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.javax.swing.txt'
-    - 'lava-test-case-attach gnu.testlet.javax.swing /tmp/mauve-gnu.testlet.javax.swing.txt text/plain'
-    - 'lava-test-case gnu.testlet.javax.print --shell mauve-test-pkg gnu/testlet/javax/print -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.javax.print.txt'
-    - 'lava-test-case-attach gnu.testlet.javax.print /tmp/mauve-gnu.testlet.javax.print.txt text/plain'
-    - 'lava-test-case gnu.testlet.java.lang --shell mauve-test-pkg gnu/testlet/java/lang -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.java.lang.txt'
-    - 'lava-test-case-attach gnu.testlet.java.lang /tmp/mauve-gnu.testlet.java.lang.txt text/plain'
-    - 'lava-test-case gnu.testlet.java.nio --shell mauve-test-pkg gnu/testlet/java/nio -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.java.nio.txt'
-    - 'lava-test-case-attach gnu.testlet.java.nio /tmp/mauve-gnu.testlet.java.nio.txt text/plain'
-    - 'lava-test-case gnu.testlet.java.text --shell mauve-test-pkg gnu/testlet/java/text -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.java.text.txt'
-    - 'lava-test-case-attach gnu.testlet.java.text /tmp/mauve-gnu.testlet.java.text.txt text/plain'
-    - 'lava-test-case gnu.testlet.BinaryCompatibility.altered --shell mauve-test-pkg gnu/testlet/BinaryCompatibility/altered -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.BinaryCompatibility.altered.txt'
-    - 'lava-test-case-attach gnu.testlet.BinaryCompatibility.altered /tmp/mauve-gnu.testlet.BinaryCompatibility.altered.txt text/plain'
-    - 'lava-test-case gnu.testlet.BinaryCompatibility --shell mauve-test-pkg gnu/testlet/BinaryCompatibility -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.BinaryCompatibility.txt'
-    - 'lava-test-case-attach gnu.testlet.BinaryCompatibility /tmp/mauve-gnu.testlet.BinaryCompatibility.txt text/plain'
-    - 'lava-test-case gnu.testlet.javax.naming --shell mauve-test-pkg gnu/testlet/javax/naming -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.javax.naming.txt'
-    - 'lava-test-case-attach gnu.testlet.javax.naming /tmp/mauve-gnu.testlet.javax.naming.txt text/plain'
-    - 'lava-test-case gnu.testlet.java.util --shell mauve-test-pkg gnu/testlet/java/util -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.java.util.txt'
-    - 'lava-test-case-attach gnu.testlet.java.util /tmp/mauve-gnu.testlet.java.util.txt text/plain'
-    - 'lava-test-case gnu.testlet.locales --shell mauve-test-pkg gnu/testlet/locales -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.locales.txt'
-    - 'lava-test-case-attach gnu.testlet.locales /tmp/mauve-gnu.testlet.locales.txt text/plain'
-    - 'lava-test-case gnu.testlet.runner --shell mauve-test-pkg gnu/testlet/runner -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.runner.txt'
-    - 'lava-test-case-attach gnu.testlet.runner /tmp/mauve-gnu.testlet.runner.txt text/plain'
-    - 'lava-test-case gnu.testlet.java.sql --shell mauve-test-pkg gnu/testlet/java/sql -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.java.sql.txt'
-    - 'lava-test-case-attach gnu.testlet.java.sql /tmp/mauve-gnu.testlet.java.sql.txt text/plain'
-    - 'lava-test-case gnu.testlet.gnu.javax --shell mauve-test-pkg gnu/testlet/gnu/javax -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.gnu.javax.txt'
-    - 'lava-test-case-attach gnu.testlet.gnu.javax /tmp/mauve-gnu.testlet.gnu.javax.txt text/plain'
-    - 'lava-test-case gnu.testlet.javax.imageio --shell mauve-test-pkg gnu/testlet/javax/imageio -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.javax.imageio.txt'
-    - 'lava-test-case-attach gnu.testlet.javax.imageio /tmp/mauve-gnu.testlet.javax.imageio.txt text/plain'
-    - 'lava-test-case gnu.testlet.javax.sound --shell mauve-test-pkg gnu/testlet/javax/sound -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.javax.sound.txt'
-    - 'lava-test-case-attach gnu.testlet.javax.sound /tmp/mauve-gnu.testlet.javax.sound.txt text/plain'
-    - 'lava-test-case gnu.testlet.java.io --shell mauve-test-pkg gnu/testlet/java/io -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.java.io.txt'
-    - 'lava-test-case-attach gnu.testlet.java.io /tmp/mauve-gnu.testlet.java.io.txt text/plain'
-    - 'lava-test-case gnu.testlet.javax.net --shell mauve-test-pkg gnu/testlet/javax/net -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.javax.net.txt'
-    - 'lava-test-case-attach gnu.testlet.javax.net /tmp/mauve-gnu.testlet.javax.net.txt text/plain'
-    - 'lava-test-case gnu.testlet.javax.crypto --shell mauve-test-pkg gnu/testlet/javax/crypto -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.javax.crypto.txt'
-    - 'lava-test-case-attach gnu.testlet.javax.crypto /tmp/mauve-gnu.testlet.javax.crypto.txt text/plain'
-    - 'lava-test-case gnu.testlet.javax.management --shell mauve-test-pkg gnu/testlet/javax/management -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.javax.management.txt'
-    - 'lava-test-case-attach gnu.testlet.javax.management /tmp/mauve-gnu.testlet.javax.management.txt text/plain'
-    - 'lava-test-case gnu.testlet.runner.compare --shell mauve-test-pkg gnu/testlet/runner/compare -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.runner.compare.txt'
-    - 'lava-test-case-attach gnu.testlet.runner.compare /tmp/mauve-gnu.testlet.runner.compare.txt text/plain'
-    - 'lava-test-case gnu.testlet.java.beans --shell mauve-test-pkg gnu/testlet/java/beans -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.java.beans.txt'
-    - 'lava-test-case-attach gnu.testlet.java.beans /tmp/mauve-gnu.testlet.java.beans.txt text/plain'
-    - 'lava-test-case gnu.testlet.java.net --shell mauve-test-pkg gnu/testlet/java/net -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.java.net.txt'
-    - 'lava-test-case-attach gnu.testlet.java.net /tmp/mauve-gnu.testlet.java.net.txt text/plain'
-    - 'lava-test-case gnu.testlet.org.w3c --shell mauve-test-pkg gnu/testlet/org/w3c -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.org.w3c.txt'
-    - 'lava-test-case-attach gnu.testlet.org.w3c /tmp/mauve-gnu.testlet.org.w3c.txt text/plain'
-    - 'lava-test-case gnu.testlet.javax.rmi --shell mauve-test-pkg gnu/testlet/javax/rmi -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.javax.rmi.txt'
-    - 'lava-test-case-attach gnu.testlet.javax.rmi /tmp/mauve-gnu.testlet.javax.rmi.txt text/plain'
-    - 'lava-test-case gnu.testlet.java.math --shell mauve-test-pkg gnu/testlet/java/math -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.java.math.txt'
-    - 'lava-test-case-attach gnu.testlet.java.math /tmp/mauve-gnu.testlet.java.math.txt text/plain'
-    - 'lava-test-case gnu.testlet.java.security --shell mauve-test-pkg gnu/testlet/java/security -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.java.security.txt'
-    - 'lava-test-case-attach gnu.testlet.java.security /tmp/mauve-gnu.testlet.java.security.txt text/plain'
-    - 'lava-test-case gnu.testlet.gnu.java --shell mauve-test-pkg gnu/testlet/gnu/java -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.gnu.java.txt'
-    - 'lava-test-case-attach gnu.testlet.gnu.java /tmp/mauve-gnu.testlet.gnu.java.txt text/plain'
-    - 'lava-test-case gnu.testlet.java.rmi --shell mauve-test-pkg gnu/testlet/java/rmi -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.java.rmi.txt'
-    - 'lava-test-case-attach gnu.testlet.java.rmi /tmp/mauve-gnu.testlet.java.rmi.txt text/plain'
-    - 'lava-test-case gnu.testlet.javax.xml --shell mauve-test-pkg gnu/testlet/javax/xml -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.javax.xml.txt'
-    - 'lava-test-case-attach gnu.testlet.javax.xml /tmp/mauve-gnu.testlet.javax.xml.txt text/plain'
-    - 'lava-test-case gnu.testlet.javax.accessibility --shell mauve-test-pkg gnu/testlet/javax/accessibility -compile no 2>&1 | tee /tmp/mauve-gnu.testlet.javax.accessibility.txt'
-    - 'lava-test-case-attach gnu.testlet.javax.accessibility /tmp/mauve-gnu.testlet.javax.accessibility.txt text/plain'
+    - 'mauve-lava-test -compile no -showpasses < /lava/tests/$TESTRUN_ID/test-definitions/openembedded/mauve/all-packages.txt'
     - 'kill $XVFBPID'
 
 # The following "parser" will correctly parse the output from the
diff --git a/openembedded/mauve/all-packages.txt b/openembedded/mauve/all-packages.txt
new file mode 100644
index 0000000..d995cc8
--- /dev/null
+++ b/openembedded/mauve/all-packages.txt
@@ -0,0 +1,31 @@
+gnu/testlet/java/lang
+gnu/testlet/java/beans
+gnu/testlet/java/io
+gnu/testlet/java/math
+gnu/testlet/java/net
+gnu/testlet/java/nio
+gnu/testlet/java/rmi
+gnu/testlet/java/security
+gnu/testlet/java/sql
+gnu/testlet/java/text
+gnu/testlet/java/util
+gnu/testlet/BinaryCompatibility
+gnu/testlet/BinaryCompatibility/altered
+gnu/testlet/gnu/java
+gnu/testlet/gnu/javax
+gnu/testlet/javax/accessibility
+gnu/testlet/javax/crypto
+gnu/testlet/javax/imageio
+gnu/testlet/javax/management
+gnu/testlet/javax/naming
+gnu/testlet/javax/net
+gnu/testlet/javax/print
+gnu/testlet/javax/rmi
+gnu/testlet/javax/security
+gnu/testlet/javax/sound
+gnu/testlet/javax/xml
+gnu/testlet/locales
+gnu/testlet/org/omg
+gnu/testlet/org/w3c
+gnu/testlet/java/awt
+gnu/testlet/javax/swing
diff --git a/openembedded/mauve/sanity-test.txt b/openembedded/mauve/sanity-test.txt
new file mode 100644
index 0000000..1cfe40f
--- /dev/null
+++ b/openembedded/mauve/sanity-test.txt
@@ -0,0 +1 @@
+gnu/testlet/java/lang/Integer
diff --git a/openembedded/scripts/mauve-lava-test b/openembedded/scripts/mauve-lava-test
new file mode 100755
index 0000000..bd815a8
--- /dev/null
+++ b/openembedded/scripts/mauve-lava-test
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Lava Mauve test harness.
+#
+# Copyright (C) 2013, Linaro Limited.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# Author: Andrew McDermott <andrew.mcdermott@linaro.org>
+#
+
+while read -r pkg; do
+    [[ "$pkg" =~ ^#.*$ ]] && continue
+    [[ "$pkg" =~ ^$    ]] && continue
+    id=${pkg//\//.}
+    lava-test-case $id --shell mauve-test-pkg $pkg "$@" 2>&1 | tee /tmp/mauve-$id.txt
+    lava-test-case-attach $id /tmp/mauve-${id}.txt text/plain
+done