summaryrefslogtreecommitdiff
path: root/automated
diff options
context:
space:
mode:
authorChase Qi <chase.qi@linaro.org>2017-02-08 08:29:48 +0800
committerMilosz Wasilewski <milosz.wasilewski@linaro.org>2017-02-20 13:19:30 +0000
commit7d317f8b0be720c10394d94283c16583dad34b06 (patch)
tree5de82fba7fe97d5b50f3335989e9c1ecc92a72e0 /automated
parent6f49a1bbb11e50165dd17c4f67f49d8f10c01144 (diff)
downloadtest-definitions-pipe-7d317f8b0be720c10394d94283c16583dad34b06.tar.gz
automated: android: add cts test
Change-Id: I75eb65bc7de03cbe612b33c4ee69d50cfdc6608d Signed-off-by: Chase Qi <chase.qi@linaro.org>
Diffstat (limited to 'automated')
-rwxr-xr-xautomated/android/cts/cts-runner.py139
-rwxr-xr-xautomated/android/cts/cts.sh84
-rw-r--r--automated/android/cts/cts.yaml41
-rwxr-xr-xautomated/lib/android-test-lib60
-rwxr-xr-xautomated/lib/py_test_lib.py4
5 files changed, 320 insertions, 8 deletions
diff --git a/automated/android/cts/cts-runner.py b/automated/android/cts/cts-runner.py
new file mode 100755
index 0000000..5bd7777
--- /dev/null
+++ b/automated/android/cts/cts-runner.py
@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+
+import datetime
+import os
+import sys
+import shlex
+import shutil
+import subprocess
+import xml.etree.ElementTree as ET
+import pexpect
+import argparse
+import logging
+
+sys.path.insert(0, '../../lib/')
+import py_test_lib # nopep8
+
+
+def result_parser(xml_file):
+ try:
+ tree = ET.parse(xml_file)
+ except ET.ParseError as e:
+ logger.error('xml.etree.ElementTree.ParseError: %s' % e)
+ logger.info('Please Check %s manually' % xml_file)
+ sys.exit(1)
+ root = tree.getroot()
+ logger.info('Test modules in %s: %s'
+ % (xml_file, str(len(root.findall('Module')))))
+ for elem in root.findall('Module'):
+ # Naming: Module Name + Test Case Name + Test Name
+ if 'abi' in elem.attrib.keys():
+ module_name = '.'.join([elem.attrib['abi'], elem.attrib['name']])
+ else:
+ module_name = elem.attrib['name']
+
+ tests_executed = len(elem.findall('.//Test'))
+ tests_passed = len(elem.findall('.//Test[@result="pass"]'))
+ tests_failed = len(elem.findall('.//Test[@result="fail"]'))
+
+ result = '%s_executed pass %s' % (module_name, str(tests_executed))
+ py_test_lib.add_result(RESULT_FILE, result)
+
+ result = '%s_passed pass %s' % (module_name, str(tests_passed))
+ py_test_lib.add_result(RESULT_FILE, result)
+
+ failed_result = 'pass'
+ if tests_failed > 0:
+ failed_result = 'fail'
+ result = '%s_failed %s %s' % (module_name, failed_result,
+ str(tests_failed))
+ py_test_lib.add_result(RESULT_FILE, result)
+
+
+OUTPUT = '%s/output' % os.getcwd()
+RESULT_FILE = '%s/result.txt' % OUTPUT
+CTS_STDOUT = '%s/cts-stdout.txt' % OUTPUT
+CTS_LOGCAT = '%s/cts-logcat.txt' % OUTPUT
+TEST_PARAMS = ''
+SN = ''
+
+parser = argparse.ArgumentParser()
+parser.add_argument('-t', dest='TEST_PARAMS', required=True,
+ help="cts test parameters")
+parser.add_argument('-n', dest='SN', required=True,
+ help='Target device serial no.')
+args = parser.parse_args()
+TEST_PARAMS = args.TEST_PARAMS
+SN = args.SN
+
+if os.path.exists(OUTPUT):
+ suffix = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
+ shutil.move(OUTPUT, '%s_%s' % (OUTPUT, suffix))
+os.makedirs(OUTPUT)
+
+# Setup logger.
+# There might be an issue in lava/local dispatcher, most likely problem of
+# pexpect. It prints the messages from print() last, not by sequence.
+# Use logging and subprocess.call() to work around this.
+logger = logging.getLogger('CTS')
+logger.setLevel(logging.DEBUG)
+ch = logging.StreamHandler()
+ch.setLevel(logging.DEBUG)
+formatter = logging.Formatter('%(asctime)s - %(name)s: %(levelname)s: %(message)s')
+ch.setFormatter(formatter)
+logger.addHandler(ch)
+
+cts_stdout = open(CTS_STDOUT, 'w')
+cts_logcat_out = open(CTS_LOGCAT, 'w')
+cts_logcat = subprocess.Popen(['adb', 'logcat'], stdout=cts_logcat_out)
+
+logger.info('Test params: %s' % TEST_PARAMS)
+logger.info('Starting CTS test...')
+
+child = pexpect.spawn('android-cts/tools/cts-tradefed', logfile=cts_stdout)
+try:
+ child.expect('cts-tf >', timeout=60)
+ child.sendline(TEST_PARAMS)
+except pexpect.TIMEOUT:
+ result = 'lunch-cts-rf-shell fail'
+ py_test_lib.add_result(RESULT_FILE, result)
+
+while child.isalive():
+ subprocess.call('echo')
+ subprocess.call(['echo', '--- line break ---'])
+ logger.info('Checking adb connectivity...')
+ adb_command = "adb -s %s shell echo OK" % SN
+ adb_check = subprocess.Popen(shlex.split(adb_command))
+ if adb_check.wait() != 0:
+ subprocess.call(['adb', 'devices'])
+ logger.error('Terminating CTS test as adb connection is lost!')
+ child.terminate(force=True)
+ result = 'check-adb-connectivity fail'
+ py_test_lib.add_result(RESULT_FILE, result)
+ break
+ else:
+ logger.info('adb device is alive')
+
+ try:
+ # Check if all tests finished every minute.
+ child.expect('I/ResultReporter: Full Result:', timeout=60)
+ # Once all tests finshed, exit from tf shell and throw EOF.
+ child.sendline('exit')
+ child.expect(pexpect.EOF, timeout=60)
+ except pexpect.TIMEOUT:
+ logger.info('Printing cts recent output...')
+ subprocess.call(['tail', CTS_STDOUT])
+
+logger.info('CTS test finished')
+cts_logcat.kill()
+cts_logcat_out.close()
+cts_stdout.close()
+
+# Locate and parse test result.
+result_dir = 'android-cts/results'
+test_result = 'test_result.xml'
+if os.path.exists(result_dir) and os.path.isdir(result_dir):
+ for root, dirs, files in os.walk(result_dir):
+ for name in files:
+ if name == test_result:
+ result_parser(xml_file=os.path.join(root, name))
diff --git a/automated/android/cts/cts.sh b/automated/android/cts/cts.sh
new file mode 100755
index 0000000..86b4dd2
--- /dev/null
+++ b/automated/android/cts/cts.sh
@@ -0,0 +1,84 @@
+#!/bin/sh -x
+
+# shellcheck disable=SC1091
+. ../../lib/sh-test-lib
+# shellcheck disable=SC1091
+. ../../lib/android-test-lib
+
+SKIP_INSTALL="false"
+TIMEOUT="300"
+JDK="openjdk-8-jdk-headless"
+PKG_DEPS="curl wget zip xz-utils python-lxml python-setuptools python-pexpect aapt android-tools-adb lib32z1-dev libc6-dev-i386 lib32gcc1 libc6:i386 libstdc++6:i386 libgcc1:i386 zlib1g:i386 libncurses5:i386"
+CTS_URL="http://testdata.validation.linaro.org/cts/android-cts-7.1_r1.zip"
+TEST_PARAMS="run cts -m CtsBionicTestCases --abi arm64-v8a --disable-reboot --skip-preconditions --skip-device-info"
+RESULT_FILE="$(pwd)/output/result.txt"
+export RESULT_FILE
+
+usage() {
+ echo "Usage: $0 [-s <true|false>] [-o timeout] [-n serialno] [-d jdk-version] [-c cts_url] [-t test_params]" 1>&2
+ exit 1
+}
+
+while getopts ':s:o:n:d:c:t:' opt; do
+ case "${opt}" in
+ s) SKIP_INSTALL="${OPTARG}" ;;
+ o) TIMEOUT="${OPTARG}" ;;
+ n) SN="${OPTARG}" ;;
+ d) JDK="${OPTARG}" ;;
+ c) CTS_URL="${OPTARG}" ;;
+ t) TEST_PARAMS="${OPTARG}" ;;
+ *) usage ;;
+ esac
+done
+
+if [ "${SKIP_INSTALL}" = "true" ] || [ "${SKIP_INSTALL}" = "True" ]; then
+ info_msg "Package installation skipped"
+else
+ dist_name
+ dist_info
+ # shellcheck disable=SC2154
+ case "${dist}" in
+ debian)
+ dpkg --add-architecture i386
+ dist_info
+ echo "deb [arch=amd64,i386] http://ftp.us.debian.org/debian ${Codename} main non-free contrib" > /etc/apt/sources.list.d/cts.list
+ echo "deb http://ftp.debian.org/debian ${Codename}-backports main" >> /etc/apt/sources.list.d/cts.list
+ cat /etc/apt/sources.list.d/cts.list
+ apt-get update || true
+ install_deps "${JDK}" || install_deps "-t ${Codename}-backports ${JDK}"
+ install_deps "${PKG_DEPS}"
+ ;;
+ *)
+ install_deps "${PKG_DEPS} ${JDK}"
+ ;;
+ esac
+fi
+
+initialize_adb
+wait_boot_completed "${TIMEOUT}"
+wait_homescreen "${TIMEOUT}"
+
+# Increase the heap size. KVM devices in LAVA default to ~250M of heap
+export _JAVA_OPTIONS="-Xmx350M"
+java -version
+
+# Download CTS test package or copy it from local disk.
+if echo "${CTS_URL}" | grep "^http" ; then
+ wget -S --progress=dot:giga "${CTS_URL}"
+else
+ cp "${CTS_URL}" ./
+fi
+file_name=$(basename "${CTS_URL}")
+unzip -q "${file_name}"
+rm -f "${file_name}"
+
+if [ -d android-cts/results ]; then
+ mv android-cts/results "android-cts/results_$(date +%Y%m%d%H%M%S)"
+fi
+
+# Run CTS test.
+./cts-runner.py -t "${TEST_PARAMS}" -n "${SN}"
+check_return "cts-test-run"
+
+# Cleanup.
+rm -f /etc/apt/sources.list.d/cts.list
diff --git a/automated/android/cts/cts.yaml b/automated/android/cts/cts.yaml
new file mode 100644
index 0000000..74e65cf
--- /dev/null
+++ b/automated/android/cts/cts.yaml
@@ -0,0 +1,41 @@
+metadata:
+ name: cts
+ format: "Lava-Test-Shell Test Definition 1.0"
+ description: "Run CTS on Linaro android."
+ maintainer:
+ - milosz.wasilewski@linaro.org
+ - chase.qi@linaro.org
+ os:
+ - debian
+ - ubuntu
+ devices:
+ - lxc
+ scope:
+ - functional
+
+params:
+ SKIP_INSTALL: "false"
+ # Specify timeout in seconds for wait_boot_completed and wait_homescreen.
+ TIMEOUT: "300"
+ # Specify adb device SN if more then one device connected.
+ SN: ""
+ JDK: "openjdk-8-jdk-headless"
+ # Download CTS package or copy it from local disk.
+ # CTS_URL: "/root/android-cts/linaro/7.1_r1/android-cts-7.1_r1.zip"
+ CTS_URL: "http://testdata.validation.linaro.org/cts/android-cts-7.1_r1.zip"
+ TEST_PARAMS: "run cts -m CtsBionicTestCases --abi arm64-v8a --disable-reboot --skip-preconditions --skip-device-info"
+ # Specify url and token for file uploading.
+ URL: "https://archive.validation.linaro.org/artifacts/team/qa/"
+ TOKEN: "4373c97b474497dbd12373689d7d492e"
+run:
+ steps:
+ - cd ./automated/android/cts
+ - ./cts.sh -s "${SKIP_INSTALL}" -o "${TIMEOUT}" -n "${SN}" -d "${JDK}" -c "${CTS_URL}" -t "${TEST_PARAMS}"
+ # Upload test log and result files to artifactorial.
+ - cp -r ./android-cts/results ./output/ || true
+ - cp -r ./android-cts/logs ./output/ || true
+ - tar caf cts-output-$(date +%Y%m%d%H%M%S).tar.xz ./output
+ - ATTACHMENT=$(ls cts-output-*.tar.xz)
+ - ../../utils/upload-to-artifactorial.sh -a "${ATTACHMENT}" -u "${URL}" -t "${TOKEN}"
+ # Send test result to LAVA.
+ - ../../utils/send-to-lava.sh ./output/result.txt
diff --git a/automated/lib/android-test-lib b/automated/lib/android-test-lib
index 034b153..b66d0da 100755
--- a/automated/lib/android-test-lib
+++ b/automated/lib/android-test-lib
@@ -1,27 +1,71 @@
#!/bin/sh
initialize_adb() {
+ adb devices
if [ -z "${SN}" ]; then
- local number="$(adb devices | grep -wc 'device')"
+ number="$(adb devices | grep -wc 'device')"
if [ "${number}" -gt 1 ]; then
warn_msg "Device not specified; define SN or use '-s'"
error_msg "More than one device or emulator found"
elif [ "${number}" -eq 1 ]; then
- SN="$(adb devices | grep -w 'device' | awk '{print $1}')"
+ SN="$(adb get-serialno)"
export SN
else
error_msg "Device NOT found"
fi
fi
- adb -s "${SN}" shell ls / > /dev/null 2>&1
- if [ $? -eq 0 ]; then
+ if adb -s "${SN}" shell echo "Testing adb connectivity"; then
info_msg "Connected to device ${SN} successfully"
else
error_msg "Unable to connect to device ${SN}"
fi
}
+wait_boot_completed() {
+ timeout="$1"
+ [ "$#" -ne 1 ] && error_msg "Usage: wait_for_boot_completed timeout_in_seconds"
+ end=$(( $(date +%s) + timeout ))
+
+ boot_completed=false
+ while [ "$(date +%s)" -lt "$end" ]; do
+ if adb -s "${SN}" shell getprop sys.boot_completed | grep "1"; then
+ boot_completed=true
+ break
+ else
+ sleep 3
+ fi
+ done
+
+ if "${boot_completed}"; then
+ info_msg "Target booted up completely."
+ else
+ error_msg "wait_boot_completed timed out after ${timeout} seconds"
+ fi
+}
+
+wait_homescreen() {
+ timeout="$1"
+ [ "$#" -ne 1 ] && error_msg "Usage: wait_homescreen timeout_in_seconds"
+ end=$(( $(date +%s) + timeout ))
+
+ homescreen_displayed=false
+ while [ "$(date +%s)" -lt "$end" ]; do
+ if adb -s "${SN}" logcat -sd ActivityManager:I | grep "Displayed com.android.launcher"; then
+ homescreen_displayed=true
+ break
+ else
+ sleep 3
+ fi
+ done
+
+ if "${homescreen_displayed}"; then
+ info_msg "Target booted to homescreen successfully."
+ else
+ error_msg "wait_homescreen timed out after ${timeout} seconds"
+ fi
+}
+
detect_abi() {
# "| tr -d '\r'" is needed here, refer to the below issue.
# https://code.google.com/p/android/issues/detail?id=2482
@@ -35,8 +79,8 @@ detect_abi() {
}
install() {
- local file_path="$1"
- local file_name="$(basename "${file_path}")"
+ file_path="$1"
+ file_name="$(basename "${file_path}")"
if adb -s "${SN}" shell mount | grep system | grep -q ro; then
# Remounts the /system partition on the device read-write
@@ -50,8 +94,8 @@ install() {
}
pull_output() {
- local device_output="$1"
- local host_output="$2"
+ device_output="$1"
+ host_output="$2"
info_msg "Pulling output from devcie ${SN}"
adb -s "${SN}" pull "${device_output}" "${host_output}"
diff --git a/automated/lib/py_test_lib.py b/automated/lib/py_test_lib.py
new file mode 100755
index 0000000..e8932bf
--- /dev/null
+++ b/automated/lib/py_test_lib.py
@@ -0,0 +1,4 @@
+def add_result(result_file, result):
+ print(result)
+ with open(result_file, 'a') as f:
+ f.write('%s\n' % result)