From 7d317f8b0be720c10394d94283c16583dad34b06 Mon Sep 17 00:00:00 2001 From: Chase Qi Date: Wed, 8 Feb 2017 08:29:48 +0800 Subject: automated: android: add cts test Change-Id: I75eb65bc7de03cbe612b33c4ee69d50cfdc6608d Signed-off-by: Chase Qi --- automated/android/cts/cts-runner.py | 139 ++++++++++++++++++++++++++++++++++++ automated/android/cts/cts.sh | 84 ++++++++++++++++++++++ automated/android/cts/cts.yaml | 41 +++++++++++ automated/lib/android-test-lib | 60 +++++++++++++--- automated/lib/py_test_lib.py | 4 ++ 5 files changed, 320 insertions(+), 8 deletions(-) create mode 100755 automated/android/cts/cts-runner.py create mode 100755 automated/android/cts/cts.sh create mode 100644 automated/android/cts/cts.yaml create mode 100755 automated/lib/py_test_lib.py (limited to 'automated') diff --git a/automated/android/cts/cts-runner.py b/automated/android/cts/cts-runner.py new file mode 100755 index 0000000..5bd7777 --- /dev/null +++ b/automated/android/cts/cts-runner.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python + +import datetime +import os +import sys +import shlex +import shutil +import subprocess +import xml.etree.ElementTree as ET +import pexpect +import argparse +import logging + +sys.path.insert(0, '../../lib/') +import py_test_lib # nopep8 + + +def result_parser(xml_file): + try: + tree = ET.parse(xml_file) + except ET.ParseError as e: + logger.error('xml.etree.ElementTree.ParseError: %s' % e) + logger.info('Please Check %s manually' % xml_file) + sys.exit(1) + root = tree.getroot() + logger.info('Test modules in %s: %s' + % (xml_file, str(len(root.findall('Module'))))) + for elem in root.findall('Module'): + # Naming: Module Name + Test Case Name + Test Name + if 'abi' in elem.attrib.keys(): + module_name = '.'.join([elem.attrib['abi'], elem.attrib['name']]) + else: + module_name = elem.attrib['name'] + + tests_executed = len(elem.findall('.//Test')) + tests_passed = len(elem.findall('.//Test[@result="pass"]')) + tests_failed = len(elem.findall('.//Test[@result="fail"]')) + + result = '%s_executed pass %s' % (module_name, str(tests_executed)) + py_test_lib.add_result(RESULT_FILE, result) + + result = '%s_passed pass %s' % (module_name, str(tests_passed)) + py_test_lib.add_result(RESULT_FILE, result) + + failed_result = 'pass' + if tests_failed > 0: + failed_result = 'fail' + result = '%s_failed %s %s' % (module_name, failed_result, + str(tests_failed)) + py_test_lib.add_result(RESULT_FILE, result) + + +OUTPUT = '%s/output' % os.getcwd() +RESULT_FILE = '%s/result.txt' % OUTPUT +CTS_STDOUT = '%s/cts-stdout.txt' % OUTPUT +CTS_LOGCAT = '%s/cts-logcat.txt' % OUTPUT +TEST_PARAMS = '' +SN = '' + +parser = argparse.ArgumentParser() +parser.add_argument('-t', dest='TEST_PARAMS', required=True, + help="cts test parameters") +parser.add_argument('-n', dest='SN', required=True, + help='Target device serial no.') +args = parser.parse_args() +TEST_PARAMS = args.TEST_PARAMS +SN = args.SN + +if os.path.exists(OUTPUT): + suffix = datetime.datetime.now().strftime('%Y%m%d%H%M%S') + shutil.move(OUTPUT, '%s_%s' % (OUTPUT, suffix)) +os.makedirs(OUTPUT) + +# Setup logger. +# There might be an issue in lava/local dispatcher, most likely problem of +# pexpect. It prints the messages from print() last, not by sequence. +# Use logging and subprocess.call() to work around this. +logger = logging.getLogger('CTS') +logger.setLevel(logging.DEBUG) +ch = logging.StreamHandler() +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s - %(name)s: %(levelname)s: %(message)s') +ch.setFormatter(formatter) +logger.addHandler(ch) + +cts_stdout = open(CTS_STDOUT, 'w') +cts_logcat_out = open(CTS_LOGCAT, 'w') +cts_logcat = subprocess.Popen(['adb', 'logcat'], stdout=cts_logcat_out) + +logger.info('Test params: %s' % TEST_PARAMS) +logger.info('Starting CTS test...') + +child = pexpect.spawn('android-cts/tools/cts-tradefed', logfile=cts_stdout) +try: + child.expect('cts-tf >', timeout=60) + child.sendline(TEST_PARAMS) +except pexpect.TIMEOUT: + result = 'lunch-cts-rf-shell fail' + py_test_lib.add_result(RESULT_FILE, result) + +while child.isalive(): + subprocess.call('echo') + subprocess.call(['echo', '--- line break ---']) + logger.info('Checking adb connectivity...') + adb_command = "adb -s %s shell echo OK" % SN + adb_check = subprocess.Popen(shlex.split(adb_command)) + if adb_check.wait() != 0: + subprocess.call(['adb', 'devices']) + logger.error('Terminating CTS test as adb connection is lost!') + child.terminate(force=True) + result = 'check-adb-connectivity fail' + py_test_lib.add_result(RESULT_FILE, result) + break + else: + logger.info('adb device is alive') + + try: + # Check if all tests finished every minute. + child.expect('I/ResultReporter: Full Result:', timeout=60) + # Once all tests finshed, exit from tf shell and throw EOF. + child.sendline('exit') + child.expect(pexpect.EOF, timeout=60) + except pexpect.TIMEOUT: + logger.info('Printing cts recent output...') + subprocess.call(['tail', CTS_STDOUT]) + +logger.info('CTS test finished') +cts_logcat.kill() +cts_logcat_out.close() +cts_stdout.close() + +# Locate and parse test result. +result_dir = 'android-cts/results' +test_result = 'test_result.xml' +if os.path.exists(result_dir) and os.path.isdir(result_dir): + for root, dirs, files in os.walk(result_dir): + for name in files: + if name == test_result: + result_parser(xml_file=os.path.join(root, name)) diff --git a/automated/android/cts/cts.sh b/automated/android/cts/cts.sh new file mode 100755 index 0000000..86b4dd2 --- /dev/null +++ b/automated/android/cts/cts.sh @@ -0,0 +1,84 @@ +#!/bin/sh -x + +# shellcheck disable=SC1091 +. ../../lib/sh-test-lib +# shellcheck disable=SC1091 +. ../../lib/android-test-lib + +SKIP_INSTALL="false" +TIMEOUT="300" +JDK="openjdk-8-jdk-headless" +PKG_DEPS="curl wget zip xz-utils python-lxml python-setuptools python-pexpect aapt android-tools-adb lib32z1-dev libc6-dev-i386 lib32gcc1 libc6:i386 libstdc++6:i386 libgcc1:i386 zlib1g:i386 libncurses5:i386" +CTS_URL="http://testdata.validation.linaro.org/cts/android-cts-7.1_r1.zip" +TEST_PARAMS="run cts -m CtsBionicTestCases --abi arm64-v8a --disable-reboot --skip-preconditions --skip-device-info" +RESULT_FILE="$(pwd)/output/result.txt" +export RESULT_FILE + +usage() { + echo "Usage: $0 [-s ] [-o timeout] [-n serialno] [-d jdk-version] [-c cts_url] [-t test_params]" 1>&2 + exit 1 +} + +while getopts ':s:o:n:d:c:t:' opt; do + case "${opt}" in + s) SKIP_INSTALL="${OPTARG}" ;; + o) TIMEOUT="${OPTARG}" ;; + n) SN="${OPTARG}" ;; + d) JDK="${OPTARG}" ;; + c) CTS_URL="${OPTARG}" ;; + t) TEST_PARAMS="${OPTARG}" ;; + *) usage ;; + esac +done + +if [ "${SKIP_INSTALL}" = "true" ] || [ "${SKIP_INSTALL}" = "True" ]; then + info_msg "Package installation skipped" +else + dist_name + dist_info + # shellcheck disable=SC2154 + case "${dist}" in + debian) + dpkg --add-architecture i386 + dist_info + echo "deb [arch=amd64,i386] http://ftp.us.debian.org/debian ${Codename} main non-free contrib" > /etc/apt/sources.list.d/cts.list + echo "deb http://ftp.debian.org/debian ${Codename}-backports main" >> /etc/apt/sources.list.d/cts.list + cat /etc/apt/sources.list.d/cts.list + apt-get update || true + install_deps "${JDK}" || install_deps "-t ${Codename}-backports ${JDK}" + install_deps "${PKG_DEPS}" + ;; + *) + install_deps "${PKG_DEPS} ${JDK}" + ;; + esac +fi + +initialize_adb +wait_boot_completed "${TIMEOUT}" +wait_homescreen "${TIMEOUT}" + +# Increase the heap size. KVM devices in LAVA default to ~250M of heap +export _JAVA_OPTIONS="-Xmx350M" +java -version + +# Download CTS test package or copy it from local disk. +if echo "${CTS_URL}" | grep "^http" ; then + wget -S --progress=dot:giga "${CTS_URL}" +else + cp "${CTS_URL}" ./ +fi +file_name=$(basename "${CTS_URL}") +unzip -q "${file_name}" +rm -f "${file_name}" + +if [ -d android-cts/results ]; then + mv android-cts/results "android-cts/results_$(date +%Y%m%d%H%M%S)" +fi + +# Run CTS test. +./cts-runner.py -t "${TEST_PARAMS}" -n "${SN}" +check_return "cts-test-run" + +# Cleanup. +rm -f /etc/apt/sources.list.d/cts.list diff --git a/automated/android/cts/cts.yaml b/automated/android/cts/cts.yaml new file mode 100644 index 0000000..74e65cf --- /dev/null +++ b/automated/android/cts/cts.yaml @@ -0,0 +1,41 @@ +metadata: + name: cts + format: "Lava-Test-Shell Test Definition 1.0" + description: "Run CTS on Linaro android." + maintainer: + - milosz.wasilewski@linaro.org + - chase.qi@linaro.org + os: + - debian + - ubuntu + devices: + - lxc + scope: + - functional + +params: + SKIP_INSTALL: "false" + # Specify timeout in seconds for wait_boot_completed and wait_homescreen. + TIMEOUT: "300" + # Specify adb device SN if more then one device connected. + SN: "" + JDK: "openjdk-8-jdk-headless" + # Download CTS package or copy it from local disk. + # CTS_URL: "/root/android-cts/linaro/7.1_r1/android-cts-7.1_r1.zip" + CTS_URL: "http://testdata.validation.linaro.org/cts/android-cts-7.1_r1.zip" + TEST_PARAMS: "run cts -m CtsBionicTestCases --abi arm64-v8a --disable-reboot --skip-preconditions --skip-device-info" + # Specify url and token for file uploading. + URL: "https://archive.validation.linaro.org/artifacts/team/qa/" + TOKEN: "4373c97b474497dbd12373689d7d492e" +run: + steps: + - cd ./automated/android/cts + - ./cts.sh -s "${SKIP_INSTALL}" -o "${TIMEOUT}" -n "${SN}" -d "${JDK}" -c "${CTS_URL}" -t "${TEST_PARAMS}" + # Upload test log and result files to artifactorial. + - cp -r ./android-cts/results ./output/ || true + - cp -r ./android-cts/logs ./output/ || true + - tar caf cts-output-$(date +%Y%m%d%H%M%S).tar.xz ./output + - ATTACHMENT=$(ls cts-output-*.tar.xz) + - ../../utils/upload-to-artifactorial.sh -a "${ATTACHMENT}" -u "${URL}" -t "${TOKEN}" + # Send test result to LAVA. + - ../../utils/send-to-lava.sh ./output/result.txt diff --git a/automated/lib/android-test-lib b/automated/lib/android-test-lib index 034b153..b66d0da 100755 --- a/automated/lib/android-test-lib +++ b/automated/lib/android-test-lib @@ -1,27 +1,71 @@ #!/bin/sh initialize_adb() { + adb devices if [ -z "${SN}" ]; then - local number="$(adb devices | grep -wc 'device')" + number="$(adb devices | grep -wc 'device')" if [ "${number}" -gt 1 ]; then warn_msg "Device not specified; define SN or use '-s'" error_msg "More than one device or emulator found" elif [ "${number}" -eq 1 ]; then - SN="$(adb devices | grep -w 'device' | awk '{print $1}')" + SN="$(adb get-serialno)" export SN else error_msg "Device NOT found" fi fi - adb -s "${SN}" shell ls / > /dev/null 2>&1 - if [ $? -eq 0 ]; then + if adb -s "${SN}" shell echo "Testing adb connectivity"; then info_msg "Connected to device ${SN} successfully" else error_msg "Unable to connect to device ${SN}" fi } +wait_boot_completed() { + timeout="$1" + [ "$#" -ne 1 ] && error_msg "Usage: wait_for_boot_completed timeout_in_seconds" + end=$(( $(date +%s) + timeout )) + + boot_completed=false + while [ "$(date +%s)" -lt "$end" ]; do + if adb -s "${SN}" shell getprop sys.boot_completed | grep "1"; then + boot_completed=true + break + else + sleep 3 + fi + done + + if "${boot_completed}"; then + info_msg "Target booted up completely." + else + error_msg "wait_boot_completed timed out after ${timeout} seconds" + fi +} + +wait_homescreen() { + timeout="$1" + [ "$#" -ne 1 ] && error_msg "Usage: wait_homescreen timeout_in_seconds" + end=$(( $(date +%s) + timeout )) + + homescreen_displayed=false + while [ "$(date +%s)" -lt "$end" ]; do + if adb -s "${SN}" logcat -sd ActivityManager:I | grep "Displayed com.android.launcher"; then + homescreen_displayed=true + break + else + sleep 3 + fi + done + + if "${homescreen_displayed}"; then + info_msg "Target booted to homescreen successfully." + else + error_msg "wait_homescreen timed out after ${timeout} seconds" + fi +} + detect_abi() { # "| tr -d '\r'" is needed here, refer to the below issue. # https://code.google.com/p/android/issues/detail?id=2482 @@ -35,8 +79,8 @@ detect_abi() { } install() { - local file_path="$1" - local file_name="$(basename "${file_path}")" + file_path="$1" + file_name="$(basename "${file_path}")" if adb -s "${SN}" shell mount | grep system | grep -q ro; then # Remounts the /system partition on the device read-write @@ -50,8 +94,8 @@ install() { } pull_output() { - local device_output="$1" - local host_output="$2" + device_output="$1" + host_output="$2" info_msg "Pulling output from devcie ${SN}" adb -s "${SN}" pull "${device_output}" "${host_output}" diff --git a/automated/lib/py_test_lib.py b/automated/lib/py_test_lib.py new file mode 100755 index 0000000..e8932bf --- /dev/null +++ b/automated/lib/py_test_lib.py @@ -0,0 +1,4 @@ +def add_result(result_file, result): + print(result) + with open(result_file, 'a') as f: + f.write('%s\n' % result) -- cgit v1.2.3