osf-linux-microplatform: add functional and benchmark tests

Change-Id: I34dea149b0a5cf5d2e24fe73f580717790511a31
Signed-off-by: Chase Qi <chase.qi@linaro.org>

9 files changed, 643 insertions, 2 deletions

diff --git a/osf-linux-microplatform/builders.sh b/osf-linux-microplatform/builders.sh
index d1a07b17..032b6fe1 100755
--- a/osf-linux-microplatform/builders.sh
+++ b/osf-linux-microplatform/builders.sh
@@ -109,12 +109,19 @@ find ${DEPLOY_DIR_IMAGE} -type f | xargs md5sum > MD5SUMS.txt
 sed -i "s|${DEPLOY_DIR_IMAGE}/||" MD5SUMS.txt
 mv MD5SUMS.txt ${DEPLOY_DIR_IMAGE}
 
+BOOT_IMG="$(find ${DEPLOY_DIR_IMAGE} -type f -name "boot-*.img" | sort | xargs -r basename)"
+ROOTFS_IMG="$(find ${DEPLOY_DIR_IMAGE} -type f -name "sparse-lmp-gateway-image*.otaimg" | xargs -r basename)"
+BASE_URL="http://snapshots.linaro.org"
+
 cat << EOF > ${WORKSPACE}/post_build_lava_parameters
-DEPLOY_DIR_IMAGE=${DEPLOY_DIR_IMAGE}
+DEPLOY_DIR_IMAGE="${DEPLOY_DIR_IMAGE}"
+BUILD_URL="${BASE_URL}/${PUB_DEST}/"
+BOOT_URL="${BUILD_URL}/${BOOT_IMG}"
+SYSTEM_URL="${BUILD_URL}/${ROOTFS_IMG}"
 EOF
 
 cat << EOF > ${WORKSPACE}/ota_params
-BUILD_URL=http://snapshots.linaro.org/${PUB_DEST}/
+BUILD_URL=${BASE_URL}/${PUB_DEST}/
 EOF
 
 # Build information
diff --git a/osf-linux-microplatform/lava-job-definitions/devices/hi6220-hikey-r2 b/osf-linux-microplatform/lava-job-definitions/devices/hi6220-hikey-r2
new file mode 100644
index 00000000..58611b10
--- /dev/null
+++ b/osf-linux-microplatform/lava-job-definitions/devices/hi6220-hikey-r2
@@ -0,0 +1,84 @@
+{% extends "microplatform-fastboot.jinja2" %}
+
+{% block device_type %}hi6220-hikey-r2{% endblock %}
+
+{% block deploy_target %}
+- deploy:
+    timeout:
+      minutes: 30
+    to: fastboot
+    namespace: hikey
+    connection: lxc
+    images:
+      ptable:
+        url: http://images.validation.linaro.org/snapshots.linaro.org/96boards/reference-platform/components/uefi-staging/70/hikey/release/ptable-linux-8g.img
+        reboot: hard-reset
+      boot:
+        url: {{BOOT_URL}}
+        reboot: hard-reset
+      system:
+        url: {{SYSTEM_URL}}
+    os: oe
+    protocols:
+      lava-lxc:
+      - action: fastboot-deploy
+        request: pre-power-command
+        timeout:
+          minutes: 2
+{% endblock deploy_target %}
+
+{% block boot_target %}
+- boot:
+    namespace: hikey
+    prompts:
+      # login as root
+      - 'root@hikey:'
+    auto_login:
+      login_prompt: 'hikey login:'
+      username: root
+    timeout:
+      minutes: 5
+    method: grub
+    commands: installed
+    transfer_overlay:
+      download_command: cd /tmp ; wget
+      unpack_command: tar -C / -xzf
+    protocols:
+      lava-lxc:
+      - action: grub-sequence-action
+        request: pre-os-command
+        timeout:
+          minutes: 2
+{% endblock boot_target %}
+
+{% block test_target %}
+- test:
+    namespace: hikey
+    timeout:
+      minutes: 15
+    definitions:
+    - from: inline
+      repository:
+        metadata:
+          format: Lava-Test Test Definition 1.0
+          name: prep-tests
+          description: Device preparation
+        run:
+          steps:
+          # the image does not raise the interface itself
+          # force a DHCP request to avoid avahi using a zero conf address.
+          - udhcpc
+          - netstat
+          - ip a
+          - ifconfig
+          - df
+          - cat /sys/class/thermal/thermal_zone0/policy || true
+          - echo "power_allocator" > /sys/class/thermal/thermal_zone0/policy || true
+          - cat /sys/class/thermal/thermal_zone0/policy || true
+          # limit max freq to avoid overheating.
+          - cat /sys/devices/system/cpu/cpufreq/policy0/scaling_available_frequencies  |
+            awk '{ print $(NF-1) }'  > /sys/devices/system/cpu/cpufreq/policy0/scaling_max_freq
+          - cat /sys/devices/system/cpu/cpufreq/policy0/scaling_max_freq || true
+      name: prep-inline
+      path: inline/prep.yaml
+{% endblock test_target %}
diff --git a/osf-linux-microplatform/lava-job-definitions/master.jinja2 b/osf-linux-microplatform/lava-job-definitions/master.jinja2
new file mode 100644
index 00000000..4a6d955c
--- /dev/null
+++ b/osf-linux-microplatform/lava-job-definitions/master.jinja2
@@ -0,0 +1,16 @@
+{% block global_settings %}
+{% endblock global_settings %}
+
+device_type: {% block device_type %}{% endblock %}
+job_name: {% block job_name %}{% endblock %}
+priority: {% block priority %}{% endblock %}
+visibility: {% block visibility %}{% endblock %}
+
+{% block metadata %}
+{% endblock metadata %}
+
+{% block protocols %}
+{% endblock protocols %}
+
+{% block actions %}
+{% endblock actions %}
diff --git a/osf-linux-microplatform/lava-job-definitions/microplatform-fastboot.jinja2 b/osf-linux-microplatform/lava-job-definitions/microplatform-fastboot.jinja2
new file mode 100644
index 00000000..0924bab0
--- /dev/null
+++ b/osf-linux-microplatform/lava-job-definitions/microplatform-fastboot.jinja2
@@ -0,0 +1,51 @@
+{% extends "microplatform.jinja2" %}
+
+{% block global_settings %}
+{{ super() }}
+context:
+  test_character_delay: 10
+{% endblock global_settings %}
+
+{% block protocols %}
+protocols:
+  lava-lxc:
+    name: lxc-hikey-test
+    template: debian
+    distribution: debian
+    release: stretch
+    mirror: http://cdn-fastly.deb.debian.org/debian
+{% endblock protocols %}
+
+{% block actions %}
+actions:
+- deploy:
+    namespace: tlxc
+    timeout:
+      minutes: 5
+    to: lxc
+    packages:
+      - fastboot
+      - sshpass
+    os: debian
+
+
+
+- boot:
+    namespace: tlxc
+    prompts:
+    - 'root@(.*):/#'
+    timeout:
+      minutes: 5
+    method: lxc
+
+
+{% block deploy_target %}
+{% endblock deploy_target %}
+
+{% block boot_target %}
+{% endblock boot_target %}
+
+{% block test_target %}
+{% endblock test_target %}
+
+{% endblock actions %}
diff --git a/osf-linux-microplatform/lava-job-definitions/microplatform.jinja2 b/osf-linux-microplatform/lava-job-definitions/microplatform.jinja2
new file mode 100644
index 00000000..896d7c95
--- /dev/null
+++ b/osf-linux-microplatform/lava-job-definitions/microplatform.jinja2
@@ -0,0 +1,22 @@
+{% extends "master.jinja2" %}
+{% block global_settings %}
+timeouts:
+  job:
+    minutes: {% if job_timeout is defined %}{{ job_timeout }}{% else %}90{% endif %}
+  action:
+    minutes: 10
+  connection:
+    minutes: 2
+
+tags:
+  - static-ip
+{% endblock global_settings %}
+
+{% block priority %}{{LAVA_JOB_PRIORITY}}{% endblock priority %}
+{% block visibility %}personal{% endblock visibility %}
+
+{% block metadata %}
+metadata:
+  series: mircroplatform
+  build-url: {{BUILD_URL}}
+{% endblock metadata %}
diff --git a/osf-linux-microplatform/lava-job-definitions/testplan/benchmark.yaml b/osf-linux-microplatform/lava-job-definitions/testplan/benchmark.yaml
new file mode 100644
index 00000000..6aeb96c4
--- /dev/null
+++ b/osf-linux-microplatform/lava-job-definitions/testplan/benchmark.yaml
@@ -0,0 +1,82 @@
+{% extends device_type %}
+
+{% set job_timeout = 180 %}
+
+{% block metadata %}
+  {{ super() }}
+  test scope: performance
+{% endblock metadata %}
+
+{% block job_name %}microplatform-{{IMAGES}}-build-{{BUILD_NUMBER}}-benchmark{% endblock job_name %}
+
+{% block test_target %}
+  {{ super() }}
+
+
+- test:
+    namespace: tlxc
+    timeout:
+      minutes: 120
+    definitions:
+    - repository: https://git.linaro.org/qa/test-definitions.git
+      from: git
+      path: automated/linux/dockerized-tests/over-ssh.yaml
+      name: linpack
+      parameters:
+        TEST: automated/linux/linpack/linpack.yaml
+        SSH_USER: "osf"
+        SSH_PASSWD: "osf"
+      timeout:
+        # The first test needs time to pull and extract docker image.
+        minutes: 30
+    - repository: https://git.linaro.org/qa/test-definitions.git
+      from: git
+      path: automated/linux/dockerized-tests/over-ssh.yaml
+      name: lmbench-memory
+      parameters:
+        TEST: automated/linux/lmbench/lmbench-memory.yaml
+        SSH_USER: "osf"
+        SSH_PASSWD: "osf"
+      timeout:
+        minutes: 10
+    - repository: https://git.linaro.org/qa/test-definitions.git
+      from: git
+      path: automated/linux/dockerized-tests/over-ssh.yaml
+      name: stream-uniprocessor
+      parameters:
+        TEST: automated/linux/stream/stream-uniprocessor.yaml
+        SSH_USER: "osf"
+        SSH_PASSWD: "osf"
+      timeout:
+        minutes: 10
+    - repository: https://git.linaro.org/qa/test-definitions.git
+      from: git
+      path: automated/linux/dockerized-tests/over-ssh.yaml
+      name: iperf
+      parameters:
+        TEST: automated/linux/iperf/iperf.yaml
+        SSH_USER: "osf"
+        SSH_PASSWD: "osf"
+      timeout:
+        minutes: 10
+    - repository: https://git.linaro.org/qa/test-definitions.git
+      from: git
+      path: automated/linux/dockerized-tests/over-ssh.yaml
+      name: hackbench
+      parameters:
+        TEST: automated/linux/hackbench/hackbench.yaml
+        SSH_USER: "osf"
+        SSH_PASSWD: "osf"
+      timeout:
+        minutes: 60
+    - repository: https://git.linaro.org/qa/test-definitions.git
+      from: git
+      path: automated/linux/dockerized-tests/over-ssh.yaml
+      name: unixbench
+      parameters:
+        TEST: automated/linux/unixbench/unixbench.yaml
+        SSH_USER: "osf"
+        SSH_PASSWD: "osf"
+      timeout:
+        minutes: 60
+{% endblock test_target %}
diff --git a/osf-linux-microplatform/lava-job-definitions/testplan/functional.yaml b/osf-linux-microplatform/lava-job-definitions/testplan/functional.yaml
new file mode 100644
index 00000000..bb79d928
--- /dev/null
+++ b/osf-linux-microplatform/lava-job-definitions/testplan/functional.yaml
@@ -0,0 +1,63 @@
+{% extends device_type %}
+
+{% set job_timeout = 90 %}
+
+{% block metadata %}
+  {{ super() }}
+  test scope: functional
+{% endblock metadata %}
+
+{% block job_name %}microplatform-{{IMAGES}}-build-{{BUILD_NUMBER}}-functional{% endblock job_name %}
+
+{% block test_target %}
+  {{ super() }}
+
+
+- test:
+    namespace: tlxc
+    timeout:
+      minutes: 60
+    definitions:
+    - repository: https://git.linaro.org/qa/test-definitions.git
+      from: git
+      path: automated/linux/dockerized-tests/over-ssh.yaml
+      name: smoke
+      parameters:
+        TEST: automated/linux/smoke/smoke.yaml
+        SSH_USER: "osf"
+        SSH_PASSWD: "osf"
+      timeout:
+        # The first test needs time to pull and extract docker image.
+        minutes: 30
+    - repository: https://git.linaro.org/qa/test-definitions.git
+      from: git
+      path: automated/linux/dockerized-tests/over-ssh.yaml
+      name: toolchain-smoke
+      parameters:
+        TEST: automated/linux/toolchain-smoke/toolchain-smoke.yaml
+        SSH_USER: "osf"
+        SSH_PASSWD: "osf"
+      timeout:
+        minutes: 10
+    - repository: https://git.linaro.org/qa/test-definitions.git
+      from: git
+      path: automated/linux/dockerized-tests/over-ssh.yaml
+      name: toolchain-smoke-static
+      parameters:
+        TEST: automated/linux/toolchain-smoke/toolchain-smoke.yaml
+        TESTDEF_PARAMS: "STATIC='True'"
+        SSH_USER: "osf"
+        SSH_PASSWD: "osf"
+      timeout:
+        minutes: 10
+    - repository: https://git.linaro.org/qa/test-definitions.git
+      from: git
+      path: automated/linux/dockerized-tests/over-ssh.yaml
+      name: pointer-tagging
+      parameters:
+        TEST: automated/linux/pointer-tagging/pointer-tagging-tests.yaml
+        SSH_USER: "osf"
+        SSH_PASSWD: "osf"
+      timeout:
+        minutes: 30
+{% endblock test_target %}
diff --git a/osf-linux-microplatform/submit_for_testing.py b/osf-linux-microplatform/submit_for_testing.py
new file mode 100644
index 00000000..59869424
--- /dev/null
+++ b/osf-linux-microplatform/submit_for_testing.py
@@ -0,0 +1,258 @@
+import argparse
+import os
+import requests
+import sys
+import StringIO
+from copy import deepcopy
+from string import Template
+from jinja2 import Environment, FileSystemLoader, StrictUndefined
+from ruamel.yaml import YAML
+
+
+try:
+    from urllib.parse import urlsplit
+except ImportError:
+    from urlparse import urlsplit
+
+
+# Templates base path
+template_base_path = 'configs/osf-linux-microplatform/lava-job-definitions'
+testplan_base_path = 'configs/osf-linux-microplatform/lava-job-definitions/'
+testplan_device_path = 'devices/'
+# Snapshots base URL
+snapshots_url = 'https://snapshots.linaro.org/openembedded/openembedded-osf-linux-microplatform'
+
+def parse_template(yaml_string):
+    '''
+        Round trip lava_job through ruamel to test parsing and
+        improve formatting. Comments are preserved.
+
+        In: yaml-formatted string
+        Out: validated yaml-formatted string
+    '''
+    yaml = YAML()
+    # ruamel does not provide a mechanism to dump to string, so use StringIO
+    # to catch it
+    output = StringIO.StringIO()
+    yaml.dump(yaml.load(yaml_string), output)
+    return output.getvalue()
+
+def _load_template(template_name, template_path, device_type):
+    template = ''
+    template_file_name = ''
+
+    if template_name:
+        template_file_name = "%s/%s/%s" % (template_path,
+                                           device_type,
+                                           template_name)
+        if os.path.exists(template_file_name):
+            with open(template_file_name, 'r') as f:
+                template = f.read()
+        else:
+            print('template (%s) was specified but not exists' %
+                  template_file_name)
+            sys.exit(1)
+
+    return template, template_file_name
+
+
+def _submit_to_squad(lava_job, lava_url_base, qa_server_api, qa_server_base, qa_token, quiet):
+    headers = {
+        "Auth-Token": qa_token
+    }
+
+    try:
+        data = {
+            "definition": lava_job,
+            "backend": urlsplit(lava_url_base).netloc  # qa-reports backends are named as lava instances
+        }
+        print("Submit to: %s" % qa_server_api)
+        results = requests.post(qa_server_api, data=data, headers=headers,
+                                timeout=31)
+        if results.status_code < 300:
+            print("%s/testjob/%s" % (qa_server_base, results.text))
+        else:
+            print(results.status_code)
+            print(results.text)
+    except requests.exceptions.RequestException as err:
+        print("QA Reports submission failed")
+        if not quiet:
+            print("offending job definition:")
+            print(lava_job)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--device-type",
+                        help="Device type in LAVA",
+                        dest="device_type",
+                        required=True)
+    parser.add_argument("--env-prefix",
+                        help="Prefix for the environment name",
+                        dest="env_prefix",
+                        default="")
+    parser.add_argument("--env-suffix",
+                        help="Suffix for the environment name",
+                        dest="env_suffix",
+                        default="")
+    parser.add_argument("--build-number",
+                        help="Build number",
+                        dest="build_number",
+                        required=True)
+    parser.add_argument("--qa-server-team",
+                        help="Team in QA Reports service",
+                        dest="qa_server_team",
+                        required=True)
+    parser.add_argument("--qa-server-project",
+                        help="Project in QA Reports service",
+                        dest="qa_server_project",
+                        required=True)
+    parser.add_argument("--qa-server",
+                        help="QA Reports server",
+                        dest="qa_server",
+                        default="https://qa-reports.linaro.org")
+    parser.add_argument("--qa-token",
+                        help="QA Reports token",
+                        dest="qa_token",
+                        default=os.environ.get('QA_REPORTS_TOKEN'))
+    parser.add_argument("--lava-server",
+                        help="LAVA server URL",
+                        dest="lava_server",
+                        required=True)
+    parser.add_argument("--git-commit",
+                        help="git commit ID",
+                        dest="git_commit",
+                        required=True)
+    parser.add_argument("--template-path",
+                        help="Path to LAVA job templates",
+                        dest="template_path",
+                        default=template_base_path)
+    parser.add_argument("--testplan-path",
+                        help="Path to Jinja2 LAVA job templates",
+                        dest="testplan_path",
+                        default=testplan_base_path)
+    parser.add_argument("--testplan-device-path",
+                        help="Relative path to Jinja2 device deployment fragments",
+                        dest="testplan_device_path",
+                        default=testplan_device_path)
+    parser.add_argument("--template-base-pre",
+                        help="base template used to construct templates, previous",
+                        dest="template_base_pre")
+    parser.add_argument("--template-base-post",
+                        help="base template used to construct templates, posterior",
+                        dest="template_base_post")
+    parser.add_argument("--template-names",
+                        help="list of the templates to submit for testing",
+                        dest="template_names",
+                        nargs="+",
+                        default=[])
+    parser.add_argument("--test-plan",
+                        help="""list of the Jinja2 templates to submit for testing.
+                        It is assumed that the templates produce valid LAVA job
+                        definitions. All varaibles are substituted using Jinja2
+                        engine. This includes environment variables.""",
+                        dest="test_plan",
+                        nargs="+",
+                        default=[])
+    parser.add_argument("--dry-run",
+                        help="""Prepare and write templates to tmp/.
+                        Don't submit to actual servers.""",
+                        action='store_true',
+                        dest="dryrun")
+    parser.add_argument("--quiet",
+                        help="Only output the final qa-reports URL",
+                        action='store_true',
+                        dest="quiet")
+
+    args, _ = parser.parse_known_args()
+
+    output_path = "tmp"
+    if args.dryrun:
+        if not os.path.exists(output_path):
+            os.mkdir(output_path)
+    if args.qa_token is None and not args.dryrun:
+        print("QA_REPORTS_TOKEN is missing")
+        sys.exit(1)
+
+    qa_server_base = args.qa_server
+    if not (qa_server_base.startswith("http://") or qa_server_base.startswith("https://")):
+        qa_server_base = "https://" + qa_server_base
+    qa_server_team = args.qa_server_team
+    qa_server_project = args.qa_server_project
+    qa_server_build = args.git_commit
+    qa_server_env = args.env_prefix + args.device_type + args.env_suffix
+    qa_server_api = "%s/api/submitjob/%s/%s/%s/%s" % (
+        qa_server_base,
+        qa_server_team,
+        qa_server_project,
+        qa_server_build,
+        qa_server_env)
+    lava_server = args.lava_server
+    if not (lava_server.startswith("http://") or lava_server.startswith("https://")):
+        lava_server = "https://" + lava_server
+    lava_url_base = "%s://%s/" % (urlsplit(lava_server).scheme, urlsplit(lava_server).netloc)
+
+    template_base_pre, _ = _load_template(args.template_base_pre,
+                                          args.template_path,
+                                          args.device_type)
+    template_base_post, _ = _load_template(args.template_base_post,
+                                           args.template_path,
+                                           args.device_type)
+    lava_jobs = []
+    for test in args.template_names:
+        test_template, template_file_name = _load_template(test,
+                                                           args.template_path,
+                                                           args.device_type)
+        if template_base_pre:
+            test_template = "%s\n%s" % (template_base_pre, test_template)
+        if template_base_post:
+            test_template = "%s\n%s" % (test_template, template_base_post)
+
+        template = Template(test_template)
+        print("using template: %s" % template_file_name)
+        lava_job = template.substitute(os.environ)
+        lava_job = parse_template(lava_job)
+        lava_jobs.append(lava_job)
+
+        if not args.quiet:
+            print(lava_job)
+        if args.dryrun:
+            testpath = os.path.join(output_path, args.device_type, test)
+            if not os.path.exists(os.path.dirname(testpath)):
+                os.makedirs(os.path.dirname(testpath))
+            with open(os.path.join(testpath), 'w') as f:
+                f.write(lava_job)
+
+    THIS_DIR = os.path.abspath(args.testplan_path)
+    # prevent creating templates when variables are missing
+    j2_env = Environment(loader=FileSystemLoader(THIS_DIR, followlinks=True), undefined=StrictUndefined)
+    context = deepcopy(os.environ)
+    context.update({"device_type": os.path.join(args.testplan_device_path, args.device_type)})
+    for test in args.test_plan:
+        ''' Prepare lava jobs '''
+        lava_job = j2_env.get_template(test).render(context)
+        lava_job = parse_template(lava_job)
+        lava_jobs.append(lava_job)
+
+        if not args.quiet:
+            print(lava_job)
+        if args.dryrun:
+            testpath = os.path.join(output_path, args.device_type, test)
+            if not os.path.exists(os.path.dirname(testpath)):
+                os.makedirs(os.path.dirname(testpath))
+            with open(os.path.join(testpath), 'w') as f:
+                f.write(lava_job)
+
+    for lava_job in lava_jobs:
+        ''' Submit lava jobs '''
+        if not args.dryrun:
+            _submit_to_squad(lava_job,
+                lava_url_base,
+                qa_server_api,
+                qa_server_base,
+                args.qa_token,
+                args.quiet)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/osf-linux-microplatform/submit_for_testing.sh b/osf-linux-microplatform/submit_for_testing.sh
new file mode 100755
index 00000000..bd592adc
--- /dev/null
+++ b/osf-linux-microplatform/submit_for_testing.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+
+set -ex
+
+[ -z "${DEVICE_TYPE}" ] && export DEVICE_TYPE="hi6220-hikey-r2"
+[ -z "${LAVA_JOB_PRIORITY}" ] && export LAVA_JOB_PRIORITY="25"
+[ -z "${SKIP_LAVA}" ] || unset DEVICE_TYPE
+[ -z "${QA_SERVER_TEAM}" ] && export QA_SERVER_TEAM="ledge"
+
+if [ -n "${DRY_RUN}" ]; then
+    export DRY_RUN="--dry-run --template-path lava-job-definitions --testplan-path lava-job-definitions/ --quiet"
+    export BASE_PATH=.
+else
+    export DRY_RUN=""
+    export BASE_PATH=configs/osf-linux-microplatform/
+fi
+
+if [ -z "${DEVICE_TYPE}" ]; then
+    echo "DEVICE_TYPE not set. Exiting"
+    exit 0
+fi
+
+if [ -z "${DRY_RUN}" ]; then
+    rm -rf configs
+
+    # Perform a shallow clone unless CONFIGS_REPO_REV_OVERRIDE is set
+    CONFIGS_REPO_CLONE_ARGS="--depth 1"
+    if [ -n ${CONFIGS_REPO_REV_OVERRIDE} ]; then
+        CONFIGS_REPO_CLONE_ARGS=""
+    fi
+
+    CONFIGS_REPO_URL=${CONFIGS_REPO_URL_OVERRIDE:-http://git.linaro.org/ci/job/configs.git}
+    git clone ${CONFIGS_REPO_CLONE_ARGS} ${CONFIGS_REPO_URL} configs
+    if [ -n ${CONFIGS_REPO_REV_OVERRIDE} ]; then
+        (cd configs && git checkout ${CONFIGS_REPO_REV_OVERRIDE})
+    fi
+fi
+
+export QA_BUILD_VERSION=${BUILD_NUMBER}
+
+[ -n ${FULL_TEST_TEMPLATES} ] && unset FULL_TEST_TEMPLATES
+TEMPLATE_PATH=""
+# Generate list of job templates for full test run
+for test in $(ls ${BASE_PATH}/lava-job-definitions/testplan/); do
+    FULL_TEST_TEMPLATES="${FULL_TEST_TEMPLATES} testplan/${test}"
+done
+
+# Submit full test run
+python2 ${BASE_PATH}/submit_for_testing.py \
+  --device-type ${DEVICE_TYPE} \
+  --build-number ${BUILD_NUMBER} \
+  --lava-server ${LAVA_SERVER} \
+  --qa-server ${QA_SERVER} \
+  --qa-server-team ${QA_SERVER_TEAM} \
+  --qa-server-project ${QA_SERVER_PROJECT} \
+  --git-commit ${QA_BUILD_VERSION} \
+  ${DRY_RUN} \
+  --test-plan ${FULL_TEST_TEMPLATES}