1. add support for submit jobs to multiple device or device types

2. refactor the lava related part in the main method

1 files changed, 269 insertions, 156 deletions

diff --git a/build-scripts/post-build-lava.py b/build-scripts/post-build-lava.py
index db0731c..a5d0876 100755
--- a/build-scripts/post-build-lava.py
+++ b/build-scripts/post-build-lava.py
@@ -54,6 +54,114 @@ TIMEOUT_SUFFIX = "_TIMEOUT"
 REBOOT_TOKEN = '[system-reboot]'
 
 
+class LAVADeviceBase(object):
+    """
+    Base class for defination of the device type and target in lava job
+    """
+
+    def __init__(self, name=None):
+        self.name = name
+
+
+class LAVADeviceType(LAVADeviceBase):
+    """
+    Representation the defination of the device type in lava job
+    """
+
+
+class LAVADeviceTarget(LAVADeviceBase):
+    """
+    Representation the defination of the device target in lava job
+    """
+
+
+def get_lava_device_type_or_target():
+    '''
+    Here we support the specification by both,
+    LAVA_DEVICE or LAVA_DEVICE_TYPE, and we can specify
+    both of them at the same time.
+    '''
+    devices = []
+
+    # allow to submit to a specific device
+    test_device = os.environ.get("LAVA_DEVICE")
+    if test_device:
+        targets = test_device.split(',')
+        for dev_target in targets:
+            dev_target = dev_target.strip()
+            if dev_target:
+                devices.append(LAVADeviceTarget(name=dev_target))
+
+    # allow overload lava device_type by build config
+    test_device_type = os.environ.get("LAVA_DEVICE_TYPE")
+
+    # only use the default device type when neither of
+    # LAVA_DEVICE or LAVA_DEVICE_TYPE is specified.
+    # or say we will not use the value when any of
+    # LAVA_DEVICE or LAVA_DEVICE_TYPE is specified.
+    if (not test_device_type) and (not test_device):
+        test_device_type = PRODUCT_MAP[get_target_product()][
+                                            "test_device_type"]
+    if test_device_type:
+        types = test_device_type.split(',')
+        for dev_type in types:
+            dev_type = dev_type.strip()
+            if dev_type:
+                devices.append(LAVADeviceType(name=dev_type))
+
+    return devices
+
+
+def get_build_url():
+    # Build url, defined by android-build, e.g.
+    # https://android-build.linaro.org/jenkins/job/linaro-android_leb-panda/61/
+    build_url = os.environ.get("BUILD_URL")
+
+    return build_url
+
+
+def get_target_product():
+    # Target product name, user defined, e.g. pandaboard
+    target_product = os.environ.get("TARGET_PRODUCT")
+    return target_product
+
+
+def get_schema_and_url():
+    lava_server = os.environ.get("LAVA_SERVER")
+    if lava_server == None:
+        schema = 'https'
+        url_no_schema = "validation.linaro.org/lava-server/RPC2/"
+    elif lava_server.find('://') >= 0:
+        schema = lava_server[:lava_server.find('://')]
+        url_no_schema = lava_server[lava_server.find('://') + len('://'):]
+    else:
+        ## the case that no https and http specified in the url
+        ## like: validation.linaro.org/lava-server/RPC2/
+        schema = 'https'
+        url_no_schema = lava_server
+    return (schema, url_no_schema)
+
+
+def get_plan_list():
+    plan_list = ["LAVA_TEST_PLAN"]
+    sec_plan_prefix = "LAVA_TEST_PLAN_SECONDARY_"
+
+    for var in os.environ.keys():
+        if var.startswith(sec_plan_prefix):
+            plan_list.append(var)
+    plan_list.sort()
+
+    return plan_list
+
+
+def check_target_product():
+    # Board-specific parameters
+    if get_target_product() not in PRODUCT_MAP:
+        # We don't know how to test this job, so skip testing.
+        print "Don't know how to test this board. Skip testing."
+        sys.exit(1)
+
+
 def gen_lava_test_shell_action(test_name=None, test_para=None):
     if not test_para or not test_name:
         return None
@@ -96,7 +204,7 @@ def gen_lava_test_shell_action(test_name=None, test_para=None):
         if key_value_hash.get('testdef'):
             repo["testdef"] = key_value_hash.get('testdef')
 
-        parameters["testdef_repos"] =  [repo]
+        parameters["testdef_repos"] = [repo]
         action = {
                     "command": "lava_test_shell",
                     "parameters": parameters
@@ -171,7 +279,7 @@ def gen_lava_android_test_actions(tests=[]):
         test_option = ''
         if test.find('(') >= 0 and test.rfind(')') > test.find('(') \
             and test[:test.find('(')].strip():
-            test_option = test[test.find('(') + 1 : test.rfind(')')]
+            test_option = test[test.find('(') + 1: test.rfind(')')]
             test = test[:test.find('(')].strip()
 
         if test.startswith('lava-test-shell'):
@@ -324,204 +432,209 @@ def gen_test_actions():
     return test_actions
 
 
-def main():
-    """Script entry point: return some JSON based on calling args.
-    We should be called from Jenkins and expect the following to
-    be defined: $TARGET_PRODUCT $JOB_NAME $BUILD_NUMBER $BUILD_URL"""
+def gen_deploy_action():
 
-    # Target product name, user defined, e.g. pandaboard
-    target_product = os.environ.get("TARGET_PRODUCT")
     # Job name, defined by android-build, e.g. linaro-android_leb-panda
     job_name = os.environ.get("JOB_NAME")
     frontend_job_name = "~" + job_name.replace("_", "/", 1)
+
     # Build number, defined by android-build, e.g. 61
     build_number = os.environ.get("BUILD_NUMBER")
-    # Build url, defined by android-build, e.g.
-    # https://android-build.linaro.org/jenkins/job/linaro-android_leb-panda/61/
-    build_url = os.environ.get("BUILD_URL")
+
     # download base URL, this may differ from job URL if we don't host
     # downloads in Jenkins any more
     download_url = "http://snapshots.linaro.org/android/%s/%s/" % \
-    (frontend_job_name, build_number)
-    # User can disable the installation of android binaries (doing this will
-    # disable hardware acceleration)
-    enable_android_install_binaries = os.environ.get("LAVA_ANDROID_BINARIES")
+                         (frontend_job_name, build_number)
+
+    # Set the file extension based on the type of artifacts
+    artifact_type = os.environ.get("MAKE_TARGETS", "tarball")
+    if artifact_type == "droidcore":
+        file_extension = "img"
+    else:
+        file_extension = "tar.bz2"
+
+    action = {"command": "deploy_linaro_android_image",
+              "parameters": {
+                    "boot": "%s%s%s" % (download_url,
+                                        "/boot.", file_extension),
+                    "system": "%s%s%s" % (download_url,
+                                         "/system.", file_extension),
+                    "data": "%s%s%s" % (download_url,
+                                       "/userdata.", file_extension)
+                  },
+              "metadata": {
+                    "android.name": job_name,
+                    "android.build": '%s' % build_number,
+                    "android.url": get_build_url()
+                  }
+             }
+    return action
+
+
+def gen_boot_action():
     # Some devices need not boot to GUI like the Tiny Android builds and builds
     # which need a proprietary binary overlay to be installed before expecting
     # GUI.
-    wait_for_homescreen = os.environ.get("LAVA_WAIT_FOR_HOMESCREEN")
-    if wait_for_homescreen == None:
-        wait_for_homescreen = True
-    elif wait_for_homescreen.lower() in ['0','false','no']:
+    boot_action = {"command": "boot_linaro_android_image"}
+
+    wait_for_homescreen = os.environ.get("LAVA_WAIT_FOR_HOMESCREEN", 'true')
+    if wait_for_homescreen.lower() in ['0', 'false', 'no']:
         wait_for_homescreen = False
-    # Not set, default to False, because this is relevant only for panda 
+    else:
+        wait_for_homescreen = True
+
+    if wait_for_homescreen == False:
+        boot_action["parameters"] = {"wait_for_home_screen": False}
+    return boot_action
+
+
+def gen_install_binaries_action():
+    # User can disable the installation of android binaries (doing this will
+    # disable hardware acceleration)
+    enable_android_install_binaries = os.environ.get("LAVA_ANDROID_BINARIES",
+                                                     'false')
+    # Not set, default to False, because this is relevant only for panda
     # from Vishal
-    if enable_android_install_binaries == None:
-        enable_android_install_binaries = False
-    elif enable_android_install_binaries.lower() in ['1', 'true', 'yes']:
+    if enable_android_install_binaries.lower() in ['1', 'true', 'yes']:
         enable_android_install_binaries = True
     else:
         enable_android_install_binaries = False
 
-    # Set the default timeout for all test,
+    if enable_android_install_binaries:
+        return {"command": "android_install_binaries"}
+    else:
+        return None
+
+
+def gen_submit_action():
+
+    (schema, url_no_schema) = get_schema_and_url()
+    schema_url = '%s://%s' % (schema, url_no_schema)
+
+    default_stream = '/private/team/linaro/android-daily/'
+
+    submit_action = {"command": "submit_results_on_host",
+                     "parameters": {
+                            "server": schema_url,
+                            "stream": PRODUCT_MAP[get_target_product()].get(
+                              "test_stream", default_stream)
+                      }
+                    }
+
+    return submit_action
+
+
+def gen_config(actions=[], device=None):
+    if not isinstance(device, LAVADeviceBase):
+        print "Not supported type of device %s" % type(device)
+        return None
+
+    # Set the default timeout for all test, also used as the timeout value of
+    # the total job when it's value bigger than 24 * 60 * 60
     # if this value is not set, then use the 18000 seconds as the default value
     default_timeout = os.environ.get("DEFAULT_TIMEOUT", 18000)
 
-    # Set the file extension based on the type of artifacts
-    artifact_type = os.environ.get("MAKE_TARGETS", "tarball")
-    if artifact_type == "droidcore":
-        file_extension = "img"
+    config_json = {"job_name": get_build_url(),
+                   "image_type": 'android',
+                   "timeout": int(default_timeout),
+                   "actions": actions
+                  }
+
+    # test_device set will win over test_device_type
+    # LAVA parameter naming could use more consistency
+    if isinstance(device, LAVADeviceType):
+        config_json["device_type"] = device.name
     else:
-        file_extension = "tar.bz2" 
+        config_json["target"] = device.name
 
-    # Board-specific parameters
-    if target_product not in PRODUCT_MAP:
-        # We don't know how to test this job, so skip testing.
-        print "Don't know how to test this board. Skip testing."
+    config = json.dumps(config_json, indent=4)
+
+    print config
+    return config
+
+def submit_job(config=None, plan=None):
+    if (not config) or (not plan):
+        print "Config or plan is None"
         return
 
-    lava_server = os.environ.get("LAVA_SERVER")
-    if lava_server == None:
-        schema = 'https'
-        url_no_schema = "validation.linaro.org/lava-server/RPC2/"
-    elif lava_server.find('://') >= 0:
-        schema = lava_server[:lava_server.find('://')]
-        url_no_schema = lava_server[lava_server.find('://') + len('://'):]
+    # get the lava token used for submit job
+    lava_token_f = os.environ.get("LAVA_TOKEN_FILE")
+    if lava_token_f == None:
+        lava_token_f = '/var/run/lava/lava-token'
     else:
-        ## the case that no https and http specified in the url
-        ## like: validation.linaro.org/lava-server/RPC2/
-        schema = 'https'
-        url_no_schema = lava_server #for compare with above condition
-    schema_url = '%s://%s' % (schema, url_no_schema)
-    lava_server = url_no_schema
+        lava_token_f = '/var/run/lava/%s' % lava_token_f
 
+    with open(lava_token_f) as fd:
+        lava_token = fd.read().strip()
+
+    # get the lava user used for submit job
     lava_user = os.environ.get("LAVA_USER")
     if lava_user == None:
         f = open('/var/run/lava/lava-user')
         lava_user = f.read().strip()
         f.close()
-    default_stream = '/private/team/linaro/android-daily/'
 
-    common_actions = [
-    {
-    "command": "deploy_linaro_android_image",
-    "parameters":
-      {
-        "boot": "%s%s%s" % (download_url,
-                            "/boot.", file_extension),
-        "system":"%s%s%s" % (download_url,
-                             "/system.", file_extension),
-        "data":"%s%s%s" % (download_url,
-                           "/userdata.", file_extension)
-      },
-    "metadata":
-      {
-        "android.name": job_name,
-        "android.build": '%s' % build_number,
-        "android.url": build_url
-      }
-    }]
+    (schema, lava_server_no_schema) = get_schema_and_url()
+    try:
+        report_url = ("%(schema)s://%(lava_user)s:%(lava_token)s@"
+                      "%(lava_server_no_schema)s") % dict(
+                                schema=schema,
+                                lava_user=lava_user,
+                                lava_token=lava_token,
+                                lava_server_no_schema=lava_server_no_schema)
+        server = xmlrpclib.ServerProxy(report_url)
+        lava_job_id = server.scheduler.submit_job(config)
+        lava_server_root = lava_server_no_schema.rstrip("/")
+        if lava_server_root.endswith("/RPC2"):
+            lava_server_root = lava_server_root[:-len("/RPC2")]
+    except xmlrpclib.ProtocolError, e:
+        print "Error making a LAVA request:", obfuscate_credentials(str(e))
+        return
 
-    if enable_android_install_binaries:
-        common_actions.append({"command": "android_install_binaries"})
+    print "LAVA Job Id: %s, URL: %s://%s/scheduler/job/%s" % \
+    (lava_job_id, schema, lava_server_root, lava_job_id)
 
-    if wait_for_homescreen == False:
-        common_actions.append({"command": "boot_linaro_android_image",
-            "parameters": {
-            "wait_for_home_screen": False
-            }
-        })
+    if plan == "LAVA_TEST_PLAN":
+        json.dump({
+                    'lava_url': "%s://%s" % (schema, lava_server_root),
+                    'job_id': lava_job_id,
+                  }, open('out/lava-job-info', 'w'))
     else:
-        common_actions.append({"command": "boot_linaro_android_image"})
+        json.dump({
+                     'lava_url': "%s://%s" % (schema, lava_server_root),
+                     'job_id': lava_job_id,
+                  }, open('out/lava-job-info-' + plan, 'w'))
 
-    plan_list = ["LAVA_TEST_PLAN"]
-    sec_plan_prefix = "LAVA_TEST_PLAN_SECONDARY_"
-    
-    for var in os.environ.keys():
-        if var.startswith(sec_plan_prefix):
-            plan_list.append(var)
-    plan_list.sort()
+
+def main():
+    """Script entry point: return some JSON based on calling args.
+    We should be called from Jenkins and expect the following to
+    be defined: $TARGET_PRODUCT $JOB_NAME $BUILD_NUMBER $BUILD_URL"""
+
+    check_target_product()
+
+    common_actions = [gen_deploy_action()]
+
+    install_binaries_action = gen_install_binaries_action()
+    if install_binaries_action:
+        common_actions.append(install_binaries_action)
+
+    common_actions.append(gen_boot_action())
+
+    plan_list = get_plan_list()
     # Create a copy of common actions
     for plan in plan_list:
         actions = copy.deepcopy(common_actions)
         if plan == "LAVA_TEST_PLAN":
-        	actions.extend(gen_test_actions())
+            actions.extend(gen_test_actions())
         else:
             actions.extend(gen_test_plan_actions(os.environ.get(plan)))
-        actions.append(
-        {
-          "command": "submit_results_on_host",
-          "parameters":
-          {
-            "server": schema_url,
-            "stream": PRODUCT_MAP[target_product].get(
-                  "test_stream", default_stream)
-          }
-        })
-    
-        config_json = {"job_name": build_url,
-                       "image_type": 'android',
-                       "timeout": int(default_timeout),
-                       "actions": actions
-                      }
-    
-        # allow overload lava device_type by build config
-        test_device_type = os.environ.get("LAVA_DEVICE_TYPE")
-        if not test_device_type:
-            test_device_type = PRODUCT_MAP[target_product]["test_device_type"]
-    
-        # allow to submit to a specific device
-        test_device = os.environ.get("LAVA_DEVICE")
-    
-        # test_device set will win over test_device_type
-        # LAVA parameter naming could use more consistency
-        if test_device:
-            config_json["target"] = test_device
-        else:
-            config_json["device_type"] = test_device_type
-    
-        config = json.dumps(config_json, indent=4)
-    
-        print config
-    
-        lava_token_f = os.environ.get("LAVA_TOKEN_FILE")
-        if lava_token_f == None:
-            lava_token_f = '/var/run/lava/lava-token'
-        else:
-            lava_token_f = '/var/run/lava/%s' % lava_token_f
-    
-        with open(lava_token_f) as fd:
-            lava_token = fd.read().strip()
-    
-        try:
-            report_url = ("%(schema)s://"
-                 "%(lava_user)s:%(lava_token)s@%(lava_server)s") % dict(
-                                        schema=schema,
-                                        lava_user=lava_user,
-                                        lava_token=lava_token,
-                                        lava_server=lava_server)
-            server = xmlrpclib.ServerProxy(report_url)
-            lava_job_id = server.scheduler.submit_job(config)
-            lava_server_root = lava_server.rstrip("/")
-            if lava_server_root.endswith("/RPC2"):
-                lava_server_root = lava_server_root[:-len("/RPC2")]
-        except xmlrpclib.ProtocolError, e:
-            print "Error making a LAVA request:", obfuscate_credentials(str(e))
-            sys.exit(1)
-
-        print "LAVA Job Id: %s, URL: %s://%s/scheduler/job/%s" % \
-        (lava_job_id, schema, lava_server_root, lava_job_id)
+        actions.append(gen_submit_action())
 
-        if plan == "LAVA_TEST_PLAN":
-            json.dump({
-                'lava_url': "%s://%s" % (schema, lava_server_root),
-                'job_id': lava_job_id,
-                 }, open('out/lava-job-info', 'w'))
-        else:
-            json.dump({
-                'lava_url': "%s://%s" % (schema, lava_server_root),
-                'job_id': lava_job_id,
-                 }, open('out/lava-job-info-' + plan  , 'w'))
+        devices = get_lava_device_type_or_target()
+        for dev in devices:
+            config = gen_config(actions=actions, device=dev)
+            submit_job(config=config, plan=plan)
 
 
 if __name__ == "__main__":