WIPsubst

Change-Id: If6ca80da65203667245048729430f3c73e6b52a6

2 files changed, 127 insertions, 34 deletions

diff --git a/driver.sh b/driver.sh
new file mode 100755
index 0000000..a8e40cf
--- /dev/null
+++ b/driver.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+#uncomment to prevent job submission
+#export SKIP_LAVA=1
+
+#this is required, but is one of those pre-programmed subs
+#this one might truly make universal sense, though
+export JOB_NAME=${JOB_NAME:-FOOBAR}
+
+#credentials as pbl expects to get them
+export LAVA_USER=bernie.ogden
+export LAVA_SERVER=validation.linaro.org/RPC2/
+
+#template file I'm using
+export CUSTOM_JSON_URL=file://${HOME}/src/lava-jobs/benchmark/template-test.json
+#export CUSTOM_JSON_URL=file://${HOME}/src/lava-jobs/hack/kvm.json
+#export CUSTOM_JSON_URL=file://${HOME}/src/abe-bernie-benchmarking/config/bench/jobdefs/cint2006-kvm_x1.json
+
+export HIMAGE='"command": "deploy_linaro_image",
+    "parameters": {
+      "role": "host",
+      "image": "http://images.validation.linaro.org/ubuntu-14-04-server-base.img.gz"
+     }'
+
+export TIMAGE='"command": "deploy_linaro_image",
+    "parameters": {
+      "role": "target",
+      "image": "http://images.validation.linaro.org/ubuntu-14-04-server-base.img.gz"
+     }'
+
+
+
+#have to use env to set the parameters, due to the %% thing
+env %%BENCHMARK%%=fakebench \
+    %%TOOLCHAIN%%=/usr/bin/gcc \
+    %%RUN_FLAGS%%= \
+    %%COMPILER_FLAGS%%= \
+    %%PUBLIC_KEY%%='ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCzlCPpuwMRSXZNq4Mut/GuitOhEturcsl9X1iNjOMGZXbtfBC8TCSXqotkziuYXpej1FCdXn3FEQsVw6Mw3I9Wq1p9okkgzVdsNzHAIob8ras/e9kOeJZGY5gSGBYf3t1MubAyaa033MJ+e3NRC4sbUmeeN3d9PD+Wc9QC3en9qfSpm6hh9EFHOHOjFk5+W0YZ83AL1PgalqL/hCN2/p9a8S1fsfDkfTgEh1XFIOiLds0fNzzmyXepxuA1n696iuSFMCIwmNcCfwRwRgt012UPgZu3avegYbDyijk8EtgFBuEA5OvX91BTHmaSjicsD9nJN63ms88l9OZMeTlPVjkD' \
+    %%LAVA_USER%%=${LAVA_USER} \
+    %%LAVA_SERVER%%=${LAVA_SERVER} \
+    %%ABE_REVISION%%=${ABE_REVISION:-bernie/benchmarking-uinstance} \
+    %%DEVICE_TYPE%%=kvm \
+    %%CONFIG%%=kvm \
+    %%TSESSION%%=config/bench/lava/target-session-kvm.yaml \
+    %%HSESSION%%=config/bench/lava/trusted-host-session.yaml \
+    %%TIMAGE%%="${TIMAGE}" \
+    %%HIMAGE%%="${HIMAGE}" \
+    %%NAME%%="${JOB_NAME}" \
+    ~/src/pbl/post-build-lava.py
+    #%%PREBUILT%%=148.251.136.42:benchsrc/fakebench-abe-built/fakebench.git.tar.xz \
diff --git a/post-build-lava.py b/post-build-lava.py
index c7506fe..ed858a8 100755
--- a/post-build-lava.py
+++ b/post-build-lava.py
@@ -10,6 +10,7 @@ import xmlrpclib
 import urllib2
 import re
 import keyring.core
+import warnings
 
 tests_timeout = {
     'bluetooth-enablement': 7200,
@@ -336,46 +337,87 @@ def submit_job_from_url():
         # Build number, defined by android-build, e.g. 61
         build_number = os.environ.get("BUILD_NUMBER")
 
-        # download base URL, this may differ from job URL if we don't host
-        # downloads in Jenkins any more
-        download_url = "%s/%s/%s/" % ('%s/android/' % snapshots_url,
-                                      frontend_job_name,
-                                      build_number)
-        # Set the file extension based on the type of artifacts
-        artifact_type = os.environ.get("MAKE_TARGETS", "tarball")
-        if artifact_type == "droidcore":
-            file_extension = "img"
-        else:
-            file_extension = "tar.bz2"
-
-        boot_subst = "%s%s%s" % (download_url, "/boot.", file_extension)
-        system_subst = "%s%s%s" % (download_url, "/system.", file_extension)
-        userdata_subst = "%s%s%s" % (download_url, "/userdata.", file_extension)
-
-        replace("job.json", "%%ANDROID_BOOT%%", boot_subst)
-        replace("job.json", "%%ANDROID_SYSTEM%%", system_subst)
-        replace("job.json", "%%ANDROID_DATA%%", userdata_subst)
-        replace("job.json", "%%ANDROID_META_NAME%%", job_name)
-        replace("job.json", "%%JOB_NAME%%", job_name)
-        replace("job.json", "%%ANDROID_META_BUILD%%",  build_number)
-        replace("job.json", "%%ANDROID_META_URL%%", os.environ.get("BUILD_URL"))
-        replace("job.json", "%%BUNDLE_STREAM%%", os.environ.get('LAVA_STREAM', '/private/team/linaro/android-daily/'))
-        replace("job.json", "%%WA2_JOB_NAME%%", build_number)
-        replace("job.json", "%%DOWNLOAD_URL%%", download_url)
-
-        devices = get_lava_device_type_or_target()
+        #This is potentially neat, but not idiomatic for the code, given the 'replace' function
+        #with open("job.json") as fd:
+          #try:
+            #re.compile('%%.*%%').sub(lambda x: os.environ.get(x.group()) or raise Exception, fd.read())
+          #except:
+            #print "dang"
+            #sys.exit(1)
+          #unreplaced = re.compile('%%.*%%').findall(preprocessed)
+          #for match in unreplaced:
+            #print "Error: No replacement for %s" % match
+          #if unreplaced:
+            #sys.exit(1)
+
+        #For every parameter in the environment, insert in place of
+        #placeholders in the template job.
+        replacement_pattern = re.compile('^%%.*%%$')
+        for replacement in filter(lambda x: replacement_pattern.match(x),
+                                  os.environ.iterkeys()):
+          replace("job.json", replacement, os.environ.get(replacement))
+
+        #If there are any remaining placeholders, either replace them with an
+        #empty string, or produce errors and exit.
+        #Ideally we would never accept undefined parameters and do the empty
+        #string thing by defining an empty environment variable. Unfortunately,
+        #when we use a properties file in Jenkins to initialize the environment
+        #we appear to get empty env vars in the Jenkins Environment Variables
+        #view, but they are not initialized at all so far as Python knows. So
+        #we have to accept undefined variables and hope that they were
+        #deliberately undefined.
+        with open("job.json") as fd:
+          unreplaced = re.compile('%%.*%%').findall(fd.read())
+        if unreplaced:
+          if os.environ.get('WARN_ON_UNDEFINED') is None:
+            for match in unreplaced:
+              sys.stderr.write("Error: No replacement for %s\n" % match)
+            sys.exit(1) #TODO: would be nicer to return, but don't think that works with the code as-is
+          else:
+            for match in unreplaced:
+              replace("job.json", match, "")
+              warnings.warn("Undefined parameter %s replaced with empty string" % match)
+
+        ## download base URL, this may differ from job URL if we don't host
+        ## downloads in Jenkins any more
+        #download_url = "%s/%s/%s/" % ('%s/android/' % snapshots_url,
+        #                              frontend_job_name,
+        #                              build_number)
+        ## Set the file extension based on the type of artifacts
+        #artifact_type = os.environ.get("MAKE_TARGETS", "tarball")
+        #if artifact_type == "droidcore":
+        #    file_extension = "img"
+        #else:
+        #    file_extension = "tar.bz2"
+#
+#        boot_subst = "%s%s%s" % (download_url, "/boot.", file_extension)
+#        system_subst = "%s%s%s" % (download_url, "/system.", file_extension)
+#        userdata_subst = "%s%s%s" % (download_url, "/userdata.", file_extension)
+#
+#        replace("job.json", "%%ANDROID_BOOT%%", boot_subst)
+#        replace("job.json", "%%ANDROID_SYSTEM%%", system_subst)
+#        replace("job.json", "%%ANDROID_DATA%%", userdata_subst)
+#        replace("job.json", "%%ANDROID_META_NAME%%", job_name)
+#        replace("job.json", "%%JOB_NAME%%", job_name)
+#        replace("job.json", "%%ANDROID_META_BUILD%%",  build_number)
+#        replace("job.json", "%%ANDROID_META_URL%%", os.environ.get("BUILD_URL"))
+#        replace("job.json", "%%BUNDLE_STREAM%%", os.environ.get('LAVA_STREAM', '/private/team/linaro/android-daily/'))
+#        replace("job.json", "%%WA2_JOB_NAME%%", build_number)
+#        replace("job.json", "%%DOWNLOAD_URL%%", download_url)
+
+        #devices = get_lava_device_type_or_target()
 
         # LAVA server URL
         lava_server = os.environ.get('LAVA_SERVER',
                                      'validation.linaro.org/RPC2/')
 
-        device = None
-        if len(devices) > 0:
-            device = devices[0];
+        #device = None
+        #if len(devices) > 0:
+        #    device = devices[0];
 
-        replace("job.json", "%%DUT_TYPE%%",  device.name)
-        with open("job.json", 'r') as fin:
-            print fin.read()
+        #replace("job.json", "%%DUT_TYPE%%",  device.name)
+        #with open("job.json", 'r') as fin:
+        #    print fin.read()
         with open("job.json") as fd:
             config = fd.read().strip()
         lava_submit(config=config, lava_server=lava_server)
@@ -403,6 +445,7 @@ def main():
     else:
         auth = None
 
+    submit_job_from_url()
     if os.environ.get('TARGET_PRODUCT') is not None:
         submit_job_from_url()