Added support for direct .json submission

Change-Id: I46ceb90b9a897d3f5f07df23a6c50c02b41ba958
Signed-off-by: Milosz Wasilewski <milosz.wasilewski@linaro.org>

2 files changed, 82 insertions, 28 deletions

diff --git a/post-build-report.py b/post-build-report.py
index 37d0a42..d7a576e 100644
--- a/post-build-report.py
+++ b/post-build-report.py
@@ -4,6 +4,7 @@ import sys
 import json
 import httplib
 import logging
+import mimetypes
 
 from urlparse import urljoin, urlsplit
 
@@ -16,37 +17,60 @@ httplib.HTTPConnection.debuglevel = 1
 RESULT_ENDPOINT = "/api/result/"
 
 
-def _push_object(auth_pw, backend_url, endpoint, params):
+def encode_multipart_formdata(fields, files):
+    LIMIT = '----------lImIt_of_THE_fIle_eW_$'
+    CRLF = '\r\n'
+    L = []
+    for key, value in fields.iteritems():
+        L.append('--' + LIMIT)
+        L.append('Content-Disposition: form-data; name="%s"' % key)
+        L.append('')
+        L.append(value)
+    for key, value in files.iteritems():
+        L.append('--' + LIMIT)
+        L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, key))
+        L.append('Content-Type: %s' % get_content_type(key))
+        L.append('')
+        L.append(value.read())
+    L.append('--' + LIMIT + '--')
+    L.append('')
+    body = CRLF.join(L)
+    content_type = 'multipart/form-data; boundary=%s' % LIMIT
+    return content_type, body
+
+def get_content_type(filename):
+    return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
+
+
+def _push_object(auth_pw, backend_url, endpoint, params, files):
     usplit = urlsplit(backend_url)
     url = urljoin(backend_url, endpoint)
 
     logger.info("Submitting to URL: %s" % url)
-
-    headers = {
-        "Content-type": "application/json",
-        "Accept": "application/json",
-        "Authorization": "Token %s" % auth_pw
-    }
-
     conn = None
     if usplit.scheme.lower() == "http":
-        conn = httplib.HTTPConnection(usplit.netloc)
+        conn = httplib.HTTP(usplit.netloc)
     if usplit.scheme.lower() == "https":
-        conn = httplib.HTTPSConnection(usplit.netloc)
+        conn = httplib.HTTPS(usplit.netloc)
 
     if conn is None:
         print "Unknown scheme: %s" % usplit.scheme
         sys.exit(1)
 
-    conn.request("POST", endpoint, json.dumps(params), headers)
-
-    response = conn.getresponse()
-    if response.status < 300:
-        return response.read()
+    content_type, body = encode_multipart_formdata(params, files)
+    conn.putrequest('POST', endpoint)
+    conn.putheader("Authorization", "Token %s" % auth_pw)
+    conn.putheader('Content-Type', content_type)
+    conn.putheader('Content-Length', str(len(body)))
+    conn.endheaders()
+    conn.send(body)
+    errcode, errmsg, headers = conn.getreply()
+
+    if errcode < 300:
+        return conn.file.read()
     else:
-        logger.warn(response.status)
-        logger.warn(response.reason)
-        logger.warn(response.read())
+        logger.warn(errcode)
+        logger.warn(errmsg)
     return []
 
 
@@ -60,6 +84,15 @@ def _get_manifest(workspace_path):
     return None
 
 
+def _get_files(workspace_path):
+    files_dict = {}
+    for dirpath, dirnames, filenames in os.walk(workspace_path):
+        for result_file_name in filenames:
+            if result_file_name.endswith(".json"):
+                files_dict.update({result_file_name: open(os.path.join(dirpath, result_file_name), 'rb')})
+    return files_dict
+
+
 def _results(workspace_path):
     benchmarks = {
         "Boot.oat size": ['boot_oat_size_ARM_32_Quick.txt',
@@ -114,7 +147,7 @@ if __name__ == '__main__':
     jenkins_project_name = os.environ.get("SOURCE_PROJECT_NAME")
 
     jenkins_build_number = os.environ.get("SOURCE_BUILD_NUMBER")
-    jenkins_build_id = os.environ.get("SOURCE_BUILD_ID")
+    jenkins_build_id = os.environ.get("SOURCE_BUILD_ID", jenkins_build_number)
     jenkins_build_url = os.environ.get("SOURCE_BUILD_URL")
 
     branch_name = os.environ.get("SOURCE_BRANCH_NAME", "")
@@ -125,9 +158,11 @@ if __name__ == '__main__':
         art_token = os.environ.get("ART_TOKEN_ART_REPORTS")
     elif urlsplit(art_url).netloc.startswith('android-qa-reports'):
         art_token = os.environ.get("ART_TOKEN_ANDROID_REPORTS")
+    else:
+        art_token = os.environ.get("ART_TOKEN")
 
     manifest = _get_manifest("./artifacts")
-    test_jobs = os.environ.get("LAVA_JOB_IDS", "")
+    test_jobs = os.environ.get("LAVA_JOB_IDS", None)
     results = _results("./artifacts")
 
     if jenkins_build_number is None:
@@ -159,15 +194,22 @@ if __name__ == '__main__':
         'manifest': manifest,
         'branch_name': branch_name,
 
-        "gerrit_change_number": os.environ.get("SOURCE_GERRIT_CHANGE_NUMBER"),
-        "gerrit_patchset_number":os.environ.get("SOURCE_GERRIT_PATCHSET_NUMBER"),
-        "gerrit_change_url": os.environ.get("SOURCE_GERRIT_CHANGE_URL"),
+        "gerrit_change_number": os.environ.get("SOURCE_GERRIT_CHANGE_NUMBER", ""),
+        "gerrit_patchset_number":os.environ.get("SOURCE_GERRIT_PATCHSET_NUMBER", ""),
+        "gerrit_change_url": os.environ.get("SOURCE_GERRIT_CHANGE_URL", ""),
         "gerrit_change_id": os.environ.get("SOURCE_GERRIT_CHANGE_ID", ""),
 
         "results": results
     }
 
-    _push_object(art_token, art_url, RESULT_ENDPOINT, params)
+    if not results:
+        params.pop("results")
+
+    files = []
+    if test_jobs is None:
+        params.pop('test_jobs')
+        files = _get_files("./artifacts")
+    _push_object(art_token, art_url, RESULT_ENDPOINT, params, files)
     params.pop('manifest')
     print params
 
diff --git a/post-build-report.yaml b/post-build-report.yaml
index 932877f..ec70009 100644
--- a/post-build-report.yaml
+++ b/post-build-report.yaml
@@ -7,7 +7,6 @@
         <li>SOURCE_BUILD_NUMBER - Jenkins build number
         <li>LAVA_JOB_IDS - comma separated list of LAVA job IDs
         <li>ART_URL - URL of the API endpoing of art-reports
-        <li>ART_TOKEN - token for art-reports
         </ul>
         Optionally the following variables might be set:
         <ul>
@@ -43,6 +42,14 @@
         execution-strategy:
         sequential: true
     builders:
+         - shell: |
+            #!/bin/bash
+            # Remove old artifacts
+            rm artifacts/pinned-manifest.xml
+            rm artifacts/boot_oat_size*
+            rm artifacts/avg_oat_size*
+            rm artifacts/*.json
+            exit 0
         - copyartifact:
             project: ${SOURCE_PROJECT_NAME}
             filter: 'pinned-manifest.xml'
@@ -67,12 +74,17 @@
             flatten: true
             which-build: specific-build
             build-number: ${SOURCE_BUILD_NUMBER}
+        - copyartifact:
+            project: ${SOURCE_PROJECT_NAME}
+            filter: '*.json'
+            target: artifacts
+            optional: true
+            flatten: true
+            which-build: specific-build
+            build-number: ${SOURCE_BUILD_NUMBER}
          - shell: |
             #!/bin/bash
             # Send to art-reports
-            rm pinned-manifest.xml
-            rm boot_oat_size*
-            rm avg_oat_size*
             rm -rf pbr
             git clone git://git.linaro.org/people/milosz.wasilewski/post-build-report.git pbr
             python pbr/post-build-report.py