#!/usr/bin/python import os import sys import json import httplib import logging import mimetypes from urlparse import urljoin, urlsplit logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) logger = logging.getLogger(__name__) httplib.HTTPConnection.debuglevel = 1 RESULT_ENDPOINT = "/api/result/" def encode_multipart_formdata(fields, files): LIMIT = '----------lImIt_of_THE_fIle_eW_$' CRLF = '\r\n' L = [] for key, value in fields.iteritems(): L.append('--' + LIMIT) L.append('Content-Disposition: form-data; name="%s"' % key) L.append('') L.append(value) for key, value in files.iteritems(): L.append('--' + LIMIT) L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, key)) L.append('Content-Type: %s' % get_content_type(key)) L.append('') L.append(value.read()) L.append('--' + LIMIT + '--') L.append('') body = CRLF.join(L) content_type = 'multipart/form-data; boundary=%s' % LIMIT return content_type, body def get_content_type(filename): return mimetypes.guess_type(filename)[0] or 'application/octet-stream' def _push_object(auth_pw, backend_url, endpoint, params, files): usplit = urlsplit(backend_url) url = urljoin(backend_url, endpoint) logger.info("Submitting to URL: %s" % url) conn = None if usplit.scheme.lower() == "http": conn = httplib.HTTP(usplit.netloc) if usplit.scheme.lower() == "https": conn = httplib.HTTPS(usplit.netloc) if conn is None: print "Unknown scheme: %s" % usplit.scheme sys.exit(1) content_type, body = encode_multipart_formdata(params, files) conn.putrequest('POST', endpoint) conn.putheader("Authorization", "Token %s" % auth_pw) conn.putheader('Content-Type', content_type) conn.putheader('Content-Length', str(len(body))) conn.endheaders() conn.send(body) errcode, errmsg, headers = conn.getreply() if errcode < 300: return conn.file.read() else: logger.warn(errcode) logger.warn(errmsg) return [] def _get_manifest(workspace_path): manifest_path = os.path.join(workspace_path, "pinned-manifest.xml") print "Searching for: %s" % manifest_path if os.path.exists(manifest_path): with open(manifest_path, "r") as manifest_file: return manifest_file.read() print "Manifest not found" return None def _get_files(workspace_path): files_dict = {} for dirpath, dirnames, filenames in os.walk(workspace_path): for result_file_name in filenames: if result_file_name.endswith(".json"): files_dict.update({result_file_name: open(os.path.join(dirpath, result_file_name), 'rb')}) return files_dict def _results(workspace_path): benchmarks = { "Boot.oat size": ['boot_oat_size_ARM_32_Quick.txt', 'boot_oat_size_ARM_64_Quick.txt', 'boot_oat_size_x86_32_Quick.txt', 'boot_oat_size_x86_64_Quick.txt', 'boot_oat_size_x86_64_Optimizing.txt', 'boot_oat_size_x86_32_Optimizing.txt', 'boot_oat_size_ARM_64_Optimizing.txt', 'boot_oat_size_ARM_32_Optimizing.txt', 'boot_oat_size_mips_64_Optimizing.txt', 'boot_oat_size_mips_32_Quick.txt'], "Oat Execution Time": ['avg_oat_time_ARM_32_Quick.txt', 'avg_oat_time_ARM_64_Quick.txt', 'avg_oat_time_x86_64_Quick.txt', 'avg_oat_time_x86_32_Quick.txt', 'avg_oat_time_x86_64_Optimizing.txt', 'avg_oat_time_x86_32_Optimizing.txt', 'avg_oat_time_ARM_32_Optimizing.txt', 'avg_oat_time_ARM_64_Optimizing.txt', 'avg_oat_time_mips_64_Optimizing.txt', 'avg_oat_time_mips_32_Quick.txt'] } val = [] for benchmark, subscores in benchmarks.items(): for subscore in subscores: path = os.path.join(workspace_path, subscore) print path, os.path.exists(path) if os.path.exists(path): raw = open(path, 'r').read().strip() measurement = float(raw.replace("YVALUE=", "")) name = (subscore .replace("avg_oat_time_", "") .replace("boot_oat_size_", "") .replace(".txt", "") .replace("_", " ")) val.append({ "benchmark": benchmark, "name": name, "measurement": measurement }) return val if __name__ == '__main__': jenkins_project_name = os.environ.get("SOURCE_PROJECT_NAME") jenkins_build_number = os.environ.get("SOURCE_BUILD_NUMBER") jenkins_build_id = os.environ.get("SOURCE_BUILD_ID", jenkins_build_number) jenkins_build_url = os.environ.get("SOURCE_BUILD_URL") branch_name = os.environ.get("SOURCE_BRANCH_NAME", "") art_url = os.environ.get("ART_URL", "http://localhost:8000/") art_token = None if urlsplit(art_url).netloc.startswith('art-reports'): art_token = os.environ.get("ART_TOKEN_ART_REPORTS") elif urlsplit(art_url).netloc.startswith('android-qa-reports'): art_token = os.environ.get("ART_TOKEN_ANDROID_REPORTS") else: art_token = os.environ.get("ART_TOKEN") manifest = _get_manifest("./artifacts") test_jobs = os.environ.get("LAVA_JOB_IDS", None) results = _results("./artifacts") if jenkins_build_number is None: print "Build number not set. Exiting!" sys.exit(1) if jenkins_project_name is None: print "Project name not set. Exiting!" sys.exit(1) if jenkins_build_url is None: print "Build URL not set. Exiting!" sys.exit(1) if art_token is None: print "ART token not set. Exiting!" sys.exit(1) if not manifest: print "Manifest missing. Exiting!" sys.exit(1) print "Registered test jobs: %s" % test_jobs params = { 'name': jenkins_project_name, 'build_id': jenkins_build_id, 'build_url': jenkins_build_url, 'build_number': jenkins_build_number, 'test_jobs': test_jobs, 'manifest': manifest, 'branch_name': branch_name, "gerrit_change_number": os.environ.get("SOURCE_GERRIT_CHANGE_NUMBER", ""), "gerrit_patchset_number":os.environ.get("SOURCE_GERRIT_PATCHSET_NUMBER", ""), "gerrit_change_url": os.environ.get("SOURCE_GERRIT_CHANGE_URL", ""), "gerrit_change_id": os.environ.get("SOURCE_GERRIT_CHANGE_ID", ""), "results": results } if not results: params.pop("results") files = {} if test_jobs is None: params.pop('test_jobs') files = _get_files("./artifacts") _push_object(art_token, art_url, RESULT_ENDPOINT, params, files) params.pop('manifest') print params