From 4b226b744bceca11d2df2159e5c48e6e1588e78d Mon Sep 17 00:00:00 2001 From: Arthur She Date: Thu, 25 Feb 2016 17:56:31 +0800 Subject: Added a feature to generate spreadsheet out of LAVA results --- functional_test_results_template.ods | Bin 0 -> 25592 bytes report_automation.py | 50 ++++++++++++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 functional_test_results_template.ods diff --git a/functional_test_results_template.ods b/functional_test_results_template.ods new file mode 100644 index 0000000..f19b652 Binary files /dev/null and b/functional_test_results_template.ods differ diff --git a/report_automation.py b/report_automation.py index b3311d8..5fa80ac 100755 --- a/report_automation.py +++ b/report_automation.py @@ -93,7 +93,7 @@ class LAVA(object): job_info['job_no'] = job_no job_info['result_link'] = job_detail['_results_link'] job_info['result_bundle'] = json.loads(self.server.dashboard.get(job_status['bundle_sha1'])['content']) - if any(t['test_id'] == 'bionic_libc_tests' for t in job_info['result_bundle']['test_runs']): + if any(t['test_id'] == 'busybox' for t in job_info['result_bundle']['test_runs']): job_info['type'] = "lava_test" elif any(t['test_id'] == 'cts-host' for t in job_info['result_bundle']['test_runs']): job_info['type'] = "cts" @@ -294,12 +294,49 @@ class Report(object): "benchmark_tests" : self.benchmark_tests } + +class FunctionalTestReport(Report): + def parse_lava_test(self, job_info): + logger.info("Parsing lava test job #%s" % job_info['job_no']) + lava_tests = {} + lava_tests['result_link'] = job_info['result_link'] + lava_tests['testsuites'] = [] + for test_run in job_info['result_bundle']['test_runs']: + if test_run['test_id'] == 'lava': + continue + # Count passed and failed test cases + test_case = {} + test_case['Pass'] = 0 + test_case['Fail'] = 0 + test_case['Skip'] = 0 + test_case['Unknown'] = 0 + test_case['name'] = test_run['test_id'] + test_case['results'] = [] + for test_result in test_run['test_results']: + result = test_result['result'].title() + name = test_result['test_case_id'] + measurement = "" + if 'measurement' in test_result.keys(): + measurement = test_result['measurement'] + test_case[result] = test_case[result] + 1 + test_case['results'].append({'name': name, 'result': result, 'measurement': measurement}) + lava_tests['testsuites'].append(test_case) + self.lava_tests = lava_tests + + def to_dict(self): + return { + "testsuites" : self.lava_tests['testsuites'] + } + + if __name__ == '__main__': JENKINS_URL = "https://android-build.linaro.org/jenkins/" JENKINS_JOB = "linaro-android_lcr-member-juno" REPORT_TEMPLATE = "py3o_report_juno_template.odt" REPORT_OUTPUT = "report_output.odt" + FUNCT_REPORT_TEMPLATE = "functional_test_results_template.ods" + FUNCT_REPORT_OUTPUT = "functional_report_output.ods" parser = argparse.ArgumentParser(formatter_class=RawTextHelpFormatter, description="""This script will collect all data and make a report. @@ -320,7 +357,11 @@ e.g. help="Specify the test jobs that don't want to be included in the report. This is optional.") parser.add_argument("-temp", "--template", dest="template_file", default=REPORT_TEMPLATE, help="Specify the report template. This is optional.\nDefault: %s" % REPORT_TEMPLATE) + parser.add_argument("-ftemp", "--functional_template", dest="functional_template_file", default=FUNCT_REPORT_TEMPLATE, + help="Specify the report template for functional tests. This is optional.\nDefault: %s" % FUNCT_REPORT_TEMPLATE) parser.add_argument("-o", "--output", dest="output_file", default=REPORT_OUTPUT, + help="Specify the output file name for functional tests. This is optional.\nDefault: %s" % FUNCT_REPORT_OUTPUT) + parser.add_argument("-fo", "--functional-output", dest="functional_output_file", default=FUNCT_REPORT_OUTPUT, help="Specify the output file name. This is optional.\nDefault: %s" % REPORT_OUTPUT) parser.add_argument("-f", "--force-report", dest="force_report", help="Generate report even there're some errors", action="store_true") @@ -338,7 +379,9 @@ e.g. exit(-1) jenkins_ligin_info = netrc.netrc().authenticators(args.jenkins_url.split('/')[2]) t = Template(args.template_file, args.output_file) + ft = Template(args.functional_template_file, args.functional_output_file) rep_obj = Report() + fun_rep_obj = FunctionalTestReport() if jenkins_ligin_info is not None: jenkins = LinaroAndroidBuildSystem(base_url=args.jenkins_url, build_job=args.jenkins_job_name, username=jenkins_ligin_info[0], password=jenkins_ligin_info[2]) @@ -369,6 +412,7 @@ e.g. try: if job_info['type'] == 'lava_test': rep_obj.parse_lava_test(job_info) + fun_rep_obj.parse_lava_test(job_info) elif job_info['type'] == 'benchmark': rep_obj.parse_benchmark_test(job_info) elif job_info['type'] == 'cts': @@ -391,6 +435,10 @@ e.g. if generate_report: logger.info("Prepare for the report!!") data = rep_obj.to_dict() + fun_data = fun_rep_obj.to_dict() + with open("backup_fun_data.json", "w") as bf: + json.dump(fun_data, bf) t.render(data) + ft.render(fun_data) logger.info("Done, the output report is \"%s\"" % REPORT_OUTPUT) -- cgit v1.2.3