aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSerge Broslavsky <serge.broslavsky@linaro.org>2020-10-29 22:59:59 +0200
committerSerge Broslavsky <serge.broslavsky@linaro.org>2020-10-29 22:59:59 +0200
commit06227dc2e14ecd6a85eca101724506f017bd484a (patch)
tree824e4e1def15c657a02718f2e43e0c42ac9732fc
parent9dda0f6a7d476653edea0d7a1056d43e0e1e2650 (diff)
downloadlkft-bucket-06227dc2e14ecd6a85eca101724506f017bd484a.tar.gz
report generation completed
-rw-r--r--stable_report.py382
-rw-r--r--stable_report_config.py25
-rw-r--r--templates/stable-build-report.jinja91
3 files changed, 274 insertions, 224 deletions
diff --git a/stable_report.py b/stable_report.py
index 8e0c4f7..1f308bc 100644
--- a/stable_report.py
+++ b/stable_report.py
@@ -14,6 +14,8 @@ from requests.compat import urljoin
from squad_client.core.api import SquadApi
from squad_client.core.models import Squad, Project
+import stable_report_config
+
DEBUG=True
#DEBUG=False
@@ -59,9 +61,8 @@ DIRECTORY_JOBS = "jobs"
# directory where all the report files are stored
DIRECTORY_REPORTS = "reports"
-
-# number of already reported builds to check for skipped builds
-ENV_SCAN_DEPTH = "LKFT_SCAN_DEPTH"
+# directory where all the summary files are stored
+DIRECTORY_SUMMARIES = "summaries"
# list of projects (and exact builds) to process
# the format is:
@@ -71,14 +72,14 @@ ENV_SCAN_DEPTH = "LKFT_SCAN_DEPTH"
# and builds that have not yet been reported will be processed, automatically finding the suitable previous and base builds
ENV_PROJECT_LIST = "LKFT_PROJECT_LIST"
+# number of already reported builds to check for skipped builds
+ENV_SCAN_DEPTH = "LKFT_SCAN_DEPTH"
+
# when scanning for not-yet-reported builds how many already reported builds should be checked if they were reported
# this is done to pick up any skipped builds that were not reported
RESULT_WHEN_FAILED_TO_CHECK_IF_PROCESSED = True
-# projects that are mathing this regexp will be scanned for builds
-REGEXP_STABLE_PROJECT = "^linux-stable-rc-linux-([\d]+)\.([\d]+)\.y$"
-
# regexp to parse builds name
REGEXP_STABLE_BUILD = "^v(?P<major>\d+)\.(?P<minor>\d+)(?:\.(?P<patch>\d+))?(?:-(?P<count>\d+)-g(?P<hash>[\da-f]+))?$"
@@ -163,6 +164,10 @@ if os.environ.get(ENV_AWS_MODE):
table.put_item(Item=record)
+ def send_email(headers, body):
+ pass
+
+
# not deployed to AWS - limited functionality
else: # not os.environ.get(ENV_AWS_MODE)
@@ -192,9 +197,13 @@ else: # not os.environ.get(ENV_AWS_MODE)
return
-def list_builds(group):
+ def send_email(headers, body):
+ pass
+
+
+def list_builds(group, list_all=False):
"""
- List all the builds for each of the projects.
+ List all the builds for each of the projects that match the regexp.
"""
group_projects = group.projects()
@@ -202,9 +211,11 @@ def list_builds(group):
for project in group_projects.values():
if project.is_archived:
continue
- match = re_stable.match(project.slug)
- if not match:
- continue
+
+ if not list_all:
+ match = re_stable.match(project.slug)
+ if not match:
+ continue
builds = project.builds(status__finished=True)
print("{0}: {1}".format(project.slug, project.name))
@@ -222,6 +233,14 @@ def list_builds(group):
print()
+def list_all_builds(group):
+ """
+ List all the builds of all the projects.
+ """
+
+ list_builds(group, True)
+
+
class BuildCollection(object):
"""
Build collection that allows working with a set of builds.
@@ -301,7 +320,7 @@ class BuildCollection(object):
def values(self):
return self.Iterator(self)
- def build(version_name):
+ def build(self, version_name):
return self.by_version.get(version_name)
def __len__(self):
@@ -414,6 +433,8 @@ def scan_builds(group):
Prepare jobs for the report generation.
"""
+ print("Scanning builds")
+
try:
os.mkdir(DIRECTORY_JOBS)
except FileExistsError:
@@ -426,20 +447,26 @@ def scan_builds(group):
)
try:
- os.mkdir(DIRECTORY_REPORTS)
+ os.mkdir(DIRECTORY_SUMMARIES)
except FileExistsError:
pass
except:
print(
- "Exception creating the job directory '{0}': {1}".format(
- DIRECTORY_JOBS, sys.exc_info()[0]
+ "Exception creating the summaries directory '{0}': {1}".format(
+ DIRECTORY_SUMMARIES, sys.exc_info()[0]
)
)
try:
- os.remove("{0}/overview.txt".format(DIRECTORY_REPORTS))
- except FileNotFoundError:
+ os.mkdir(DIRECTORY_REPORTS)
+ except FileExistsError:
pass
+ except:
+ print(
+ "Exception creating the report directory '{0}': {1}".format(
+ DIRECTORY_REPORTS, sys.exc_info()[0]
+ )
+ )
group_projects = group.projects()
group_projects_by_slug = {
@@ -452,7 +479,7 @@ def scan_builds(group):
# (<project>(':'<build>','<prev-build>(','<base-build>)?':')*';')+
if project_defs:
print("List of projects provided in the envvar {0}".format(ENV_PROJECT_LIST))
- for project_definition in projects.split(";"):
+ for project_definition in project_defs.split(";"):
project_definition = project_definition.strip()
project_name, _, project_builds = project_definition.partition(":")
@@ -471,7 +498,7 @@ def scan_builds(group):
project_builds = project_builds.strip()
- for project_build in project_builds:
+ for project_build in project_builds.split(":"):
project_build = project_build.strip()
if not project_build:
break
@@ -491,7 +518,7 @@ def scan_builds(group):
if not build:
print(
"Envvar {0} has {1} listed as build for project {2} and that is not a known build. Project skipped.".format(
- ENV_PROJECT_LIST, build_version[0], project.slug
+ ENV_PROJECT_LIST, build_versions[0], project.slug
)
)
continue
@@ -499,7 +526,7 @@ def scan_builds(group):
report_setup = {
"group": group.slug,
"project": project.slug,
- "build": build_version[0],
+ "build": build.version,
"prev_build": None,
"base_build": None,
}
@@ -513,7 +540,7 @@ def scan_builds(group):
)
)
continue
- report_setup.prev_build = build
+ report_setup["prev_build"] = build.version
if len(build_versions) > 2:
build = project.build_collection.build(build_versions[2])
@@ -524,12 +551,11 @@ def scan_builds(group):
)
)
continue
- report_setup.base_build = build
+ report_setup["base_build"] = build.version
- project["builds_to_process"].append(report_setup)
+ project.builds_to_process.append(report_setup)
- if project.builds_to_process:
- projects_to_process.append(project)
+ projects_to_process.append(project)
# get projects from the group projects matching the regexp
else:
@@ -601,7 +627,7 @@ def scan_builds(group):
# create job files
for project in projects_to_process:
if not project.builds_to_process:
- with open("{0}/overview.txt".format(DIRECTORY_REPORTS), "a") as fp:
+ with open("{0}/{1}.txt".format(DIRECTORY_SUMMARIES, project.slug), "w") as fp:
fp.write("Project {0} has no unreported builds\n".format(project.slug))
try:
os.remove(job_file_name)
@@ -620,7 +646,7 @@ def scan_builds(group):
)
job_file_counter += 1
with open(job_file_name, "w") as fp:
- json.dump(project.builds_to_process, fp)
+ json.dump(project.builds_to_process, fp, indent=2)
def generate_reports(group):
@@ -632,6 +658,8 @@ def generate_reports(group):
each worker does their "column" of jobs (% wise)
"""
+ print("Generating reports")
+
try:
worker_index = int(os.environ.get(ENV_PARALLEL_JOB_ID, "1"))
try:
@@ -683,6 +711,7 @@ def generate_reports(group):
)
return
+ job_files.sort()
job_index = int(worker_index)
# run all jobs for the worker
@@ -722,6 +751,14 @@ def process_job_file(group, job_file_name, job_index=None):
)
return
+ print(
+ "{0}Processing job {1}:\n{2}".format(
+ "({0}) ".format(job_index) if job_index != None else "",
+ job_file_name,
+ json.dumps(setup_list, indent=2)
+ )
+ )
+
for setup in setup_list:
build_report(setup)
@@ -731,89 +768,122 @@ def increment_build_counter(stats, counter_type):
stats["build"][counter_type] = 0
stats["build"][counter_type] += 1
+
def increment_environment_counter(stats, environment, counter_type):
env_record = stats["environments"].get(environment)
if not env_record:
- env_record = stats["environments"][environment] = {}
+ env_record = stats["environments"][environment] = {"suites": {}, "pass": 0, "fail": 0, "skip": 0, "xfail": 0}
if env_record.get(counter_type) == None:
env_record[counter_type] = 0
env_record[counter_type] += 1
-def increment_suite_counter(stats, suite, counter_type):
+
+def increment_suite_counter(stats, environment, suite, counter_type):
+ # suite counter
suite_record = stats["suites"].get(suite)
if not suite_record:
- suite_record = stats["suites"][suite] = {}
+ suite_record = stats["suites"][suite] = {"pass": 0, "fail": 0, "skip": 0, "xfail": 0}
if suite_record.get(counter_type) == None:
suite_record[counter_type] = 0
suite_record[counter_type] += 1
+ # suite-in-environment counter
+ env_record = stats["environments"].get(environment)
+ if not env_record:
+ env_record = stats["environments"][environment] = {"suites": {}, "pass": 0, "fail": 0, "skip": 0, "xfail": 0}
+ suite_record = env_record["suites"].get(suite)
+ if not suite_record:
+ suite_record = env_record["suites"][suite] = {"pass": 0, "fail": 0, "skip": 0, "xfail": 0}
+ if suite_record.get(counter_type) == None:
+ suite_record[counter_type] = 0
+
+
+# in memory cache of suite names retrieved by suite id from suite urls
+g_suite_name_by_url = {}
+
+def suite_name_from_url(group, project, url):
+ global g_suite_name_by_url
+
+ lookup_result = g_suite_name_by_url.get(url)
+ if lookup_result:
+ return lookup_result
+
+ match = re.match(REGEXP_SUITE_URL, url)
+ if not match:
+ suite_name = url
+ else:
+ suite = project.suites(id=match.group("id"))
+ if suite:
+ suite = list(suite.values())[0]
+ if suite:
+ suite_name = suite.slug
+ else:
+ suite_name = "suite-{0}".format(match.group("id"))
+ else:
+ suite_name = "suite-{0}".format(match.group("id"))
+
+ g_suite_name_by_url[url] = suite_name
+
+ return suite_name
+
def get_build_stats(group, project, build):
- build_stats = {"build": {},
+ build_stats = {"build": {"pass": 0, "fail": 0, "skip": 0, "xfail": 0},
"environments": {},
- "suites": {}
+ "suites": {},
+ "failures": [],
+ "skips": []
}
testruns = build.testruns(count=1000)
for testrun_id, testrun in testruns.items():
-
- #print("\n - Testrun ({0}):".format(testrun_id))
- #dump_object(testrun, " ")
- #summary = testrun.summary()
- #print(" - Summary:")
- #dump_object(summary, " ")
-
tests = testrun.tests()
- #print("\n Tests ({0}):".format(len(tests)))
for test_id, test in tests.items():
- #print("\n - Test ({0}):".format(test_id))
- #dump_object(test, " ")
-
increment_build_counter(build_stats, test.status)
-
increment_environment_counter(build_stats, testrun.environment.slug, test.status)
+ increment_suite_counter(build_stats, testrun.environment.slug, test.suite, test.status)
+
+ if test.status == "fail":
+ build_stats["failures"].append((testrun.environment.slug, suite_name_from_url(group, project, test.suite), test.name))
+ build_stats["build"][test.status] += 1
+
+ elif test.status == "skip":
+ build_stats["skips"].append((testrun.environment.slug, suite_name_from_url(group, project, test.suite), test.name))
+ build_stats["build"][test.status] += 1
+
+ elif test.status == "pass":
+ build_stats["build"][test.status] += 1
- for suite in testrun.test_suites:
- increment_suite_counter(build_stats, suite, test.status)
+ elif test.status == "xfail":
+ build_stats["build"][test.status] += 1
- #print("\nSuites ({0}):".format(len(stats["suites"].keys())))
sorted_suites = []
- re_url = re.compile(REGEXP_SUITE_URL)
for suite_url, stats in build_stats["suites"].items():
- match = re_url.match(suite_url)
- # breakpoint()
- if not match:
- sorted_suites.append((suite_url, suite_url, stats))
- else:
- suite = project.suites(id=match.group("id"))
- if suite:
- suite = list(suite.values())[0]
- if suite:
- sorted_suites.append((suite.slug, suite.slug, stats))
- else:
- sorted_suites.append((suite_url, suite_url, stats))
- else:
- sorted_suites.append((suite_url, suite_url, stats))
+ suite = suite_name_from_url(group, project, suite_url)
+ sorted_suites.append((suite, stats))
sorted_suites = sorted(
- ((suite, stats) for suite, _, stats in sorted_suites),
- key=lambda item: item[1]
+ ((suite, stats) for suite, stats in sorted_suites),
+ key=lambda item: item[0]
)
build_stats["suites"] = sorted_suites
- #for suite, test_count in sorted_suites:
- #print(" - {0}: {1} tests".format(suite.slug, test_count))
+ # sort all suites under environments
+ for environment, stats in build_stats["environments"].items():
+ suites = stats["suites"]
+ sorted_suites = sorted(
+ ((suite_name_from_url(group, project, url), stats) for url, stats in suites.items()),
+ key=lambda item: item[0],
+ )
+ stats["suites"] = sorted_suites
- #print("\nEnvironments ({0}):".format(len(environments.keys())))
sorted_environments = sorted(
((name, stats) for name, stats in build_stats["environments"].items()),
key=lambda item: item[0],
)
build_stats["environments"] = sorted_environments
- #for environment, test_count in sorted_environments:
- #print(" - {0}: {1} tests".format(environment, test_count))
return build_stats
@@ -846,20 +916,24 @@ def build_report(report_setup):
)
return None
- print("Generating report for build {0} of {1}".format(build.version, project.slug))
+ print("Generating report for build {0} of {1} ...".format(build.version, project.slug))
- prev_build = project.build(report_setup["prev_build"])
- if not prev_build:
- print(
- "Unable to get build named {0} while generating report for {1} of {2}".format(
- report_setup["prev_build"],
- report_setup["build"],
- report_setup["project"],
+ if not report_setup["prev_build"]:
+ prev_build = None
+ compared_to_prev = None
+ else:
+ prev_build = project.build(report_setup["prev_build"])
+ if not prev_build:
+ print(
+ "Unable to get build named {0} while generating report for {1} of {2}".format(
+ report_setup["prev_build"],
+ report_setup["build"],
+ report_setup["project"],
+ )
)
- )
- return None
+ return None
- compared_to_prev = Project.compare_builds(prev_build.id, build.id)
+ compared_to_prev = Project.compare_builds(prev_build.id, build.id)
if (
report_setup["base_build"]
@@ -879,7 +953,7 @@ def build_report(report_setup):
compared_to_base = Project.compare_builds(base_build.id, build.id)
else:
- base_report = None
+ base_build = None
compared_to_base = None
build_details_url = urljoin(
@@ -905,17 +979,55 @@ def build_report(report_setup):
build_stats=build_stats
)
- file_name = "{0}/{1}-{2}.txt".format(DIRECTORY_REPORTS, project.slug, build.version)
+ file_name = "{0}/{1}--{2}.txt".format(DIRECTORY_REPORTS, project.slug, build.version)
with open(file_name, "w",) as f:
f.write(report)
- print("Wrote the report into {0}".format(file_name))
+ print("... wrote the report into {0}".format(file_name))
+
+ summary = template_env.get_template("stable-build-summary.jinja").render(
+ build=build,
+ prev_build=prev_build,
+ base_build=base_build,
+ compared_to_prev=compared_to_prev,
+ compared_to_base=compared_to_base,
+ build_details_url=build_details_url,
+ build_stats=build_stats
+ )
+ with open("{0}/{1}.txt".format(DIRECTORY_SUMMARIES, project.slug), "a") as fp:
+ fp.write(summary)
+
+
+def send_notifications(group):
+ try:
+ summary_files = os.listdir(DIRECTORY_SUMMARIES)
+ except FileNotFoundError as e:
+ print(
+ "Summary directory '{1}' not found".format(DIRECTORY_JOBS)
+ )
+ return
+
+ if not summary_files:
+ print("No summaries found")
+ return
+
+ summary_files.sort()
+
+ body = ""
+ for summary_file in summary_files:
+ with open(summary_file, "r") as f:
+ body += f.read()
+
+ with open("{0}/summary.txt".format(DIRECTORY_SUMMARIES), "w") as f:
+ f.write(body)
+
+
def send_reports(group):
pass
-def dump_object(obj, margin=""):
+def dump_object(obj, margin="", escape_nls=True):
functions = []
for var_name in dir(obj):
if var_name.startswith("_"):
@@ -926,7 +1038,7 @@ def dump_object(obj, margin=""):
functions.append(var_name)
continue
- if type(value) == str:
+ if type(value) == str and escape_nls:
value = value.replace("\r", "").replace("\n", "↵")
print(
@@ -938,96 +1050,6 @@ def dump_object(obj, margin=""):
print("{0} - functions: {1}".format(margin, ", ".join(functions)))
-def debug(group):
- project = group.project("linux-stable-rc-linux-5.8.y")
- print("Project:")
- dump_object(project)
- suites = project.suites(count=1000)
-
- # print("Suites ({0}):".format(len(suites)))
- # for suite_id, suite in suites.items():
- # print(" - {0} ({1}):".format(suite.slug, suite_id))
- # dump_object(suite, " ")
-
- environments = {}
- suites = {}
-
-
- builds = list(project.builds().values())
- new_build = builds[0]
- #breakpoint()
- for baseline_build in builds[1:]:
- print("\n{0} -- {1}".format(baseline_build.version, new_build.version))
- bc = Project.compare_builds(baseline_build.id, new_build.id)
- print(json.dumps(bc, sort_keys=True, indent=2))
- #new_build = baseline_build
-
-
-def other():
- testruns = build.testruns(count=1000)
- print("\n Test runs ({0}):".format(len(testruns)))
-
- for testrun_id, testrun in testruns.items():
-
- print("\n - Testrun ({0}):".format(testrun_id))
- dump_object(testrun, " ")
- summary = testrun.summary()
- print(" - Summary:")
- dump_object(summary, " ")
-
- tests = testrun.tests()
- print("\n Tests ({0}):".format(len(tests)))
-
- for test_id, test in tests.items():
- print("\n - Test ({0}):".format(test_id))
- dump_object(test, " ")
-
- if testrun.environment.slug not in environments:
- environments[testrun.environment.slug] = 1
- else:
- environments[testrun.environment.slug] += 1
-
- if test.suite not in suites:
- suites[test.suite] = 1
- else:
- suites[test.suite] += 1
-
- print("\nSuites ({0}):".format(len(suites.keys())))
- sorted_suites = []
- re_url = re.compile(REGEXP_SUITE_URL)
- for suite_url, test_count in suites.items():
- match = re_url.match(suite_url)
- # breakpoint()
- if not match:
- sorted_suites.append((suite_url, suite_url, test_count))
- else:
- suite = project.suites(id=match.group("id"))
- if suite:
- suite = list(suite.values())[0]
- if suite:
- sorted_suites.append((suite, suite.slug, test_count))
- else:
- sorted_suites.append((suite_url, suite_url, test_count))
- else:
- sorted_suites.append((suite_url, suite_url, test_count))
-
- sorted_suites = sorted(
- ((name, count) for name, _, count in sorted_suites),
- key=lambda item: item[1]
- )
-
- for suite, test_count in sorted_suites:
- print(" - {0}: {1} tests".format(suite.slug, test_count))
-
- print("\nEnvironments ({0}):".format(len(environments.keys())))
- sorted_environments = sorted(
- ((name, count) for name, count in environments.items()),
- key=lambda item: item[0],
- )
- for environment, test_count in sorted_environments:
- print(" - {0}: {1} tests".format(environment, test_count))
-
-
def main():
ap = argparse.ArgumentParser(description="LKFT stable report")
ap.add_argument(
@@ -1035,7 +1057,7 @@ def main():
metavar="stage",
type=str,
nargs="+",
- choices=("scan", "generate", "send", "list", "debug"),
+ choices=("scan", "generate", "notify", "send", "list", "list-all"),
help="mention which stage(s) to run",
)
@@ -1048,9 +1070,10 @@ def main():
function = {
"scan": scan_builds,
"generate": generate_reports,
+ "notify": send_notifications,
"send": send_reports,
"list": list_builds,
- "debug": debug,
+ "list-all": list_all_builds,
}.get(stage)
if not function:
@@ -1059,9 +1082,6 @@ def main():
function(group)
- # list_builds(group)
- # scan_builds(group)
- # generate_reports(group)
return
diff --git a/stable_report_config.py b/stable_report_config.py
new file mode 100644
index 0000000..fcfd073
--- /dev/null
+++ b/stable_report_config.py
@@ -0,0 +1,25 @@
+# projects that are mathing this regexp will be scanned for builds
+REGEXP_STABLE_PROJECT = "^linux-stable-rc-linux-([\d]+)\.([\d]+)\.y(?:-sanity)?$"
+
+
+# notification email config
+notification_email_setup = {
+ "From": "LKFT <lkft@linaro.org>",
+ "To": [
+ "Serge Broslavsky <serge.broslavsky@linaro.org>"
+ ],
+ "Cc": [
+ ],
+ "Subject": "Stable RC reports have been generated"
+}
+
+
+# report email config
+report_email_setup = {
+ "From": "LKFT <lkft@linaro.org>",
+ "To": [
+ "Serge Broslavsky <serge.broslavsky@linaro.org>"
+ ],
+ "Cc": [
+ ]
+}
diff --git a/templates/stable-build-report.jinja b/templates/stable-build-report.jinja
index 805e99e..41635b6 100644
--- a/templates/stable-build-report.jinja
+++ b/templates/stable-build-report.jinja
@@ -51,7 +51,7 @@ Environment "{{ environment }}":
{% if base_build %}
-## Compared to base build: {{ prev_build.version }}
+## Compared to base build: {{ base_build.version }}
### Regressions
{% if compared_to_base.regressions|length == 0 %}
@@ -92,57 +92,62 @@ Environment "{{ environment }}":
## Build Summary for {{ build.version }}
-{% for environment in environments %}
-{% for suite in suites %}
-{% set total = summary[environment.slug][suite.slug].pass
- + summary[environment.slug][suite.slug].fail
- + summary[environment.slug][suite.slug].skip
- + summary[environment.slug][suite.slug].xfail %}
+
+Test stats per environment and test suite:
+{% for environment, env_stats in build_stats.environments %}
+{% for suite, suite_stats in build_stats.suites %}
+{% set total = suite_stats.pass + suite_stats.fail + suite_stats.skip + suite_stats.xfail %}
{% if total == 0 %}{% continue %}{% endif %}
-### {{ environment.slug }}, {{ suite.slug }}
-* total: {{ total }}
-* pass: {{ summary[environment.slug][suie.slug].pass }}
-* fail: {{ summary[environment.slug][suie.slug].fail }}
-* skip: {{ summary[environment.slug][suie.slug].skip }}
-* xfail: {{ summary[environment.slug][suie.slug].xfail }}
+ * {{ environment }}, {{ suite }}: {{ total }} total{{ ", {0} passed".format(suite_stats.pass) if suite_stats.pass else "" }}{{ ", {0} failed".format(suite_stats.fail) if suite_stats.fail else ""}}{{ ", {0} skipped".format(suite_stats.skip) if suite_stats.skip else ""}}{{ ", {0} xfailed".format(suite_stats.xfail) if suite_stats.xfail else ""}}
{% endfor %}
{% endfor %}
-## Environments
-{% for environment, stats in build_stats.environments %}
-* {{ environment }}: p:{{ stats.pass if stats.pass else 0}}/f:{{ stats.fail if stats.fail else 0}}/s:{{ stats.skip if stats.skip else 0}}/x:{{ stats.xfail if stats.xfail else 0}}
-{% endfor %}
+## Environments for {{ build.version }}
-## Suites
-{% for suite, stats in build_stats.suites %}
-* {{ suite }}: p:{{ stats.pass if stats.pass else 0}}/f:{{ stats.fail if stats.fail else 0}}/s:{{ stats.skip if stats.skip else 0}}/x:{{ stats.xfail if stats.xfail else 0}}
-{% endfor %}
+{% if build_stats.environments|length > 0 %}
+Test stats per environment:
+{% for environment, stats in build_stats.environments %}
+{% set total = stats.pass + stats.fail + stats.skip + stats.xfail %}
+{% if total == 0 %}{% continue %}{% endif %}
+ * {{ environment }}: {{ total }} total{{ ", {0} passed".format(stats.pass) if stats.pass else ""}}{{ ", {0} failed".format(stats.fail) if stats.fail else ""}}{{ ", {0} skipped".format(stats.skip) if stats.skip else ""}}{{ ", {0} xfailed".format(stats.xfail) if stats.xfail else ""}}
+{% endfor %}
+{% else %}
+None.
+{% endif %}
-## Failures
-{% for environment in results %}
-{% for suite in results[environment] %}
-{% if results[environment][suite]|count == 0%}{% continue %}{% endif %}
-### {{ environment }}, {{ suite }}
-{% for test in results[environment][suite] %}
-{% if test.status == 'fail' %}
-* {{ test.short_name }}
-{% endif %}
-{% endfor %}
+## Suites for {{ build.version }}
+
+{% if build_stats.suites|length > 0 %}
+Test stats per suite:
+{% for suite, stats in build_stats.suites %}
+{% set total = stats.pass + stats.fail + stats.skip + stats.xfail %}
+ * {{ suite }}: {{ total }} total{{ ", {0} passed".format(stats.pass) if stats.pass else ""}}{{ ", {0} failed".format(stats.fail) if stats.fail else ""}}{{ ", {0} skipped".format(stats.skip) if stats.skip else ""}}{{ ", {0} xfailed".format(stats.xfail) if stats.xfail else ""}}
{% endfor %}
-{% endfor %}
+{% else %}
+None.
+{% endif %}
-## Skips
-{% for environment in results %}
-{% for suite in results[e] %}
-{% if results[environment][suite]|count == 0%}{% continue %}{% endif %}
-### {{ environment }}, {{ suite }}
-{% for test in results[environment][suite] %}
-{% if test.status == 'skip' %}
-* {{ test.short_name }}
-{% endif %}
-{% endfor %}
+## Failures for {{ build.version }}
+
+{% if build_stats.failures|length > 0 %}
+Failed tests ({{ build_stats.failures|length }}):
+{% for environment, suite, test in build_stats.failures %}
+ * {{ environment }}, {{ suite }}: {{ test }}
{% endfor %}
-{% endfor %}
+{% else %}
+None.
+{% endif %}
+
+## Skips for {{ build.version }}
+
+{% if build_stats.skips|length > 0 %}
+Skipped tests ({{ build_stats.skips|length }}):
+{% for environment, suite, test in build_stats.skips %}
+ * {{ environment }}, {{ suite }}: {{ test }}
+{% endfor %}
+{% else %}
+None.
+{% endif %}
--
Linaro LKFT