aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSerge Broslavsky <serge.broslavsky@linaro.org>2020-10-30 11:47:16 +0200
committerSerge Broslavsky <serge.broslavsky@linaro.org>2020-10-30 11:47:16 +0200
commit367334f94efdc1b9d4e1319b85d71fd7a3e2951c (patch)
tree9c7a0dc6f94a5f50e4b34abd3f54ad5b6239d02c
parent06227dc2e14ecd6a85eca101724506f017bd484a (diff)
downloadlkft-bucket-367334f94efdc1b9d4e1319b85d71fd7a3e2951c.tar.gz
WIP report sending
-rw-r--r--stable_report.py98
-rw-r--r--stable_report_config.py34
2 files changed, 101 insertions, 31 deletions
diff --git a/stable_report.py b/stable_report.py
index 1f308bc..580cf0a 100644
--- a/stable_report.py
+++ b/stable_report.py
@@ -9,12 +9,13 @@ import sys
import json
import jinja2
import datetime
-import mailbox
+import email
+from email.message import EmailMessage
from requests.compat import urljoin
from squad_client.core.api import SquadApi
from squad_client.core.models import Squad, Project
-import stable_report_config
+import stable_report_config as config
DEBUG=True
#DEBUG=False
@@ -55,6 +56,9 @@ ENV_PARALLEL_JOB_ID = "CI_NODE_INDEX"
# total number of parallel jobs that is set by gitlab for parallel jobs
ENV_PARALLEL_JOB_TOTAL = "CI_NODE_TOTAL"
+# url of the job details
+ENV_JOB_DETAILS_URL = "CI_JOB_URL"
+
# directory where all the reporting jobs are stored
DIRECTORY_JOBS = "jobs"
@@ -68,7 +72,7 @@ DIRECTORY_SUMMARIES = "summaries"
# the format is:
# (<project>(':'<build>','<prev-build>(','<base-build>)?':')*';')+
#
-# if not set - all the projects matching REGEXP_STABLE_PROJECT will be processed
+# if not set - all the projects matching config.REGEXP_STABLE_PROJECT will be processed
# and builds that have not yet been reported will be processed, automatically finding the suitable previous and base builds
ENV_PROJECT_LIST = "LKFT_PROJECT_LIST"
@@ -207,7 +211,7 @@ def list_builds(group, list_all=False):
"""
group_projects = group.projects()
- re_stable = re.compile(REGEXP_STABLE_PROJECT)
+ re_stable = re.compile(config.REGEXP_STABLE_PROJECT)
for project in group_projects.values():
if project.is_archived:
continue
@@ -561,7 +565,7 @@ def scan_builds(group):
else:
print("Building the list of projects using pattern matching")
unsorted_projects = []
- re_stable = re.compile(REGEXP_STABLE_PROJECT)
+ re_stable = re.compile(config.REGEXP_STABLE_PROJECT)
for project in group_projects.values():
if project.is_archived:
continue
@@ -979,9 +983,37 @@ def build_report(report_setup):
build_stats=build_stats
)
- file_name = "{0}/{1}--{2}.txt".format(DIRECTORY_REPORTS, project.slug, build.version)
+ msg = email.message_from_bytes(bytearray(report, "utf-8"))
+
+ # remove and readd after other headers to get it as a last header
+ subject = msg["Subject"]
+ del msg["Subject"]
+
+ setup = {
+ "From": "LKFT <lkft@linaro.org>",
+ "To": "LKFT <lkft@linaro.org>"
+ }
+ for regexp, a_setup in config.report_email_by_project:
+ match = re.match(regexp, project.slug)
+ if match:
+ setup.update(a_setup)
+ break
+
+ msg["From"]= setup.get("From", config.default_report_from)
+
+ values = setup.get("To", config.default_report_to)
+ msg["To"] = ", ".join(values)
+
+ values = setup.get("Cc", config.default_report_cc)
+ if values and len(values):
+ msg["Cc"] = ", ".join(values)
+
+ msg["Subject"] = subject
+
+ file_name = "{0}/{1}--{2}.eml".format(DIRECTORY_REPORTS, project.slug, build.version)
+
with open(file_name, "w",) as f:
- f.write(report)
+ f.write(msg.as_string())
print("... wrote the report into {0}".format(file_name))
summary = template_env.get_template("stable-build-summary.jinja").render(
@@ -991,7 +1023,8 @@ def build_report(report_setup):
compared_to_prev=compared_to_prev,
compared_to_base=compared_to_base,
build_details_url=build_details_url,
- build_stats=build_stats
+ build_stats=build_stats,
+ details_url=os.environ.get(ENV_JOB_DETAILS_URL)
)
with open("{0}/{1}.txt".format(DIRECTORY_SUMMARIES, project.slug), "a") as fp:
fp.write(summary)
@@ -1014,13 +1047,31 @@ def send_notifications(group):
body = ""
for summary_file in summary_files:
- with open(summary_file, "r") as f:
+ with open(os.path.join(DIRECTORY_SUMMARIES, summary_file), "r") as f:
body += f.read()
+ body += "\n-- \nLKFT"
+
+ msg = EmailMessage()
+ msg.set_content(body)
- with open("{0}/summary.txt".format(DIRECTORY_SUMMARIES), "w") as f:
- f.write(body)
+ msg["From"]= config.notification_email.get("From", config.default_notification_from)
+
+ values = config.notification_email.get("To")
+ if len(values):
+ msg["To"] = ", ".join(values)
+ else:
+ msg["To"] = config.default_notification_to
+ values = config.notification_email.get("Cc")
+ if values and len(values):
+ msg["Cc"] = ", ".join(values)
+ elif config.default_notification_cc and len(config.default_notification_cc):
+ msg["Cc"] = ", ".join(config.default_notification_cc)
+ msg["Subject"] = config.notification_email.get("Subject", config.default_notification_subject)
+
+ with open("{0}/summary.eml".format(DIRECTORY_SUMMARIES), "w") as f:
+ f.write(msg.as_string())
def send_reports(group):
@@ -1050,15 +1101,26 @@ def dump_object(obj, margin="", escape_nls=True):
print("{0} - functions: {1}".format(margin, ", ".join(functions)))
+g_stages = {
+ "scan": scan_builds,
+ "generate": generate_reports,
+ "notify": send_notifications,
+ "send": send_reports,
+ "list": list_builds,
+ "list-all": list_all_builds,
+}
+
def main():
+ global g_stages
+
ap = argparse.ArgumentParser(description="LKFT stable report")
ap.add_argument(
"stages",
metavar="stage",
type=str,
nargs="+",
- choices=("scan", "generate", "notify", "send", "list", "list-all"),
- help="mention which stage(s) to run",
+ choices=g_stages.keys(),
+ help="mention which stage(s) to run, one or more of: {0}".format(", ".join(g_stages.keys())),
)
args = ap.parse_args()
@@ -1067,15 +1129,7 @@ def main():
SquadApi.configure(url="https://qa-reports.linaro.org/")
group = Squad().group("lkft")
for stage in args.stages:
- function = {
- "scan": scan_builds,
- "generate": generate_reports,
- "notify": send_notifications,
- "send": send_reports,
- "list": list_builds,
- "list-all": list_all_builds,
- }.get(stage)
-
+ function = g_stages.get(stage)
if not function:
print("Unknown stage: {0}, skipped".format(stage))
continue
diff --git a/stable_report_config.py b/stable_report_config.py
index fcfd073..599c0d0 100644
--- a/stable_report_config.py
+++ b/stable_report_config.py
@@ -1,9 +1,18 @@
+# This is a configuration file for LKFT stable report
+# Edit this file to adjust the tool to your needs
+#
+
# projects that are mathing this regexp will be scanned for builds
REGEXP_STABLE_PROJECT = "^linux-stable-rc-linux-([\d]+)\.([\d]+)\.y(?:-sanity)?$"
# notification email config
-notification_email_setup = {
+default_notification_from = "LKFT <lkft@linaro.org>"
+default_notification_to = "LKFT <lkft@linaro.org>"
+default_notification_cc = None
+default_notification_subject = "Stable RC reports have been generated"
+
+notification_email = {
"From": "LKFT <lkft@linaro.org>",
"To": [
"Serge Broslavsky <serge.broslavsky@linaro.org>"
@@ -15,11 +24,18 @@ notification_email_setup = {
# report email config
-report_email_setup = {
- "From": "LKFT <lkft@linaro.org>",
- "To": [
- "Serge Broslavsky <serge.broslavsky@linaro.org>"
- ],
- "Cc": [
- ]
-}
+default_report_from = "LKFT <lkft@linaro.org>"
+default_report_to = "LKFT <lkft@linaro.org>"
+default_report_cc = None
+
+# project name matching regexp => email setup
+report_email_by_project = [
+ ("^.*$", {
+ "From": "LKFT <lkft@linaro.org>",
+ "To": [
+ "Serge Broslavsky <serge.broslavsky@linaro.org>"
+ ],
+ "Cc": [
+ ]
+ }),
+]