diff options
Diffstat (limited to 'wmfphablib')
-rwxr-xr-x | wmfphablib/__init__.py | 42 | ||||
-rwxr-xr-x | wmfphablib/bzlib.py | 90 | ||||
-rwxr-xr-x | wmfphablib/phabapi.py | 25 | ||||
-rwxr-xr-x | wmfphablib/phabdb.py | 41 | ||||
-rw-r--r-- | wmfphablib/util.py | 48 |
5 files changed, 201 insertions, 45 deletions
diff --git a/wmfphablib/__init__.py b/wmfphablib/__init__.py index c41bde8..28ee7b6 100755 --- a/wmfphablib/__init__.py +++ b/wmfphablib/__init__.py @@ -2,15 +2,20 @@ import os import sys import syslog import datetime -import bzwmfphab as bzlib +import bzlib import rtlib import fablib +import time +from util import log +from util import vlog +from util import errorlog +from util import datetime_to_epoch +from util import epoch_to_datetime from phabapi import phabapi as Phab from phabdb import phdb from phabdb import mailinglist_phid from phabdb import set_project_icon from config import cfile as configfile -import time def now(): return int(time.time()) @@ -42,40 +47,7 @@ def return_bug_list(): log("Bugs count: %d" % (len(bugs))) return bugs -def datetime_to_epoch(date_time): - return str((date_time - datetime.datetime(1970,1,1)).total_seconds()) - -def epoch_to_datetime(epoch, timezone='UTC'): - return str((datetime.datetime.fromtimestamp(int(float(epoch)) - ).strftime('%Y-%m-%d %H:%M:%S'))) + " (%s)" % (timezone,) - -def errorlog(msg): - msg = unicode(msg) - try: - syslog.syslog(msg) - print >> sys.stderr, msg - except: - print 'error logging, well...error output' - -def log(msg): - msg = unicode(msg) - if '-v' in ''.join(sys.argv): - try: - syslog.syslog(msg) - print msg - except: - print 'error logging output' - -def vlog(msg): - msg = unicode(msg) - if '-vv' in ''.join(sys.argv): - try: - print '-> ', msg - except: - print 'error logging output' - def save_attachment(name, data): f = open(name, 'wb') f.write(data) f.close() - diff --git a/wmfphablib/bzlib.py b/wmfphablib/bzlib.py new file mode 100755 index 0000000..ffce4ff --- /dev/null +++ b/wmfphablib/bzlib.py @@ -0,0 +1,90 @@ +import re + +prepend = 'bz' +security_mask = '_hidden_' + +def build_comment(c): + """ takes a native bz comment dict and outputs + a dict ready for processing into phab + """ + clean_c = {} + clean_c['author'] = c['author'].split('@')[0] + clean_c['creation_time'] = str(c['creation_time']) + clean_c['creation_time'] = int(float(c['creation_time'])) + if c['author'] != c['creator']: + clean_c['creator'] = c['creator'].split('@')[0] + + clean_c['count'] = c['count'] + if c['count'] == 0: + clean_c['bug_id'] = c['bug_id'] + + if c['is_private']: + c['text'] = security_mask + + attachment = find_attachment_in_comment(c['text']) + if attachment: + fmt_text = [] + text = c['text'].splitlines() + for t in text: + if not t.startswith('Created attachment'): + fmt_text.append(t) + c['text'] = '\n'.join(fmt_text) + clean_c['attachment'] = attachment + clean_c['text'] = c['text'] + return clean_c + +def find_attachment_in_comment(text): + a = re.search('Created\sattachment\s(\d+)', text) + if a: + return a.group(1) + else: + return '' + +def status_convert(bz_status): + """ + UNCONFIRMED (default) Open + Needs Triage (default) + NEW Open + ASSIGNED open + PATCH_TO_REVIEW open + NEED_INFO needs_info + RESOLVED FIXED resolved + RESOLVED INVALID invalid + RESOLVED WONTFIX declined + RESOLVED WORKSFORME resolved + RESOLVED DUPLICATE closed + + needs_info stalled + resolved closed + invalid no historical value will be purged eventually (spam, etc) + declined we have decided not too -- even though we could + """ + + statuses = {'new': 'open', + 'resolved': 'resolved', + 'reopened': 'open', + 'closed': 'resolved', + 'need_info': 'needs_info', + 'verified': 'resolved', + 'assigned': 'open', + 'unconfirmed': 'open', + 'patch_to_review': 'open'} + + return statuses[bz_status.lower()] + +def priority_convert(bz_priority): + """ + "100" : "Unbreak Now!", + "90" : "Needs Triage", + "80" : "High", + "50" : "Normal", + "25" : "Low", + "10" : "Needs Volunteer", + """ + priorities = {'unprioritized': 90, + 'immediate': 100, + 'highest': 100, + 'high': 80, + 'normal': 50, + 'low': 25, + 'lowest': 10} + return priorities[bz_priority.lower()] diff --git a/wmfphablib/phabapi.py b/wmfphablib/phabapi.py index 20c79f3..e6c1be1 100755 --- a/wmfphablib/phabapi.py +++ b/wmfphablib/phabapi.py @@ -1,15 +1,19 @@ -def log(m): - pass - import base64 import phabricator from phabricator import Phabricator import phabdb +from . import log +from . import vlog +from . import errorlog as elog class phabapi: def __init__(self, user, cert, host): + self.user = user + self.cert = cert + self.host = host + if host: self.con = Phabricator(username=user, certificate=cert, @@ -17,6 +21,17 @@ class phabapi: else: self.con = None + + def blog_update(self, botname, title, body): + blogphid = phabdb.get_bot_blog(botname) + if blogphid is None: + elog('blogphid is none') + return + return self.con.phame.createpost(blogPHID=blogphid, + body=body, + title=title, + phameTitle=title) + def sync_assigned(self, userphid, id, prepend): refs = phabdb.reference_ticket('%s%s' % (prepend, id)) if not refs: @@ -27,7 +42,7 @@ class phabapi: log('current owner found for => %s' % (str(id),)) return current log('assigning T%s to %s' % (str(id), userphid)) - return self.con.maniphest.update(phid=refs[0], ownerPHID=userphid) + return phabdb.set_issue_assigned(refs[0], userphid) def synced_authored(self, phid, id, ref): refs = phabdb.reference_ticket('%s%s' % (ref, id)) @@ -65,7 +80,7 @@ class phabapi: existing_proj = self.con.project.query(names=[project_name]) if not existing_proj['data']: - log('need to make: ' + project_name) + log('need to create project(s) ' + project_name) try: new_proj = self.con.project.create(name=project_name, members=pmembers) #XXX: Bug where we have to specify a members array! diff --git a/wmfphablib/phabdb.py b/wmfphablib/phabdb.py index 33d1305..d1f721c 100755 --- a/wmfphablib/phabdb.py +++ b/wmfphablib/phabdb.py @@ -16,20 +16,20 @@ from config import fabmigrate_passwd def get_user_relations_last_finish(dbcon): #get_user_relations_last_finish(pmig) - fin = dbcon.sql_x("SELECT max(finish_epoch) from user_relations_job", ()) + fin = dbcon.sql_x("SELECT max(finish_epoch) from user_relations_jobs", ()) try: return int(fin[0][0]) except: return 1 -def user_relations_start(pid, start, status, start_epoch, user_count, issue_count, dbcon): - insert_values = (pid, start, status, start_epoch, user_count, issue_count, int(time.time())) - query = "INSERT INTO user_relations_job (pid, start, status, start_epoch, user_count, issue_count, modified) VALUES (%s, %s, %s, %s, %s, %s, %s)" +def user_relations_start(pid, source, start, status, start_epoch, user_count, issue_count, dbcon): + insert_values = (pid, source, start, status, start_epoch, user_count, issue_count, int(time.time())) + query = "INSERT INTO user_relations_jobs (pid, source, start, status, start_epoch, user_count, issue_count, modified) VALUES (%s, %s, %s, %s, %s, %s, %s, %s)" return dbcon.sql_x(query, insert_values) def user_relations_finish(pid, finish, status, finish_epoch, completed, failed, dbcon): update_values = (finish, status, finish_epoch, completed, failed, int(time.time()), pid) - return dbcon.sql_x("UPDATE user_relations_job SET finish=%s, status=%s, finish_epoch=%s, completed=%s , failed=%s, modified=%s WHERE pid = %s", + return dbcon.sql_x("UPDATE user_relations_jobs SET finish=%s, status=%s, finish_epoch=%s, completed=%s , failed=%s, modified=%s WHERE pid = %s", update_values) def get_user_relations_priority(user, dbcon): @@ -80,6 +80,13 @@ def get_user_migration_comment_history(user, dbcon): return () return saved_history[0] +def get_bot_blog(botname): + p = phdb(db='phabricator_phame', user=phuser_user, passwd=phuser_passwd) + _ = p.sql_x("SELECT phid from phame_blog where name=%s", ('%s_updates' % (botname,)), limit=1) + p.close() + if _ is not None and len(_[0]) > 0: + return _[0][0] + def is_bot(userphid): p = phdb(db='phabricator_user', user=phuser_user, passwd=phuser_passwd) isbot = p.sql_x("SELECT isSystemAgent from user where phid=%s", (userphid,), limit=1) @@ -104,6 +111,11 @@ def set_issue_status(taskphid, status): p.sql_x("UPDATE maniphest_task SET status=%s WHERE phid=%s", (status, taskphid)) p.close() +def set_issue_assigned(taskphid, userphid): + p = phdb(db='phabricator_maniphest', user=phuser_user, passwd=phuser_passwd) + p.sql_x("UPDATE maniphest_task SET ownerPHID=%s WHERE phid=%s", (userphid, taskphid)) + p.close() + def set_comment_content(transxphid, content): """set manual content for a comment :param transxphid: str @@ -219,6 +231,15 @@ def get_user_relations(): return '' return _ +def get_verified_user(email): + phid, email, is_verified = get_user_email_info(email) + log("Single specified user: %s, %s, %s" % (phid, email, is_verified)) + if is_verified: + return [(phid, email)] + else: + log("%s is not a verified email" % (email,)) + return [()] + def get_user_email_info(emailaddress): p = phdb(db='phabricator_user', user=phuser_user, passwd=phuser_passwd) sql = "SELECT userPHID, address, isVerified from user_email where address=%s" @@ -227,6 +248,16 @@ def get_user_email_info(emailaddress): p.close() return _[0] or '' +def get_verified_users(modtime, limit=None): + #Find the task in new Phabricator that matches our lookup + verified = get_verified_emails(modtime=modtime, limit=limit) + create_times = [v[2] for v in verified] + try: + newest = max(create_times) + except ValueError: + newest = modtime + return verified, newest + def get_verified_emails(modtime=0, limit=None): p = phdb(db='phabricator_user', user=phuser_user, passwd=phuser_passwd) sql = "SELECT userPHID, address, dateModified from user_email where dateModified > %s and isVerified = 1" diff --git a/wmfphablib/util.py b/wmfphablib/util.py index a34f601..302146a 100644 --- a/wmfphablib/util.py +++ b/wmfphablib/util.py @@ -1,6 +1,54 @@ +import os import sys import json import subprocess +import config +import time +import datetime +import syslog + +def datetime_to_epoch(date_time): + return str((date_time - datetime.datetime(1970,1,1)).total_seconds()) + +def epoch_to_datetime(epoch, timezone='UTC'): + return str((datetime.datetime.fromtimestamp(int(float(epoch)) + ).strftime('%Y-%m-%d %H:%M:%S'))) + " (%s)" % (timezone,) + +def errorlog(msg): + msg = unicode(msg) + try: + syslog.syslog(msg) + print >> sys.stderr, msg + except: + print 'error logging, well...error output' + +def log(msg): + msg = unicode(msg) + if '-v' in ''.join(sys.argv): + try: + syslog.syslog(msg) + print msg + except: + print 'error logging output' + +def vlog(msg): + msg = unicode(msg) + if '-vv' in ''.join(sys.argv): + try: + print '-> ', msg + except: + print 'error logging output' + +def update_blog(source, complete, failed, user_count, issue_count, apicon): + title = "%s completed %s / failed %s" % (epoch_to_datetime(time.time()), + complete, + failed) + print title + body = "%s:\nUsers updated: %s\nIssues affected: %s" % (source, user_count, issue_count) + return apicon.blog_update(apicon.user, title, body) + +def source_name(path): + return os.path.basename(path.strip('.py')) def can_edit_ref(): f = open('/srv/phab/phabricator/conf/local/local.json', 'r').read() |