path: root/bugzilla_create.py

#!/usr/bin/env python
import multiprocessing
import time
import yaml
import ast
import json
import sys
import xmlrpclib
import os
import re
from phabricator import Phabricator
from wmfphablib import Phab as phabmacros
from wmfphablib import return_bug_list
from wmfphablib import phdb
from wmfphablib import now
from wmfphablib import mailinglist_phid
from wmfphablib import set_project_icon
from wmfphablib import phabdb
from wmfphablib import Phab
from wmfphablib import log
from wmfphablib import notice
from wmfphablib import vlog
from wmfphablib import errorlog as elog
from wmfphablib import bzlib
from wmfphablib import config
from wmfphablib import bzlib
from wmfphablib import util
from wmfphablib import datetime_to_epoch
from wmfphablib import epoch_to_datetime
from wmfphablib import ipriority

def create(bugid):
    phab = Phabricator(token=config.phab_cert, 
                       host=config.phab_host)

    phabm = phabmacros('', '', '')
    phabm.con = phab

    pmig = phabdb.phdb(db=config.bzmigrate_db,
                       user=config.bzmigrate_user,
                       passwd=config.bzmigrate_passwd)

    current = pmig.sql_x("SELECT priority, \
                          header, \
                          comments, \
                          created, \
                          modified \
                          FROM bugzilla_meta WHERE id = %s",
                          (bugid,))
    if current:
        import_priority, buginfo, com, created, modified = current[0]
    else:    
        pmig.close()
        elog('%s not present for migration' % (bugid,))
        return False

    def get_ref(id):
        refexists = phabdb.reference_ticket('%s%s' % (bzlib.prepend,
                                                      id))
        if refexists:
            pmig.close()
            return refexists

    if get_ref(bugid):
        log('reference ticket %s already exists' % (bugid,))
        return True

    buginfo = json.loads(buginfo)
    com = json.loads(com)
    bugid = int(bugid)
    vlog(bugid)
    vlog(buginfo)

    buginfo["secstate"] = 'none'
    # And the relevant herald rule must be in place.
    if bzlib.is_sensitive(buginfo["product"]):
        buginfo["secstate"] = 'security-bug'
        log("found security-bug issue %s" % (bugid,))

        #this allows testing while respecting security queue
        if config.bz_security.lower() != 'true':
            log("ignoring security issue %s" % (bugid,))
            return True

    server = xmlrpclib.ServerProxy(config.Bugzilla_url, use_datetime=True)
    token_data = server.User.login({'login': config.Bugzilla_login,
                                    'password': config.Bugzilla_password})

    token = token_data['token']
    #http://www.bugzilla.org/docs/tip/en/html/api/Bugzilla/WebService/Bug.html#attachments
    kwargs = { 'ids': [bugid], 'Bugzilla_token': token }

    bzdata= open("data/bugzilla.yaml", 'r')
    bzdata_yaml = yaml.load(bzdata)
    tag_keys = bzdata_yaml['keywords_to_tags'].split(' ')
    mlists = bzdata_yaml['assigned_to_lists'].split(' ')
    vlog("Mailinglists: " + str(mlists))

    attached = server.Bug.attachments(kwargs)['bugs'][str(bugid)]
    vlog("bz attached items %s" % (str(attached)))

    #process ticket uploads to map attach id to phab file id
    uploads = {}
    for a in attached:
        vlog("processing bz attachment %s" % (str(a)))
        if a['is_private']:
            vlog('ignoring private attachment: %s' %  (str(a)))
            a['ignore'] = 'private'
        elif a['is_obsolete'] == 1:
            vlog('ignoring obsolete attachment: %s' %  (str(a)))
            a['ignore'] = 'obsolete'
        else:
            if buginfo["secstate"] == 'none':
                viewpolicy = 'public'
            else:
                viewpolicy = phabdb.get_project_phid('security')
            try:
                #upload = phabm.upload_file(a['file_name'], a['data'].data)
                upload = phabm.upload_file(a['file_name'], a['data'].data, viewpolicy)
                a['phid'] = upload['phid']
                a['name'] = upload['name']
                a['objectName'] = upload['objectName']
            except Exception as e:
                 print "\n%s ATTACHMENT CORRUPTED -- %s\n" % (str(bugid), str(a))
                 print "%s --" % (str(e))
                 a['ignore'] = 'corrupt on retrieval'

        uploads[a['id']] = a

    log('%s attachment count: %s' % (bugid, str(len(uploads.keys()))))
    vlog("phab upload details: %s" % (str(uploads)))

    #list of projects to add to ticket
    ptags = []

    #mask emails for public consumption
    buginfo['cc'] = [c.split('@')[0] for c in buginfo['cc']]

    # Convert bugzilla source to phabricator
    buginfo['status'] = bzlib.status_convert(buginfo['status'],
                                             buginfo['resolution'])

    buginfo['priority'] = bzlib.priority_convert(buginfo['priority'])

    if '-d' in sys.argv:
        with open('dump', 'w') as d:
            d.write(str(json.dumps(buginfo)))

    if buginfo['status'].lower() == 'patch_to_review':
        ptags.append(('patch_to_review', 'tags', 'green'))

    if buginfo['status'] == 'verified':
        ptags.append(('verified', 'tags'))

    #kspoon: I think this is a custom WM thing... might need to lift it
    #  for any bugzilla customizations linaro has added
    #if buginfo['cf_browser'] not in ['---', "Other"]:
    #    btag = "Browser_Support_%s" % (buginfo['cf_browser'].replace(' ', '-'),)
    #    log('Adding browser tag: %s' % (btag,))
    #    ptags.append((btag, 'tags'))

    if buginfo['target_milestone'] != '---':
        log('Creating milestone: %s' % (buginfo['target_milestone'],))
        ptags.append((buginfo['target_milestone'], 'truck'))

    #set defaults to be overridden by sec if needed
    buginfo['viewPolicy'] = 'public'
    buginfo['editPolicy'] = 'users'
    #buginfo['project'] = bzlib.sanitize_project_name(buginfo["product"],
    #                                                 buginfo["component"])

    #kspoon: for linaro, we want to create the product as a container/parent
    # project, and the components as subprojects of the parent whereas wm just
    # flattened it.
    buginfo['parent_project'] = bzlib.sanitize_project_name(buginfo["product"])
    parent_phid = phabm.get_project_phid(buginfo['parent_project'])
    if not parent_phid:
        parent_phid = phabm.ensure_project(buginfo['parent_project'], parent_phid=None, edit='public', view='public')

    buginfo['bugzilla_id'] = "bugzilla-%s" % buginfo['id']

    print "parent_phid set to %s" % parent_phid

    buginfo['project'] = bzlib.sanitize_project_name(buginfo["component"])
    vlog(buginfo['project'])
    ptags.append((buginfo['project'], None))
    ptags.append((buginfo['parent_project'], None))

    title = buginfo['summary']

    clean_com = []
    for c in com:
        if not isinstance(c, dict):
            c = ast.literal_eval(c)
        clean_c = bzlib.build_comment(c, buginfo['secstate'])
        clean_com.append(clean_c)

    log('project: ' + buginfo['project'])

    try:
        # strip out comment 0 as description
        description = clean_com[0]
        del clean_com[0]
    except IndexError:
        log("%s has no comment 0" % (str(bugid)))
        # some random tasks were created at a point in bugzilla
        # history with metadata but no comment 0
        # https://bugzilla.wikimedia.org/show_bug.cgi?id=32056
        description = {'author': buginfo['creator'].split('@')[0],
                       'text': '//this issue has no description//',
                       'creation_time': buginfo['creation_time']}

    created = epoch_to_datetime(description['creation_time'])
    desc_block = "**Creator:** `%s`\n\n**Description:**\n%s\n" % (description['creator'],
                                                                 description['text'])

    # https://phabricator.wikimedia.org/T694
    desc_tail = '--------------------------'
    desc_tail += "\n**Version**: %s" % (buginfo['version'])
    desc_tail += "\n**Severity**: %s" % (buginfo['severity'] or 'none')

    if buginfo['op_sys'] != 'All':
        desc_tail += "\n**OS**: %s" % (buginfo['op_sys'])

    if "platform" in buginfo and buginfo['platform'] != 'All':
        desc_tail += "\n**Platform**: %s" % (buginfo['platform'])

    if buginfo['whiteboard']:
        desc_tail += "\n**Whiteboard**: %s" % (buginfo['whiteboard'])

    if buginfo['url']:
        desc_tail += "\n**URL**: %s" % (buginfo['url'])

    if buginfo['see_also']:
        desc_tail += "\n**See Also**:\n%s" % ('\n'.join(buginfo['see_also']))

    attachments = ''
    if 'attachment' in description:
        attached = int(description['attachment'])
        if attached in uploads:
            cattached = uploads[int(description['attachment'])]
            if 'objectName' in cattached:
               attachments = "\n\n**Attached**: {%s}" % (cattached['objectName'])

            if 'ignore' in cattached:
                attachments = "\n\n//attachment %s ignored as %s//" % (cattached['file_name'],
                                                                       cattached['ignore'])
        else:
            attachments = "\n\n//attachment missing in source//"

    desc_tail += attachments
    full_description = desc_block + '\n' + desc_tail

    keys = buginfo['keywords']
    for k in keys:
        if k in tag_keys:
            if k == 'ops':
                k = 'operations'
            ptags.append((k, 'tags'))

    def project_security_settings(pname):
        if bzlib.is_sensitive(pname):
            ephid = phabdb.get_project_phid('security')
            edit = ephid
        else:
            edit = 'public'
        view = 'public'
        return edit, view

    phids = []
    for p in ptags:
        edit, view = project_security_settings(p[0])
        phid = phabm.ensure_project(p[0], parent_phid=parent_phid, edit=edit, view=view)
        phids.append(phid)
        if p[1] is not None:
            vlog("setting project %s icon to %s" % (p[0], p[1]))
            set_project_icon(p[0], icon=p[1])

    log("ptags: " + str(ptags))
    vlog("phids: " + str(phids))

    #buginfo'assigned_to': u'wikibugs-l@lists.wikimedia.org'
    assignee = buginfo['assigned_to']

    ccphids = []
    if assignee in mlists:
        ccphids.append(mailinglist_phid(assignee))

    # viewPolicy = buginfo['viewPolicy'],
    # editPolicy = buginfo['editPolicy'],
    vlog("Ticket Info: %s" % (desc_block,))
    #kspoon: see if ticket already exists by checking title... not a great
    # idea.  Should search for bugzilla-<id> tag instead
    print "looking up tickets..."
    tickets = phab.maniphest.search(constraints={'query':'"%s"' % buginfo['summary']}, queryKey='all', order='newest') 

    # if 
    if len( tickets['data'] ) > 0:
      log("Task exists: T%s (%s)" % (tickets['data'][0]['id'], tickets['data'][0]['phid']))
      return True
    ticket = phab.maniphest.createtask(title=buginfo['summary'],
                                 description=full_description,
                                 projectPHIDs=phids,
                                 ccPHIDs=ccphids,
                                 priority=buginfo['priority'],
                                 auxiliary={"std:maniphest:external_reference":"bz%s" % (bugid,),
                                            "std:maniphest:security_topic":"%s" % (buginfo["secstate"],)})

    log("Created task: T%s (%s)" % (ticket['id'], ticket['phid']))
    botphid = phabdb.get_phid_by_username(config.phab_user)
    phabdb.set_task_title_transaction(ticket['phid'],
                                      botphid,
                                      buginfo['viewPolicy'],
                                      buginfo['editPolicy'])

    phabdb.set_task_ctime(ticket['phid'],
                          int(buginfo['creation_time'].split('.')[0]))

    fmt_comments = {}
    for c in clean_com:
        fmt_comment = {}
        created = epoch_to_datetime(c['creation_time'])
        comment_header = "**%s** wrote:\n\n" % (c['creator'],)
        comment_body = c['text']
        attachments = ''
        if 'attachment' in c:
            attached = int(c['attachment'])
            if attached in uploads:
                cattached = uploads[int(c['attachment'])]
                if 'objectName' in cattached:
                    attachments += "\n\n**Attached**: {%s}" % (cattached['objectName'])

                if 'ignore' in cattached:
                    attachments += "\n\n//attachment %s ignored as %s//" % (cattached['file_name'],
                                                                          cattached['ignore'])
            else:
                attachments += "\n\n//attachment missing in source//"
        fmt_comment['xpreamble'] = comment_header
        fmt_comment['xattached'] = attachments
        phabm.task_comment(ticket['id'],
                           comment_header + comment_body + attachments)
        ctransaction = phabdb.last_comment(ticket['phid'])
        phabdb.set_comment_time(ctransaction, c['creation_time'])
        fmt_comment['xctransaction'] = ctransaction
        fmt_comments[c['count']] = fmt_comment

    if buginfo['status'] != 'open':
        log("setting status for T%s to %s" % (ticket['id'], buginfo['status']))
        phabdb.set_issue_status(ticket['phid'], buginfo['status'])

    phabdb.set_task_mtime(ticket['phid'],
                          int(buginfo['last_change_time'].split('.')[0]))
    xcomments = json.dumps(fmt_comments)
    pmig.sql_x("UPDATE bugzilla_meta \
                SET xcomments=%s WHERE id = %s", (xcomments, bugid))
    pmig.sql_x("UPDATE bugzilla_meta \
                SET priority=%s, modified=%s WHERE id = %s",
               (ipriority['creation_success'], now(), bugid))
    pmig.close()
    return True


def run_create(bugid, tries=1):
    if tries == 0:
        pmig = phabdb.phdb(db=config.bzmigrate_db,
                       user=config.bzmigrate_user,
                       passwd=config.bzmigrate_passwd)
        import_priority = pmig.sql_x("SELECT priority \
                                      FROM bugzilla_meta \
                                      WHERE id = %s", (bugid,))
        if import_priority:
            pmig.sql_x("UPDATE bugzilla_meta \
                        SET priority=%s \
                        WHERE id = %s", (ipriority['creation_failed'],
                                         bugid))
        else:
            elog("%s does not seem to exist" % (bugid))
        pmig.close()
        elog('failed to create %s' % (bugid,))
        return False
    try:
        return create(bugid)
    except Exception as e:
        import traceback
        tries -= 1
        time.sleep(5)
        traceback.print_exc(file=sys.stdout)
        elog('failed to create %s (%s)' % (bugid, e))
        return run_create(bugid, tries=tries)

def main():

    def remove(bugid):
        notice("Removing bugid %s" % (bugid,))
        log(util.remove_issue_by_bugid(bugid, bzlib.prepend))

    if not util.can_edit_ref:
        elog('%s reference field not editable on this install' % (bugid,))
        sys.exit(1)

    if 'failed' in sys.argv:
        priority = ipriority['creation_failed']
    else:
        priority = 0

    pmig = phabdb.phdb(db=config.bzmigrate_db,
                       user=config.bzmigrate_user,
                       passwd=config.bzmigrate_passwd)
    bugs = return_bug_list(dbcon=pmig, priority=priority)
    pmig.close()

    #Serious business
    if 'failed' in sys.argv or '-r' in sys.argv:
        for b in bugs:
            remove(b)

    from multiprocessing import Pool
    pool = Pool(processes=int(config.bz_createmulti))
    _ =  pool.map(run_create, bugs)
    complete = len(filter(bool, _))
    failed = len(_) - complete
    print '%s completed %s, failed %s' % (sys.argv[0], complete, failed)

if __name__ == '__main__':
    main()