phab lots of random improvements :)

12 files changed, 391 insertions, 53 deletions

diff --git a/README b/README
index cab1be1..b67e740 100755
--- a/README
+++ b/README
@@ -12,13 +12,15 @@ Preflight check for migrations:
 
     set up security group
     set up security herald rule
-    make sure mailing lists are in place
-    make sure ext ref key is editable
+    -- make sure ext ref key is editable --
+    -- make sure icons are excluded
     make sure bot is in security group
-    ensure mysql file size
+    ensure file size (mysql / php)
     bot creation
+    set up hosts file entry for db on main (dns is slow)
     update /etc/phabtools
 
+
 IMPORTANT:
 
     * bot user must be a member of importbots group
@@ -99,10 +101,10 @@ CREATE TABLE user_relations_comments
 );
 
 
-create table user_relations_jobs
+CREATE table user_relations_jobs
 (
                                   id INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
-			          pid INT(12),
+			          pid INT,
                                   source TEXT(30),
 			          start INT(12),
 			          finish INT(12),
diff --git a/bugzilla_create.py b/bugzilla_create.py
index 5711e25..255cd14 100755
--- a/bugzilla_create.py
+++ b/bugzilla_create.py
@@ -60,7 +60,7 @@ def create(bugid):
 
     if get_ref(bugid):
         log('reference ticket %s already exists' % (bugid,))
-        #return True
+        return True
 
     buginfo = json.loads(buginfo)
     com = json.loads(com)
@@ -102,15 +102,26 @@ def create(bugid):
     #process ticket uploads to map attach id to phab file id
     uploads = {}
     for a in attached:
-        if a['is_private']:
-            continue
         vlog("processing bz attachment %s" % (str(a)))
+        if a['is_private']:
+            vlog('ignoring private attachment: %s' %  (str(a)))
+            a['ignore'] = 'private'
+        elif a['is_obsolete'] == 1:
+            vlog('ignoring obsolete attachment: %s' %  (str(a)))
+            a['ignore'] = 'obsolete'
+        else:
+            try:
+                upload = phabm.upload_file(a['file_name'], a['data'].data)
+                a['phid'] = upload['phid']
+                a['name'] = upload['name']
+                a['objectName'] = upload['objectName']
+            except Exception as e:
+                 print "\n%s ATTACHMENT CORRUPTED -- %s\n" % (str(bugid), str(a))
+                 print "%s --" % (str(e))
+                 a['ignore'] = 'corrupt on retrieval'
 
-        upload = phabm.upload_file(a['file_name'], a['data'].data)
-        a['phid'] = upload['phid']
-        a['name'] = upload['name']
-        a['objectName'] = upload['objectName']
         uploads[a['id']] = a
+
     log('%s attachment count: %s' % (bugid, str(len(uploads.keys()))))
     vlog("phab upload details: %s" % (str(uploads)))
 
@@ -123,6 +134,7 @@ def create(bugid):
     # Convert bugzilla source to phabricator
     buginfo['status'] = bzlib.status_convert(buginfo['status'],
                                              buginfo['resolution'])
+
     buginfo['priority'] = bzlib.priority_convert(buginfo['priority'])
 
     if '-d' in sys.argv:
@@ -130,9 +142,9 @@ def create(bugid):
             d.write(str(json.dumps(buginfo)))
 
     if buginfo['status'].lower() == 'patch_to_review':
-        ptags.append(('Patch-For-Review', 'tags', 'green'))
+        ptags.append(('patch_to_review', 'tags', 'green'))
 
-    if buginfo['status'].lower() == 'verified':
+    if buginfo['status'] == 'verified':
         ptags.append(('verified', 'tags'))
 
     if buginfo['cf_browser'] not in ['---', "Other"]:
@@ -163,9 +175,18 @@ def create(bugid):
 
     log('project: ' + buginfo['project'])
 
-    # strip out comment 0 as description
-    description = clean_com[0]
-    del clean_com[0]
+    try:
+        # strip out comment 0 as description
+        description = clean_com[0]
+        del clean_com[0]
+    except IndexError:
+        log("%s has no comment 0" % (str(bugid)))
+        # some random tasks were created at a point in bugzilla
+        # history with metadata but no comment 0
+        # https://bugzilla.wikimedia.org/show_bug.cgi?id=32056
+        description = {'author': buginfo['creator'].split('@')[0],
+                       'text': '//this issue has no description//',
+                       'creation_time': buginfo['creation_time']}
 
     created = epoch_to_datetime(description['creation_time'])
     desc_block = "**Author:** `%s`\n\n**Description:**\n%s\n" % (description['author'],
@@ -191,12 +212,21 @@ def create(bugid):
     if buginfo['see_also']:
         desc_tail += "\n**See Also**:\n%s" % ('\n'.join(buginfo['see_also']))
 
+    attachments = ''
     if 'attachment' in description:
         attached = int(description['attachment'])
         if attached in uploads:
             cattached = uploads[int(description['attachment'])]
-            desc_tail += "\n\n**Attached**: {%s}" % (cattached['objectName'])
+            if 'objectName' in cattached:
+               attachments = "\n\n**Attached**: {%s}" % (cattached['objectName'])
+
+            if 'ignore' in cattached:
+                attachments = "\n\n//attachment %s ignored as %s//" % (cattached['file_name'],
+                                                                       cattached['ignore'])
+        else:
+            attachments = "\n\n//attachment missing in source//"
 
+    desc_tail += attachments
     full_description = desc_block + '\n' + desc_tail
 
     keys = buginfo['keywords']
@@ -234,14 +264,14 @@ def create(bugid):
     if assignee in mlists:
         ccphids.append(mailinglist_phid(assignee))
 
+    # viewPolicy = buginfo['viewPolicy'],
+    # editPolicy = buginfo['editPolicy'],
     vlog("Ticket Info: %s" % (desc_block,))
     ticket = phab.maniphest.createtask(title=buginfo['summary'],
                                  description=full_description,
                                  projectPHIDs=phids,
                                  ccPHIDs=ccphids,
                                  priority=buginfo['priority'],
-                                 viewPolicy = buginfo['viewPolicy'],
-                                 editPolicy = buginfo['editPolicy'],
                                  auxiliary={"std:maniphest:external_reference":"bz%s" % (bugid,),
                                             "std:maniphest:security_topic":"%s" % (buginfo["secstate"],)})
 
@@ -264,16 +294,20 @@ def create(bugid):
         attachments = ''
         if 'attachment' in c:
             attached = int(c['attachment'])
-            #some comments match the attachment regex but the attachment was deleted
-            # by an admin from bugzilla and so is now missing.
-            if attached not in uploads:
-                attachments += "\n\n //attachment %s missing in source//" % (attached,)
-            else:
+            if attached in uploads:
                 cattached = uploads[int(c['attachment'])]
-                attachments += "\n\n**Attached**: {%s}" % (cattached['objectName'])
+                if 'objectName' in cattached:
+                    attachments += "\n\n**Attached**: {%s}" % (cattached['objectName'])
+
+                if 'ignore' in cattached:
+                    attachments += "\n\n//attachment %s ignored as %s//" % (cattached['file_name'],
+                                                                          cattached['ignore'])
+            else:
+                attachments += "\n\n//attachment missing in source//"
         fmt_comment['xpreamble'] = comment_header
         fmt_comment['xattached'] = attachments
-        phabm.task_comment(ticket['id'], comment_header + comment_body + attachments)
+        phabm.task_comment(ticket['id'],
+                           comment_header + comment_body + attachments)
         ctransaction = phabdb.last_comment(ticket['phid'])
         phabdb.set_comment_time(ctransaction, c['creation_time'])
         fmt_comment['xctransaction'] = ctransaction
@@ -283,10 +317,13 @@ def create(bugid):
         log("setting status for T%s to %s" % (ticket['id'], buginfo['status']))
         phabdb.set_issue_status(ticket['phid'], buginfo['status'])
 
-    phabdb.set_task_mtime(ticket['phid'], int(buginfo['last_change_time'].split('.')[0]))
+    phabdb.set_task_mtime(ticket['phid'],
+                          int(buginfo['last_change_time'].split('.')[0]))
     xcomments = json.dumps(fmt_comments)
-    pmig.sql_x("UPDATE bugzilla_meta SET xcomments=%s WHERE id = %s", (xcomments, bugid))
-    pmig.sql_x("UPDATE bugzilla_meta SET priority=%s, modified=%s WHERE id = %s",
+    pmig.sql_x("UPDATE bugzilla_meta \
+                SET xcomments=%s WHERE id = %s", (xcomments, bugid))
+    pmig.sql_x("UPDATE bugzilla_meta \
+                SET priority=%s, modified=%s WHERE id = %s",
                (ipriority['creation_success'], now(), bugid))
     pmig.close()
     return True
@@ -295,10 +332,14 @@ def create(bugid):
 def run_create(bugid, tries=1):
     if tries == 0:
         pmig = phabdb.phdb(db=config.bzmigrate_db)
-        import_priority = pmig.sql_x("SELECT priority FROM bugzilla_meta WHERE id = %s", (bugid,))
+        import_priority = pmig.sql_x("SELECT priority \
+                                      FROM bugzilla_meta \
+                                      WHERE id = %s", (bugid,))
         if import_priority:
-            pmig.sql_x("UPDATE bugzilla_meta SET priority=%s WHERE id = %s", (ipriority['creation_failed'],
-                                                                              bugid))
+            pmig.sql_x("UPDATE bugzilla_meta \
+                        SET priority=%s \
+                        WHERE id = %s", (ipriority['creation_failed'],
+                                         bugid))
         else:
             elog("%s does not seem to exist" % (bugid))
         pmig.close()
@@ -316,19 +357,27 @@ def run_create(bugid, tries=1):
 
 def main():
 
+    def remove(bugid):
+        notice("Removing bugid %s" % (bugid,))
+        log(util.remove_issue_by_bugid(bugid, bzlib.prepend))
+
     if not util.can_edit_ref:
         elog('%s reference field not editable on this install' % (bugid,))
         sys.exit(1)
 
+    if 'failed' in sys.argv:
+        priority = ipriority['creation_failed']
+    else:
+        priority = 0
+
     pmig = phdb(db=config.bzmigrate_db)
-    bugs = return_bug_list(dbcon=pmig)
+    bugs = return_bug_list(dbcon=pmig, priority=priority)
     pmig.close()
 
     #Serious business
-    if 'failed' in sys.argv:
+    if 'failed' in sys.argv or '-r' in sys.argv:
         for b in bugs:
-            notice("Removing bugid %s" % (b,))
-            log(util.remove_issue_by_bugid(b, bzlib.prepend))
+            remove(b)
 
     from multiprocessing import Pool
     pool = Pool(processes=int(config.bz_createmulti))
diff --git a/bugzilla_fetch.py b/bugzilla_fetch.py
index 9102bc4..ad867f1 100755
--- a/bugzilla_fetch.py
+++ b/bugzilla_fetch.py
@@ -32,8 +32,9 @@ def fetch(bugid):
     #grabbing one bug at a time for now
     buginfo = server.Bug.get(kwargs)['bugs']	
     buginfo =  buginfo[0]
+    # some bugs have header data but no actual content
+    # https://bugzilla.wikimedia.org/show_bug.cgi?id=32738
     com = server.Bug.comments(kwargs)['bugs'][str(bugid)]['comments']
-    bug_id = com[0]['bug_id']
 
     #have to do for json
     buginfo['last_change_time'] = datetime_to_epoch(buginfo['last_change_time'])
@@ -79,15 +80,22 @@ def fetch(bugid):
 def run_fetch(bugid, tries=1):
     if tries == 0:
         pmig = phabdb.phdb(db=config.bzmigrate_db)
-        current = pmig.sql_x("SELECT * from bugzilla_meta where id = %s", bugid)
+        current = pmig.sql_x("SELECT * from bugzilla_meta \
+                              where id = %s", bugid)
         if current:
             update_values =  (ipriority['fetch_failed'], '', '', now(), bugid)
-            pmig.sql_x("UPDATE bugzilla_meta SET priority=%s, header=%s, comments=%s modified=%s WHERE id = %s",
+            pmig.sql_x("UPDATE bugzilla_meta SET priority=%s, \
+                                                 header=%s, \
+                                                 comments=%s, \
+                                                 modified=%s \
+                                                 WHERE id = %s",
                        update_values)
         else:
             insert_values =  (bugid, ipriority['fetch_failed'], '', '', now(), now())
-            pmig.sql_x("INSERT INTO bugzilla_meta (id, priority, header, comments, modified, created) VALUES (%s, %s, %s, %s, %s, %s)",
-                       insert_values)
+            pmig.sql_x("INSERT INTO bugzilla_meta \
+                        (id, priority, header, comments, modified, created) \
+                        VALUES (%s, %s, %s, %s, %s, %s)",
+                        insert_values)
             pmig.close()
         elog('failed to grab %s' % (bugid,))
         return False
diff --git a/bugzilla_populate_user_relations_comments_table.py b/bugzilla_populate_user_relations_comments_table.py
index cefc5e3..dc66cfb 100755
--- a/bugzilla_populate_user_relations_comments_table.py
+++ b/bugzilla_populate_user_relations_comments_table.py
@@ -77,7 +77,7 @@ def populate(bugid):
 
 def run_populate(bugid, tries=1):
     if tries == 0:
-        elog('failed to populate for %s' % (bugid,))
+        elog('user relations comments failed to populate %s' % (bugid,))
         return False
     try:
         return populate(bugid)
@@ -86,7 +86,7 @@ def run_populate(bugid, tries=1):
         tries -= 1
         time.sleep(5)
         traceback.print_exc(file=sys.stdout)
-        elog('failed to populate %s' % (bugid,))
+        elog('user relations comments failed to populate %s' % (bugid,))
         return run_populate(bugid, tries=tries)
 
 def main():
diff --git a/bugzilla_populate_user_relations_table.py b/bugzilla_populate_user_relations_table.py
index 889f8fa..ba69b78 100755
--- a/bugzilla_populate_user_relations_table.py
+++ b/bugzilla_populate_user_relations_table.py
@@ -155,7 +155,7 @@ def populate(bugid):
 
 def run_populate(bugid, tries=1):
     if tries == 0:
-        elog('failed to populate for %s' % (bugid,))
+        elog('user relations failed to populate for %s' % (bugid,))
         return False
     try:
         return populate(bugid)
@@ -164,7 +164,7 @@ def run_populate(bugid, tries=1):
         tries -= 1
         time.sleep(5)
         traceback.print_exc(file=sys.stdout)
-        elog('failed to populate %s (%s)' % (bugid, e))
+        elog('user relations failed to populate %s (%s)' % (bugid, e))
         return run_populate(bugid, tries=tries)
 
 def main():
diff --git a/bugzilla_reorder.py b/bugzilla_reorder.py
index 56d51a2..859a428 100755
--- a/bugzilla_reorder.py
+++ b/bugzilla_reorder.py
@@ -35,6 +35,7 @@ def reorder(first, start, end, placeholder=300001):
         print "placeholder %s not empty (%s)" % (placeholder, pphid)
         return 
 
+
     for t in issues:
         print "Reassigning reference: %s to %s" % (t, newid)
         # Find PHID of the first ticket in our lineup
diff --git a/bugzilla_update_user_header.py b/bugzilla_update_user_header.py
index 613488f..cf38607 100755
--- a/bugzilla_update_user_header.py
+++ b/bugzilla_update_user_header.py
@@ -28,7 +28,8 @@ def update(user):
                        config.phab_cert,
                        config.phab_host)
 
-    pmig = phabdb.phdb(db=config.bzmigrate_db,
+    pmig = phabdb.phdb(host=config.dbhost,
+                       db=config.bzmigrate_db,
                        user=config.bzmigrate_user,
                        passwd=config.bzmigrate_passwd)
 
diff --git a/rt_update_user_comments.py b/rt_update_user_comments.py
new file mode 100755
index 0000000..838c592
--- /dev/null
+++ b/rt_update_user_comments.py
@@ -0,0 +1,199 @@
+#!/usr/bin/env python
+import os
+import argparse
+import time
+import json
+import multiprocessing
+import sys
+import collections
+from phabricator import Phabricator
+from wmfphablib import Phab as phabmacros
+from wmfphablib import phabdb
+from wmfphablib import log
+from wmfphablib import util
+from wmfphablib import bzlib
+from wmfphablib import config
+from wmfphablib import vlog
+from wmfphablib import errorlog as elog
+from wmfphablib import epoch_to_datetime
+from wmfphablib import ipriority
+from wmfphablib import now
+from wmfphablib import return_bug_list
+from wmfphablib import ipriority
+
+
+def update(user):
+
+    phab = Phabricator(config.phab_user,
+                       config.phab_cert,
+                       config.phab_host)
+
+    pmig = phabdb.phdb(db=config.rtmigrate_db,
+                       user=config.rtmigrate_user,
+                       passwd=config.rtmigrate_passwd)
+
+    phabm = phabmacros('', '', '')
+    phabm.con = phab
+
+    if phabdb.is_bot(user['userphid']):
+        log("%s is a bot no action" % (user['user']))
+        return True
+
+    epriority = phabdb.get_user_relations_comments_priority(user['user'], pmig)
+    if epriority and len(epriority[0]) > 0:
+        if epriority[0][0] == ipriority['update_success']:
+            log('Skipping %s as already updated' % (user['user']))
+            #return True
+
+    if not user['issues']:
+        log("%s has no issues to update" % (user['user'],))
+        return True
+
+    for i in user['issues']:
+        comdetails = pmig.sql_x("SELECT comments, xcomments FROM bugzilla_meta WHERE id = %s", (int(i),))
+        jcom, jxcom = comdetails[0]
+        coms = json.loads(jcom)
+        xcoms = json.loads(jxcom)
+
+        for key, xi in xcoms.iteritems():
+            com = coms[util.get_index(coms, "count", int(key))]
+            content = com['text']
+            if com["creator"] == user['user']:
+                log("Updating comment %s for %s" % (xi['xctransaction'], user['user']))
+                phabdb.set_comment_author(xi['xctransaction'], user['userphid'])
+                phabdb.set_comment_content(xi['xctransaction'], content + xi['xattached'])
+
+    current = phabdb.get_user_migration_comment_history(user['user'], pmig)
+    if current:
+        log(phabdb.set_user_relations_comments_priority(ipriority['update_success'], user['user'], pmig))
+    else:
+        log('%s user does not exist to update' % (user['user']))
+        return False
+    pmig.close()
+    log(util.purge_cache())
+    return True
+
+def run_update(user, tries=1):
+    if tries == 0:
+        pmig = phabdb.phdb(db=config.rtmigrate_db,
+                       user=config.rtmigrate_user,
+                       passwd=config.rtmigrate_passwd)
+        current = phabdb.get_user_migration_history(user['user'], pmig)
+        if current:
+           log(phabdb.set_user_relations_priority(ipriority['update_failed'], user['user'], pmig))
+        else:
+            log('%s user does not exist to update' % (user['user']))
+        pmig.close()
+        elog('final fail to update %s' % (user['user'],))
+        return False
+    try:
+        return update(user)
+    except Exception as e:
+        import traceback
+        tries -= 1
+        time.sleep(5)
+        traceback.print_exc(file=sys.stdout)
+        elog('failed to update %s' % (user,))
+        return run_update(user, tries=tries)
+
+def get_user_histories(verified):
+    histories = []
+    pmig = phabdb.phdb(db=config.rtmigrate_db,
+                       user=config.rtmigrate_user,
+                       passwd=config.rtmigrate_passwd)
+
+    for v in verified:
+        vlog(str(v))
+        saved_history = phabdb.get_user_migration_comment_history(v[1], pmig)
+        if not saved_history:
+            log('%s verified email has no saved history' % (v[1],))
+            continue
+        log('%s is being processed' % (v[1],))
+        history = {}
+        history['user'] = v[1]
+        history['userphid'] = v[0]
+        history['issues'] = saved_history[0]
+        history['created'] = saved_history[1]
+        history['modified'] = saved_history[2]
+        histories.append(history)
+
+    pmig.close()
+    return [util.translate_json_dict_items(d) for d in histories]
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Updates user header metadata from bugzilla')
+    parser.add_argument('-a', action="store_true", default=False)
+    parser.add_argument('-e', action="store", dest='email')
+    parser.add_argument('-m', action="store", dest="starting_epoch", default=None)
+    parser.add_argument('-v', action="store_true", default=False)
+    args =  parser.parse_args()
+
+    pmig = phabdb.phdb(db=config.rtmigrate_db,
+                       user=config.rtmigrate_user,
+                       passwd=config.rtmigrate_passwd)
+
+    if args.a:
+        starting_epoch = phabdb.get_user_relations_comments_last_finish(pmig)
+        users, finish_epoch = phabdb.get_verified_users(starting_epoch, config.bz_updatelimit)
+    elif args.email:
+        users = phabdb.get_verified_user(args.email)
+        starting_epoch = 0
+        finish_epoch = 0
+    elif args.starting_epoch:
+        users, finish_epoch = phabdb.get_verified_users(args.starting_epoch)
+        starting_epoch = args.starting_epoch
+    else:
+        parser.print_help()
+        sys.exit(1)
+
+    if not any(users):
+        log("Existing as there are no new verified users")
+        sys.exit()
+
+    histories = get_user_histories(filter(bool, users))
+    user_count = len(histories)
+
+    icounts = []
+    for u in histories:
+        c = 0
+        if u['issues']:
+            c += len(u['issues'])
+        icounts.append(c)
+    issue_count = sum(icounts)
+
+    log("User Count %s" % (str(user_count)))
+    log("Issue Count %s" % (str(issue_count)))
+
+    pid = os.getpid()
+    source = util.source_name(sys.argv[0])
+    phabdb.user_relations_start(pid,
+                                source,
+                                int(time.time()),
+                                ipriority['na'],
+                                starting_epoch,
+                                user_count, issue_count, pmig)
+
+
+    from multiprocessing import Pool
+
+    pool = Pool(processes=int(config.bz_updatemulti))
+    _ =  pool.map(run_update, histories)
+    complete = len(filter(bool, _))
+    failed = len(_) - complete
+    phabdb.user_relations_finish(pid,
+                                 int(time.time()),
+                                 ipriority['update_success'],
+                                 finish_epoch,
+                                 complete,
+                                 failed,
+                                 pmig)
+
+    pm = phabmacros(config.phab_user, config.phab_cert, config.phab_host)
+    vlog(util.update_blog(source, complete, failed, user_count, issue_count, pm))
+
+    pmig.close()
+    print '%s completed %s, failed %s' % (sys.argv[0], complete, failed)
+
+if __name__ == '__main__':
+    main()
diff --git a/wmfphablib/__init__.py b/wmfphablib/__init__.py
index d32ef23..a83f8d7 100755
--- a/wmfphablib/__init__.py
+++ b/wmfphablib/__init__.py
@@ -31,6 +31,7 @@ ipriority = {'creation_failed': 6,
              'fetch_failed': 5,
              'na': 0,
              'denied': 2,
+             'missing': 3,
              'update_success': 8,
              'update_failed': 9,
              'unresolved': 1}
diff --git a/wmfphablib/bzlib.py b/wmfphablib/bzlib.py
index 8b7f733..6e1f8b0 100755
--- a/wmfphablib/bzlib.py
+++ b/wmfphablib/bzlib.py
@@ -1,7 +1,10 @@
 import re
 
 prepend = 'bz'
-security_mask = '_hidden_'
+security_mask = '//**content hidden as private in Bugzilla**//'
+
+# Some issues are just missing instead of constant failures we skip
+missing = [15368, 15369, 15370, 15371, 15372, 15373, 15374]
 
 def sanitize_project_name(product, component):
     """ translate bz product/component into valid project
@@ -23,8 +26,12 @@ def build_comment(c):
     a dict ready for processing into phab
     """
 
+    # these indicate textual metadata history and should be
+    # preserved as literal
     dupe_literals = ['This bug has been marked as a duplicate of bug',
+                     'This bug has been marked as a duplicate of',
                      'has been marked as a duplicate of this bug']
+
     clean_c = {}
     clean_c['author'] =  c['author'].split('@')[0]
     clean_c['creation_time'] = str(c['creation_time'])
diff --git a/wmfphablib/phabdb.py b/wmfphablib/phabdb.py
index c4df32e..b96d7c1 100755
--- a/wmfphablib/phabdb.py
+++ b/wmfphablib/phabdb.py
@@ -44,6 +44,23 @@ def get_user_relations_comments_last_finish(dbcon):
     except:
         return 1
 
+def get_issues_by_priority(dbcon, priority):
+    """ get failed creations
+    :param dbcon: db connector
+    :param priority: int
+    :returns: list
+    """
+    _ = dbcon.sql_x("SELECT id \
+                    from bugzilla_meta \
+                    where priority=%s",
+                    (priority,),
+                    limit=None)
+    if _ is None:
+        return
+    f_ = list(util.tflatten(_))
+    if f_:
+        return f_
+
 def get_failed_creations(dbcon):
     """ get failed creations
     :param dbcon: db connector
@@ -350,6 +367,7 @@ def set_task_mtime(taskphid, mtime):
              (mtime, taskphid))
     p.close()
 
+
 def set_task_ctime(taskphid, ctime):
     """set manual epoch ctime for task
     :param taskphid: str
@@ -366,6 +384,33 @@ def set_task_ctime(taskphid, ctime):
 
     p.close()
 
+def get_task_description(taskphid):
+    """get task description
+    :param taskphid: str
+    """
+    p = phdb(db='phabricator_maniphest',
+             user=phuser_user,
+             passwd=phuser_passwd)
+    _ = p.sql_x("SELECT description \
+                 from maniphest_task \
+                 WHERE phid=%s", (phid,))
+    p.close()
+    if _ is not None and len(_[0]) > 0:
+        return _[0][0]
+
+def set_task_description(taskphid, text):
+    """set task description
+    :param taskphid: str
+    :param mtime: int of modtime
+    """
+    p = phdb(db='phabricator_maniphest',
+             user=phuser_user,
+             passwd=phuser_passwd)
+    p.sql_x("UPDATE maniphest_task \
+             SET description=%s \
+             WHERE phid=%s", (text, taskphid))
+    p.close()
+
 def get_emails(modtime=0):
     p = phdb(db='phabricator_user',
              user=phuser_user,
@@ -894,6 +939,24 @@ def set_project_policy(projphid, view, edit):
                               projphid))
     p.close()
 
+def set_project_policy(projphid, view, edit):
+    """set a project as view policy
+    :param projphid: str
+    :param view: str
+    :param edit: str
+    """
+    p = phdb(db='phabricator_project',
+             user=phuser_user,
+             passwd=phuser_passwd)
+
+    p.sql_x("UPDATE project \
+             SET viewPolicy=%s, \
+             editPolicy=%s \
+             WHERE phid=%s", (view,
+                              edit,
+                              projphid))
+    p.close()
+
 def get_project_phid(project):
     p = phdb(db='phabricator_project',
              user=phuser_user,
@@ -1007,8 +1070,8 @@ def set_task_subscriber(taskphid, userphid):
 
 class phdb:
     def __init__(self, host = dbhost,
-                       user = "root",
-                       passwd = "labspass",
+                       user = phuser_user,
+                       passwd = phuser_passwd,
                        db = "phab_migration",
                        charset = 'utf8',):
 
diff --git a/wmfphablib/util.py b/wmfphablib/util.py
index 9f77cf5..7ec1e36 100644
--- a/wmfphablib/util.py
+++ b/wmfphablib/util.py
@@ -145,18 +145,20 @@ def remove_issue_by_bugid(bugid, ref):
     out += phabdb.reference_ticket("%s%s" % (ref, bugid))
     return out
 
-def return_bug_list(dbcon=None):
+def return_bug_list(dbcon=None, priority=None):
 
     if sys.stdin.isatty():
         bugs = sys.argv[1:]
     else:
         bugs = sys.stdin.read().strip('\n').strip().split()
 
-    if 'failed' in ''.join(sys.argv):
+    #if 'failed' in ''.join(sys.argv):
+    if priority:
         if dbcon == None:
             print "cant find dbcon for priority buglist"
             return []
-        bugs = phabdb.get_failed_creations(dbcon)
+        bugs = phabdb.get_issues_by_priority(dbcon, priority)
+        #bugs = phabdb.get_failed_creations(dbcon)
     elif '-' in bugs[0]:
         start, stop = bugs[0].split('-')
 
@@ -175,5 +177,10 @@ def return_bug_list(dbcon=None):
         print "Bug list not built"
         return
 
+    #exclude known bad
+    bugs = [b for b in bugs if b not in bzlib.missing]
+
     log("Bugs count: %d" % (len(bugs)))
+    if bugs is None:
+        return []
     return bugs