comment metadata works

17 files changed, 1356 insertions, 244 deletions

diff --git a/README b/README
index 1b2da21..1195225 100755
--- a/README
+++ b/README
@@ -27,6 +27,7 @@ CREATE TABLE bugzilla_meta (
                              priority INT,
                              header TEXT(10000),
                              comments TEXT(100000), 
+                             xcomments TEXT(100000), 
                              created int(12) NOT NULL,
                              modified int(12) NOT NULL
                             );
@@ -66,6 +67,16 @@ CREATE TABLE user_relations
                              modified int(12) NOT NULL
 );
 
+CREATE TABLE user_relations_comments
+(
+                             id INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+                             priority INT,
+                             user TEXT(1000),
+                             issues TEXT(10000),
+                             created int(12) NOT NULL,
+                             modified int(12) NOT NULL
+);
+
 
 create table user_relations_job
 (
diff --git a/bugrun.sh b/bugrun.sh
new file mode 100755
index 0000000..bbc3928
--- /dev/null
+++ b/bugrun.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+python bugzilla_fetch.py $1 -v
+python bugzilla_create.py $1 -v
+python bugzilla_update_tasks.py $1 -v
+python bugzilla_populate_user_relations_comments_table.py $1
+python bugzilla_populate_user_relations_table.py $1
+python bugzilla_update_user_header.py -m 1
+python bugzilla_update_user_comments.py -m 1
diff --git a/bugzilla_create.py b/bugzilla_create.py
index daf1431..81f8d08 100755
--- a/bugzilla_create.py
+++ b/bugzilla_create.py
@@ -1,13 +1,4 @@
 #!/usr/bin/env python
-"""
-
-2014 Chase Pettet
-
-
-This script is a WIP for getting Bugzilla information
-with the end goal of it living in phabricator
-
-"""
 import multiprocessing
 import time
 import yaml
@@ -17,68 +8,83 @@ import json
 import sys
 import xmlrpclib
 import os
-import MySQLdb
-
+from phabricator import Phabricator
+from wmfphablib import Phab as phabmacros
+from wmfphablib import return_bug_list
 from wmfphablib import phdb
 from wmfphablib import mailinglist_phid
 from wmfphablib import set_project_icon
 from wmfphablib import phabdb
 from wmfphablib import Phab
 from wmfphablib import log
+from wmfphablib import vlog
+from wmfphablib import errorlog as elog
 from wmfphablib import bzlib
+from wmfphablib import config
+from wmfphablib import bzlib
+from wmfphablib import util
 from wmfphablib import datetime_to_epoch
 from wmfphablib import epoch_to_datetime
 from wmfphablib import ipriority
 from email.parser import Parser
-from wmfphablib import get_config_file
-import ConfigParser
-
-configfile = get_config_file()
-
-def fetch(bugid):
-    parser = ConfigParser.SafeConfigParser()
-    parser_mode = 'phab'
-    parser.read(configfile)
-    phab = Phab(user=parser.get(parser_mode, 'username'),
-                cert=parser.get(parser_mode, 'certificate'),
-                host=parser.get(parser_mode, 'host'))
-    parser_mode = 'bz'
-    server = xmlrpclib.ServerProxy(parser.get(parser_mode, 'url'), use_datetime=True)
+
+def create(bugid):
+
+    phab = Phabricator(config.phab_user,
+                       config.phab_cert,
+                       config.phab_host)
+
+    phabm = phabmacros('', '', '')
+    phabm.con = phab
+
+    pmig = phdb(db=config.bzmigrate_db)
+    current = pmig.sql_x("SELECT priority, header, comments, created, modified FROM bugzilla_meta WHERE id = %s", (bugid,))
+    if current:
+        import_priority, buginfo, com, created, modified = current[0]
+    else:    
+        elog('%s not present for migration' % (bugid,))
+        return False
+
+    refexists = phabdb.reference_ticket('%s%s' % (bzlib.prepend, bugid))
+    if refexists:
+        log('reference ticket %s already exists' % (bugid,))
+        return True
+
+    buginfo = json.loads(buginfo)
+    com = json.loads(com)
+    bugid = int(bugid)
+    vlog(bugid)
+    vlog(buginfo)
+
+    server = xmlrpclib.ServerProxy(config.Bugzilla_url, use_datetime=True)
+    token_data = server.User.login({'login': config.Bugzilla_login,
+                                    'password': config.Bugzilla_password})
+
+    token = token_data['token']
+    #http://www.bugzilla.org/docs/tip/en/html/api/Bugzilla/WebService/Bug.html#attachments
+    kwargs = { 'ids': [bugid], 'Bugzilla_token': token }
+
     bzdata= open("data/bugzilla.yaml", 'r')
     bzdata_yaml = yaml.load(bzdata)
     tag_keys = bzdata_yaml['keywords_to_tags'].split(' ')
     mlists = bzdata_yaml['assigned_to_lists'].split(' ')
-    log("Mailinglists: " + str(mlists))
+    vlog("Mailinglists: " + str(mlists))
 
-    #http://www.bugzilla.org/docs/tip/en/html/api/Bugzilla/WebService/Bug.html#attachments
-    kwargs = { 'ids': [bugid],
-               'Bugzilla_login': parser.get(parser_mode, 'Bugzilla_login'),
-               'Bugzilla_password': parser.get(parser_mode, 'Bugzilla_password')}
-    attached = server.Bug.attachments(kwargs)['bugs'][bugid]
+    #print server.Bug.attachments(kwargs)['bugs']
+    attached = server.Bug.attachments(kwargs)['bugs'][str(bugid)]
 
     #process ticket uploads to map attach id to phab file id
     uploads = {}
     for a in attached:
         if a['is_private']:
             continue
-        upload = phab.upload_file(a['file_name'], str(a['data']))
+        upload = phabm.upload_file(a['file_name'], str(a['data']))
         a['phid'] = upload['phid']
         a['name'] = upload['name']
         a['objectName'] = upload['objectName']
         uploads[a['id']] = a
     log('Attachment count: ' + str(len(uploads.keys())))
 
-    pmig = phdb(db='bugzilla_migration')
-    bugid, import_priority, buginfo, com, created, modified = pmig.sql_x("SELECT * FROM bugzilla_meta WHERE id = %s",
-                                     (bugid,))
-    pmig.close()
-
-    buginfo = json.loads(buginfo)
-    com = json.loads(com)
-    bugid = int(bugid)
-    log(bugid)
-    log(buginfo)
-
     #list of projects to add to ticket
     ptags = []
 
@@ -89,18 +95,19 @@ def fetch(bugid):
     buginfo['status'] = bzlib.status_convert(buginfo['status'])
     buginfo['priority'] = bzlib.priority_convert(buginfo['priority'])
 
-    with open('dump', 'w') as d:
-        d.write(str(json.dumps(buginfo)))
+    if '-d' in sys.argv:
+        with open('dump', 'w') as d:
+            d.write(str(json.dumps(buginfo)))
 
     if buginfo['status'].lower() == 'patch_to_review':
-        ptags.append(('patch_to_review', 'tag', 'green'))
+        ptags.append(('patch_to_review', 'tags', 'green'))
 
     if buginfo['status'] == 'verified':
-        ptags.append(('verified', 'tag'))
+        ptags.append(('verified', 'tags'))
 
     if buginfo['cf_browser'] not in ['---', "Other"]:
         log('Adding browser tag: %s' % (buginfo['cf_browser'],))
-        ptags.append((buginfo['cf_browser'], 'tag'))
+        ptags.append((buginfo['cf_browser'], 'tags'))
 
     if buginfo['target_milestone'] != '---':
         log('Creating milestone: %s' % (buginfo['target_milestone'],))
@@ -125,7 +132,7 @@ def fetch(bugid):
                          buginfo["component"])
 
     buginfo['project'] = project
-    log(buginfo['project'])
+    vlog(buginfo['project'])
     ptags.append((buginfo['project'], None))
 
     title = buginfo['summary']
@@ -134,30 +141,7 @@ def fetch(bugid):
     for c in com:
         if not isinstance(c, dict):
             c = ast.literal_eval(c)
-        clean_c = {}
-        clean_c['author'] =  c['author'].split('@')[0]
-
-        clean_c['creation_time'] = str(c['creation_time'])
-        if c['author'] != c['creator']:
-            clean_c['creator'] =  c['creator'].split('@')[0]
-
-        if c['count'] == 0:
-            clean_c['bug_id'] = c['bug_id']
-
-        if c['is_private']:
-            c['text'] = '_hidden_'
-
-        attachment = bzlib.find_attachment_in_comment(c['text'])
-        if attachment:
-            fmt_text = []
-            text = c['text'].splitlines()
-            for t in text:
-                if not t.startswith('Created attachment'):
-                    fmt_text.append(t)
-            c['text'] = '\n'.join(fmt_text)
-            clean_c['attachment'] = attachment
-
-        clean_c['text'] = c['text']
+        clean_c = bzlib.build_comment(c)
         clean_com.append(clean_c)
 
     log('project: ' + buginfo['project'])
@@ -167,9 +151,8 @@ def fetch(bugid):
     del clean_com[0]
 
     created = epoch_to_datetime(description['creation_time'])
-    desc_block = "**Created**: `%s`\n\n**Author:** `%s`\n\n**Description:**\n%s\n" % (created,
-                                                                                        description['author'],
-                                                                                        description['text'])
+    desc_block = "**Author:** `%s`\n\n**Description:**\n%s\n" % (description['author'],
+                                                                   description['text'])
     desc_tail = '--------------------------'
     desc_tail += "\n**URL**: %s" % (buginfo['url'].lower() or 'none')
     desc_tail += "\n**Version**: %s" % (buginfo['version'].lower())
@@ -195,17 +178,13 @@ def fetch(bugid):
 
     phids = []
     for p in ptags:
-        phids.append(phab.ensure_project(p[0]))
+        phids.append(phabm.ensure_project(p[0]))
         if p[1] is not None:
-            if len(p) > 2:
-                color = p[2]
-            else:
-                color = 'blue'
-            log("Setting project %s icon to %s" % (p[0], p[1]))
-            set_project_icon(p[0], icon=p[1], color=color)
+            vlog("setting project %s icon to %s" % (p[0], p[1]))
+            set_project_icon(p[0], icon=p[1])
 
     log("ptags: " + str(ptags))
-    log("phids: " + str(phids))
+    vlog("phids: " + str(phids))
 
     #buginfo'assigned_to': u'wikibugs-l@lists.wikimedia.org'
     assignee = buginfo['assigned_to']
@@ -214,79 +193,87 @@ def fetch(bugid):
     if assignee in mlists:
         ccphids.append(mailinglist_phid(assignee))
 
-    log("Ticket Info: %s" % (desc_block,))
-    ticket = phab.task_create(title,
-                    full_description,
-                    description['bug_id'],
-                    buginfo['priority'],
-                    buginfo["secstate"],
-                    ccPHIDs=ccphids,
-                    projects=phids,
-                    refcode=bzlib.prepend)
+    vlog("Ticket Info: %s" % (desc_block,))
+    ticket =  phab.maniphest.createtask(title=buginfo['summary'],
+                                 description=full_description,
+                                 projectPHIDs=phids,
+                                 ccPHIDs=ccphids,
+                                 priority=buginfo['priority'],
+                                 auxiliary={"std:maniphest:external_reference":"bz%s" % (bugid,)})
 
-    print "Created: ", ticket['id']
+    log("Created task: T%s (%s)" % (ticket['id'], ticket['phid']))
+    phabdb.set_task_ctime(ticket['phid'], int(buginfo['creation_time'].split('.')[0]))
 
-    comment_block = "**%s** `%s` \n\n %s"
+    fmt_comments = {}
     for c in clean_com:
-        log('-------------------------------------')
+        fmt_comment = {}
         created = epoch_to_datetime(c['creation_time'])
-        comment_body = "**%s** wrote on `%s`\n\n%s" % (c['author'], created, c['text'])
+        comment_header = "**%s** wrote:\n\n" % (c['author'],)
+        comment_body = c['text']
+        attachments = ''
         if 'attachment' in c:
             attached = int(c['attachment'])
-
             #some comments match the attachment regex but the attachment was deleted
             # by an admin from bugzilla and so is now missing.
             if attached not in uploads:
-                comment_body += "\n\n //attachment %s missing in source//" % (attached,)
+                attachments += "\n\n //attachment %s missing in source//" % (attached,)
             else:
                 cattached = uploads[int(c['attachment'])]
-                comment_body += "\n\n**Attached**: {%s}" % (cattached['objectName'])
-        phab.task_comment(ticket['id'], comment_body)
-
-    log(str(ticket['id']) + str(buginfo['status'])) 
+                attachments += "\n\n**Attached**: {%s}" % (cattached['objectName'])
+        fmt_comment['xpreamble'] = comment_header
+        fmt_comment['xattached'] = attachments
+        phabm.task_comment(ticket['id'], comment_header + comment_body + attachments)
+        ctransaction = phabdb.last_comment(ticket['phid'])
+        phabdb.set_comment_time(ctransaction, c['creation_time'])
+        fmt_comment['xctransaction'] = ctransaction
+        fmt_comments[c['count']] = fmt_comment
 
     if buginfo['status'] != 'open':
-        phab.task_comment(ticket['id'], '//importing issue status//')
-        phab.set_status(ticket['id'], buginfo['status'])
+        log("setting status for T%s to %s" % (ticket['id'], buginfo['status']))
+        phabdb.set_issue_status(ticket['phid'], buginfo['status'])
 
+    phabdb.set_task_mtime(ticket['phid'], int(buginfo['last_change_time'].split('.')[0]))
+    xcomments = json.dumps(fmt_comments)
+    pmig.sql_x("UPDATE bugzilla_meta SET xcomments=%s WHERE id = %s", (xcomments, bugid))
+    pmig.close()
     return True
 
-def run_fetch(bugid, tries=1):
+
+def run_create(bugid, tries=1):
     if tries == 0:
-        pmig = phabdb.phdb(db='bugzilla_migration')
+        pmig = phabdb.phdb(db=config.bzmigrate_db)
         import_priority = pmig.sql_x("SELECT priority FROM bugzilla_meta WHERE id = %s", (bugid,))
         if import_priority:
-            log('updating existing record')
             pmig.sql_x("UPDATE bugzilla_meta SET priority=%s WHERE id = %s", (ipriority['creation_failed'],
                                                                               bugid))
         else:
-            print "%s does not seem to exist" % (bugid)
+            elog("%s does not seem to exist" % (bugid))
         pmig.close()
-        print 'failed to grab %s' % (bugid,)
+        elog('failed to create %s' % (bugid,))
         return False
     try:
-        if fetch(bugid):
-            print time.time()
-            print 'done with %s' % (bugid,)
-            return True
+        return create(bugid)
     except Exception as e:
         import traceback
         tries -= 1
         time.sleep(5)
         traceback.print_exc(file=sys.stdout)
-        print 'failed to grab %s (%s)' % (bugid, e)
-        return run_fetch(bugid, tries=tries)
-
-if sys.stdin.isatty():
-    bugs = sys.argv[1:]
-else:
-    bugs = sys.stdin.read().strip('\n').strip().split()
-
-bugs = [i for i in bugs if i.isdigit()]
-print len(bugs)
-from multiprocessing import Pool
-pool = Pool(processes=10)
-_ =  pool.map(run_fetch, bugs)
-complete = len(filter(bool, _))
-failed = len(_) - complete
-print 'completed %s, failed %s' % (complete, failed)
+        elog('failed to create %s (%s)' % (bugid, e))
+        return run_create(bugid, tries=tries)
+
+def main():
+
+    if not util.can_edit_ref:
+        elog('%s reference field not editable on this install' % (bugid,))
+        sys.exit(1)
+
+    bugs = return_bug_list()
+    from multiprocessing import Pool
+    pool = Pool(processes=10)
+    _ =  pool.map(run_create, bugs)
+    complete = len(filter(bool, _))
+    failed = len(_) - complete
+    print '%s completed %s, failed %s' % (sys.argv[0], complete, failed)
+
+if __name__ == '__main__':
+    main()
diff --git a/bugzilla_fetch.py b/bugzilla_fetch.py
index f06c6e1..abb5a3b 100755
--- a/bugzilla_fetch.py
+++ b/bugzilla_fetch.py
@@ -1,13 +1,4 @@
 #!/usr/bin/env python
-"""
-
-2014 Chase Pettet
-
-
-This script is a WIP for getting Bugzilla information
-with the end goal of it living in phabricator
-
-"""
 import time
 import yaml
 import json
@@ -15,26 +6,24 @@ import sys
 import xmlrpclib
 import os
 from wmfphablib import log
+from wmfphablib import vlog
+from wmfphablib import errorlog as elog
 from wmfphablib import bzlib
+from wmfphablib import config
 from wmfphablib import epoch_to_datetime
 from wmfphablib import datetime_to_epoch
 from wmfphablib import phabdb
 from wmfphablib import ipriority
-from wmfphablib import get_config_file
 from wmfphablib import now
-import ConfigParser
+from wmfphablib import return_bug_list
 
-configfile = get_config_file()
 
 def fetch(bugid):
 
-    parser = ConfigParser.SafeConfigParser()
-    parser_mode = 'bz'
-    parser.read(configfile)
-    server = xmlrpclib.ServerProxy(parser.get(parser_mode, 'url'), use_datetime=True)
-
-    token_data = server.User.login({'login': parser.get(parser_mode, 'Bugzilla_login'),
-                             'password': parser.get(parser_mode, 'Bugzilla_password')})
+    pmig = phabdb.phdb(db=config.bzmigrate_db)
+    server = xmlrpclib.ServerProxy(config.Bugzilla_url, use_datetime=True)
+    token_data = server.User.login({'login': config.Bugzilla_login,
+                                    'password': config.Bugzilla_password})
 
     token = token_data['token']
     kwargs = { 'ids': [bugid], 'Bugzilla_token': token }
@@ -61,20 +50,19 @@ def fetch(bugid):
     else:
         creation_priority = ipriority['unresolved']
 
-    pmig = phabdb.phdb(db='bugzilla_migration')
     current = pmig.sql_x("SELECT * from bugzilla_meta where id = %s", bugid)
     if current:
-        log('updating current record')
         update_values = (creation_priority,
                          json.dumps(buginfo),
                          json.dumps(com),
                          now(),
                          bugid)
-        pmig.sql_x("UPDATE bugzilla_meta SET priority=%s, header=%s, comments=%s modified=%s WHERE id = %s",
+        vlog('update: ' + str(update_values))
+        pmig.sql_x("UPDATE bugzilla_meta SET priority=%s, header=%s, comments=%s, modified=%s WHERE id = %s",
                    update_values)
     else:
-        log('inserting new record')
         insert_values =  (bugid, creation_priority, json.dumps(buginfo), json.dumps(com), now(), now())
+        vlog('insert: ' + str(insert_values))
         sql = "INSERT INTO bugzilla_meta (id, priority, header, comments, created, modified) VALUES (%s, %s, %s, %s, %s, %s)"
         pmig.sql_x(sql,
                    insert_values)
@@ -83,34 +71,39 @@ def fetch(bugid):
 
 def run_fetch(bugid, tries=1):
     if tries == 0:
-        pmig = phabdb.phdb(db='bugzilla_migration')
-        insert_values =  (bugid, ipriority['fetch_failed'], '', '', now(), now())
-        pmig.sql_x("INSERT INTO bugzilla_meta (id, priority, header, comments, modified, created) VALUES (%s, %s, %s, %s, %s, %s)",
-                   insert_values)
-        pmig.close()
-        print 'failed to grab %s' % (bugid,)
+        pmig = phabdb.phdb(db=config.bzmigrate_db)
+        current = pmig.sql_x("SELECT * from bugzilla_meta where id = %s", bugid)
+        if current:
+            update_values =  (ipriority['fetch_failed'], '', '', now(), bugid)
+            pmig.sql_x("UPDATE bugzilla_meta SET priority=%s, header=%s, comments=%s modified=%s WHERE id = %s",
+                       update_values)
+        else:
+            insert_values =  (bugid, ipriority['fetch_failed'], '', '', now(), now())
+            pmig.sql_x("INSERT INTO bugzilla_meta (id, priority, header, comments, modified, created) VALUES (%s, %s, %s, %s, %s, %s)",
+                       insert_values)
+            pmig.close()
+        elog('failed to grab %s' % (bugid,))
         return False
     try:
-        if fetch(bugid):
-            print time.time()
-            print 'done with %s' % (bugid,)
-            return True
+        return fetch(bugid)
     except Exception as e:
+        import traceback
         tries -= 1
         time.sleep(5)
-        print 'failed to grab %s (%s)' % (bugid, e)
+        traceback.print_exc(file=sys.stdout)
+        elog('failed to fetch %s (%s)' % (bugid, e))
         return run_fetch(bugid, tries=tries)
 
-if sys.stdin.isatty():
-    bugs = sys.argv[1:]
-else:
-    bugs = sys.stdin.read().strip('\n').strip().split()
 
-bugs = [i for i in bugs if i.isdigit()]    
-print len(bugs)
-from multiprocessing import Pool
-pool = Pool(processes=10)
-_ =  pool.map(run_fetch, bugs)
-complete = len(filter(bool, _))
-failed = len(_) - complete
-print 'completed %s, failed %s' % (complete, failed)
+def main():
+
+    bugs = return_bug_list()
+    from multiprocessing import Pool
+    pool = Pool(processes=10)
+    _ =  pool.map(run_fetch, bugs)
+    complete = len(filter(bool, _))
+    failed = len(_) - complete
+    print '%s completed %s, failed %s' % (sys.argv[0], complete, failed)
+
+if __name__ == '__main__':
+    main()
diff --git a/bugzilla_populate_user_relations_comments_table.py b/bugzilla_populate_user_relations_comments_table.py
new file mode 100755
index 0000000..af6e380
--- /dev/null
+++ b/bugzilla_populate_user_relations_comments_table.py
@@ -0,0 +1,102 @@
+#!/usr/bin/env python
+import time
+import json
+import multiprocessing
+import sys
+import collections
+from phabricator import Phabricator
+from wmfphablib import Phab as phabmacros
+from wmfphablib import phabdb
+from wmfphablib import log
+from wmfphablib import vlog
+from wmfphablib import errorlog as elog
+from wmfphablib import config
+from wmfphablib import epoch_to_datetime
+from wmfphablib import ipriority
+from wmfphablib import now
+from wmfphablib import tflatten
+from wmfphablib import return_bug_list
+
+
+def populate(bugid):
+
+    def add_comment_ref(owner):    
+        """ adds an issue reference to a user
+        """
+        ouser = pmig.sql_x("SELECT user FROM user_relations_comments WHERE user = %s", (owner,))
+        if ouser:
+            jcommed = pmig.sql_x("SELECT issues FROM user_relations_comments WHERE user = %s", (owner,))
+            if jcommed and any(tflatten(jcommed)):
+                issues = json.loads(jcommed[0][0])
+            else:
+                issues = []
+
+            if bugid not in issues:
+                log("Comment reference %s to %s" % (str(bugid), owner))
+                issues.append(bugid)
+            pmig.sql_x("UPDATE user_relations_comments SET issues=%s, modified=%s WHERE user = %s", (json.dumps(issues),
+                                                                                                     now(),
+                                                                                                     owner))
+        else:
+            issues = json.dumps([bugid])
+            insert_values =  (owner,
+                              issues,
+                              now(),
+                              now())
+
+            pmig.sql_x("INSERT INTO user_relations_comments (user, issues, created, modified) VALUES (%s, %s, %s, %s)",
+                       insert_values)
+
+    pmig = phabdb.phdb(db=config.bzmigrate_db)
+    issue = pmig.sql_x("SELECT id FROM bugzilla_meta WHERE id = %s", bugid)
+    if not issue:
+        log('issue %s does not exist for user population' % (bugid,))
+        return True
+
+    fpriority= pmig.sql_x("SELECT priority FROM bugzilla_meta WHERE id = %s", bugid)
+    if fpriority[0] == ipriority['fetch_failed']:
+        log('issue %s does not fetched successfully for user population (failed fetch)' % (bugid,))
+        return True
+
+    current = pmig.sql_x("SELECT comments, xcomments, modified FROM bugzilla_meta WHERE id = %s", bugid)
+    if current:
+        comments, xcomments, modified = current[0]
+    else:
+        log('%s not present for migration' % (bugid,))
+        return True
+
+    com = json.loads(comments)
+    xcom = json.loads(xcomments)
+    commenters = [c['author'] for c in com if c['count'] > 0]
+    commenters = set(commenters)
+    log("Commenters for issue %s: %s" % (bugid, str(commenters)))
+    for c in commenters:
+        add_comment_ref(c)
+    pmig.close()
+    return True
+
+def run_populate(bugid, tries=1):
+    if tries == 0:
+        elog('failed to populate for %s' % (bugid,))
+        return False
+    try:
+        return populate(bugid)
+    except Exception as e:
+        import traceback
+        tries -= 1
+        time.sleep(5)
+        traceback.print_exc(file=sys.stdout)
+        elog('failed to populate %s' % (bugid,))
+        return run_populate(bugid, tries=tries)
+
+def main():
+    bugs = return_bug_list()
+    from multiprocessing import Pool
+    pool = Pool(processes=10)
+    _ =  pool.map(run_populate, bugs)
+    complete = len(filter(bool, _))
+    failed = len(_) - complete
+    print '%s completed %s, failed %s' % (sys.argv[0], complete, failed)
+
+if __name__ == '__main__':
+    main()
diff --git a/bugzilla_populate_user_relations_table.py b/bugzilla_populate_user_relations_table.py
new file mode 100755
index 0000000..6b586a7
--- /dev/null
+++ b/bugzilla_populate_user_relations_table.py
@@ -0,0 +1,168 @@
+#!/usr/bin/env python
+import time
+import json
+import multiprocessing
+import sys
+import collections
+from phabricator import Phabricator
+from wmfphablib import Phab as phabmacros
+from wmfphablib import phabdb
+from wmfphablib import log
+from wmfphablib import vlog
+from wmfphablib import errorlog as elog
+from wmfphablib import config
+from wmfphablib import epoch_to_datetime
+from wmfphablib import ipriority
+from wmfphablib import now
+from wmfphablib import tflatten
+from wmfphablib import return_bug_list
+
+
+def populate(bugid):
+    pmig = phabdb.phdb(db=config.bzmigrate_db)
+
+    issue = pmig.sql_x("SELECT id FROM bugzilla_meta WHERE id = %s", bugid)
+    if not issue:
+        log('issue %s does not exist for user population' % (bugid,))
+        return True
+
+    fpriority= pmig.sql_x("SELECT priority FROM bugzilla_meta WHERE id = %s", bugid)
+    if fpriority[0] == ipriority['fetch_failed']:
+        log('issue %s does not fetched successfully for user population (failed fetch)' % (bugid,))
+        return True
+
+    current = pmig.sql_x("SELECT priority, header, comments, created, modified FROM bugzilla_meta WHERE id = %s", bugid)
+    if current:
+        import_priority, buginfo, com, created, modified = current[0]
+    else:
+        log('%s not present for migration' % (bugid,))
+        return True
+
+    header = json.loads(buginfo)
+    vlog(str(header))
+    relations = {}
+    relations['author'] = header["creator"]
+    relations['cc'] = header['cc']
+    relations['owner'] = header['assigned_to']
+
+    for k, v in relations.iteritems():
+        if relations[k]:
+            relations[k] = filter(bool, v)
+
+    def add_owner(owner):    
+        ouser = pmig.sql_x("SELECT user FROM user_relations WHERE user = %s", (owner,))
+        if ouser:
+            jassigned = pmig.sql_x("SELECT assigned FROM user_relations WHERE user = %s", (owner,))
+            jflat = tflatten(jassigned)
+            if any(jflat):
+                assigned = json.loads(jassigned[0][0])
+            else:
+                assigned = []
+            if bugid not in assigned:
+                log("Assigning %s to %s" % (str(bugid), owner))
+                assigned.append(bugid)
+            vlog("owner %s" % (str(assigned),))
+            pmig.sql_x("UPDATE user_relations SET assigned=%s, modified=%s WHERE user = %s", (json.dumps(assigned),
+                                                                                              now(),
+                                                                                              owner))
+        else:
+            vlog('inserting new record')
+            assigned = json.dumps([bugid])
+            insert_values =  (owner,
+                              assigned,
+                              now(),
+                              now())
+
+            pmig.sql_x("INSERT INTO user_relations (user, assigned, created, modified) VALUES (%s, %s, %s, %s)",
+                       insert_values)
+
+
+    def add_author(author):
+        euser = pmig.sql_x("SELECT user FROM user_relations WHERE user = %s", (relations['author'],))
+        if euser:
+            jauthored = pmig.sql_x("SELECT author FROM user_relations WHERE user = %s", (relations['author'],))
+            jflat = tflatten(jauthored)
+            if any(jflat):
+                authored = json.loads(jauthored[0][0])
+            else:
+               authored = []
+            if bugid not in authored:
+                authored.append(bugid)
+            vlog("author %s" % (str(authored),))
+            pmig.sql_x("UPDATE user_relations SET author=%s, modified=%s WHERE user = %s", (json.dumps(authored),
+                                                                                        now(),
+                                                                                        relations['author']))
+        else:
+            vlog('inserting new record')
+            authored = json.dumps([bugid])
+            insert_values =  (relations['author'],
+                              authored,
+                              now(),
+                              now())
+            pmig.sql_x("INSERT INTO user_relations (user, author, created, modified) VALUES (%s, %s, %s, %s)",
+                       insert_values)
+
+
+    def add_cc(ccuser):
+        eccuser = pmig.sql_x("SELECT user FROM user_relations WHERE user = %s", (ccuser,))
+        if eccuser:
+            jcc = pmig.sql_x("SELECT cc FROM user_relations WHERE user = %s", (ccuser,))
+            jflat = tflatten(jcc)
+            if any(jflat):
+               cc = json.loads(jcc[0][0])
+            else:
+               cc = []
+            if bugid not in cc:
+                cc.append(bugid)
+            vlog("cc %s" % (str(cc),))
+            pmig.sql_x("UPDATE user_relations SET cc=%s, modified=%s WHERE user = %s", (json.dumps(cc),
+                                                                                        now(),
+                                                                                        ccuser))
+        else:
+            vlog('inserting new record')
+            cc = json.dumps([bugid])
+            insert_values =  (ccuser,
+                              cc,
+                              now(),
+                              now())
+            pmig.sql_x("INSERT INTO user_relations (user, cc, created, modified) VALUES (%s, %s, %s, %s)",
+                   insert_values)
+
+    if relations['author']:
+        add_author(relations['author'])
+
+    if relations['owner']:
+        add_owner(relations['owner'])
+
+    if relations['cc']:
+        for u in filter(bool, relations['cc']):
+            add_cc(u)
+
+    pmig.close()
+    return True
+
+def run_populate(bugid, tries=1):
+    if tries == 0:
+        elog('failed to populate for %s' % (bugid,))
+        return False
+    try:
+        return populate(bugid)
+    except Exception as e:
+        import traceback
+        tries -= 1
+        time.sleep(5)
+        traceback.print_exc(file=sys.stdout)
+        elog('failed to populate %s (%s)' % (bugid, e))
+        return run_populate(bugid, tries=tries)
+
+def main():
+    bugs = return_bug_list()
+    from multiprocessing import Pool
+    pool = Pool(processes=10)
+    _ =  pool.map(run_populate, bugs)
+    complete = len(filter(bool, _))
+    failed = len(_) - complete
+    print '%s completed %s, failed %s' % (sys.argv[0], complete, failed)
+
+if __name__ == '__main__':
+    main()
diff --git a/bugzilla_product_info.py b/bugzilla_product_info.py
index b7a1dc0..91303a0 100755
--- a/bugzilla_product_info.py
+++ b/bugzilla_product_info.py
@@ -1,12 +1,4 @@
 #!/usr/bin/env python
-"""
-
-WIP for fetching information about products to migrate
-as project state to phabricator
-
-"""
-
-
 import yaml
 import ast
 import base64
@@ -20,19 +12,11 @@ import re
 import MySQLdb
 from phabdb import archive_project
 from wmfphablib import config
+from wmfphablib import log
+from wmfphablib import vlog
 import phabricator
 from phabricator import Phabricator
 
-def log(msg):
-    import syslog
-    msg = unicode(msg)
-    if '-v' in sys.argv:
-        try:
-            syslog.syslog(msg)
-            print '-> ', msg
-        except:
-            print 'error logging output'
-
 def main(bugid):
 
     phab = Phabricator(config.phab_user,
diff --git a/bugzilla_update_tasks.py b/bugzilla_update_tasks.py
new file mode 100755
index 0000000..d6bae45
--- /dev/null
+++ b/bugzilla_update_tasks.py
@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+import sys
+import time
+import json
+import multiprocessing
+from wmfphablib import phabdb
+from wmfphablib import log
+from wmfphablib import vlog
+from wmfphablib import errorlog as elog
+from wmfphablib import now
+from wmfphablib import return_bug_list
+from wmfphablib import bzlib
+from wmfphablib import ipriority
+from wmfphablib import config
+
+
+def update(bugid):
+
+    pmig = phabdb.phdb(db=config.bzmigrate_db)
+
+    epriority = pmig.sql_x("SELECT priority from task_relations where id = %s", bugid)
+    if epriority and epriority[0] == ipriority['update_success']:
+        log('skipping %s as blockers already updated' % (bugid,))
+        return True
+
+    hq = "SELECT header FROM bugzilla_meta WHERE id = %s"
+    header = pmig.sql_x(hq, (bugid,))
+    if not header:
+       elog('no header found for %s' % (bugid,))
+       return False
+
+    def extref(ticket):
+        refid = phabdb.reference_ticket("%s%s" % (bzlib.prepend, ticket))
+        if not refid:
+            return ''
+        return refid[0]
+
+    blocker_ref = extref(bugid)
+
+    tinfo = json.loads(header[0][0])
+    if not tinfo['blocks']:
+        log("%s doesn't block anything" % (str(bugid),))
+        return True
+
+    for b in tinfo["blocks"]:
+        blocked_ref = extref(b)
+        log("%s is blocking %s" % (blocker_ref, blocked_ref))
+        if blocked_ref:
+            log(phabdb.set_blocked_task(blocker_ref, blocked_ref))
+        else:
+            log('%s is missing blocker %s' % (blocked_ref, blocker_ref))
+
+    blocks = phabdb.get_tasks_blocked(blocker_ref)
+    vlog('%s is blocking %s' % (blocker_ref, str(blocks)))
+    current = pmig.sql_x("SELECT * from task_relations where id = %s", bugid)
+    if current:
+        pmig.sql_x("UPDATE task_relations SET priority=%s, blocks=%s, modified=%s WHERE id = %s",
+                   (ipriority['update_success'], json.dumps(blocks), now(), bugid))
+    else:
+        sql = "INSERT INTO task_relations (id, priority, blocks, modified) VALUES (%s, %s, %s, %s)"
+        pmig.sql_x(sql, (bugid, ipriority['update_success'], json.dumps(blocks), now()))
+    pmig.close()
+    return True
+ 
+
+def run_update(bugid, tries=1):
+    if tries == 0:
+        pmig = phabdb.phdb(db=config.bzmigrate_db)
+        current = pmig.sql_x("SELECT * from task_relations where id = %s", bugid)
+        if current:
+            pmig.sql_x("UPDATE task_relations SET priority=%s, blocks=%s, modified=%s WHERE id = %s",
+                      (ipriority['creation_failed'], json.dumps([]), now(), bugid))
+        else:
+            sql = "INSERT INTO task_relations (id, priority, blocks, modified) VALUES (%s, %s, %s, %s)"
+            pmig.sql_x(sql, (bugid, ipriority['creation_failed'], json.dumps([]), now()))
+        pmig.close()
+        elog('final fail to update %s' % (bugid,))
+        return False
+    try:
+        return update(bugid)
+    except Exception as e:
+        import traceback
+        tries -= 1
+        time.sleep(5)
+        traceback.print_exc(file=sys.stdout)
+        elog('failed to update %s' % (bugid,))
+        return run_update(bugid, tries=tries)
+
+def main():
+    bugs = return_bug_list()
+    from multiprocessing import Pool
+    pool = Pool(processes=2)
+    _ =  pool.map(run_update, bugs)
+    complete = len(filter(bool, _))
+    failed = len(_) - complete
+    print '%s completed %s, failed %s' % (sys.argv[0], complete, failed)
+
+if __name__ == '__main__':
+    main()
diff --git a/bugzilla_update_user_comments.py b/bugzilla_update_user_comments.py
new file mode 100755
index 0000000..764db05
--- /dev/null
+++ b/bugzilla_update_user_comments.py
@@ -0,0 +1,210 @@
+#!/usr/bin/env python
+import os
+import argparse
+import time
+import json
+import multiprocessing
+import sys
+import collections
+from phabricator import Phabricator
+from wmfphablib import Phab as phabmacros
+from wmfphablib import phabdb
+from wmfphablib import log
+from wmfphablib import util
+from wmfphablib import bzlib
+from wmfphablib import config
+from wmfphablib import vlog
+from wmfphablib import errorlog as elog
+from wmfphablib import epoch_to_datetime
+from wmfphablib import ipriority
+from wmfphablib import now
+from wmfphablib import return_bug_list
+from wmfphablib import ipriority
+
+
+def update(user):
+
+    phab = Phabricator(config.phab_user,
+                       config.phab_cert,
+                       config.phab_host)
+
+    pmig = phabdb.phdb(db=config.bzmigrate_db,
+                       user=config.bzmigrate_user,
+                       passwd=config.bzmigrate_passwd)
+
+    phabm = phabmacros('', '', '')
+    phabm.con = phab
+
+    if phabdb.is_bot(user['userphid']):
+        log("%s is a bot no action" % (user['user']))
+        return True
+
+    epriority = phabdb.get_user_relations_priority(user['user'], pmig)
+    if epriority and len(epriority[0]) > 0:
+        if epriority[0][0] == ipriority['update_success']:
+            log('Skipping %s as already updated' % (user['user']))
+            return True
+
+    if not user['issues']:
+        log("%s has no issues to update" % (user['user'],))
+        return True
+
+    for i in user['issues']:
+        comdetails = pmig.sql_x("SELECT comments, xcomments FROM bugzilla_meta WHERE id = %s", (int(i),))
+        jcom, jxcom = comdetails[0]
+        coms = json.loads(jcom)
+        xcoms = json.loads(jxcom)
+
+        for key, xi in xcoms.iteritems():
+            com = coms[util.get_index(coms, "count", int(key))]
+            content = com['text']
+            log("Updating comment %s for %s" % (xi['xctransaction'], user['user']))
+            phabdb.set_comment_author(xi['xctransaction'], user['userphid'])
+            phabdb.set_comment_content(xi['xctransaction'], content + xi['xattached'])
+
+    current = phabdb.get_user_migration_comment_history(user['user'], pmig)
+    if current:
+        log(phabdb.set_user_relations_comments_priority(ipriority['update_success'], user['user'], pmig))
+    else:
+        log('%s user does not exist to update' % (user['user']))
+        return False
+    pmig.close()
+    log(util.purge_cache())
+    return True
+
+def run_update(user, tries=1):
+    if tries == 0:
+        pmig = phabdb.phdb(db=config.bzmigrate_db,
+                       user=config.bzmigrate_user,
+                       passwd=config.bzmigrate_passwd)
+        current = phabdb.get_user_migration_history(user['user'], pmig)
+        if current:
+           log(phabdb.set_user_relations_priority(ipriority['update_failed'], user['user'], pmig))
+        else:
+            log('%s user does not exist to update' % (user['user']))
+        pmig.close()
+        elog('final fail to update %s' % (user['user'],))
+        return False
+    try:
+        return update(user)
+    except Exception as e:
+        import traceback
+        tries -= 1
+        time.sleep(5)
+        traceback.print_exc(file=sys.stdout)
+        elog('failed to update %s' % (user,))
+        return run_update(user, tries=tries)
+
+def get_user_histories(verified):
+    histories = []
+    pmig = phabdb.phdb(db=config.bzmigrate_db,
+                       user=config.bzmigrate_user,
+                       passwd=config.bzmigrate_passwd)
+
+    for v in verified:
+        vlog(str(v))
+        saved_history = phabdb.get_user_migration_comment_history(v[1], pmig)
+        if not saved_history:
+            log('%s verified email has no saved history' % (v[1],))
+            continue
+        log('%s is being processed' % (v[1],))
+        history = {}
+        history['user'] = v[1]
+        history['userphid'] = v[0]
+        history['issues'] = saved_history[0]
+        history['created'] = saved_history[1]
+        history['modified'] = saved_history[2]
+        histories.append(history)
+
+    pmig.close()
+    return [util.translate_json_dict_items(d) for d in histories]
+
+def get_verified_users(modtime, limit=None):
+    #Find the task in new Phabricator that matches our lookup
+    verified = phabdb.get_verified_emails(modtime=modtime, limit=limit)
+    create_times = [v[2] for v in verified]
+    try:
+        newest = max(create_times)
+    except ValueError:
+        newest = modtime
+    return verified, newest
+
+def get_verified_user(email):
+    phid, email, is_verified = phabdb.get_user_email_info(email)
+    log("Single verified user: %s, %s, %s" % (phid, email, is_verified))
+    if is_verified:
+        return [(phid, email)]
+    else:
+        log("%s is not a verified email" % (email,))
+        return [()]
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Updates user header metadata from bugzilla')
+    parser.add_argument('-a', action="store_true", default=False)
+    parser.add_argument('-e', action="store", dest='email')
+    parser.add_argument('-m', action="store", dest="starting_epoch", default=None)
+    parser.add_argument('-v', action="store_true", default=False)
+    args =  parser.parse_args()
+
+    pmig = phabdb.phdb(db=config.bzmigrate_db,
+                       user=config.bzmigrate_user,
+                       passwd=config.bzmigrate_passwd)
+
+    if args.a:
+        starting_epoch = phabdb.get_user_relations_last_finish(pmig)
+        users, finish_epoch = get_verified_users(starting_epoch, config.fab_limit)
+    elif args.email:
+        users = get_verified_user(args.email)
+        starting_epoch = 0
+        finish_epoch = 0
+    elif args.starting_epoch:
+        users, finish_epoch = get_verified_users(args.starting_epoch)
+        starting_epoch = args.starting_epoch
+    else:
+        parser.print_help()
+        sys.exit(1)
+
+    if not any(users):
+        log("Existing as there are no new verified users")
+        sys.exit()
+
+    histories = get_user_histories(filter(bool, users))
+    user_count = len(histories)
+
+    icounts = []
+    for u in histories:
+        c = 0
+        if u['issues']:
+            c += len(u['issues'])
+        icounts.append(c)
+    issue_count = sum(icounts)
+
+    log("User Count %s" % (str(user_count)))
+    log("Issue Count %s" % (str(issue_count)))
+
+    pid = os.getpid()
+    phabdb.user_relations_start(pid,
+                                int(time.time()),
+                                0,
+                                starting_epoch,
+                                user_count, issue_count, pmig)
+
+
+    from multiprocessing import Pool
+    pool = Pool(processes=config.fab_multi)
+    _ =  pool.map(run_update, histories)
+    complete = len(filter(bool, _))
+    failed = len(_) - complete
+    phabdb.user_relations_finish(pid,
+                                 int(time.time()),
+                                 ipriority['update_success'],
+                                 finish_epoch,
+                                 complete,
+                                 failed,
+                                 pmig)
+    print '%s completed %s, failed %s' % (sys.argv[0], complete, failed)
+    pmig.close()
+
+if __name__ == '__main__':
+    main()
diff --git a/bugzilla_update_user_header.py b/bugzilla_update_user_header.py
new file mode 100755
index 0000000..deba274
--- /dev/null
+++ b/bugzilla_update_user_header.py
@@ -0,0 +1,216 @@
+#!/usr/bin/env python
+import os
+import argparse
+import time
+import json
+import multiprocessing
+import sys
+import collections
+from phabricator import Phabricator
+from wmfphablib import Phab as phabmacros
+from wmfphablib import phabdb
+from wmfphablib import log
+from wmfphablib import vlog
+from wmfphablib import errorlog as elog
+from wmfphablib import bzlib
+from wmfphablib import util
+from wmfphablib import config
+from wmfphablib import epoch_to_datetime
+from wmfphablib import ipriority
+from wmfphablib import now
+from wmfphablib import return_bug_list
+from wmfphablib import ipriority
+
+
+def update(user):
+
+    phab = Phabricator(config.phab_user,
+                       config.phab_cert,
+                       config.phab_host)
+
+    pmig = phabdb.phdb(db=config.bzmigrate_db,
+                       user=config.bzmigrate_user,
+                       passwd=config.bzmigrate_passwd)
+
+    phabm = phabmacros('', '', '')
+    phabm.con = phab
+
+    if phabdb.is_bot(user['userphid']):
+        log("%s is a bot no action" % (user['user']))
+        return True
+
+    epriority = phabdb.get_user_relations_priority(user['user'], pmig)
+    if epriority and len(epriority[0]) > 0:
+        if epriority[0][0] == ipriority['update_success']:
+            log('Skipping %s as already updated' % (user['user']))
+            return True
+
+    # 'author': [409, 410, 411, 404, 412],
+    # 'cc': [221, 69, 203, 268, 261, 8],
+    # 'created': 1410276037L,
+    # 'modified': 1410276060L,
+    # 'assigned': [97, 64, 150, 59, 6],
+    # 'userphid': 'PHID-USER-4hsexplytovmqmcb7tq2',
+    # 'user': u'chase.mp@xxx.com'}
+
+    if user['assigned']:
+        for ag in user['assigned']:
+             vlog(phabm.sync_assigned(user['userphid'], ag, bzlib.prepend))
+
+    if user['author']:
+        for a in user['author']:
+            vlog(phabm.synced_authored(user['userphid'], a, bzlib.prepend))
+
+    if user['cc']:
+        for ccd in user['cc']:
+            vlog(phabdb.add_task_cc_by_ref(user['userphid'], ccd))
+
+    current = phabdb.get_user_migration_history(user['user'], pmig)
+    if current:
+        log(phabdb.set_user_relations_priority(ipriority['update_success'], user['user'], pmig))
+    else:
+        elog('%s user does not exist to update' % (user['user']))
+        return False
+    pmig.close()
+    return True
+
+def run_update(user, tries=1):
+    if tries == 0:
+        pmig = phabdb.phdb(db=config.bzmigrate_db,
+                       user=config.bzmigrate_user,
+                       passwd=config.bzmigrate_passwd)
+        current = phabdb.get_user_migration_history(user['user'], pmig)
+        if current:
+           elog(phabdb.set_user_relations_priority(ipriority['update_failed'], user['user'], pmig))
+        else:
+            elog('%s user does not exist to update' % (user['user']))
+        pmig.close()
+        elog('final fail to update %s' % (user['user'],))
+        return False
+    try:
+        return update(user)
+    except Exception as e:
+        import traceback
+        tries -= 1
+        time.sleep(5)
+        traceback.print_exc(file=sys.stdout)
+        elog('failed to update %s' % (user,))
+        return run_update(user, tries=tries)
+
+def get_user_histories(verified):
+    histories = []
+    pmig = phabdb.phdb(db=config.bzmigrate_db,
+                       user=config.bzmigrate_user,
+                       passwd=config.bzmigrate_passwd)
+
+    for v in verified:
+        vlog(str(v))
+        saved_history = phabdb.get_user_migration_history(v[1], pmig)
+        if not saved_history:
+            log('%s verified email has no saved history' % (v[1],))
+            continue
+        log('%s is being processed' % (v[1],))
+        history = {}
+        history['user'] = v[1]
+        history['userphid'] = v[0]
+        history['assigned'] = saved_history[0]
+        history['cc'] = saved_history[1]
+        history['author'] = saved_history[2]
+        history['created'] = saved_history[3]
+        history['modified'] = saved_history[4]
+        histories.append(history)
+    pmig.close()
+    return [util.translate_json_dict_items(d) for d in histories]
+
+def get_verified_users(modtime, limit=None):
+    #Find the task in new Phabricator that matches our lookup
+    verified = phabdb.get_verified_emails(modtime=modtime, limit=limit)
+    create_times = [v[2] for v in verified]
+    try:
+        newest = max(create_times)
+    except ValueError:
+        newest = modtime
+    return verified, newest
+
+def get_verified_user(email):
+    phid, email, is_verified = phabdb.get_user_email_info(email)
+    log("Single specified user: %s, %s, %s" % (phid, email, is_verified))
+    if is_verified:
+        return [(phid, email)]
+    else:
+        log("%s is not a verified email" % (email,))
+        return [()]
+
+def main():
+    parser = argparse.ArgumentParser(description='Updates user header metadata from bugzilla')
+    parser.add_argument('-a', action="store_true", default=False)
+    parser.add_argument('-e', action="store", dest='email')
+    parser.add_argument('-m', action="store", dest="starting_epoch", default=None)
+    parser.add_argument('-v', action="store_true", default=False)
+    args =  parser.parse_args()
+
+    pmig = phabdb.phdb(db=config.bzmigrate_db,
+                       user=config.bzmigrate_user,
+                       passwd=config.bzmigrate_passwd)
+
+    if args.a:
+        starting_epoch = phabdb.get_user_relations_last_finish(pmig)
+        users, finish_epoch = get_verified_users(starting_epoch, config.fab_limit)
+    elif args.email:
+        users = get_verified_user(args.email)
+        starting_epoch = 0
+        finish_epoch = 0
+    elif args.starting_epoch:
+        users, finish_epoch = get_verified_users(args.starting_epoch)
+        starting_epoch = args.starting_epoch
+    else:
+        parser.print_help()
+        sys.exit(1)
+
+    if not any(users):
+        log("Existing as there are no new verified users")
+        sys.exit()
+
+    histories = get_user_histories(filter(bool, users))
+    user_count = len(histories)
+
+    icounts = []
+    for u in histories:
+        c = 0
+        if u['cc']:
+            c += len(u['cc'])
+        if u['author']:
+            c += len(u['author'])
+        if u['assigned']:
+            c += len(u['assigned'])
+        icounts.append(c)
+    issue_count = sum(icounts)
+
+    log("User Count %s" % (str(user_count)))
+    log("Issue Count %s" % (str(issue_count)))
+
+    pid = os.getpid()
+    phabdb.user_relations_start(pid,
+                                int(time.time()),
+                                0,
+                                starting_epoch,
+                                user_count, issue_count, pmig)
+
+
+    from multiprocessing import Pool
+    pool = Pool(processes=config.fab_multi)
+    _ =  pool.map(run_update, histories)
+    complete = len(filter(bool, _))
+    failed = len(_) - complete
+    phabdb.user_relations_finish(pid,
+                                 int(time.time()),
+                                 ipriority['update_success'],
+                                 finish_epoch,
+                                 complete,
+                                 failed,
+                                 pmig)
+    pmig.close()
+    print '%s completed %s, failed %s' % (sys.argv[0], complete, failed)
+
+if __name__ == '__main__':
+    main()
diff --git a/bz_populate_user_relations_table.py b/bz_populate_user_relations_table.py
new file mode 100755
index 0000000..732b149
--- /dev/null
+++ b/bz_populate_user_relations_table.py
@@ -0,0 +1,165 @@
+import time
+import json
+import multiprocessing
+import sys
+import collections
+from phabricator import Phabricator
+from wmfphablib import Phab as phabmacros
+from wmfphablib import phabdb
+from wmfphablib import log
+from wmfphablib import vlog
+from wmfphablib import epoch_to_datetime
+from wmfphablib import ipriority
+from wmfphablib import get_config_file
+from wmfphablib import now
+from wmfphablib import return_bug_list
+import ConfigParser
+
+
+configfile = get_config_file()
+
+
+def fetch(fabid):
+    ausers = {}
+    pmig = phabdb.phdb(db=config.bzmigrate_db)
+    issue = pmig.sql_x("SELECT id FROM bugzilla_meta WHERE id = %s", fabid)
+
+    if not issue:
+        log('issue %s does not exist for user population' % (fabid,))
+        return True
+
+    fpriority= pmig.sql_x("SELECT priority FROM bugzilla_meta WHERE id = %s", fabid)
+    if fpriority[0] == ipriority['fetch_failed']:
+        log('issue %s does not fetched successfully for user population (failed fetch)' % (fabid,))
+        return True
+
+
+    tid, import_priority, jheader, com, created, modified = pmig.sql_x("SELECT * FROM bugzilla_meta WHERE id = %s", fabid)
+    header = json.loads(jheader)
+    vlog(str(header))
+    relations = {}
+    relations['author'] = header['xauthor']
+    relations['cc'] = header['xccs']
+    relations['owner'] = header['xowner']
+
+    for k, v in relations.iteritems():
+        if relations[k]:
+            relations[k] = filter(bool, v)
+
+    def add_owner(owner):    
+        ouser = pmig.sql_x("SELECT user FROM user_relations WHERE user = %s", (owner,))
+        if ouser:
+            jassigned = pmig.sql_x("SELECT assigned FROM user_relations WHERE user = %s", (owner,))
+
+            if jassigned[0]:
+                assigned = json.loads(jassigned[0])
+            else:
+                assigned = []
+            if fabid not in assigned:
+                log("Assigning %s to %s" % (str(fabid), owner))
+                assigned.append(fabid)
+            vlog("owner %s" % (str(assigned),))
+            pmig.sql_x("UPDATE user_relations SET assigned=%s, modified=%s WHERE user = %s", (json.dumps(assigned),
+                                                                                              now(),
+                                                                                              owner))
+        else:
+            vlog('inserting new record')
+            assigned = json.dumps([fabid])
+            insert_values =  (owner,
+                              assigned,
+                              now(),
+                              now())
+
+            pmig.sql_x("INSERT INTO user_relations (user, assigned, created, modified) VALUES (%s, %s, %s, %s)",
+                       insert_values)
+
+
+    def add_author(author):
+        euser = pmig.sql_x("SELECT user FROM user_relations WHERE user = %s", (relations['author'],))
+        if euser:
+            jauthored = pmig.sql_x("SELECT author FROM user_relations WHERE user = %s", (relations['author'],))
+            if jauthored[0]:
+                authored = json.loads(jauthored[0])
+            else:
+               authored = []
+            if fabid not in authored:
+                authored.append(fabid)
+            vlog("author %s" % (str(authored),))
+            pmig.sql_x("UPDATE user_relations SET author=%s, modified=%s WHERE user = %s", (json.dumps(authored),
+                                                                                        now(),
+                                                                                        relations['author']))
+        else:
+            vlog('inserting new record')
+            authored = json.dumps([fabid])
+            insert_values =  (relations['author'],
+                              authored,
+                              now(),
+                              now())
+            pmig.sql_x("INSERT INTO user_relations (user, author, created, modified) VALUES (%s, %s, %s, %s)",
+                       insert_values)
+
+
+    def add_cc(ccuser):
+        eccuser = pmig.sql_x("SELECT user FROM user_relations WHERE user = %s", (ccuser,))
+        if eccuser:
+            jcc = pmig.sql_x("SELECT cc FROM user_relations WHERE user = %s", (ccuser,))
+            if jcc[0]:
+               cc = json.loads(jcc[0])
+            else:
+               cc = []
+            if fabid not in cc:
+                cc.append(fabid)
+            vlog("cc %s" % (str(cc),))
+            pmig.sql_x("UPDATE user_relations SET cc=%s, modified=%s WHERE user = %s", (json.dumps(cc),
+                                                                                        now(),
+                                                                                        ccuser))
+        else:
+            vlog('inserting new record')
+            cc = json.dumps([fabid])
+            insert_values =  (ccuser,
+                              cc,
+                              now(),
+                              now())
+            pmig.sql_x("INSERT INTO user_relations (user, cc, created, modified) VALUES (%s, %s, %s, %s)",
+                   insert_values)
+
+    if relations['author']:
+        add_author(relations['author'])
+
+    if relations['owner']:
+        add_owner(relations['owner'])
+
+    if relations['cc']:
+        for u in filter(bool, relations['cc']):
+            add_cc(u)
+
+    pmig.close()
+    return True
+
+def run_fetch(fabid, tries=1):
+    if tries == 0:
+        log('failed to populate for %s' % (fabid,))
+        return False
+    try:
+        if fetch(fabid):
+            vlog(str(time.time()))
+            log('done with %s' % (fabid,))
+            return True
+    except Exception as e:
+        import traceback
+        tries -= 1
+        time.sleep(5)
+        traceback.print_exc(file=sys.stdout)
+        log('failed to grab %s (%s)' % (fabid, e))
+        return run_fetch(fabid, tries=tries)
+
+
+bugs = return_bug_list()
+vlog(bugs)
+log("Count %s" % (str(len(bugs))))
+from multiprocessing import Pool
+pool = Pool(processes=10)
+_ =  pool.map(run_fetch, bugs)
+complete = len(filter(bool, _))
+failed = len(_) - complete
+print 'completed %s, failed %s' % (complete, failed)
diff --git a/fab_update_user.py b/fab_update_user.py
index 644ae4d..1e84909 100755
--- a/fab_update_user.py
+++ b/fab_update_user.py
@@ -11,9 +11,9 @@ from wmfphablib import Phab as phabmacros
 from wmfphablib import phabdb
 from wmfphablib import log
 from wmfphablib import config
+from wmfphablib import util
 from wmfphablib import vlog
 from wmfphablib import epoch_to_datetime
-from wmfphablib import ipriority
 from wmfphablib import now
 from wmfphablib import return_bug_list
 from wmfphablib import ipriority
@@ -32,36 +32,15 @@ def update(user):
     phabm = phabmacros('', '', '')
     phabm.con = phab
 
+    if phabdb.is_bot(user['userphid']):
+        log("%s is a bot no action" % (user['user']))
+        return True
+
     epriority = phabdb.get_user_relations_priority(user['user'], pmig)
-    if epriority and epriority[0] == ipriority['creation_success']:
+    if epriority and epriority[0] == ipriority['update_success']:
         log('Skipping %s as already updated' % (user['user']))
         return True
 
-    def sync_assigned(userphid, id):
-        refs = phabdb.reference_ticket('fl%s' % (id,))
-        if not refs:
-            log('reference ticket not found for %s' % ('fl%s' % (id,),))
-            return 
-        current = phab.maniphest.query(phids=[refs[0]])
-        if current[current.keys()[0]]['ownerPHID']:
-            log('current owner found for => %s' % (str(id),))
-            return current
-
-        log('assigning %s to %s' % (str(id), userphid))
-        return phab.maniphest.update(phid=refs[0], ownerPHID=userphid)
-
-
-    def add_cc(userphid, id):
-        refs = phabdb.reference_ticket('fl%s' % (id,))
-        if not refs:
-            log('reference ticket not found for %s' % ('fl%s' % (id,),))
-            return
-        current = phab.maniphest.query(phids=[refs[0]])
-        cclist = current[current.keys()[0]]['ccPHIDs']
-        cclist.append(userphid)
-        log('updating cc list for issue %s with %s' % (str(id), userphid))
-        return phab.maniphest.update(ccPHIDs=cclist, phid=refs[0])
-
     # 'author': [409, 410, 411, 404, 412],
     # 'cc': [221, 69, 203, 268, 261, 8],
     # 'created': 1410276037L,
@@ -72,15 +51,15 @@ def update(user):
 
     if user['assigned']:
         for ag in user['assigned']:
-             vlog(sync_assigned(user['userphid'], ag))
+             vlog(phabm.sync_assigned(user['userphid'], ag, 'fl'))
 
     if user['author']:
         for a in user['author']:
-            vlog(phabm.synced_authored(user['userphid'], a))
+            vlog(phabm.synced_authored(user['userphid'], a, 'fl'))
 
     if user['cc']:
         for ccd in user['cc']:
-            vlog(add_cc(user['userphid'], ccd))
+            vlog(phabdb.add_task_cc_by_ref(user['userphid'], ccd))
 
     current = phabdb.get_user_migration_history(user['user'], pmig)
     if current:
@@ -139,19 +118,8 @@ def get_user_histories(verified):
         history['created'] = saved_history[3]
         history['modified'] = saved_history[4]
         histories.append(history)
-
-    # types of history are broken into a dict
-    # many of these are json objects we need decode
-    for i, user in enumerate(histories):
-    #for email, item in histories.iteritems():
-        for t in user.keys():
-            if user[t]:
-                try:
-                    user[t] = json.loads(user[t])
-                except (TypeError, ValueError):
-                    pass
     pmig.close()
-    return histories
+    return [util.translate_json_dict_items(d) for d in histories]
 
 def get_verified_users(modtime, limit=None):
     #Find the task in new Phabricator that matches our lookup
@@ -206,8 +174,11 @@ def main():
         parser.print_help()
         sys.exit(1)
 
+    if not any(users):
+        log("Existing as there are no new verified users")
+        sys.exit()
 
-    histories = get_user_histories(users)
+    histories = get_user_histories(filter(bool, users))
     user_count = len(histories)
 
     icounts = []
@@ -248,7 +219,7 @@ def main():
                                  complete,
                                  failed,
                                  pmig)
-    print 'completed %s, failed %s' % (complete, failed)
+    print '%s completed %s, failed %s' % (sys.argv[0], complete, failed)
     pmig.close()
 
 if __name__ == '__main__':
diff --git a/wmfphablib/__init__.py b/wmfphablib/__init__.py
index a456f3d..c41bde8 100755
--- a/wmfphablib/__init__.py
+++ b/wmfphablib/__init__.py
@@ -15,6 +15,9 @@ import time
 def now():
     return int(time.time())
 
+def tflatten(t_of_tuples):
+    return [element for tupl in t_of_tuples for element in tupl]
+
 #import priority status meanings
 ipriority = {'creation_failed': 6,
              'creation_success': 7,
@@ -32,9 +35,11 @@ def return_bug_list():
 
     if '-' in bugs[0]:
         start, stop = bugs[0].split('-')
-        bugs = range(int(start), int(stop) + 1)
+        bugrange = range(int(start), int(stop) + 1)
+        bugs = [str(b) for b in bugrange]
     else:
         bugs = [i for i in bugs if i.isdigit()]
+    log("Bugs count: %d" % (len(bugs)))
     return bugs
 
 def datetime_to_epoch(date_time):
@@ -44,6 +49,14 @@ def epoch_to_datetime(epoch, timezone='UTC'):
     return str((datetime.datetime.fromtimestamp(int(float(epoch))
            ).strftime('%Y-%m-%d %H:%M:%S'))) + " (%s)" % (timezone,)
 
+def errorlog(msg):
+    msg = unicode(msg)
+    try:
+        syslog.syslog(msg)
+        print >> sys.stderr, msg
+    except:
+        print 'error logging, well...error output'
+
 def log(msg):
     msg = unicode(msg)
     if '-v' in ''.join(sys.argv):
diff --git a/wmfphablib/bzwmfphab.py b/wmfphablib/bzwmfphab.py
index 2125c81..ffce4ff 100755
--- a/wmfphablib/bzwmfphab.py
+++ b/wmfphablib/bzwmfphab.py
@@ -1,6 +1,37 @@
 import re
 
 prepend = 'bz'
+security_mask = '_hidden_'
+
+def build_comment(c):
+    """ takes a native bz comment dict and outputs
+    a dict ready for processing into phab
+    """
+    clean_c = {}
+    clean_c['author'] =  c['author'].split('@')[0]
+    clean_c['creation_time'] = str(c['creation_time'])
+    clean_c['creation_time'] = int(float(c['creation_time']))
+    if c['author'] != c['creator']:
+        clean_c['creator'] =  c['creator'].split('@')[0]
+
+    clean_c['count'] = c['count']
+    if c['count'] == 0:
+        clean_c['bug_id'] = c['bug_id']
+
+    if c['is_private']:
+        c['text'] = security_mask
+
+    attachment = find_attachment_in_comment(c['text'])
+    if attachment:
+        fmt_text = []
+        text = c['text'].splitlines()
+        for t in text:
+            if not t.startswith('Created attachment'):
+                fmt_text.append(t)
+        c['text'] = '\n'.join(fmt_text)
+        clean_c['attachment'] = attachment
+    clean_c['text'] = c['text']
+    return clean_c
 
 def find_attachment_in_comment(text):
     a = re.search('Created\sattachment\s(\d+)', text)
diff --git a/wmfphablib/phabapi.py b/wmfphablib/phabapi.py
index d6ca7a0..20c79f3 100755
--- a/wmfphablib/phabapi.py
+++ b/wmfphablib/phabapi.py
@@ -17,12 +17,24 @@ class phabapi:
         else:
             self.con = None
 
-    def synced_authored(self, phid, id):
-        refs = phabdb.reference_ticket('fl%s' % (id,))
+    def sync_assigned(self, userphid, id, prepend):
+        refs = phabdb.reference_ticket('%s%s' % (prepend, id))
         if not refs:
-            log('reference ticket not found for %s' % ('fl%s' % (id,),))
+            log('reference ticket not found for %s' % ('%s%s' % (prepend, id),))
             return
-        log('reference ticket found for %s' % ('fl%s' % (id,),))
+        current = self.con.maniphest.query(phids=[refs[0]])
+        if current[current.keys()[0]]['ownerPHID']:
+            log('current owner found for => %s' % (str(id),))
+            return current
+        log('assigning T%s to %s' % (str(id), userphid))
+        return self.con.maniphest.update(phid=refs[0], ownerPHID=userphid)
+
+    def synced_authored(self, phid, id, ref):
+        refs = phabdb.reference_ticket('%s%s' % (ref, id))
+        if not refs:
+            log('reference ticket not found for %s' % ('%s%s' % (ref, id),))
+            return
+        log('reference ticket found for %s' % ('%s%s' % (ref, id),))
         newid = self.ticket_id_by_phid(refs[0])
         log("Updating author for %s to %s" % (refs, phid))
         phabdb.set_task_author(phid, newid)
diff --git a/wmfphablib/phabdb.py b/wmfphablib/phabdb.py
index d5beb7c..33d1305 100755
--- a/wmfphablib/phabdb.py
+++ b/wmfphablib/phabdb.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 import sys
+import json
 import MySQLdb
 import traceback
 import syslog
@@ -34,6 +35,9 @@ def user_relations_finish(pid, finish, status, finish_epoch, completed, failed,
 def get_user_relations_priority(user, dbcon):
     return dbcon.sql_x("SELECT priority from user_relations where user = %s", user)
 
+def get_user_relations_comments_priority(user, dbcon):
+    return dbcon.sql_x("SELECT priority from user_relations_comments where user = %s", user)
+
 def set_user_relations_priority(priority, user, dbcon):
     """Set user status for data import
     :param priority: int
@@ -43,6 +47,15 @@ def set_user_relations_priority(priority, user, dbcon):
     return dbcon.sql_x("UPDATE user_relations SET priority=%s, modified=%s WHERE user = %s",
                   (priority, time.time(), user))
 
+def set_user_relations_comments_priority(priority, user, dbcon):
+    """Set user status for data import
+    :param priority: int
+    :param user: user email
+    :param dbcon: db connection
+     """
+    return dbcon.sql_x("UPDATE user_relations_comments SET priority=%s, modified=%s WHERE user = %s",
+                       (priority, time.time(), user))
+
 def get_user_migration_history(user, dbcon):
     """ get user history from import source
     :param user: user email
@@ -55,6 +68,76 @@ def get_user_migration_history(user, dbcon):
         return ()
     return saved_history[0]
 
+def get_user_migration_comment_history(user, dbcon):
+    """ get user comment history from import source
+    :param user: user email
+    :param dbcon: db connection
+    :returns: saved history output
+     """
+    hq = "SELECT issues, created, modified FROM user_relations_comments WHERE user = %s"
+    saved_history = dbcon.sql_x(hq, user)
+    if saved_history is None:
+        return ()
+    return saved_history[0]
+
+def is_bot(userphid):
+    p = phdb(db='phabricator_user', user=phuser_user, passwd=phuser_passwd)
+    isbot = p.sql_x("SELECT isSystemAgent from user where phid=%s", (userphid,), limit=1)
+    p.close()
+    if not isbot:
+        raise Exception("user is not a present")
+    if int(isbot[0][0]) > 0:
+        return True
+    return False
+
+def last_comment(phid):
+    p = phdb(db='phabricator_maniphest', user=phuser_user, passwd=phuser_passwd)
+    com = "SELECT phid from maniphest_transaction where objectPHID=%s and transactionType='core:comment' ORDER BY dateCreated DESC"
+    _ = p.sql_x(com, (phid,), limit=1)
+    p.close()
+    if not _:
+        return ''
+    return _[0][0]
+
+def set_issue_status(taskphid, status):
+    p = phdb(db='phabricator_maniphest', user=phuser_user, passwd=phuser_passwd)
+    p.sql_x("UPDATE maniphest_task SET status=%s WHERE phid=%s", (status, taskphid))
+    p.close()
+
+def set_comment_content(transxphid, content):
+    """set manual content for a comment
+    :param transxphid: str
+    :param userphid: str
+    """
+    p = phdb(db='phabricator_maniphest', user=phuser_user, passwd=phuser_passwd)
+    p.sql_x("UPDATE maniphest_transaction_comment SET content=%s WHERE transactionPHID=%s", (content, transxphid))
+    p.close()
+    return
+
+def set_comment_time(transxphid, metatime):
+    """set manual epoch modtime for task
+    :param taskphid: str
+    :param mtime: int of modtime
+    """
+    p = phdb(db='phabricator_maniphest', user=phuser_user, passwd=phuser_passwd)
+    p.sql_x("UPDATE maniphest_transaction SET dateModified=%s WHERE phid=%s", (metatime, transxphid))
+    p.sql_x("UPDATE maniphest_transaction SET dateCreated=%s WHERE phid=%s", (metatime, transxphid))
+    p.sql_x("UPDATE maniphest_transaction_comment SET dateModified=%s WHERE transactionPHID=%s", (metatime, transxphid))
+    p.sql_x("UPDATE maniphest_transaction_comment SET dateCreated=%s WHERE transactionPHID=%s", (metatime, transxphid))
+    p.close()
+    return
+
+def set_comment_author(transxphid, userphid):
+    """set manual owner for a comment
+    :param transxphid: str
+    :param userphid: str
+    """
+    p = phdb(db='phabricator_maniphest', user=phuser_user, passwd=phuser_passwd)
+    p.sql_x("UPDATE maniphest_transaction SET authorPHID=%s WHERE phid=%s", (userphid, transxphid))
+    p.sql_x("UPDATE maniphest_transaction_comment SET authorPHID=%s WHERE transactionPHID=%s", (userphid, transxphid))
+    p.close()
+    return
+
 def set_task_mtime(taskphid, mtime):
     """set manual epoch modtime for task
     :param taskphid: str
@@ -161,7 +244,9 @@ def reference_ticket(reference):
     p = phdb(db='phabricator_maniphest', user=phmanifest_user, passwd=phmanifest_passwd)
     _ = p.sql_x("SELECT objectPHID FROM maniphest_customfieldstringindex WHERE indexValue = %s", reference)
     p.close()
-    return _[0] or ''
+    if not _:
+        return ''
+    return _[0]
 
 def email_by_userphid(userphid):
     """
@@ -242,6 +327,13 @@ def set_project_icon(project, icon='briefcase', color='blue'):
         people    = users
         truck     = releases
     """
+    if icon == 'tags':
+        color = 'yellow'
+    elif icon == 'people':
+        color = 'violet'
+    elif icon == 'truck':
+        color == 'orange'
+
     p = phdb(db='phabricator_project', user=phuser_user, passwd=phuser_passwd)
     _ = p.sql_x("UPDATE project SET icon=%s, color=%s WHERE name=%s", ('fa-' + icon, color, project))
     p.close()
@@ -253,6 +345,25 @@ def set_task_author(authorphid, id):
     p.close()
     return _
 
+def add_task_cc_by_ref(userphid, oldid):
+    refs = reference_ticket('bz%s' % (oldid,))
+    if not refs:
+        log('reference ticket not found for %s' % ('bz%s' % (oldid,),))
+        return
+    return add_task_cc(refs[0], userphid)
+
+def add_task_cc(ticketphid, userphid):
+    p = phdb(db='phabricator_maniphest', user=phuser_user, passwd=phuser_passwd)
+    ccq = "SELECT ccPHIDs FROM maniphest_task WHERE phid = %s"
+    jcc_list = p.sql_x(ccq, ticketphid)[0]
+    cc_list = json.loads(jcc_list[0])
+    if userphid not in cc_list:
+        cc_list.append(userphid)
+    p.sql_x("UPDATE maniphest_task SET ccPHIDs=%s WHERE phid=%s", (json.dumps(cc_list), ticketphid))
+    final_jcclist = p.sql_x(ccq, ticketphid)[0]
+    p.close()
+    return json.loads(final_jcclist[0])
+
 
 class phdb:
     def __init__(self, host = dbhost,
diff --git a/wmfphablib/util.py b/wmfphablib/util.py
new file mode 100644
index 0000000..a34f601
--- /dev/null
+++ b/wmfphablib/util.py
@@ -0,0 +1,31 @@
+import sys
+import json
+import subprocess
+
+def can_edit_ref():
+    f = open('/srv/phab/phabricator/conf/local/local.json', 'r').read()
+    settings = json.loads(f)
+    try:
+        return settings['maniphest.custom-field-definitions']['external_reference']['edit']
+    except:
+        return False
+
+def runBash(cmd):
+   p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
+   out = p.stdout.read().strip()
+   return out
+
+def translate_json_dict_items(dict_of_j):
+    for t in dict_of_j.keys():
+        if dict_of_j[t]:
+            try:
+                dict_of_j[t] = json.loads(dict_of_j[t])
+            except (TypeError, ValueError):
+                pass
+    return dict_of_j
+
+def get_index(seq, attr, value):
+    return next(index for (index, d) in enumerate(seq) if d[attr] == value)
+
+def purge_cache():
+    return runBash('/srv/phab/phabricator/bin/cache purge --purge-remarkup')