getting through RT populate tables

4 files changed, 60 insertions, 25 deletions

diff --git a/rt_create.py b/rt_create.py
index 62c6434..0211dfe 100755
--- a/rt_create.py
+++ b/rt_create.py
@@ -92,7 +92,7 @@ def create(rtid):
             elif line.strip().startswith('#'):
                 return uob('%%%') + uob(line.strip()) + uob('%%%')
             else:
-                return uob(line).strip()
+                return uob('%%%') + uob(line).strip() + uob('%%%')
         elif line.strip().startswith('>'):
             quoted_content = line.lstrip('>').strip()
             if not quoted_content.lstrip('>').strip():
@@ -436,6 +436,7 @@ def create(rtid):
             value_update = ''
             value_update_text = rtlib.shadow_emails(dbody['description'])
             value_update_text = value_update_text.replace('fsck.com-rt', 'https')
+
             relations = ['Reference by ticket',
                          'Dependency by',
                          'Reference to ticket',
@@ -478,13 +479,17 @@ def create(rtid):
 
         phabdb.set_comment_time(ctransaction,
                                 created)
+
         fmt_comment['xctransaction'] = ctransaction
         fmt_comment['preamble'] = preamble
         fmt_comment['content'] = cbody
         fmt_comment['created'] = created
+        fmt_comment['author'] = dbody['creator']
+
         # XXX TRX both ways?
         #fmt_comment['creator'] = dbody['creator']user_lookup(name)
         cid =  len(fmt_comments.keys()) + 1
+        fmt_comment['count'] = cid
         fmt_comments[cid] = fmt_comment
 
     if rtinfo['Status'].lower() != 'open':
diff --git a/rt_populate_user_relations_comments_table.py b/rt_populate_user_relations_comments_table.py
index cefc5e3..a76b792 100755
--- a/rt_populate_user_relations_comments_table.py
+++ b/rt_populate_user_relations_comments_table.py
@@ -7,6 +7,7 @@ import collections
 from phabricator import Phabricator
 from wmfphablib import Phab as phabmacros
 from wmfphablib import phabdb
+from wmfphablib import rtlib
 from wmfphablib import log
 from wmfphablib import vlog
 from wmfphablib import errorlog as elog
@@ -47,30 +48,37 @@ def populate(bugid):
             pmig.sql_x("INSERT INTO user_relations_comments (user, issues, created, modified) VALUES (%s, %s, %s, %s)",
                        insert_values)
 
-    pmig = phabdb.phdb(db=config.bzmigrate_db)
-    issue = pmig.sql_x("SELECT id FROM bugzilla_meta WHERE id = %s", bugid)
+
+    pmig = phabdb.phdb(db=config.rtmigrate_db,
+                       user=config.rtmigrate_user,
+                       passwd=config.rtmigrate_passwd)
+
+    issue = pmig.sql_x("SELECT id FROM rt_meta WHERE id = %s", bugid)
     if not issue:
         log('issue %s does not exist for user population' % (bugid,))
         return True
 
-    fpriority= pmig.sql_x("SELECT priority FROM bugzilla_meta WHERE id = %s", bugid)
+    fpriority= pmig.sql_x("SELECT priority FROM rt_meta WHERE id = %s", bugid)
     if fpriority[0] == ipriority['fetch_failed']:
         log('issue %s does not fetched successfully for user population (failed fetch)' % (bugid,))
         return True
 
-    current = pmig.sql_x("SELECT comments, xcomments, modified FROM bugzilla_meta WHERE id = %s", bugid)
+    current = pmig.sql_x("SELECT comments, xcomments, modified FROM rt_meta WHERE id = %s", bugid)
     if current:
         comments, xcomments, modified = current[0]
     else:
         log('%s not present for migration' % (bugid,))
-        return True
+        return 'missing'
 
     com = json.loads(comments)
     xcom = json.loads(xcomments)
-    commenters = [c['author'] for c in com if c['count'] > 0]
-    commenters = set(commenters)
+    # rtlib.user_lookup(header["Creator"])
+    commenters = [rtlib.user_lookup(c['author']) for c in xcom.values() if c['count'] > 0]
+    commenters = set(filter(bool, commenters))
+    print commenters
     log("commenters for issue %s: %s" % (bugid, str(commenters)))
     for c in commenters:
+        print c
         add_comment_ref(c)
     pmig.close()
     return True
@@ -89,14 +97,22 @@ def run_populate(bugid, tries=1):
         elog('failed to populate %s' % (bugid,))
         return run_populate(bugid, tries=tries)
 
+
 def main():
     bugs = return_bug_list()
-    from multiprocessing import Pool
-    pool = Pool(processes=10)
-    _ =  pool.map(run_populate, bugs)
-    complete = len(filter(bool, _))
-    failed = len(_) - complete
-    print '%s completed %s, failed %s' % (sys.argv[0], complete, failed)
+    result = []
+    for b in bugs:
+        result.append(run_populate(b))
 
+    missing = len([i for i in result if i == 'missing'])
+    complete = len(filter(bool, [i for i in result if i not in ['missing']]))
+    failed = (len(result) - missing) - complete
+    print '-----------------------------\n \
+          %s Total %s (missing %s)\n \
+          completed %s, failed %s' % (sys.argv[0],
+                                                          len(bugs),
+                                                          missing,
+                                                          complete,
+                                                          failed)
 if __name__ == '__main__':
     main()
diff --git a/rt_populate_user_relations_table.py b/rt_populate_user_relations_table.py
index 23f3063..278cf96 100755
--- a/rt_populate_user_relations_table.py
+++ b/rt_populate_user_relations_table.py
@@ -20,12 +20,15 @@ from wmfphablib import return_bug_list
 
 
 def populate(rtid):
-    pmig = phabdb.phdb(db=config.rtmigrate_db)
+
+    pmig = phabdb.phdb(db=config.rtmigrate_db,
+                       user=config.rtmigrate_user,
+                       passwd=config.rtmigrate_passwd)
 
     issue = pmig.sql_x("SELECT id FROM rt_meta WHERE id = %s", rtid)
     if not issue:
         log('issue %s does not exist for user population' % (rtid,))
-        return True
+        return 'missing'
 
     fpriority= pmig.sql_x("SELECT priority FROM rt_meta WHERE id = %s", rtid)
     if fpriority[0] == ipriority['fetch_failed']:
@@ -45,18 +48,18 @@ def populate(rtid):
     relations['author'] = rtlib.user_lookup(header["Creator"])
     ccusers = header['AdminCc'].split(',') + header['Cc'].split(',')
     relations['cc'] = ccusers
+    relations['cc'] = [cc.strip() for cc in relations['cc'] if cc]
     # RT uses a literal nobody for no assigned
     if header['Owner'] == 'Nobody':
         relations['owner'] = ''
     else:
         relations['owner'] = rtlib.user_lookup(header['Owner'])
 
-
     for k, v in relations.iteritems():
         if relations[k]:
             relations[k] = filter(bool, v)
 
-    def add_owner(owner):    
+    def add_owner(owner):
         ouser = pmig.sql_x("SELECT user FROM user_relations WHERE user = %s", (owner,))
         if ouser:
             jassigned = pmig.sql_x("SELECT assigned FROM user_relations WHERE user = %s", (owner,))
@@ -164,12 +167,21 @@ def run_populate(rtid, tries=1):
 
 def main():
     bugs = return_bug_list()
-    from multiprocessing import Pool
-    pool = Pool(processes=10)
-    _ =  pool.map(run_populate, bugs)
-    complete = len(filter(bool, _))
-    failed = len(_) - complete
-    print '%s completed %s, failed %s' % (sys.argv[0], complete, failed)
+    result = []
+    for b in bugs:
+        result.append(run_populate(b))
+
+    missing = len([i for i in result if i == 'missing'])
+    complete = len(filter(bool, [i for i in result if i not in ['missing']]))
+    failed = (len(result) - missing) - complete
+    print '-----------------------------\n \
+          %s Total %s (missing %s)\n \
+          completed %s, failed %s' % (sys.argv[0],
+                                                          len(bugs),
+                                                          missing,
+                                                          complete,
+                                                          failed)
 
 if __name__ == '__main__':
     main()
+
diff --git a/wmfphablib/rtlib.py b/wmfphablib/rtlib.py
index 8e3a714..53fdb32 100755
--- a/wmfphablib/rtlib.py
+++ b/wmfphablib/rtlib.py
@@ -50,7 +50,9 @@ def project_translate(pname):
 
 def user_lookup(name):
     """ match user name in rt to user email"""
-    return users.get(name, None) or name
+    if not users:
+        raise Exception("RT PPL must be present to translate users!")
+    return users.get(name, '').strip()
 
 def shadow_emails(text):
     emails =  re.findall('([^@|\s]+@[^@]+\.[^@|\s]+)', text)