RT fetch reports only _unexpected_

failures as actual failures and counts
other exclusions separately.

5 files changed, 79 insertions, 31 deletions

diff --git a/README b/README
index b67e740..01fa68c 100755
--- a/README
+++ b/README
@@ -101,7 +101,7 @@ CREATE TABLE user_relations_comments
 );
 
 
-CREATE table user_relations_jobs
+CREATE TABLE user_relations_jobs
 (
                                   id INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
 			          pid INT,
diff --git a/rt_fetch.py b/rt_fetch.py
index 5ee9e13..020907a 100755
--- a/rt_fetch.py
+++ b/rt_fetch.py
@@ -7,7 +7,7 @@ import getpass
 import ConfigParser
 import json
 sys.path.append('/home/rush/python-rtkit/')
-from wmfphablib import phdb
+from wmfphablib import phabdb
 from wmfphablib import rtlib
 from wmfphablib import log
 from wmfphablib import vlog
@@ -27,16 +27,26 @@ def fetch(tid):
                                    config.rt_login,
                                    config.rt_passwd,
                                    authenticators.CookieAuthenticator)
+
     log("fetching issue %s" % (tid,))
     tinfo = response.get(path="ticket/%s" % (tid,))
     history = response.get(path="ticket/%s/history?format=l" % (tid,))
     links = response.get(path="ticket/%s/links/show" % (tid,))
     vlog(tinfo)
 
+    if re.search('\#\sTicket\s\d+\sdoes\snot\sexist.$', tinfo.strip()):
+        log("Skipped as source missing for %s" % (tid,))
+        return 'missing'
+
     # some private todo's and such
     if 'You are not allowed to display' in tinfo:
-        log("Access denied for %s" % (tid,))
-        return False
+        log("Skipped as access denied for %s" % (tid,))
+        return 'denied'
+
+    #breaking detailed history into posts
+    #23/23 (id/114376/total)
+    comments = re.split("\d+\/\d+\s+\(id\/.\d+\/total\)", history)
+    comments = [c.rstrip('#').rstrip('--') for c in comments]
 
     # we get back freeform text and create a dict
     dtinfo = {}
@@ -51,10 +61,10 @@ def fetch(tid):
             v = cv_kv[1]
             dtinfo[k.strip()] = v.strip()
 
-    #breaking detailed history into posts
-    #23/23 (id/114376/total)
-    comments = re.split("\d+\/\d+\s+\(id\/.\d+\/total\)", history)
-    comments = [c.rstrip('#').rstrip('--') for c in comments]
+    vlog("Enabled queues: %s" % (str(rtlib.enabled)))
+    if dtinfo['Queue'] not in rtlib.enabled:
+        log("Skipped as disabled queue for %s (%s)" % (str(tid), dtinfo['Queue']))
+        return 'disabled'
 
     if dtinfo['Status'] == 'resolved':
         creation_priority = ipriority['na']
@@ -63,9 +73,27 @@ def fetch(tid):
 
     com = json.dumps(comments)
     tinfo = json.dumps(dtinfo)
-    pmig = phdb(db=config.rtmigrate_db)
-    insert_values =  (tid, creation_priority, tinfo, com, now(), now())
-    pmig.sql_x("INSERT INTO rt_meta \
+
+    pmig = phabdb.phdb(db=config.rtmigrate_db,
+                       user=config.rtmigrate_user,
+                       passwd=config.rtmigrate_passwd)
+
+
+    current = pmig.sql_x("SELECT * from rt_meta where id = %s", tid)
+    if current:
+        update_values =  (creation_priority, tinfo, com, now(), now())
+        pmig.sql_x("UPDATE rt_meta SET priority=%s, \
+                                       header=%s, \
+                                       comments=%s, \
+                                       modified=%s \
+                                       WHERE id = %s",
+                   update_values)
+        vlog('update: ' + str(update_values))
+
+    else:
+        insert_values =  (tid, creation_priority, tinfo, com, now(), now())
+
+        pmig.sql_x("INSERT INTO rt_meta \
                 (id, priority, header, comments, created, modified) \
                 VALUES (%s, %s, %s, %s, %s, %s)",
                 insert_values)
@@ -74,10 +102,15 @@ def fetch(tid):
 
 def run_fetch(tid, tries=1):
     if tries == 0:
-        pmig = phdb(db=config.rtmigrate_db)
+        pmig = phabdb.phdb(db=config.rtmigrate_db,
+                       user=config.rtmigrate_user,
+                       passwd=config.rtmigrate_passwd)
         insert_values =  (tid, ipriority['fetch_failed'], '', '', now(), now())
-        pmig.sql_x("INSERT INTO rt_meta (id, priority, header, comments, created, modified) VALUES (%s, %s, %s, %s, %s, %s)",
-                   insert_values)
+
+        pmig.sql_x("INSERT INTO rt_meta \
+                (id, priority, header, comments, created, modified) \
+                VALUES (%s, %s, %s, %s, %s, %s)",
+                insert_values)
         pmig.close()
         elog('failed to grab %s' % (tid,))
         return False
@@ -92,13 +125,29 @@ def run_fetch(tid, tries=1):
         return run_fetch(tid, tries=tries)
 
 def main():
+
     bugs = return_bug_list()
     from multiprocessing import Pool
     pool = Pool(processes=int(config.bz_fetchmulti))
     _ =  pool.map(run_fetch, bugs)
-    complete = len(filter(bool, _))
-    failed = len(_) - complete
-    print '%s completed %s, failed %s' % (sys.argv[0], complete, failed)
+    vlog(_)
+    denied = len([i for i in _ if i == 'denied'])
+    disabled = len([i for i in _ if i == 'disabled'])
+    missing = len([i for i in _ if i == 'missing'])
+    complete = len(filter(bool, [i for i in _ if i not in ['denied', 'disabled', 'missing']]))
+    known_bad = denied + disabled + missing
+    failed = (len(_) - known_bad) - complete
+    print '-----------------------------\n \
+          %s Total %s\n \
+          known bad %s (denied %s, disabled %s, missing %s)\n\n \
+          completed %s, failed %s' % (sys.argv[0],
+                                                          len(bugs),
+                                                          known_bad,
+                                                          denied,
+                                                          disabled,
+                                                          missing,
+                                                          complete,
+                                                          failed)
 
 if __name__ == '__main__':
     main()
diff --git a/wmfphablib/phabdb.py b/wmfphablib/phabdb.py
index 456aeb8..e0bf645 100755
--- a/wmfphablib/phabdb.py
+++ b/wmfphablib/phabdb.py
@@ -662,10 +662,10 @@ def add_task_policy_users(taskPHID,
     return policyPHID
 
 def get_task_edit_policyPHID(taskPHID):
-   """ retrive task edit policy
-   :param taskPHID: str
-   :returns: str
-   """
+    """ retrive task edit policy
+    :param taskPHID: str
+    :returns: str
+    """
 
     p = phdb(db='phabricator_maniphest',
              user=phuser_user,
@@ -679,10 +679,10 @@ def get_task_edit_policyPHID(taskPHID):
         return _[0][0]
 
 def get_task_view_policy(phid):
-   """ retrive task view policy
-   :param taskPHID: str
-   :returns: str
-   """
+    """ retrive task view policy
+    :param taskPHID: str
+    :returns: str
+    """
 
     p = phdb(db='phabricator_maniphest',
              user=phuser_user,
diff --git a/wmfphablib/rtlib.py b/wmfphablib/rtlib.py
index d7670f8..c4f3001 100755
--- a/wmfphablib/rtlib.py
+++ b/wmfphablib/rtlib.py
@@ -1,5 +1,4 @@
-enabled = ['codfw', 'ulsfo', 'pmtpa', 'ops-requests', 'network', 'esams', 'eqiad',
-                'core-ops',]
+enabled = ['codfw', 'ulsfo', 'pmtpa', 'ops-requests', 'network', 'esams', 'eqiad', 'core-ops',]
 
 import util
 try:
diff --git a/wmfphablib/util.py b/wmfphablib/util.py
index c22c649..9c4369b 100644
--- a/wmfphablib/util.py
+++ b/wmfphablib/util.py
@@ -60,17 +60,17 @@ def epoch_to_datetime(epoch, timezone='UTC'):
            ).strftime('%Y-%m-%d %H:%M:%S'))) + " (%s)" % (timezone,)
 
 def errorlog(msg):
-    msg = unicode(msg)
     try:
+        msg = unicode(msg)
         syslog.syslog(msg)
         print >> sys.stderr, msg
     except:
         print 'error logging, well...error output'
 
 def log(msg):
-    msg = unicode(msg)
     if '-v' in ''.join(sys.argv):
         try:
+            msg = unicode(msg)
             syslog.syslog(msg)
             print msg
         except:
@@ -82,12 +82,12 @@ def notice(msg):
     log(msg)
 
 def vlog(msg):
-    msg = unicode(msg)
     if '-vv' in ''.join(sys.argv):
         try:
+            msg = unicode(msg)
             print '-> ', msg
         except:
-            print 'error logging output'
+            print 'error logging verbose output'
 
 def update_blog(source, complete, failed, user_count, issue_count, apicon):
     title = "%s completed %s / failed %s" % (epoch_to_datetime(time.time()),