reframe backlog processing for user metadata

I was doing something clever to throttle a large backlog
over the long term, but I ended up just batch processsing.
This is much simpler to debug for days to come.  Look at last
n seconds and fix new users.

3 files changed, 20 insertions, 16 deletions

diff --git a/bugzilla_update_user_comments.py b/bugzilla_update_user_comments.py
index 3acc4c5..aa224d9 100755
--- a/bugzilla_update_user_comments.py
+++ b/bugzilla_update_user_comments.py
@@ -126,7 +126,7 @@ def main():
     parser = argparse.ArgumentParser(description='Updates user header metadata from bugzilla')
     parser.add_argument('-a', action="store_true", default=False)
     parser.add_argument('-e', action="store", dest='email')
-    parser.add_argument('-m', action="store", dest="starting_epoch", default=None)
+    parser.add_argument('-m', action="store", dest="backlog", default=None)
     parser.add_argument('-v', action="store_true", default=False)
     args =  parser.parse_args()
 
@@ -135,15 +135,16 @@ def main():
                        passwd=config.bzmigrate_passwd)
 
     if args.a:
-        starting_epoch = phabdb.get_user_relations_comments_last_finish(pmig)
-        users, finish_epoch = phabdb.get_verified_users(starting_epoch, config.bz_updatelimit)
+        backlog = time.time() - config.bz_update_lookback
+        log("Backlog since %d" % (backlog),)
+        users, finish_epoch = phabdb.get_verified_users(backlog, config.bz_updatelimit)
     elif args.email:
         users = phabdb.get_verified_user(args.email)
-        starting_epoch = 0
+        backlog = 0
         finish_epoch = 0
-    elif args.starting_epoch:
-        users, finish_epoch = phabdb.get_verified_users(args.starting_epoch)
-        starting_epoch = args.starting_epoch
+    elif args.backlog:
+        users, finish_epoch = phabdb.get_verified_users(args.backlog)
+        backlog = args.backlog
     else:
         parser.print_help()
         sys.exit(1)
@@ -172,7 +173,7 @@ def main():
                                 source,
                                 int(time.time()),
                                 ipriority['na'],
-                                starting_epoch,
+                                backlog,
                                 user_count, issue_count, pmig)
 
     results = []
diff --git a/bugzilla_update_user_header.py b/bugzilla_update_user_header.py
index ab5d409..f9932ff 100755
--- a/bugzilla_update_user_header.py
+++ b/bugzilla_update_user_header.py
@@ -128,7 +128,7 @@ def main():
     parser = argparse.ArgumentParser(description='Updates user header metadata from bugzilla')
     parser.add_argument('-a', action="store_true", default=False)
     parser.add_argument('-e', action="store", dest='email')
-    parser.add_argument('-m', action="store", dest="starting_epoch", default=None)
+    parser.add_argument('-m', action="store", dest="backlog", default=None)
     parser.add_argument('-v', action="store_true", default=False)
     args =  parser.parse_args()
 
@@ -137,15 +137,16 @@ def main():
                        passwd=config.bzmigrate_passwd)
 
     if args.a:
-        starting_epoch = phabdb.get_user_relations_last_finish(pmig)
-        users, finish_epoch = phabdb.get_verified_users(starting_epoch, config.bz_updatelimit)
+        backlog = time.time() - config.bz_update_lookback
+        log("Backlog since %d" % (backlog),)
+        users, finish_epoch = phabdb.get_verified_users(backlog, config.bz_updatelimit)
     elif args.email:
         users = phabdb.get_verified_user(args.email)
-        starting_epoch = 0
+        backlog = 0
         finish_epoch = 0
-    elif args.starting_epoch:
-        users, finish_epoch = phabdb.get_verified_users(args.starting_epoch)
-        starting_epoch = args.starting_epoch
+    elif args.backlog:
+        users, finish_epoch = phabdb.get_verified_users(args.backlog)
+        backlog = args.backlog
     else:
         parser.print_help()
         sys.exit(1)
@@ -178,7 +179,7 @@ def main():
                                 source,
                                 int(time.time()),
                                 ipriority['na'],
-                                starting_epoch,
+                                backlog,
                                 user_count, issue_count, pmig)
 
     results = []
diff --git a/wmfphablib/config.py b/wmfphablib/config.py
index ffd6384..4d46d6f 100755
--- a/wmfphablib/config.py
+++ b/wmfphablib/config.py
@@ -39,6 +39,7 @@ bz_createmulti = int(parser.get(parser_mode, 'create_multi'))
 bz_fetchmulti = int(parser.get(parser_mode, 'fetch_multi'))
 bz_updatelimit = int(parser.get(parser_mode, 'update_limit'))
 bz_updatemulti = int(parser.get(parser_mode, 'update_multi'))
+bz_update_lookback = int(parser.get(parser_mode, 'update_lookback'))
 bz_populate_multi = int(parser.get(parser_mode, 'populate_multi'))
 bz_security = parser.get(parser_mode, 'security')
 parser_mode = 'rtmigrate'
@@ -84,6 +85,7 @@ if __name__ == '__main__':
     print bz_updatelimit
     print bz_updatemulti
     print bz_populate_multi
+    print bz_update_lookback
     print bz_security
     print 'RT\n'
     print rtmigrate_db