diff options
Diffstat (limited to 'lava_dashboard_tool/commands.py')
| -rw-r--r-- | lava_dashboard_tool/commands.py | 239 |
1 files changed, 149 insertions, 90 deletions
diff --git a/lava_dashboard_tool/commands.py b/lava_dashboard_tool/commands.py index 1159681..1d6a25e 100644 --- a/lava_dashboard_tool/commands.py +++ b/lava_dashboard_tool/commands.py @@ -37,11 +37,11 @@ import simplejson from json_schema_validator.extensions import datetime_extension from lava_tool.authtoken import AuthenticatingServerProxy, KeyringAuthBackend -from lava.tool.commands import ExperimentalCommandMixIn from lava.tool.command import Command, CommandGroup class dashboard(CommandGroup): + """ Commands for interacting with LAVA Dashboard """ @@ -50,16 +50,19 @@ class dashboard(CommandGroup): class InsufficientServerVersion(Exception): + """ Exception raised when server version that a command interacts with is too old to support required features. """ + def __init__(self, server_version, required_version): self.server_version = server_version self.required_version = required_version class DataSetRenderer(object): + """ Support class for rendering a table out of list of dictionaries. @@ -77,8 +80,10 @@ class DataSetRenderer(object): Each dictionary must have the same keys. In particular the first row is used to determine columns. """ + def __init__(self, column_map=None, row_formatter=None, empty=None, - order=None, caption=None, separator=" ", header_separator=None): + order=None, caption=None, separator=" ", + header_separator=None): if column_map is None: column_map = {} if row_formatter is None: @@ -179,11 +184,11 @@ class DataSetRenderer(object): if column in self.row_formatter: row[column] = self.row_formatter[column](row[column]) maxlen = dict( - [(column, max( - len(self.column_map.get(column, column)), - max([ - len(str(row[column])) for row in dataset_out]))) - for column in columns]) + [(column, max( + len(self.column_map.get(column, column)), + max([ + len(str(row[column])) for row in dataset_out]))) + for column in columns]) return dataset_out, columns, maxlen def _render_header(self, dataset, columns, maxlen): @@ -244,7 +249,7 @@ class DataSetRenderer(object): # Now print the coulum names print self.separator.join([ "{0:^{1}}".format(self.column_map.get(column, column), - maxlen[column]) for column in columns]) + maxlen[column]) for column in columns]) # Finally print the header separator if self.header_separator: print "-" * total_len @@ -265,7 +270,7 @@ class DataSetRenderer(object): >>> maxlen = {'a': 13, 'bee': 3} Now a plain table. Note! To really understand this test - you should check out the length of the strings below. There + you should check out the whitespace in the strings below. There are two more spaces after 'b' in the second row >>> DataSetRenderer()._render_rows(dataset, columns, maxlen) shorter @@ -306,6 +311,7 @@ class DataSetRenderer(object): class XMLRPCCommand(Command): + """ Abstract base class for commands that interact with dashboard server over XML-RPC. @@ -344,7 +350,7 @@ class XMLRPCCommand(Command): '0.4.0b2' """ try: - major, minor, micro, releaselevel, serial = version.split(".") + major, minor, micro, releaselevel, serial = version.split(".") except ValueError: raise ValueError( ("version %r does not follow pattern " @@ -377,7 +383,7 @@ class XMLRPCCommand(Command): def __init__(self, parser, args): super(XMLRPCCommand, self).__init__(parser, args) - xml_rpc_url = self._construct_xml_rpc_url(self.args.dashboard_url) + xml_rpc_url = self._construct_xml_rpc_url(self.args.dashboard_url) self.server = AuthenticatingServerProxy( xml_rpc_url, verbose=args.verbose_xml_rpc, @@ -393,19 +399,20 @@ class XMLRPCCommand(Command): @classmethod def register_arguments(cls, parser): - dashboard_group = parser.add_argument_group("dashboard specific arguments") + dashboard_group = parser.add_argument_group( + "dashboard specific arguments") default_dashboard_url = os.getenv("DASHBOARD_URL") if default_dashboard_url: dashboard_group.add_argument("--dashboard-url", - metavar="URL", help="URL of your validation dashboard (currently %(default)s)", - default=default_dashboard_url) + metavar="URL", help="URL of your validation dashboard (currently %(default)s)", + default=default_dashboard_url) else: dashboard_group.add_argument("--dashboard-url", required=True, - metavar="URL", help="URL of your validation dashboard") + metavar="URL", help="URL of your validation dashboard") debug_group = parser.add_argument_group("debugging arguments") debug_group.add_argument("--verbose-xml-rpc", - action="store_true", default=False, - help="Show XML-RPC data") + action="store_true", default=False, + help="Show XML-RPC data") return dashboard_group @contextlib.contextmanager @@ -414,7 +421,7 @@ class XMLRPCCommand(Command): yield except socket.error as ex: print >> sys.stderr, "Unable to connect to server at %s" % ( - self.args.dashboard_url,) + self.args.dashboard_url,) # It seems that some errors are reported as -errno # while others as +errno. ex.errno = abs(ex.errno) @@ -429,12 +436,13 @@ class XMLRPCCommand(Command): print >> sys.stderr, "Socket %d: %s" % (ex.errno, ex.strerror) except xmlrpclib.ProtocolError as ex: print >> sys.stderr, "Unable to exchange XML-RPC message with dashboard server" - print >> sys.stderr, "HTTP error code: %d/%s" % (ex.errcode, ex.errmsg) + print >> sys.stderr, "HTTP error code: %d/%s" % ( + ex.errcode, ex.errmsg) except xmlrpclib.Fault as ex: self.handle_xmlrpc_fault(ex.faultCode, ex.faultString) except InsufficientServerVersion as ex: print >> sys.stderr, ("This command requires at least server version " - "%s, actual server version is %s" % + "%s, actual server version is %s" % (ex.required_version, ex.server_version)) def invoke(self): @@ -447,13 +455,15 @@ class XMLRPCCommand(Command): print >> sys.stderr, "Dashboard server has experienced internal error" print >> sys.stderr, faultString else: - print >> sys.stderr, "XML-RPC error %d: %s" % (faultCode, faultString) + print >> sys.stderr, "XML-RPC error %d: %s" % ( + faultCode, faultString) def invoke_remote(self): raise NotImplementedError() class server_version(XMLRPCCommand): + """ Display dashboard server version """ @@ -463,6 +473,7 @@ class server_version(XMLRPCCommand): class put(XMLRPCCommand): + """ Upload a bundle on the server """ @@ -471,11 +482,11 @@ class put(XMLRPCCommand): def register_arguments(cls, parser): super(put, cls).register_arguments(parser) parser.add_argument("LOCAL", - type=argparse.FileType("rb"), - help="pathname on the local file system") + type=argparse.FileType("rb"), + help="pathname on the local file system") parser.add_argument("REMOTE", - default="/anonymous/", nargs='?', - help="pathname on the server") + default="/anonymous/", nargs='?', + help="pathname on the server") def invoke_remote(self): content = self.args.LOCAL.read() @@ -487,7 +498,7 @@ class put(XMLRPCCommand): def handle_xmlrpc_fault(self, faultCode, faultString): if faultCode == 404: print >> sys.stderr, "Bundle stream %s does not exist" % ( - self.args.REMOTE) + self.args.REMOTE) elif faultCode == 409: print >> sys.stderr, "You have already uploaded this bundle to the dashboard" else: @@ -495,6 +506,7 @@ class put(XMLRPCCommand): class get(XMLRPCCommand): + """ Download a bundle from the server """ @@ -503,15 +515,15 @@ class get(XMLRPCCommand): def register_arguments(cls, parser): super(get, cls).register_arguments(parser) parser.add_argument("SHA1", - type=str, - help="SHA1 of the bundle to download") + type=str, + help="SHA1 of the bundle to download") parser.add_argument("--overwrite", - action="store_true", - help="Overwrite files on the local disk") + action="store_true", + help="Overwrite files on the local disk") parser.add_argument("--output", "-o", - type=argparse.FileType("wb"), - default=None, - help="Alternate name of the output file") + type=argparse.FileType("wb"), + default=None, + help="Alternate name of the output file") def invoke_remote(self): response = self.server.get(self.args.SHA1) @@ -519,7 +531,7 @@ class get(XMLRPCCommand): filename = self.args.SHA1 if os.path.exists(filename) and not self.args.overwrite: print >> sys.stderr, "File {filename!r} already exists".format( - filename=filename) + filename=filename) print >> sys.stderr, "You may pass --overwrite to write over it" return -1 stream = open(filename, "wb") @@ -528,17 +540,18 @@ class get(XMLRPCCommand): filename = self.args.output.name stream.write(response['content']) print "Downloaded bundle {0} to file {1!r}".format( - self.args.SHA1, filename) + self.args.SHA1, filename) def handle_xmlrpc_fault(self, faultCode, faultString): if faultCode == 404: print >> sys.stderr, "Bundle {sha1} does not exist".format( - sha1=self.args.SHA1) + sha1=self.args.SHA1) else: super(get, self).handle_xmlrpc_fault(faultCode, faultString) class deserialize(XMLRPCCommand): + """ Deserialize a bundle on the server """ @@ -547,8 +560,8 @@ class deserialize(XMLRPCCommand): def register_arguments(cls, parser): super(deserialize, cls).register_arguments(parser) parser.add_argument("SHA1", - type=str, - help="SHA1 of the bundle to deserialize") + type=str, + help="SHA1 of the bundle to deserialize") def invoke_remote(self): response = self.server.deserialize(self.args.SHA1) @@ -558,13 +571,17 @@ class deserialize(XMLRPCCommand): def handle_xmlrpc_fault(self, faultCode, faultString): if faultCode == 404: print >> sys.stderr, "Bundle {sha1} does not exist".format( - sha1=self.args.SHA1) + sha1=self.args.SHA1) elif faultCode == 409: print >> sys.stderr, "Unable to deserialize bundle {sha1}".format( sha1=self.args.SHA1) print >> sys.stderr, faultString else: - super(deserialize, self).handle_xmlrpc_fault(faultCode, faultString) + super( + deserialize, + self).handle_xmlrpc_fault( + faultCode, + faultString) def _get_pretty_renderer(**kwargs): @@ -576,6 +593,7 @@ def _get_pretty_renderer(**kwargs): class streams(XMLRPCCommand): + """ Show streams you have access to """ @@ -596,33 +614,34 @@ class streams(XMLRPCCommand): class bundles(XMLRPCCommand): + """ Show bundles in the specified stream """ renderer = _get_pretty_renderer( - column_map={ - 'uploaded_by': 'Uploader', - 'uploaded_on': 'Upload date', - 'content_filename': 'File name', - 'content_sha1': 'SHA1', - 'is_deserialized': "Deserialized?"}, - row_formatter={ - 'is_deserialized': lambda x: "yes" if x else "no", - 'uploaded_by': lambda x: x or "(anonymous)", - 'uploaded_on': lambda x: x}, - order=('content_sha1', 'content_filename', 'uploaded_by', - 'uploaded_on', 'is_deserialized'), - empty="There are no bundles in this stream", - caption="Bundles", - separator=" | ") + column_map={ + 'uploaded_by': 'Uploader', + 'uploaded_on': 'Upload date', + 'content_filename': 'File name', + 'content_sha1': 'SHA1', + 'is_deserialized': "Deserialized?"}, + row_formatter={ + 'is_deserialized': lambda x: "yes" if x else "no", + 'uploaded_by': lambda x: x or "(anonymous)", + 'uploaded_on': lambda x: x}, + order=('content_sha1', 'content_filename', 'uploaded_by', + 'uploaded_on', 'is_deserialized'), + empty="There are no bundles in this stream", + caption="Bundles", + separator=" | ") @classmethod def register_arguments(cls, parser): super(bundles, cls).register_arguments(parser) parser.add_argument("PATHNAME", - default="/anonymous/", nargs='?', - help="pathname on the server (defaults to %(default)s)") + default="/anonymous/", nargs='?', + help="pathname on the server (defaults to %(default)s)") def invoke_remote(self): self.renderer.render(self.server.bundles(self.args.PATHNAME)) @@ -630,12 +649,13 @@ class bundles(XMLRPCCommand): def handle_xmlrpc_fault(self, faultCode, faultString): if faultCode == 404: print >> sys.stderr, "Bundle stream %s does not exist" % ( - self.args.PATHNAME) + self.args.PATHNAME) else: super(bundles, self).handle_xmlrpc_fault(faultCode, faultString) class make_stream(XMLRPCCommand): + """ Create a bundle stream on the server """ @@ -660,9 +680,9 @@ class make_stream(XMLRPCCommand): class backup(XMLRPCCommand): + """ Backup data uploaded to a dashboard instance. - Not all data is preserved. The following data is lost: identity of the user that uploaded each bundle, time of uploading and deserialization on the server, name of the bundle stream that contained the data @@ -679,7 +699,9 @@ class backup(XMLRPCCommand): os.mkdir(self.args.BACKUP_DIR) for bundle_stream in self.server.streams(): print "Processing stream %s" % bundle_stream["pathname"] - bundle_stream_dir = os.path.join(self.args.BACKUP_DIR, urllib.quote_plus(bundle_stream["pathname"])) + bundle_stream_dir = os.path.join( + self.args.BACKUP_DIR, + urllib.quote_plus(bundle_stream["pathname"])) if not os.path.exists(bundle_stream_dir): os.mkdir(bundle_stream_dir) with open(os.path.join(bundle_stream_dir, "metadata.json"), "wt") as stream: @@ -692,14 +714,18 @@ class backup(XMLRPCCommand): for bundle in self.server.bundles(bundle_stream["pathname"]): print " * Backing up bundle %s" % bundle["content_sha1"] data = self.server.get(bundle["content_sha1"]) - bundle_pathname = os.path.join(bundle_stream_dir, bundle["content_sha1"]) - # Note: we write bundles as binary data to preserve anything the user might have dumped on us + bundle_pathname = os.path.join( + bundle_stream_dir, + bundle["content_sha1"]) + # Note: we write bundles as binary data to preserve anything + # the user might have dumped on us with open(bundle_pathname + ".json", "wb") as stream: stream.write(data["content"]) with open(bundle_pathname + ".metadata.json", "wt") as stream: simplejson.dump({ "uploaded_by": bundle["uploaded_by"], - "uploaded_on": datetime_extension.to_json(bundle["uploaded_on"]), + "uploaded_on": + datetime_extension.to_json(bundle["uploaded_on"]), "content_filename": bundle["content_filename"], "content_sha1": bundle["content_sha1"], "content_size": bundle["content_size"], @@ -707,6 +733,7 @@ class backup(XMLRPCCommand): class restore(XMLRPCCommand): + """ Restore a dashboard instance from backup """ @@ -720,7 +747,9 @@ class restore(XMLRPCCommand): def invoke_remote(self): self._check_server_version(self.server, "0.3") for stream_pathname_quoted in os.listdir(self.args.BACKUP_DIR): - filesystem_stream_pathname = os.path.join(self.args.BACKUP_DIR, stream_pathname_quoted) + filesystem_stream_pathname = os.path.join( + self.args.BACKUP_DIR, + stream_pathname_quoted) if not os.path.isdir(filesystem_stream_pathname): continue stream_pathname = urllib.unquote(stream_pathname_quoted) @@ -731,12 +760,18 @@ class restore(XMLRPCCommand): stream_metadata = {} print "Processing stream %s" % stream_pathname try: - self.server.make_stream(stream_pathname, stream_metadata.get("name", "Restored from backup")) + self.server.make_stream( + stream_pathname, + stream_metadata.get( + "name", + "Restored from backup")) except xmlrpclib.Fault as ex: if ex.faultCode != 409: raise for content_sha1 in [item[:-len(".json")] for item in os.listdir(filesystem_stream_pathname) if item.endswith(".json") and not item.endswith(".metadata.json") and item != "metadata.json"]: - filesystem_content_filename = os.path.join(filesystem_stream_pathname, content_sha1 + ".json") + filesystem_content_filename = os.path.join( + filesystem_stream_pathname, + content_sha1 + ".json") if not os.path.isfile(filesystem_content_filename): continue with open(os.path.join(filesystem_stream_pathname, content_sha1) + ".metadata.json", "rt") as stream: @@ -745,16 +780,18 @@ class restore(XMLRPCCommand): content = stream.read() print " * Restoring bundle %s" % content_sha1 try: - self.server.put(content, bundle_metadata["content_filename"], stream_pathname) + self.server.put( + content, + bundle_metadata["content_filename"], + stream_pathname) except xmlrpclib.Fault as ex: if ex.faultCode != 409: raise - -class pull(ExperimentalCommandMixIn, XMLRPCCommand): + +class pull(XMLRPCCommand): """ Copy bundles and bundle streams from one dashboard to another. - This command checks for two environment varialbes: The value of DASHBOARD_URL is used as a replacement for --dashbard-url. The value of REMOTE_DASHBOARD_URL as a replacement for FROM. @@ -785,7 +822,10 @@ class pull(ExperimentalCommandMixIn, XMLRPCCommand): group.add_argument( "FROM", help="URL of the remote validation dashboard)") - group.add_argument("STREAM", nargs="*", help="Streams to pull from (all by default)") + group.add_argument( + "STREAM", + nargs="*", + help="Streams to pull from (all by default)") @staticmethod def _filesizeformat(num_bytes): @@ -810,7 +850,7 @@ class pull(ExperimentalCommandMixIn, XMLRPCCommand): def invoke_remote(self): self._check_server_version(self.server, "0.3") - + print "Checking local and remote streams" remote = self.remote_server.streams() if self.args.STREAM: @@ -819,28 +859,37 @@ class pull(ExperimentalCommandMixIn, XMLRPCCommand): remote_set = frozenset((stream["pathname"] for stream in remote)) unavailable_set = requested_set - remote_set if unavailable_set: - print >> sys.stderr, "Remote stream not found: %s" % ", ".join(unavailable_set) + print >> sys.stderr, "Remote stream not found: %s" % ", ".join( + unavailable_set) return -1 # Limit to requested streams if necessary - remote = [stream for stream in remote if stream["pathname"] in requested_set] + remote = [ + stream for stream in remote if stream[ + "pathname"] in requested_set] local = self.server.streams() - missing_pathnames = set([stream["pathname"] for stream in remote]) - set([stream["pathname"] for stream in local]) + missing_pathnames = set([stream["pathname"] + for stream in remote]) - set([stream["pathname"] for stream in local]) for stream in remote: if stream["pathname"] in missing_pathnames: self.server.make_stream(stream["pathname"], stream["name"]) local_bundles = [] else: - local_bundles = [bundle for bundle in self.server.bundles(stream["pathname"])] - remote_bundles = [bundle for bundle in self.remote_server.bundles(stream["pathname"])] - missing_bundles = set((bundle["content_sha1"] for bundle in remote_bundles)) - missing_bundles -= set((bundle["content_sha1"] for bundle in local_bundles)) + local_bundles = [ + bundle for bundle in self.server.bundles(stream["pathname"])] + remote_bundles = [ + bundle for bundle in self.remote_server.bundles(stream["pathname"])] + missing_bundles = set( + (bundle["content_sha1"] for bundle in remote_bundles)) + missing_bundles -= set( + (bundle["content_sha1"] for bundle in local_bundles)) try: missing_bytes = sum( (bundle["content_size"] for bundle in remote_bundles if bundle["content_sha1"] in missing_bundles)) except KeyError as ex: - # Older servers did not return content_size so this part is optional + # Older servers did not return content_size so this part is + # optional missing_bytes = None if missing_bytes: print "Stream %s needs update (%s)" % (stream["pathname"], self._filesizeformat(missing_bytes)) @@ -855,17 +904,20 @@ class pull(ExperimentalCommandMixIn, XMLRPCCommand): print "got %s, storing" % (self._filesizeformat(len(data["content"]))), sys.stdout.flush() try: - self.server.put(data["content"], data["content_filename"], stream["pathname"]) + self.server.put( + data["content"], + data["content_filename"], + stream["pathname"]) except xmlrpclib.Fault as ex: if ex.faultCode == 409: # duplicate print "already present (in another stream)" else: raise - else: + else: print "done" -class data_views(ExperimentalCommandMixIn, XMLRPCCommand): +class data_views(XMLRPCCommand): """ Show data views defined on the server """ @@ -885,7 +937,7 @@ class data_views(ExperimentalCommandMixIn, XMLRPCCommand): print "Tip: to invoke a data view try `lc-tool query-data-view`" -class query_data_view(ExperimentalCommandMixIn, XMLRPCCommand): +class query_data_view(XMLRPCCommand): """ Invoke a specified data view """ @@ -893,7 +945,8 @@ class query_data_view(ExperimentalCommandMixIn, XMLRPCCommand): def register_arguments(cls, parser): super(query_data_view, cls).register_arguments(parser) parser.add_argument("QUERY", metavar="QUERY", nargs="...", - help="Data view name and any optional and required arguments") + help="Data view name and any optional \ + and required arguments") def _probe_data_views(self): """ @@ -914,7 +967,7 @@ class query_data_view(ExperimentalCommandMixIn, XMLRPCCommand): del parser._actions[-1] subparsers = parser.add_subparsers( title="Data views available on the server") - for data_view in self.data_views: + for data_view in self.data_views: data_view_parser = subparsers.add_parser( data_view["name"], help=data_view["summary"], @@ -924,14 +977,16 @@ class query_data_view(ExperimentalCommandMixIn, XMLRPCCommand): for argument in data_view["arguments"]: if argument["default"] is None: group.add_argument( - "--{name}".format(name=argument["name"].replace("_", "-")), + "--{name}".format( + name=argument["name"].replace("_", "-")), dest=argument["name"], help=argument["help"], type=str, required=True) else: group.add_argument( - "--{name}".format(name=argument["name"].replace("_", "-")), + "--{name}".format( + name=argument["name"].replace("_", "-")), dest=argument["name"], help=argument["help"], type=str, @@ -953,9 +1008,11 @@ class query_data_view(ExperimentalCommandMixIn, XMLRPCCommand): for argument in self.args.data_view["arguments"]: arg_name = argument["name"] if arg_name in self.args: - data_view_args[arg_name] = getattr(self.args, arg_name) + data_view_args[arg_name] = getattr(self.args, arg_name) # Invoke the data view - response = self.server.query_data_view(self.args.data_view["name"], data_view_args) + response = self.server.query_data_view( + self.args.data_view["name"], + data_view_args) # Create a pretty-printer renderer = _get_pretty_renderer( caption=self.args.data_view["summary"], @@ -971,9 +1028,11 @@ class query_data_view(ExperimentalCommandMixIn, XMLRPCCommand): class version(Command): + """ Show dashboard client version """ + def invoke(self): import versiontools from lava_dashboard_tool import __version__ |