# Copyright (C) 2010,2011 Linaro Limited # # Author: Zygmunt Krynicki # # This file is part of lava-dashboard-tool. # # lava-dashboard-tool is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License version 3 # as published by the Free Software Foundation # # lava-dashboard-tool is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License # along with lava-dashboard-tool. If not, see . """ Module with command-line tool commands that interact with the dashboard server. All commands listed here should have counterparts in the launch_control.dashboard_app.xml_rpc package. """ import argparse import contextlib import errno import os import re import socket import sys import urllib import urlparse import xmlrpclib import simplejson from json_schema_validator.extensions import datetime_extension from lava_tool.authtoken import AuthenticatingServerProxy, KeyringAuthBackend from lava.tool.command import Command, CommandGroup class dashboard(CommandGroup): """ Commands for interacting with LAVA Dashboard """ namespace = "lava.dashboard.commands" class InsufficientServerVersion(Exception): """ Exception raised when server version that a command interacts with is too old to support required features. """ def __init__(self, server_version, required_version): self.server_version = server_version self.required_version = required_version class DataSetRenderer(object): """ Support class for rendering a table out of list of dictionaries. It supports several features that make printing tabular data easier. * Automatic layout * Custom column headers * Custom cell formatting * Custom table captions * Custom column ordering * Custom Column separators * Custom dataset notification The primary method is render() which does all of the above. You need to pass a dataset argument which is a list of dictionaries. Each dictionary must have the same keys. In particular the first row is used to determine columns. """ def __init__(self, column_map=None, row_formatter=None, empty=None, order=None, caption=None, separator=" ", header_separator=None): if column_map is None: column_map = {} if row_formatter is None: row_formatter = {} if empty is None: empty = "There is no data to display" self.column_map = column_map self.row_formatter = row_formatter self.empty = empty self.order = order self.separator = separator self.caption = caption self.header_separator = header_separator def _analyze_dataset(self, dataset): """ Determine the columns that will be displayed and the maximum length of each of those columns. Returns a tuple (dataset, columms, maxlen) where columns is a list of column names and maxlen is a dictionary mapping from column name to maximum length of any value in the row or the column header and the dataset is a copy of the dataset altered as necessary. Some examples: First the dataset, an array of dictionaries >>> dataset = [ ... {'a': 'shorter', 'bee': ''}, ... {'a': 'little longer', 'bee': 'b'}] Note that column 'bee' is actually three characters long as the column name made it wider. >>> dataset_out, columns, maxlen = DataSetRenderer( ... )._analyze_dataset(dataset) Unless you format rows with a custom function the data is not altered. >>> dataset_out is dataset True Columns come out in sorted alphabetic order >>> columns ['a', 'bee'] Maximum length determines the width of each column. Note that the header affects the column width. >>> maxlen {'a': 13, 'bee': 3} You can constrain or reorder columns. In that case columns you decided to ignore are simply left out of the output. >>> dataset_out, columns, maxlen = DataSetRenderer( ... order=['bee'])._analyze_dataset(dataset) >>> columns ['bee'] >>> maxlen {'bee': 3} You can format values anyway you like: >>> dataset_out, columns, maxlen = DataSetRenderer(row_formatter={ ... 'bee': lambda value: "%10s" % value} ... )._analyze_dataset(dataset) Dataset is altered to take account of the row formatting function. The original dataset argument is copied. >>> dataset_out [{'a': 'shorter', 'bee': ' '}, {'a': 'little longer', 'bee': ' b'}] >>> dataset_out is not dataset True Columns stay the same though: >>> columns ['a', 'bee'] Note how formatting altered the width of the column 'bee' >>> maxlen {'a': 13, 'bee': 10} You can also format columns (with nice aliases).Note how column 'bee' maximum width is now dominated by the long column name: >>> dataset_out, columns, maxlen = DataSetRenderer(column_map={ ... 'bee': "Column B"})._analyze_dataset(dataset) >>> maxlen {'a': 13, 'bee': 8} """ if self.order: columns = self.order else: columns = sorted(dataset[0].keys()) if self.row_formatter: dataset_out = [dict(row) for row in dataset] else: dataset_out = dataset for row in dataset_out: for column in row: if column in self.row_formatter: row[column] = self.row_formatter[column](row[column]) maxlen = dict( [(column, max( len(self.column_map.get(column, column)), max([ len(str(row[column])) for row in dataset_out]))) for column in columns]) return dataset_out, columns, maxlen def _render_header(self, dataset, columns, maxlen): """ Render a header, possibly with a caption string Caption is controlled by the constructor. >>> dataset = [ ... {'a': 'shorter', 'bee': ''}, ... {'a': 'little longer', 'bee': 'b'}] >>> columns = ['a', 'bee'] >>> maxlen = {'a': 13, 'bee': 3} By default there is no caption, just column names: >>> DataSetRenderer()._render_header( ... dataset, columns, maxlen) a bee If you enable the header separator then column names will be visually separated from the first row of data. >>> DataSetRenderer(header_separator=True)._render_header( ... dataset, columns, maxlen) a bee ----------------- If you provide a caption it gets rendered as a centered underlined text before the data: >>> DataSetRenderer(caption="Dataset")._render_header( ... dataset, columns, maxlen) Dataset ================= a bee You can use both caption and header separator >>> DataSetRenderer(caption="Dataset", header_separator=True)._render_header( ... dataset, columns, maxlen) Dataset ================= a bee ----------------- Observe how the total length of the output horizontal line depends on the separator! Also note the columns labels are aligned to the center of the column >>> DataSetRenderer(caption="Dataset", separator=" | ")._render_header( ... dataset, columns, maxlen) Dataset =================== a | bee """ total_len = sum(maxlen.itervalues()) if len(columns): total_len += len(self.separator) * (len(columns) - 1) # Print the caption if self.caption: print "{0:^{1}}".format(self.caption, total_len) print "=" * total_len # Now print the coulum names print self.separator.join([ "{0:^{1}}".format(self.column_map.get(column, column), maxlen[column]) for column in columns]) # Finally print the header separator if self.header_separator: print "-" * total_len def _render_rows(self, dataset, columns, maxlen): """ Render rows of the dataset. Each row is printed on one line using the maxlen argument to determine correct column size. Text is aligned left. First the dataset, columns and maxlen as produced by _analyze_dataset() >>> dataset = [ ... {'a': 'shorter', 'bee': ''}, ... {'a': 'little longer', 'bee': 'b'}] >>> columns = ['a', 'bee'] >>> maxlen = {'a': 13, 'bee': 3} Now a plain table. Note! To really understand this test you should check out the whitespace in the strings below. There are two more spaces after 'b' in the second row >>> DataSetRenderer()._render_rows(dataset, columns, maxlen) shorter little longer b """ for row in dataset: print self.separator.join([ "{0!s:{1}}".format(row[column], maxlen[column]) for column in columns]) def _render_dataset(self, dataset): """ Render the header followed by the rows of data. """ dataset, columns, maxlen = self._analyze_dataset(dataset) self._render_header(dataset, columns, maxlen) self._render_rows(dataset, columns, maxlen) def _render_empty_dataset(self): """ Render empty dataset. By default it just prints out a fixed sentence: >>> DataSetRenderer()._render_empty_dataset() There is no data to display This can be changed by passing an argument to the constructor >>> DataSetRenderer(empty="there is no data")._render_empty_dataset() there is no data """ print self.empty def render(self, dataset): if len(dataset) > 0: self._render_dataset(dataset) else: self._render_empty_dataset() class XMLRPCCommand(Command): """ Abstract base class for commands that interact with dashboard server over XML-RPC. The only difference is that you should implement invoke_remote() instead of invoke(). The provided implementation catches several socket and XML-RPC errors and prints a pretty error message. """ @staticmethod def _construct_xml_rpc_url(url): """ Construct URL to the XML-RPC service out of the given URL """ parts = urlparse.urlsplit(url) if not parts.path.endswith("/RPC2/"): path = parts.path.rstrip("/") + "/xml-rpc/" else: path = parts.path return urlparse.urlunsplit( (parts.scheme, parts.netloc, path, "", "")) @staticmethod def _strict_server_version(version): """ Calculate strict server version (as defined by distutils.version.StrictVersion). This works by discarding .candidate and .dev release-levels. >>> XMLRPCCommand._strict_server_version("0.4.0.candidate.5") '0.4.0' >>> XMLRPCCommand._strict_server_version("0.4.0.dev.126") '0.4.0' >>> XMLRPCCommand._strict_server_version("0.4.0.alpha.1") '0.4.0a1' >>> XMLRPCCommand._strict_server_version("0.4.0.beta.2") '0.4.0b2' """ try: major, minor, micro, releaselevel, serial = version.split(".") except ValueError: raise ValueError( ("version %r does not follow pattern " "'major.minor.micro.releaselevel.serial'") % version) if releaselevel in ["dev", "candidate", "final"]: return "%s.%s.%s" % (major, minor, micro) elif releaselevel == "alpha": return "%s.%s.%sa%s" % (major, minor, micro, serial) elif releaselevel == "beta": return "%s.%s.%sb%s" % (major, minor, micro, serial) else: raise ValueError( ("releaselevel %r is not one of 'final', 'alpha', 'beta', " "'candidate' or 'final'") % releaselevel) def _check_server_version(self, server_obj, required_version): """ Check that server object has is at least required_version. This method may raise InsufficientServerVersion. """ from distutils.version import StrictVersion, LooseVersion # For backwards compatibility the server reports # major.minor.micro.releaselevel.serial which is not PEP-386 compliant server_version = StrictVersion( self._strict_server_version(server_obj.version())) required_version = StrictVersion(required_version) if server_version < required_version: raise InsufficientServerVersion(server_version, required_version) def __init__(self, parser, args): super(XMLRPCCommand, self).__init__(parser, args) xml_rpc_url = self._construct_xml_rpc_url(self.args.dashboard_url) self.server = AuthenticatingServerProxy( xml_rpc_url, verbose=args.verbose_xml_rpc, allow_none=True, use_datetime=True, auth_backend=KeyringAuthBackend()) def use_non_legacy_api_if_possible(self, name='server'): # Legacy APIs are registered in top-level object, non-legacy APIs are # prefixed with extension name. if "dashboard.version" in getattr(self, name).system.listMethods(): setattr(self, name, getattr(self, name).dashboard) @classmethod def register_arguments(cls, parser): dashboard_group = parser.add_argument_group( "dashboard specific arguments") default_dashboard_url = os.getenv("DASHBOARD_URL") if default_dashboard_url: dashboard_group.add_argument("--dashboard-url", metavar="URL", help="URL of your validation dashboard (currently %(default)s)", default=default_dashboard_url) else: dashboard_group.add_argument("--dashboard-url", required=True, metavar="URL", help="URL of your validation dashboard") debug_group = parser.add_argument_group("debugging arguments") debug_group.add_argument("--verbose-xml-rpc", action="store_true", default=False, help="Show XML-RPC data") return dashboard_group @contextlib.contextmanager def safety_net(self): try: yield except socket.error as ex: print >> sys.stderr, "Unable to connect to server at %s" % ( self.args.dashboard_url,) # It seems that some errors are reported as -errno # while others as +errno. ex.errno = abs(ex.errno) if ex.errno == errno.ECONNREFUSED: print >> sys.stderr, "Connection was refused." parts = urlparse.urlsplit(self.args.dashboard_url) if parts.netloc == "localhost:8000": print >> sys.stderr, "Perhaps the server is not running?" elif ex.errno == errno.ENOENT: print >> sys.stderr, "Unable to resolve address" else: print >> sys.stderr, "Socket %d: %s" % (ex.errno, ex.strerror) except xmlrpclib.ProtocolError as ex: print >> sys.stderr, "Unable to exchange XML-RPC message with dashboard server" print >> sys.stderr, "HTTP error code: %d/%s" % ( ex.errcode, ex.errmsg) except xmlrpclib.Fault as ex: self.handle_xmlrpc_fault(ex.faultCode, ex.faultString) except InsufficientServerVersion as ex: print >> sys.stderr, ("This command requires at least server version " "%s, actual server version is %s" % (ex.required_version, ex.server_version)) def invoke(self): with self.safety_net(): self.use_non_legacy_api_if_possible() return self.invoke_remote() def handle_xmlrpc_fault(self, faultCode, faultString): if faultCode == 500: print >> sys.stderr, "Dashboard server has experienced internal error" print >> sys.stderr, faultString else: print >> sys.stderr, "XML-RPC error %d: %s" % ( faultCode, faultString) def invoke_remote(self): raise NotImplementedError() class server_version(XMLRPCCommand): """ Display dashboard server version """ def invoke_remote(self): print "Dashboard server version: %s" % (self.server.version(),) class put(XMLRPCCommand): """ Upload a bundle on the server """ @classmethod def register_arguments(cls, parser): super(put, cls).register_arguments(parser) parser.add_argument("LOCAL", type=argparse.FileType("rb"), help="pathname on the local file system") parser.add_argument("REMOTE", default="/anonymous/", nargs='?', help="pathname on the server") def invoke_remote(self): content = self.args.LOCAL.read() filename = self.args.LOCAL.name pathname = self.args.REMOTE content_sha1 = self.server.put(content, filename, pathname) print "Stored as bundle {0}".format(content_sha1) def handle_xmlrpc_fault(self, faultCode, faultString): if faultCode == 404: print >> sys.stderr, "Bundle stream %s does not exist" % ( self.args.REMOTE) elif faultCode == 409: print >> sys.stderr, "You have already uploaded this bundle to the dashboard" else: super(put, self).handle_xmlrpc_fault(faultCode, faultString) class get(XMLRPCCommand): """ Download a bundle from the server """ @classmethod def register_arguments(cls, parser): super(get, cls).register_arguments(parser) parser.add_argument("SHA1", type=str, help="SHA1 of the bundle to download") parser.add_argument("--overwrite", action="store_true", help="Overwrite files on the local disk") parser.add_argument("--output", "-o", type=argparse.FileType("wb"), default=None, help="Alternate name of the output file") def invoke_remote(self): response = self.server.get(self.args.SHA1) if self.args.output is None: filename = self.args.SHA1 if os.path.exists(filename) and not self.args.overwrite: print >> sys.stderr, "File {filename!r} already exists".format( filename=filename) print >> sys.stderr, "You may pass --overwrite to write over it" return -1 stream = open(filename, "wb") else: stream = self.args.output filename = self.args.output.name stream.write(response['content']) print "Downloaded bundle {0} to file {1!r}".format( self.args.SHA1, filename) def handle_xmlrpc_fault(self, faultCode, faultString): if faultCode == 404: print >> sys.stderr, "Bundle {sha1} does not exist".format( sha1=self.args.SHA1) else: super(get, self).handle_xmlrpc_fault(faultCode, faultString) class deserialize(XMLRPCCommand): """ Deserialize a bundle on the server """ @classmethod def register_arguments(cls, parser): super(deserialize, cls).register_arguments(parser) parser.add_argument("SHA1", type=str, help="SHA1 of the bundle to deserialize") def invoke_remote(self): response = self.server.deserialize(self.args.SHA1) print "Bundle {sha1} deserialized".format( sha1=self.args.SHA1) def handle_xmlrpc_fault(self, faultCode, faultString): if faultCode == 404: print >> sys.stderr, "Bundle {sha1} does not exist".format( sha1=self.args.SHA1) elif faultCode == 409: print >> sys.stderr, "Unable to deserialize bundle {sha1}".format( sha1=self.args.SHA1) print >> sys.stderr, faultString else: super( deserialize, self).handle_xmlrpc_fault( faultCode, faultString) def _get_pretty_renderer(**kwargs): if "separator" not in kwargs: kwargs["separator"] = " | " if "header_separator" not in kwargs: kwargs["header_separator"] = True return DataSetRenderer(**kwargs) class streams(XMLRPCCommand): """ Show streams you have access to """ renderer = _get_pretty_renderer( order=('pathname', 'bundle_count', 'name'), column_map={ 'pathname': 'Pathname', 'bundle_count': 'Number of bundles', 'name': 'Name'}, row_formatter={ 'name': lambda name: name or "(not set)"}, empty="There are no streams you can access on the server", caption="Bundle streams") def invoke_remote(self): self.renderer.render(self.server.streams()) class bundles(XMLRPCCommand): """ Show bundles in the specified stream """ renderer = _get_pretty_renderer( column_map={ 'uploaded_by': 'Uploader', 'uploaded_on': 'Upload date', 'content_filename': 'File name', 'content_sha1': 'SHA1', 'is_deserialized': "Deserialized?"}, row_formatter={ 'is_deserialized': lambda x: "yes" if x else "no", 'uploaded_by': lambda x: x or "(anonymous)", 'uploaded_on': lambda x: x}, order=('content_sha1', 'content_filename', 'uploaded_by', 'uploaded_on', 'is_deserialized'), empty="There are no bundles in this stream", caption="Bundles", separator=" | ") @classmethod def register_arguments(cls, parser): super(bundles, cls).register_arguments(parser) parser.add_argument("PATHNAME", default="/anonymous/", nargs='?', help="pathname on the server (defaults to %(default)s)") def invoke_remote(self): self.renderer.render(self.server.bundles(self.args.PATHNAME)) def handle_xmlrpc_fault(self, faultCode, faultString): if faultCode == 404: print >> sys.stderr, "Bundle stream %s does not exist" % ( self.args.PATHNAME) else: super(bundles, self).handle_xmlrpc_fault(faultCode, faultString) class make_stream(XMLRPCCommand): """ Create a bundle stream on the server """ @classmethod def register_arguments(cls, parser): super(make_stream, cls).register_arguments(parser) parser.add_argument( "pathname", type=str, help="Pathname of the bundle stream to create") parser.add_argument( "--name", type=str, default="", help="Name of the bundle stream (description)") def invoke_remote(self): self._check_server_version(self.server, "0.3") pathname = self.server.make_stream(self.args.pathname, self.args.name) print "Bundle stream {pathname} created".format(pathname=pathname) class backup(XMLRPCCommand): """ Backup data uploaded to a dashboard instance. Not all data is preserved. The following data is lost: identity of the user that uploaded each bundle, time of uploading and deserialization on the server, name of the bundle stream that contained the data """ @classmethod def register_arguments(cls, parser): super(backup, cls).register_arguments(parser) parser.add_argument("BACKUP_DIR", type=str, help="Directory to backup to") def invoke_remote(self): if not os.path.exists(self.args.BACKUP_DIR): os.mkdir(self.args.BACKUP_DIR) for bundle_stream in self.server.streams(): print "Processing stream %s" % bundle_stream["pathname"] bundle_stream_dir = os.path.join( self.args.BACKUP_DIR, urllib.quote_plus(bundle_stream["pathname"])) if not os.path.exists(bundle_stream_dir): os.mkdir(bundle_stream_dir) with open(os.path.join(bundle_stream_dir, "metadata.json"), "wt") as stream: simplejson.dump({ "pathname": bundle_stream["pathname"], "name": bundle_stream["name"], "user": bundle_stream["user"], "group": bundle_stream["group"], }, stream) for bundle in self.server.bundles(bundle_stream["pathname"]): print " * Backing up bundle %s" % bundle["content_sha1"] data = self.server.get(bundle["content_sha1"]) bundle_pathname = os.path.join( bundle_stream_dir, bundle["content_sha1"]) # Note: we write bundles as binary data to preserve anything # the user might have dumped on us with open(bundle_pathname + ".json", "wb") as stream: stream.write(data["content"]) with open(bundle_pathname + ".metadata.json", "wt") as stream: simplejson.dump({ "uploaded_by": bundle["uploaded_by"], "uploaded_on": datetime_extension.to_json(bundle["uploaded_on"]), "content_filename": bundle["content_filename"], "content_sha1": bundle["content_sha1"], "content_size": bundle["content_size"], }, stream) class restore(XMLRPCCommand): """ Restore a dashboard instance from backup """ @classmethod def register_arguments(cls, parser): super(restore, cls).register_arguments(parser) parser.add_argument("BACKUP_DIR", type=str, help="Directory to backup from") def invoke_remote(self): self._check_server_version(self.server, "0.3") for stream_pathname_quoted in os.listdir(self.args.BACKUP_DIR): filesystem_stream_pathname = os.path.join( self.args.BACKUP_DIR, stream_pathname_quoted) if not os.path.isdir(filesystem_stream_pathname): continue stream_pathname = urllib.unquote(stream_pathname_quoted) if os.path.exists(os.path.join(filesystem_stream_pathname, "metadata.json")): with open(os.path.join(filesystem_stream_pathname, "metadata.json"), "rt") as stream: stream_metadata = simplejson.load(stream) else: stream_metadata = {} print "Processing stream %s" % stream_pathname try: self.server.make_stream( stream_pathname, stream_metadata.get( "name", "Restored from backup")) except xmlrpclib.Fault as ex: if ex.faultCode != 409: raise for content_sha1 in [item[:-len(".json")] for item in os.listdir(filesystem_stream_pathname) if item.endswith(".json") and not item.endswith(".metadata.json") and item != "metadata.json"]: filesystem_content_filename = os.path.join( filesystem_stream_pathname, content_sha1 + ".json") if not os.path.isfile(filesystem_content_filename): continue with open(os.path.join(filesystem_stream_pathname, content_sha1) + ".metadata.json", "rt") as stream: bundle_metadata = simplejson.load(stream) with open(filesystem_content_filename, "rb") as stream: content = stream.read() print " * Restoring bundle %s" % content_sha1 try: self.server.put( content, bundle_metadata["content_filename"], stream_pathname) except xmlrpclib.Fault as ex: if ex.faultCode != 409: raise class pull(XMLRPCCommand): """ Copy bundles and bundle streams from one dashboard to another. This command checks for two environment varialbes: The value of DASHBOARD_URL is used as a replacement for --dashbard-url. The value of REMOTE_DASHBOARD_URL as a replacement for FROM. Their presence automatically makes the corresponding argument optional. """ def __init__(self, parser, args): super(pull, self).__init__(parser, args) remote_xml_rpc_url = self._construct_xml_rpc_url(self.args.FROM) self.remote_server = AuthenticatingServerProxy( remote_xml_rpc_url, verbose=args.verbose_xml_rpc, use_datetime=True, allow_none=True, auth_backend=KeyringAuthBackend()) self.use_non_legacy_api_if_possible('remote_server') @classmethod def register_arguments(cls, parser): group = super(pull, cls).register_arguments(parser) default_remote_dashboard_url = os.getenv("REMOTE_DASHBOARD_URL") if default_remote_dashboard_url: group.add_argument( "FROM", nargs="?", help="URL of the remote validation dashboard (currently %(default)s)", default=default_remote_dashboard_url) else: group.add_argument( "FROM", help="URL of the remote validation dashboard)") group.add_argument( "STREAM", nargs="*", help="Streams to pull from (all by default)") @staticmethod def _filesizeformat(num_bytes): """ Formats the value like a 'human-readable' file size (i.e. 13 KB, 4.1 MB, 102 num_bytes, etc). """ try: num_bytes = float(num_bytes) except (TypeError, ValueError, UnicodeDecodeError): return "%(size)d byte", "%(size)d num_bytes" % {'size': 0} filesize_number_format = lambda value: "%0.2f" % (round(value, 1),) if num_bytes < 1024: return "%(size)d bytes" % {'size': num_bytes} if num_bytes < 1024 * 1024: return "%s KB" % filesize_number_format(num_bytes / 1024) if num_bytes < 1024 * 1024 * 1024: return "%s MB" % filesize_number_format(num_bytes / (1024 * 1024)) return "%s GB" % filesize_number_format(num_bytes / (1024 * 1024 * 1024)) def invoke_remote(self): self._check_server_version(self.server, "0.3") print "Checking local and remote streams" remote = self.remote_server.streams() if self.args.STREAM: # Check that all requested streams are available remotely requested_set = frozenset(self.args.STREAM) remote_set = frozenset((stream["pathname"] for stream in remote)) unavailable_set = requested_set - remote_set if unavailable_set: print >> sys.stderr, "Remote stream not found: %s" % ", ".join( unavailable_set) return -1 # Limit to requested streams if necessary remote = [ stream for stream in remote if stream[ "pathname"] in requested_set] local = self.server.streams() missing_pathnames = set([stream["pathname"] for stream in remote]) - set([stream["pathname"] for stream in local]) for stream in remote: if stream["pathname"] in missing_pathnames: self.server.make_stream(stream["pathname"], stream["name"]) local_bundles = [] else: local_bundles = [ bundle for bundle in self.server.bundles(stream["pathname"])] remote_bundles = [ bundle for bundle in self.remote_server.bundles(stream["pathname"])] missing_bundles = set( (bundle["content_sha1"] for bundle in remote_bundles)) missing_bundles -= set( (bundle["content_sha1"] for bundle in local_bundles)) try: missing_bytes = sum( (bundle["content_size"] for bundle in remote_bundles if bundle["content_sha1"] in missing_bundles)) except KeyError as ex: # Older servers did not return content_size so this part is # optional missing_bytes = None if missing_bytes: print "Stream %s needs update (%s)" % (stream["pathname"], self._filesizeformat(missing_bytes)) elif missing_bundles: print "Stream %s needs update (no estimate available)" % (stream["pathname"],) else: print "Stream %s is up to date" % (stream["pathname"],) for content_sha1 in missing_bundles: print "Getting %s" % (content_sha1,), sys.stdout.flush() data = self.remote_server.get(content_sha1) print "got %s, storing" % (self._filesizeformat(len(data["content"]))), sys.stdout.flush() try: self.server.put( data["content"], data["content_filename"], stream["pathname"]) except xmlrpclib.Fault as ex: if ex.faultCode == 409: # duplicate print "already present (in another stream)" else: raise else: print "done" class data_views(XMLRPCCommand): """ Show data views defined on the server """ renderer = _get_pretty_renderer( column_map={ 'name': 'Name', 'summary': 'Summary', }, order=('name', 'summary'), empty="There are no data views defined yet", caption="Data Views") def invoke_remote(self): self._check_server_version(self.server, "0.4") self.renderer.render(self.server.data_views()) print print "Tip: to invoke a data view try `lc-tool query-data-view`" class query_data_view(XMLRPCCommand): """ Invoke a specified data view """ @classmethod def register_arguments(cls, parser): super(query_data_view, cls).register_arguments(parser) parser.add_argument("QUERY", metavar="QUERY", nargs="...", help="Data view name and any optional \ and required arguments") def _probe_data_views(self): """ Probe the server for information about data views """ with self.safety_net(): self.use_non_legacy_api_if_possible() self._check_server_version(self.server, "0.4") return self.server.data_views() def reparse_arguments(self, parser, raw_args): self.data_views = self._probe_data_views() if self.data_views is None: return # Here we hack a little, the last actuin is the QUERY action added # in register_arguments above. By removing it we make the output # of lc-tool query-data-view NAME --help more consistent. del parser._actions[-1] subparsers = parser.add_subparsers( title="Data views available on the server") for data_view in self.data_views: data_view_parser = subparsers.add_parser( data_view["name"], help=data_view["summary"], epilog=data_view["documentation"]) data_view_parser.set_defaults(data_view=data_view) group = data_view_parser.add_argument_group("Data view parameters") for argument in data_view["arguments"]: if argument["default"] is None: group.add_argument( "--{name}".format( name=argument["name"].replace("_", "-")), dest=argument["name"], help=argument["help"], type=str, required=True) else: group.add_argument( "--{name}".format( name=argument["name"].replace("_", "-")), dest=argument["name"], help=argument["help"], type=str, default=argument["default"]) self.args = self.parser.parse_args(raw_args) def invoke(self): # Override and _not_ call 'use_non_legacy_api_if_possible' as we # already did this reparse_arguments with self.safety_net(): return self.invoke_remote() def invoke_remote(self): if self.data_views is None: return -1 self._check_server_version(self.server, "0.4") # Build a collection of arguments for data view data_view_args = {} for argument in self.args.data_view["arguments"]: arg_name = argument["name"] if arg_name in self.args: data_view_args[arg_name] = getattr(self.args, arg_name) # Invoke the data view response = self.server.query_data_view( self.args.data_view["name"], data_view_args) # Create a pretty-printer renderer = _get_pretty_renderer( caption=self.args.data_view["summary"], order=[item["name"] for item in response["columns"]]) # Post-process the data so that it fits the printer data_for_renderer = [ dict(zip( [column["name"] for column in response["columns"]], row)) for row in response["rows"]] # Print the data renderer.render(data_for_renderer) class version(Command): """ Show dashboard client version """ def invoke(self): import versiontools from lava_dashboard_tool import __version__ print "Dashboard client version: {version}".format( version=versiontools.format_version(__version__))