diff options
| -rw-r--r-- | setup.py | 2 | ||||
| -rw-r--r-- | wlauto/commands/run.py | 6 | ||||
| -rw-r--r-- | wlauto/core/command.py | 2 | ||||
| -rw-r--r-- | wlauto/core/configuration/__init__.py | 1 | ||||
| -rw-r--r-- | wlauto/core/configuration/configuration.py | 104 | ||||
| -rw-r--r-- | wlauto/core/configuration/parsers.py | 14 | ||||
| -rw-r--r-- | wlauto/core/entry_point.py | 10 | ||||
| -rw-r--r-- | wlauto/core/host.py | 33 | ||||
| -rw-r--r-- | wlauto/core/plugin.py | 8 | ||||
| -rw-r--r-- | wlauto/core/pluginloader.py | 3 | ||||
| -rw-r--r-- | wlauto/exceptions.py | 4 | ||||
| -rw-r--r-- | wlauto/tests/test_parsers.py | 45 | ||||
| -rw-r--r-- | wlauto/utils/misc.py | 2 | ||||
| -rw-r--r-- | wlauto/utils/serializer.py | 9 | ||||
| -rw-r--r-- | wlauto/utils/types.py | 1 |
15 files changed, 105 insertions, 139 deletions
@@ -78,7 +78,7 @@ params = dict( 'pyYAML', # YAML-formatted agenda parsing 'requests', # Fetch assets over HTTP 'devlib', # Interacting with devices - 'louie' # Handles signal callbacks + 'louie' # callbacks dispatch ], extras_require={ 'other': ['jinja2', 'pandas>=0.13.1'], diff --git a/wlauto/commands/run.py b/wlauto/commands/run.py index afea7a10..197a2fe9 100644 --- a/wlauto/commands/run.py +++ b/wlauto/commands/run.py @@ -22,9 +22,9 @@ import wlauto from wlauto import Command, settings from wlauto.core.execution import Executor from wlauto.utils.log import add_log_file -from wlauto.core.configuration import RunConfiguration, WAConfiguration +from wlauto.core.configuration import RunConfiguration from wlauto.core import pluginloader -from wlauto.core.configuration_parsers import Agenda, ConfigFile, EnvrironmentVars, CommandLineArgs +from wlauto.core.configuration.parsers import AgendaParser, ConfigParser, CommandLineArgsParser class RunCommand(Command): @@ -74,7 +74,7 @@ class RunCommand(Command): # STAGE 1: Gather configuratation - env = EnvrironmentVars() + env = EnvironmentVars() args = CommandLineArgs(args) # STAGE 2.1a: Early WAConfiguration, required to find config files diff --git a/wlauto/core/command.py b/wlauto/core/command.py index bad6b615..47f00c2e 100644 --- a/wlauto/core/command.py +++ b/wlauto/core/command.py @@ -21,7 +21,7 @@ from wlauto.core.version import get_wa_version def init_argument_parser(parser): - parser.add_argument('-c', '--config', help='specify an additional config.py', action='append') + parser.add_argument('-c', '--config', help='specify an additional config.py', action='append', default=[]) parser.add_argument('-v', '--verbose', action='count', help='The scripts will produce verbose output.') parser.add_argument('--version', action='version', version='%(prog)s {}'.format(get_wa_version())) diff --git a/wlauto/core/configuration/__init__.py b/wlauto/core/configuration/__init__.py index 87c7c08c..a3593794 100644 --- a/wlauto/core/configuration/__init__.py +++ b/wlauto/core/configuration/__init__.py @@ -13,7 +13,6 @@ # limitations under the License. # from wlauto.core.configuration.configuration import (settings, - WAConfiguration, RunConfiguration, JobGenerator, ConfigurationPoint) diff --git a/wlauto/core/configuration/configuration.py b/wlauto/core/configuration/configuration.py index d42deef6..837a8191 100644 --- a/wlauto/core/configuration/configuration.py +++ b/wlauto/core/configuration/configuration.py @@ -492,13 +492,26 @@ class CpuFreqParameters(object): class Configuration(object): config_points = [] - name = "" + name = '' + # The below line must be added to all subclasses configuration = {cp.name: cp for cp in config_points} + @classmethod + # pylint: disable=unused-argument + def from_pod(cls, pod, plugin_cache): + instance = cls() + for name, cfg_point in cls.configuration.iteritems(): + if name in pod: + cfg_point.set_value(instance, pod.pop(name)) + if pod: + msg = 'Invalid entry(ies) for "{}": "{}"' + raise ConfigError(msg.format(cls.name, '", "'.join(pod.keys()))) + instance.validate() + return instance + def __init__(self): - # Load default values for configuration points - for confpoint in self.configuration.itervalues(): + for confpoint in self.config_points: confpoint.set_value(self, check_mandatory=False) def set(self, name, value, check_mandatory=True): @@ -511,7 +524,7 @@ class Configuration(object): self.set(k, v, check_mandatory=check_mandatory) def validate(self): - for cfg_point in self.configuration.itervalues(): + for cfg_point in self.config_points: cfg_point.validate(self) def to_pod(self): @@ -522,24 +535,21 @@ class Configuration(object): pod[cfg_point_name] = value return pod - @classmethod - # pylint: disable=unused-argument - def from_pod(cls, pod, plugin_cache): - instance = cls() - for name, cfg_point in cls.configuration.iteritems(): - if name in pod: - cfg_point.set_value(instance, pod.pop(name)) - if pod: - msg = 'Invalid entry(ies) for "{}": "{}"' - raise ConfigError(msg.format(cls.name, '", "'.join(pod.keys()))) - instance.validate() - return instance - # This configuration for the core WA framework class WAConfiguration(Configuration): name = "WA Configuration" + + plugin_packages = [ + 'wlauto.commands', + 'wlauto.workloads', + 'wlauto.instrumentation', + 'wlauto.result_processors', + 'wlauto.managers', + 'wlauto.resource_getters', + ] + config_points = [ ConfigurationPoint( 'user_directory', @@ -551,48 +561,6 @@ class WAConfiguration(Configuration): default=os.path.join(os.path.expanduser('~'), '.workload_automation'), ), ConfigurationPoint( - 'plugin_packages', - kind=list_of_strings, - default=[ - 'wlauto.commands', - 'wlauto.workloads', - 'wlauto.instrumentation', - 'wlauto.result_processors', - 'wlauto.managers', - 'wlauto.resource_getters', - ], - description=""" - List of packages that will be scanned for WA plugins. - """, - ), - ConfigurationPoint( - 'plugin_paths', - kind=list_of_strings, - default=[ - 'workloads', - 'instruments', - 'targets', - 'processors', - - # Legacy - 'managers', - 'result_processors', - ], - description=""" - List of paths that will be scanned for WA plugins. - """, - merge=True - ), - ConfigurationPoint( - 'plugin_ignore_paths', - kind=list_of_strings, - default=[], - description=""" - List of (sub)paths that will be ignored when scanning - ``plugin_paths`` for WA plugins. - """, - ), - ConfigurationPoint( 'assets_repository', description=""" The local mount point for the filer hosting WA assets. @@ -623,7 +591,7 @@ class WAConfiguration(Configuration): Verbosity of console output. """, ), - ConfigurationPoint( # TODO: Needs some format for dates ect/ comes from cfg + ConfigurationPoint( # TODO: Needs some format for dates etc/ comes from cfg 'default_output_directory', default="wa_output", description=""" @@ -636,7 +604,19 @@ class WAConfiguration(Configuration): @property def dependencies_directory(self): - return "{}/dependencies/".format(self.user_directory) + return os.path.join(self.user_directory, 'dependencies') + + @property + def plugins_directory(self): + return os.path.join(self.user_directory, 'plugins') + + + def __init__(self, environ): + super(WAConfiguration, self).__init__() + user_directory = environ.pop('WA_USER_DIRECTORY', '') + if user_directory: + self.set('user_directory', user_directory) + # This is generic top-level configuration for WA runs. @@ -1029,4 +1009,4 @@ class JobGenerator(object): yield job_spec -settings = WAConfiguration() +settings = WAConfiguration(os.environ) diff --git a/wlauto/core/configuration/parsers.py b/wlauto/core/configuration/parsers.py index aa1dccda..6c5af0d9 100644 --- a/wlauto/core/configuration/parsers.py +++ b/wlauto/core/configuration/parsers.py @@ -283,23 +283,11 @@ class AgendaParser(object): raise ConfigError('Error in "{}":\n\t{}'.format(source, str(e))) -class EnvironmentVarsParser(object): - def __init__(self, wa_config, environ): - user_directory = environ.pop('WA_USER_DIRECTORY', '') - if user_directory: - wa_config.set('user_directory', user_directory) - plugin_paths = environ.pop('WA_PLUGIN_PATHS', '') - if plugin_paths: - wa_config.set('plugin_paths', plugin_paths.split(os.pathsep)) - ext_paths = environ.pop('WA_EXTENSION_PATHS', '') - if ext_paths: - wa_config.set('plugin_paths', ext_paths.split(os.pathsep)) - - # Command line options are parsed in the "run" command. This is used to send # certain arguments to the correct configuration points and keep a record of # how WA was invoked class CommandLineArgsParser(object): + def __init__(self, cmd_args, wa_config, jobs_config): wa_config.set("verbosity", cmd_args.verbosity) # TODO: Is this correct? Does there need to be a third output dir param diff --git a/wlauto/core/entry_point.py b/wlauto/core/entry_point.py index 09cbfece..8855a55d 100644 --- a/wlauto/core/entry_point.py +++ b/wlauto/core/entry_point.py @@ -24,6 +24,7 @@ import warnings from wlauto.core.configuration import settings from wlauto.core import pluginloader from wlauto.core.command import init_argument_parser +from wlauto.core.host import init_user_directory from wlauto.exceptions import WAError, ConfigError from wlauto.utils.misc import get_traceback from wlauto.utils.log import init_logging @@ -45,7 +46,11 @@ def load_commands(subparsers): def main(): + if not os.path.exists(settings.user_directory): + init_user_directory() + try: + description = ("Execute automated workloads on a remote device and process " "the resulting output.\n\nUse \"wa <subcommand> -h\" to see " "help for individual subcommands.") @@ -57,10 +62,7 @@ def main(): commands = load_commands(parser.add_subparsers(dest='command')) # each command will add its own subparser args = parser.parse_args() - #TODO: Set this stuff properly, i.e dont use settings (if possible) - #settings.set("verbosity", args.verbose) - #settings.load_user_config() - #settings.debug = args.debug + settings.set("verbosity", args.verbose) for config in args.config: if not os.path.exists(config): diff --git a/wlauto/core/host.py b/wlauto/core/host.py new file mode 100644 index 00000000..33810b93 --- /dev/null +++ b/wlauto/core/host.py @@ -0,0 +1,33 @@ +import os + +from wlauto.core.configuration import settings + +def init_user_directory(overwrite_existing=False): # pylint: disable=R0914 + """ + Initialise a fresh user directory. + """ + if os.path.exists(settings.user_directory): + if not overwrite_existing: + raise RuntimeError('Environment {} already exists.'.format(settings.user_directory)) + shutil.rmtree(settings.user_directory) + + os.makedirs(settings.user_directory) + os.makedirs(settings.dependencies_directory) + os.makedirs(settings.plugins_directory) + + # TODO: generate default config.yaml here + + if os.getenv('USER') == 'root': + # If running with sudo on POSIX, change the ownership to the real user. + real_user = os.getenv('SUDO_USER') + if real_user: + import pwd # done here as module won't import on win32 + user_entry = pwd.getpwnam(real_user) + uid, gid = user_entry.pw_uid, user_entry.pw_gid + os.chown(settings.user_directory, uid, gid) + # why, oh why isn't there a recusive=True option for os.chown? + for root, dirs, files in os.walk(settings.user_directory): + for d in dirs: + os.chown(os.path.join(root, d), uid, gid) + for f in files: + os.chown(os.path.join(root, f), uid, gid) diff --git a/wlauto/core/plugin.py b/wlauto/core/plugin.py index 2d737560..f614169b 100644 --- a/wlauto/core/plugin.py +++ b/wlauto/core/plugin.py @@ -25,13 +25,14 @@ from collections import OrderedDict, defaultdict from itertools import chain from copy import copy -from wlauto.exceptions import NotFoundError, LoaderError, ValidationError, ConfigError +from wlauto.exceptions import NotFoundError, LoaderError, ValidationError, ConfigError, HostError from wlauto.utils.misc import (ensure_directory_exists as _d, walk_modules, load_class, merge_dicts_simple, get_article) from wlauto.core.configuration import settings from wlauto.utils.types import identifier, boolean from wlauto.core.configuration.configuration import ConfigurationPoint as Parameter + MODNAME_TRANS = string.maketrans(':/\\.', '____') @@ -697,10 +698,9 @@ class PluginLoader(object): for package in packages: for module in walk_modules(package): self._discover_in_module(module) - except ImportError as e: - source = getattr(e, 'path', package) + except HostError as e: message = 'Problem loading plugins from {}: {}' - raise LoaderError(message.format(source, e.message)) + raise LoaderError(message.format(e.module, str(e.orig_exc))) def _discover_from_paths(self, paths, ignore_paths): paths = paths or [] diff --git a/wlauto/core/pluginloader.py b/wlauto/core/pluginloader.py index 0aa8dd3f..dde6b828 100644 --- a/wlauto/core/pluginloader.py +++ b/wlauto/core/pluginloader.py @@ -38,8 +38,7 @@ class __LoaderWrapper(object): from wlauto.core.plugin import PluginLoader from wlauto.core.configuration import settings self._loader = PluginLoader(settings.plugin_packages, - settings.plugin_paths, - settings.plugin_ignore_paths) + [settings.plugins_directory], []) def update(self, packages=None, paths=None, ignore_paths=None): if not self._loader: diff --git a/wlauto/exceptions.py b/wlauto/exceptions.py index 67999e57..bd4a0bb6 100644 --- a/wlauto/exceptions.py +++ b/wlauto/exceptions.py @@ -14,7 +14,9 @@ # -from wlauto.utils.misc import get_traceback, TimeoutError # NOQA pylint: disable=W0611 +from wlauto.utils.misc import get_traceback + +from devlib.exception import DevlibError, HostError, TargetError, TimeoutError class WAError(Exception): diff --git a/wlauto/tests/test_parsers.py b/wlauto/tests/test_parsers.py index 6f9e75eb..763d2c7f 100644 --- a/wlauto/tests/test_parsers.py +++ b/wlauto/tests/test_parsers.py @@ -8,8 +8,8 @@ from mock.mock import Mock, MagicMock, call from wlauto.exceptions import ConfigError from wlauto.core.configuration.parsers import * # pylint: disable=wildcard-import from wlauto.core.configuration.parsers import _load_file, _collect_valid_id, _resolve_params_alias -from wlauto.core.configuration import (WAConfiguration, RunConfiguration, JobGenerator, - PluginCache, ConfigurationPoint) +from wlauto.core.configuration import RunConfiguration, JobGenerator, PluginCache, ConfigurationPoint +from wlauto.core.configuration.configuration import WAConfiguration from wlauto.utils.types import toggle_set, reset_counter @@ -125,9 +125,6 @@ class TestFunctions(TestCase): with self.assertRaises(ConfigError): _resolve_params_alias(test, "new_name") - def test_construct_valid_entry(self): - raise Exception() - class TestConfigParser(TestCase): @@ -362,44 +359,6 @@ class TestAgendaParser(TestCase): assert_equal(workload['workload_name'], "test") -class TestEnvironmentVarsParser(TestCase): - - def test_environmentvarsparser(self): - wa_config = Mock(spec=WAConfiguration) - calls = [call('user_directory', '/testdir'), - call('plugin_paths', ['/test', '/some/other/path', '/testy/mc/test/face'])] - - # Valid env vars - valid_environ = {"WA_USER_DIRECTORY": "/testdir", - "WA_PLUGIN_PATHS": "/test:/some/other/path:/testy/mc/test/face"} - EnvironmentVarsParser(wa_config, valid_environ) - wa_config.set.assert_has_calls(calls) - - # Alternative env var name - wa_config.reset_mock() - alt_valid_environ = {"WA_USER_DIRECTORY": "/testdir", - "WA_EXTENSION_PATHS": "/test:/some/other/path:/testy/mc/test/face"} - EnvironmentVarsParser(wa_config, alt_valid_environ) - wa_config.set.assert_has_calls(calls) - - # Test that WA_EXTENSION_PATHS gets merged with WA_PLUGIN_PATHS. - # Also checks that other enviroment variables don't cause errors - wa_config.reset_mock() - calls = [call('user_directory', '/testdir'), - call('plugin_paths', ['/test', '/some/other/path']), - call('plugin_paths', ['/testy/mc/test/face'])] - ext_and_plgin = {"WA_USER_DIRECTORY": "/testdir", - "WA_PLUGIN_PATHS": "/test:/some/other/path", - "WA_EXTENSION_PATHS": "/testy/mc/test/face", - "RANDOM_VAR": "random_value"} - EnvironmentVarsParser(wa_config, ext_and_plgin) - # If any_order=True then the calls can be in any order, but they must all appear - wa_config.set.assert_has_calls(calls, any_order=True) - - # No WA enviroment variables present - wa_config.reset_mock() - EnvironmentVarsParser(wa_config, {"RANDOM_VAR": "random_value"}) - wa_config.set.assert_not_called() class TestCommandLineArgsParser(TestCase): diff --git a/wlauto/utils/misc.py b/wlauto/utils/misc.py index cf170810..3f6d7e8b 100644 --- a/wlauto/utils/misc.py +++ b/wlauto/utils/misc.py @@ -492,7 +492,7 @@ def merge_config_values(base, other): are treated as atomic, and not mergeable. s: A sequence. Anything iterable that is not a dict or a string (strings are considered scalars). - m: A key-value mapping. ``dict`` and it's derivatives. + m: A key-value mapping. ``dict`` and its derivatives. n: ``None``. o: A mergeable object; this is an object that implements both ``merge_with`` and ``merge_into`` methods. diff --git a/wlauto/utils/serializer.py b/wlauto/utils/serializer.py index 821e01b1..eb2d893e 100644 --- a/wlauto/utils/serializer.py +++ b/wlauto/utils/serializer.py @@ -51,7 +51,7 @@ import yaml as _yaml import dateutil.parser from wlauto.exceptions import SerializerSyntaxError -from wlauto.utils.types import regex_type +from wlauto.utils.types import regex_type, none_type from wlauto.utils.misc import isiterable @@ -70,12 +70,14 @@ POD_TYPES = [ tuple, dict, set, - basestring, + str, + unicode, int, float, bool, datetime, - regex_type + regex_type, + none_type, ] class WAJSONEncoder(_json.JSONEncoder): @@ -257,3 +259,4 @@ def _read_pod(fh, fmt=None): def is_pod(obj): return type(obj) in POD_TYPES + diff --git a/wlauto/utils/types.py b/wlauto/utils/types.py index 94f257f2..c23f8215 100644 --- a/wlauto/utils/types.py +++ b/wlauto/utils/types.py @@ -169,6 +169,7 @@ list_or_bool = list_or(boolean) regex_type = type(re.compile('')) +none_type = type(None) def regex(value): |