path: root/kci_build

#!/usr/bin/env python
#
# Copyright (C) 2018, 2019 Collabora Limited
# Author: Guillaume Tucker <guillaume.tucker@collabora.com>
#
# This module is free software; you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation; either version 2.1 of the License, or (at your option)
# any later version.
#
# This library is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this library; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

import argparse
import os
import subprocess
import sys
import time

import kernelci.build
import kernelci.config.build


# -----------------------------------------------------------------------------
# Standard arguments that can be used in sub-commands
#

class Args(object):
    config= {
        'name': '--config',
        'help': "Build config name",
    }

    variant = {
        'name': '--variant',
        'help': "Build config variant name",
    }

    build_env = {
        'name': '--build-env',
        'help': "Build environment name",
    }

    storage = {
        'name': '--storage',
        'help': "Storage URL",
        'default': "https://storage.kernelci.org",
        'required': False,
    }

    api = {
        'name': '--api',
        'help': "Backend API URL",
        'default': "https://api.kernelci.org",
        'required': False,
    }

    token = {
        'name': '--token',
        'help': "Backend API token",
    }

    commit = {
        'name': '--commit',
        'help': "Git commit checksum",
    }

    describe = {
        'name': '--describe',
        'help': "Git describe",
    }

    describe_verbose = {
        'name': '--describe-verbose',
        'help': "Verbose version of git describe",
    }

    tree_name = {
        'name': '--tree-name',
        'help': "Name of a kernel tree",
    }

    tree_url = {
        'name': '--tree-url',
        'help': "URL of a kernel tree",
    }

    branch = {
        'name': '--branch',
        'help': "Name of a kernel branch in a tree",
    }

    mirror = {
        'name': '--mirror',
        'help': "Path to the local kernel git mirror",
    }

    kdir = {
        'name': '--kdir',
        'help': "Path to the kernel checkout directory",
    }

    arch = {
        'name': '--arch',
        'help': "CPU architecture name",
    }

    defconfig = {
        'name': '--defconfig',
        'help': "Kernel defconfig name",
    }

    j = {
        'name': '-j',
        'help': "Number of parallel build processes",
    }

    verbose = {
        'name': '--verbose',
        'help': "Verbose output",
        'action': 'store_true',
    }

    output = {
        'name': '--output',
        'help': "Path the output directory",
    }

    json_path = {
        'name': '--json-path',
        'help': "Path to the JSON file",
    }


# -----------------------------------------------------------------------------
# Commands
#

class Command(object):
    help = None
    args = None
    opt_args = None

    def __init__(self, sub_parser, name):
        if not self.help:
            raise AttributeError("Missing help message for {}".format(name))
        self._parser = sub_parser.add_parser(name, help=self.help)
        if self.args:
            for arg in self.args:
                self._add_arg(arg, True)
        if self.opt_args:
            for arg in self.opt_args:
                self._add_arg(arg, False)
        self._parser.set_defaults(func=self)

    def __call__(self, *args, **kw):
        raise NotImplementedError("Command not implemented")

    def _add_arg(self, arg, required=True):
        kw = dict(arg)
        arg_name = kw.pop('name')
        if required:
            kw.setdefault('required', True)
        self._parser.add_argument(arg_name, **kw)


class cmd_list_configs(Command):
    help = "List the build configurations"

    def __call__(self, configs, args):
        for conf_name in configs['build_configs'].keys():
            print(conf_name)
        return True


class cmd_check_new_commit(Command):
    help = "Check if a new commit is available on a branch"
    args = [Args.config, Args.storage]

    def __call__(self, configs, args):
        conf = configs['build_configs'][args.config]
        update = kernelci.build.check_new_commit(conf, args.storage)
        if update is False or update is True:
            return update
        print(update)
        return True


class cmd_update_last_commit(Command):
    help = "Update the last commit file on the remote storage server"
    args = [Args.config, Args.api, Args.token, Args.commit]

    def __call__(self, configs, args):
        conf = configs['build_configs'][args.config]
        kernelci.build.set_last_commit(conf, args.api, args.token, args.commit)
        return True


class cmd_tree_branch(Command):
    help = "Print the name of the tree and branch for a given build config"
    args = [Args.config]

    def __call__(self, configs, args):
        conf = configs['build_configs'][args.config]
        print(conf.tree.name)
        print(conf.tree.url)
        print(conf.branch)
        return True


class cmd_update_mirror(Command):
    help = "Update the local kernel git mirror"
    args = [Args.config, Args.mirror]

    def __call__(self, configs, args):
        conf = configs['build_configs'][args.config]
        kernelci.build.update_mirror(conf, args.mirror)
        return True


class cmd_update_repo(Command):
    help = "Update the local kernel repository checkout"
    args = [Args.config, Args.kdir]
    opt_args = [Args.mirror]

    def __call__(self, configs, args):
        conf = configs['build_configs'][args.config]
        kernelci.build.update_repo(conf, args.kdir, args.mirror)
        return True



class cmd_describe(Command):
    help = "Print the git commit, describe and verbose describe from kdir"
    args = [Args.config, Args.kdir]

    def __call__(self, configs, args):
        conf = configs['build_configs'][args.config]
        commit = kernelci.build.head_commit(args.kdir)
        describe = kernelci.build.git_describe(conf.tree.name, args.kdir)
        verbose = kernelci.build.git_describe_verbose(args.kdir)
        print(commit)
        print(describe)
        print(verbose or describe)
        return True


class cmd_generate_fragments(Command):
    help = "Generate the config fragment files in kdir"
    args = [Args.config, Args.kdir]

    def __call__(self, configs, args):
        conf = configs['build_configs'][args.config]
        kernelci.build.generate_fragments(conf, args.kdir)
        return True


class cmd_generate_defconfig_fragments(Command):
    help = "Generate the config fragment files for a given defconfig"
    args = [Args.defconfig, Args.kdir]

    def __call__(self, configs, args):
        fragments = configs['fragments']
        split = args.defconfig.split('+')
        defconfig = split.pop(0)
        for part in split:
            frag = fragments.get(part)
            if frag and frag.configs:
                kernelci.build.generate_config_fragment(frag, args.kdir)
        return True


class cmd_expand_fragments(Command):
    help = "Expand a defconfig string with full fragment paths"
    args = [Args.defconfig]

    def __call__(self, configs, args):
        frags = configs['fragments']
        split = args.defconfig.split('+')
        expanded = [split.pop(0)]
        for part in split:
            frag = frags.get(part)
            if frag:
                expanded.append(frag.path)
            else:
                expanded.append(part)
        print('+'.join(expanded))
        return True


class cmd_push_tarball(Command):
    help = "Create and up a source tarball to the remote storage server"
    args = [Args.config, Args.kdir, Args.storage, Args.api, Args.token]

    def __call__(self, configs, args):
        conf = configs['build_configs'][args.config]
        func_args = (conf, args.kdir, args.storage, args.api, args.token)
        if not all(func_args):
            print("Invalid arguments")
            return False
        tarball_url = kernelci.build.push_tarball(*func_args)
        if not tarball_url:
            return False
        print(tarball_url)
        return True


class cmd_list_variants(Command):
    help = "Print the list of build variants for a given build configuration"
    args = [Args.config]

    def __call__(self, configs, args):
        conf = configs['build_configs'][args.config]
        for variant in conf.variants:
            print(variant.name)
        return True


class cmd_arch_list(Command):
    help = "Print the list of CPU architecture names for a given build variant"
    args = [Args.config, Args.variant]

    def __call__(self, configs, args):
        conf = configs['build_configs'][args.config]
        variant = conf.get_variant(args.variant)
        for arch in variant.arch_list:
            print(arch)
        return True


def _show_build_env(build_env, arch=None):
    print(build_env.name)
    print(build_env.cc)
    print(build_env.cc_version)
    if arch:
        print(build_env.get_arch_name(args.arch))
        xc = build_env.get_cross_compile(args.arch)
        if xc:
            print(xc)


class cmd_get_build_env(Command):
    help = "Print the build environment parameters for a given build variant"
    args = [Args.config, Args.variant]
    opt_args = [Args.arch]

    def __call__(self, configs, args):
        conf = configs['build_configs'][args.config]
        variant = conf.get_variant(args.variant)
        _show_build_env(variant.build_environment, args.arch)
        return True


class cmd_show_build_env(Command):
    help = "Show parameters of a given build environment"
    args = [Args.build_env]
    opt_args = [Args.arch]

    def __call__(self, configs, args):
        build_env = configs['build_environments'][args.build_env]
        _show_build_env(build_env, args.arch)
        return True


class cmd_list_kernel_configs(Command):
    help = "List the kernel configs to build for a given build configuration"
    args = [Args.config, Args.kdir]
    opt_args = [Args.variant, Args.arch]

    def __call__(self, configs, args):
        conf = configs['build_configs'][args.config]
        configs = kernelci.build.list_kernel_configs(
            conf, args.kdir, args.variant, args.arch)
        for item in configs:
            print(' '.join(item))
        return True


class cmd_build_kernel(Command):
    help = "Build a kernel"
    args = [Args.build_env, Args.arch, Args.kdir]
    opt_args = [Args.defconfig, Args.j, Args.verbose, Args.output]

    def __call__(self, configs, args):
        build_env = configs['build_environments'][args.build_env]
        return kernelci.build.build_kernel(
            build_env, args.kdir, args.arch, args.defconfig, args.j,
            args.verbose, args.output)


class cmd_install_kernel(Command):
    help = "Install the kernel binaries and build.json locally"
    args = [Args.kdir]
    opt_args = [Args.config, Args.tree_name, Args.tree_url, Args.branch,
                Args.commit, Args.describe, Args.describe_verbose,
                Args.output]

    def __call__(self, configs, args):
        if args.config:
            conf = configs['build_configs'][args.config]
            tree_name = conf.tree.name
            tree_url = conf.tree.url
            branch = conf.branch
        else:
            tree_name = args.tree_name
            tree_url = args.tree_url
            branch = args.branch
        if not all((tree_name, tree_url, branch)):
            print("""\
Invalid arguments, either of these 2 sets are possible:
   --tree-name, --tree-url and --branch
or
   --config (in which case the tree and branch are read from the YAML config)\
""")
            return False
        return kernelci.build.install_kernel(
            args.kdir, tree_name, tree_url, branch, args.commit, args.describe,
            args.describe_verbose, args.output)


class cmd_push_kernel(Command):
    help = "Push the kernel binaries"
    args = [Args.kdir, Args.api, Args.token]

    def __call__(self, configs, args):
        return kernelci.build.push_kernel(args.kdir, args.api, args.token)


class cmd_publish_kernel(Command):
    help = "Publish the kernel meta-data"
    args = [Args.kdir]
    opt_args = [Args.api, Args.token, Args.json_path]

    def __call__(self, configs, args):
        if not ((args.api and args.token) or args.json_path):
            print("""\
Invalid arguments, please provide at least one of these sets of options:
    --token, --api to publish to the backend server
    --json-path to save the data in a local JSON file\
""")
            return False
        return kernelci.build.publish_kernel(
            args.kdir, api=args.api, token=args.token,
            json_path=args.json_path)

# -----------------------------------------------------------------------------
# Main
#

if __name__ == '__main__':
    parser = argparse.ArgumentParser("kci_build")
    parser.add_argument("--build-configs", default="build-configs.yaml",
                        help="Path to build configs file")
    sub_parser = parser.add_subparsers(title="Commands",
                                       help="List of available commands")

    commands = dict()
    for k in globals().keys():
        split = k.split('cmd_')
        if len(split) == 2:
            obj = globals().get(k)
            if issubclass(obj, Command):
                cmd_name = split[1]
                commands[cmd_name] = obj(sub_parser, cmd_name)

    args = parser.parse_args()
    configs = kernelci.config.build.from_yaml(args.build_configs)
    status = args.func(configs, args)
    sys.exit(0 if status is True else 1)