Add a new command for submiting tuxbuild data to squad

Signed-off-by: Justin Cook <justin.cook@linaro.org>

11 files changed, 429 insertions, 114 deletions

diff --git a/.flake8 b/.flake8
new file mode 100644
index 0000000..6678c93
--- /dev/null
+++ b/.flake8
@@ -0,0 +1,3 @@
+[flake8]
+extend-exclude =
+    .venv
diff --git a/.gitignore b/.gitignore
index d35765d..ce9454c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,3 +4,4 @@ dist/
 *.egg-info/
 *.sqlite3
 squad_client_cache.sqlite
+.venv
diff --git a/requirements.txt b/requirements.txt
index 6b02026..2e7d319 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,6 @@
 requests
 requests_cache
 jinja2
+jsonschema
 ipython
 pyyaml
diff --git a/squad_client/commands/submit.py b/squad_client/commands/submit.py
index 078e10c..216988f 100644
--- a/squad_client/commands/submit.py
+++ b/squad_client/commands/submit.py
@@ -1,4 +1,3 @@
-import hashlib
 import json
 import logging
 import os
@@ -31,11 +30,6 @@ class SubmitCommand(SquadClientCommand):
             choices=["pass", "fail", "skip"],
         )
         parser.add_argument(
-            "--results-layout",
-            help="Layout of the results file, if any",
-            choices=["tuxbuild"],
-        )
-        parser.add_argument(
             "--metrics",
             help="File with metrics(benchmarsk) to submit. Max 5MB. JSON and YAML formats are supported",
         )
@@ -95,48 +89,6 @@ class SubmitCommand(SquadClientCommand):
 
         return output_dict
 
-    def _load_tuxbuild(self, path):
-        if not self.__check_file(path):
-            return None
-
-        data = None
-        try:
-            with open(path) as f:
-                tb = {}
-                builds = json.load(f)
-
-                for b in builds:
-                    name = self._get_tuxbuild_test_name(b)
-                    tb[name] = b["build_status"]
-
-                data = tb
-
-        except IndexError as ie:
-            logger.error("Failed to load tuxbuild json due to a missing kconfig value: %s", ie)
-
-        except KeyError as ke:
-            logger.error("Failed to load tuxbuild json due to a missing variable: %s", ke)
-
-        except json.JSONDecodeError as jde:
-            logger.error("Failed to load json: %s", jde)
-
-        except OSError as ose:
-            logger.error("Failed to open file: %s", ose)
-
-        return data
-
-    def _get_tuxbuild_test_name(self, build):
-        suite = "build"
-
-        if len(build["kconfig"][1:]):
-            kconfig = "%s-%s" % (build["kconfig"][0], hashlib.sha1(json.dumps(build["kconfig"][1:]).encode()).hexdigest()[0:8])
-        else:
-            kconfig = build["kconfig"][0]
-
-        return "%s/%s-%s" % (
-            suite, build["toolchain"], kconfig,
-        )
-
     def run(self, args):
         results_dict = {}
         metrics_dict = {}
@@ -149,10 +101,7 @@ class SubmitCommand(SquadClientCommand):
             results_dict = {args.result_name: args.result_value}
 
         if args.results:
-            if args.results_layout == 'tuxbuild':
-                results_dict = self._load_tuxbuild(args.results)
-            else:
-                results_dict = self.__read_input_file(args.results)
+            results_dict = self.__read_input_file(args.results)
 
             if results_dict is None:
                 return False
diff --git a/squad_client/commands/submit_tuxbuild.py b/squad_client/commands/submit_tuxbuild.py
new file mode 100644
index 0000000..f562bed
--- /dev/null
+++ b/squad_client/commands/submit_tuxbuild.py
@@ -0,0 +1,126 @@
+import hashlib
+import json
+import jsonschema
+import logging
+from squad_client.shortcuts import submit_results
+from squad_client.core.command import SquadClientCommand
+
+logger = logging.getLogger()
+
+tuxbuild_schema = {
+    "type": "object",
+    "properties": {
+        "build_status": {
+            "type": "string",
+            "enum": ["fail", "pass"],
+        },
+        "git_describe": {
+            "type": "string",
+        },
+        "kconfig": {
+            "type": "array",
+            "minItems": 1,
+            "uniqueItems": True,
+            "items": [{"type": "string"}],
+        },
+        "target_arch": {
+            "type": "string",
+        },
+        "toolchain": {
+            "type": "string",
+        },
+    },
+    "required": [
+        "build_status",
+        "git_describe",
+        "kconfig",
+        "target_arch",
+        "toolchain",
+    ],
+}
+
+
+class SubmitTuxbuildCommand(SquadClientCommand):
+    command = "submit-tuxbuild"
+    help_text = "submit tuxbuild results to SQUAD"
+
+    def register(self, subparser):
+        parser = super(SubmitTuxbuildCommand, self).register(subparser)
+        parser.add_argument(
+            "--group", help="SQUAD group where results are stored", required=True
+        )
+        parser.add_argument(
+            "--project", help="SQUAD project where results are stored", required=True
+        )
+        parser.add_argument(
+            "tuxbuild",
+            help="File with tuxbuild results to submit",
+        )
+
+    def _load_builds(self, path):
+        builds = None
+        try:
+            with open(path) as f:
+                builds = json.load(f)
+
+        except json.JSONDecodeError as jde:
+            logger.error("Failed to load json: %s", jde)
+
+        except OSError as ose:
+            logger.error("Failed to open file: %s", ose)
+
+        return builds
+
+    def _get_test_name(self, kconfig, toolchain):
+        if len(kconfig[1:]):
+            kconfig_hash = "%s-%s" % (
+                kconfig[0],
+                hashlib.sha1(json.dumps(kconfig[1:]).encode()).hexdigest()[0:8],
+            )
+        else:
+            kconfig_hash = kconfig[0]
+
+        return "build/%s-%s" % (toolchain, kconfig_hash)
+
+    def run(self, args):
+        builds = self._load_builds(args.tuxbuild)
+
+        # log
+        if builds is None:
+            return False
+
+        data = {}
+        for build in builds:
+            try:
+                jsonschema.validate(instance=build, schema=tuxbuild_schema)
+
+                arch = build["target_arch"]
+                description = build["git_describe"]
+                kconfig = build["kconfig"]
+                status = build["build_status"]
+                toolchain = build["toolchain"]
+                test = self._get_test_name(kconfig, toolchain)
+
+                multi_key = "%s.%s" % (description, arch)
+                if multi_key not in data:
+                    data[multi_key] = {}
+
+                data[multi_key].update({test: status})
+
+            except jsonschema.exceptions.ValidationError as ve:
+                logger.error("Failed to validate tuxbuild data: %s", ve)
+                return False
+
+        for key, result in data.items():
+            description, arch = key.split(".", 1)
+            submit_results(
+                group_project_slug="%s/%s" % (args.group, args.project),
+                build_version=description,
+                env_slug=arch,
+                tests=result,
+                metrics={},
+                log={},
+                metadata={},
+            )
+
+        return True
diff --git a/tests/data/submit/tuxbuild/empty_build_status.json b/tests/data/submit/tuxbuild/empty_build_status.json
new file mode 100644
index 0000000..853c8bd
--- /dev/null
+++ b/tests/data/submit/tuxbuild/empty_build_status.json
@@ -0,0 +1,29 @@
+[
+    {
+        "build_key": "t8NSUfTBZiSPbBVaXLH7kw",
+        "build_status": "",
+        "client_token": "3cffcab5-aaf8-446d-8dec-861da6814788",
+        "download_url": "https://builds.tuxbuild.com/t8NSUfTBZiSPbBVaXLH7kw/",
+        "errors_count": 0,
+        "git_describe": "next-20201021",
+        "git_repo": "https://gitlab.com/Linaro/lkft/mirrors/next/linux-next",
+        "git_sha": "5302568121ba345f5c22528aefd72d775f25221e",
+        "git_short_log": "5302568121ba (\"Add linux-next specific files for 20201021\")",
+        "kconfig": [
+            "defconfig",
+            "https://raw.githubusercontent.com/Linaro/meta-lkft/sumo/recipes-kernel/linux/files/lkft.config",
+            "https://raw.githubusercontent.com/Linaro/meta-lkft/sumo/recipes-kernel/linux/files/lkft-crypto.config",
+            "https://raw.githubusercontent.com/Linaro/meta-lkft/sumo/recipes-kernel/linux/files/distro-overrides.config",
+            "https://raw.githubusercontent.com/Linaro/meta-lkft/sumo/recipes-kernel/linux/files/systemd.config",
+            "https://raw.githubusercontent.com/Linaro/meta-lkft/sumo/recipes-kernel/linux/files/virtio.config",
+            "CONFIG_IGB=y",
+            "CONFIG_UNWINDER_FRAME_POINTER=y"
+        ],
+        "kernel_version": "5.9.0",
+        "status_message": "build completed",
+        "target_arch": "x86",
+        "toolchain": "gcc-9",
+        "tuxbuild_status": "complete",
+        "warnings_count": 1
+    }
+]
diff --git a/tests/data/submit/tuxbuild/malformed_build_status.json b/tests/data/submit/tuxbuild/malformed_build_status.json
new file mode 100644
index 0000000..b22ee88
--- /dev/null
+++ b/tests/data/submit/tuxbuild/malformed_build_status.json
@@ -0,0 +1,29 @@
+[
+    {
+        "build_key": "t8NSUfTBZiSPbBVaXLH7kw",
+        "build_status": {"build":"pass"},
+        "client_token": "3cffcab5-aaf8-446d-8dec-861da6814788",
+        "download_url": "https://builds.tuxbuild.com/t8NSUfTBZiSPbBVaXLH7kw/",
+        "errors_count": 0,
+        "git_describe": "next-20201021",
+        "git_repo": "https://gitlab.com/Linaro/lkft/mirrors/next/linux-next",
+        "git_sha": "5302568121ba345f5c22528aefd72d775f25221e",
+        "git_short_log": "5302568121ba (\"Add linux-next specific files for 20201021\")",
+        "kconfig": [
+            "defconfig",
+            "https://raw.githubusercontent.com/Linaro/meta-lkft/sumo/recipes-kernel/linux/files/lkft.config",
+            "https://raw.githubusercontent.com/Linaro/meta-lkft/sumo/recipes-kernel/linux/files/lkft-crypto.config",
+            "https://raw.githubusercontent.com/Linaro/meta-lkft/sumo/recipes-kernel/linux/files/distro-overrides.config",
+            "https://raw.githubusercontent.com/Linaro/meta-lkft/sumo/recipes-kernel/linux/files/systemd.config",
+            "https://raw.githubusercontent.com/Linaro/meta-lkft/sumo/recipes-kernel/linux/files/virtio.config",
+            "CONFIG_IGB=y",
+            "CONFIG_UNWINDER_FRAME_POINTER=y"
+        ],
+        "kernel_version": "5.9.0",
+        "status_message": "build completed",
+        "target_arch": "x86",
+        "toolchain": "gcc-9",
+        "tuxbuild_status": "complete",
+        "warnings_count": 1
+    }
+]
diff --git a/tests/data/submit/tuxbuild/malformed_kconfig.json b/tests/data/submit/tuxbuild/malformed_kconfig.json
new file mode 100644
index 0000000..2c16d54
--- /dev/null
+++ b/tests/data/submit/tuxbuild/malformed_kconfig.json
@@ -0,0 +1,22 @@
+[
+    {
+        "build_key": "t8NSUfTBZiSPbBVaXLH7kw",
+        "build_status": "pass",
+        "client_token": "3cffcab5-aaf8-446d-8dec-861da6814788",
+        "download_url": "https://builds.tuxbuild.com/t8NSUfTBZiSPbBVaXLH7kw/",
+        "errors_count": 0,
+        "git_describe": "next-20201021",
+        "git_repo": "https://gitlab.com/Linaro/lkft/mirrors/next/linux-next",
+        "git_sha": "5302568121ba345f5c22528aefd72d775f25221e",
+        "git_short_log": "5302568121ba (\"Add linux-next specific files for 20201021\")",
+        "kconfig": [
+            {"CONFIG_ARM64_MODULE_PLTS": "y" }
+        ],
+        "kernel_version": "5.9.0",
+        "status_message": "build completed",
+        "target_arch": "x86",
+        "toolchain": "gcc-9",
+        "tuxbuild_status": "complete",
+        "warnings_count": 1
+    }
+]
diff --git a/tests/data/submit/tuxbuild/missing_build_status.json b/tests/data/submit/tuxbuild/missing_build_status.json
new file mode 100644
index 0000000..1915339
--- /dev/null
+++ b/tests/data/submit/tuxbuild/missing_build_status.json
@@ -0,0 +1,28 @@
+[
+    {
+        "build_key": "t8NSUfTBZiSPbBVaXLH7kw",
+        "client_token": "3cffcab5-aaf8-446d-8dec-861da6814788",
+        "download_url": "https://builds.tuxbuild.com/t8NSUfTBZiSPbBVaXLH7kw/",
+        "errors_count": 0,
+        "git_describe": "next-20201021",
+        "git_repo": "https://gitlab.com/Linaro/lkft/mirrors/next/linux-next",
+        "git_sha": "5302568121ba345f5c22528aefd72d775f25221e",
+        "git_short_log": "5302568121ba (\"Add linux-next specific files for 20201021\")",
+        "kconfig": [
+            "defconfig",
+            "https://raw.githubusercontent.com/Linaro/meta-lkft/sumo/recipes-kernel/linux/files/lkft.config",
+            "https://raw.githubusercontent.com/Linaro/meta-lkft/sumo/recipes-kernel/linux/files/lkft-crypto.config",
+            "https://raw.githubusercontent.com/Linaro/meta-lkft/sumo/recipes-kernel/linux/files/distro-overrides.config",
+            "https://raw.githubusercontent.com/Linaro/meta-lkft/sumo/recipes-kernel/linux/files/systemd.config",
+            "https://raw.githubusercontent.com/Linaro/meta-lkft/sumo/recipes-kernel/linux/files/virtio.config",
+            "CONFIG_IGB=y",
+            "CONFIG_UNWINDER_FRAME_POINTER=y"
+        ],
+        "kernel_version": "5.9.0",
+        "status_message": "build completed",
+        "target_arch": "x86",
+        "toolchain": "gcc-9",
+        "tuxbuild_status": "complete",
+        "warnings_count": 1
+    }
+]
diff --git a/tests/test_submit.py b/tests/test_submit.py
index d4cdfd4..613869a 100644
--- a/tests/test_submit.py
+++ b/tests/test_submit.py
@@ -71,7 +71,7 @@ class SubmitCommandTest(unittest.TestCase):
         self.squad = Squad()
         SquadApi.configure(url=self.testing_server, token=self.testing_token)
 
-    def manage_submit(self, results=None, results_layout=None, result_name=None, result_value=None, metrics=None,
+    def manage_submit(self, results=None, result_name=None, result_value=None, metrics=None,
                       metadata=None, attachments=None, logs=None, environment=None):
         argv = ['./manage.py', '--squad-host', self.testing_server, '--squad-token', self.testing_token,
                 'submit', '--group', 'my_group', '--project', 'my_project', '--build', 'my_build6', '--environment', 'test_submit_env']
@@ -80,8 +80,6 @@ class SubmitCommandTest(unittest.TestCase):
             argv += ['--logs', logs]
         if results:
             argv += ['--results', results]
-        if results_layout:
-            argv += ['--results-layout', results_layout]
         if metrics:
             argv += ['--metrics', metrics]
         if metadata:
@@ -146,65 +144,6 @@ class SubmitCommandTest(unittest.TestCase):
         self.assertFalse(proc.ok)
         self.assertIn('Failed parsing file', proc.err)
 
-    def test_submit_results_tuxbuild(self):
-        proc = self.manage_submit(results='tests/data/submit/tuxbuild/build.json', results_layout='tuxbuild')
-        self.assertTrue(proc.ok, msg=proc.err)
-        self.assertIn("Submitting 2 tests", proc.err)
-
-        test = first(self.squad.tests(name="gcc-9-defconfig-b9979cfa"))
-        self.assertEqual("build/gcc-9-defconfig-b9979cfa", test.name)
-        self.assertEqual("pass", test.status)
-
-        test = first(self.squad.tests(name="gcc-9-defconfig-5b09568e"))
-        self.assertEqual("build/gcc-9-defconfig-5b09568e", test.name)
-        self.assertEqual("fail", test.status)
-
-    def test_submit_results_tuxbuild_buildset_json(self):
-        proc = self.manage_submit(results='tests/data/submit/tuxbuild/buildset.json', results_layout='tuxbuild')
-        self.assertIn("Submitting 3 tests", proc.err)
-
-        test = first(self.squad.tests(name="gcc-8-allnoconfig"))
-        self.assertEqual("build/gcc-8-allnoconfig", test.name)
-        self.assertEqual("pass", test.status)
-
-        test = first(self.squad.tests(name="gcc-8-tinyconfig"))
-        self.assertEqual("build/gcc-8-tinyconfig", test.name)
-        self.assertEqual("pass", test.status)
-
-        test = first(self.squad.tests(name="gcc-8-x86_64_defconfig"))
-        self.assertEqual("build/gcc-8-x86_64_defconfig", test.name)
-        self.assertEqual("pass", test.status)
-
-    def test_submit_results_tuxbuild_malformed(self):
-        proc = self.manage_submit(results='tests/data/submit/tuxbuild/malformed.json', results_layout='tuxbuild')
-        self.assertFalse(proc.ok, msg=proc.err)
-        self.assertIn("Failed to load json", proc.err)
-
-    def test_submit_results_tuxbuild_missing(self):
-        proc = self.manage_submit(results="tests/data/submit/tuxbuild/missing.json", results_layout="tuxbuild")
-        self.assertFalse(proc.ok)
-        self.assertIn("Requested file tests/data/submit/tuxbuild/missing.json doesn't exist", proc.err)
-
-    def test_submit_results_tuxbuild_results_opt_missing(self):
-        proc = self.manage_submit(results_layout="tuxbuild")
-        self.assertFalse(proc.ok)
-        self.assertIn("At least one of --result-name, --results, --metrics is required", proc.err)
-
-    def test_submit_results_tuxbuild_layout_arg_bad(self):
-        proc = self.manage_submit(results="tests/data/submit/tuxbuild/build.json", results_layout="bad_layout")
-        self.assertFalse(proc.ok)
-        self.assertIn("argument --results-layout: invalid choice: 'bad_layout'", proc.err)
-
-    def test_submit_results_tuxbuild_empty_kconfig(self):
-        proc = self.manage_submit(results="tests/data/submit/tuxbuild/empty_kconfig.json", results_layout="tuxbuild")
-        self.assertFalse(proc.ok)
-        self.assertIn("Failed to load tuxbuild json due to a missing kconfig value: list index out of range", proc.err)
-
-    def test_submit_results_tuxbuild_missing_kconfig(self):
-        proc = self.manage_submit(results="tests/data/submit/tuxbuild/missing_kconfig.json", results_layout="tuxbuild")
-        self.assertFalse(proc.ok)
-        self.assertIn("Failed to load tuxbuild json due to a missing variable: 'kconfig'", proc.err)
-
     def test_submit_results_yaml(self):
         proc = self.manage_submit(results='tests/submit_results/sample_results.yaml')
         self.assertTrue(proc.ok)
diff --git a/tests/test_submit_tuxbuild.py b/tests/test_submit_tuxbuild.py
new file mode 100644
index 0000000..aac74c5
--- /dev/null
+++ b/tests/test_submit_tuxbuild.py
@@ -0,0 +1,188 @@
+import unittest
+import subprocess as sp
+
+from . import settings
+from squad_client.core.api import SquadApi
+from squad_client.core.models import Squad
+from squad_client.utils import first
+
+
+class SubmitTuxbuildCommandTest(unittest.TestCase):
+
+    testing_server = "http://localhost:%s" % settings.DEFAULT_SQUAD_PORT
+    testing_token = "193cd8bb41ab9217714515954e8724f651ef8601"
+
+    def setUp(self):
+        self.squad = Squad()
+        SquadApi.configure(url=self.testing_server, token=self.testing_token)
+
+    def submit_tuxbuild(self, tuxbuild):
+        argv = [
+            "./manage.py",
+            "--squad-host",
+            self.testing_server,
+            "--squad-token",
+            self.testing_token,
+            "submit-tuxbuild",
+            "--group",
+            "my_group",
+            "--project",
+            "my_project",
+            tuxbuild,
+        ]
+
+        proc = sp.Popen(argv, stdout=sp.PIPE, stderr=sp.PIPE)
+        proc.ok = False
+
+        try:
+            out, err = proc.communicate()
+            proc.ok = proc.returncode == 0
+        except sp.TimeoutExpired:
+            self.logger.error(
+                'Running "%s" time out after %i seconds!' % " ".join(argv)
+            )
+            proc.kill()
+            out, err = proc.communicate()
+
+        proc.out = out.decode("utf-8")
+        proc.err = err.decode("utf-8")
+        return proc
+
+    def test_submit_tuxbuild_build(self):
+        proc = self.submit_tuxbuild("tests/data/submit/tuxbuild/build.json")
+        self.assertTrue(proc.ok, msg=proc.err)
+        self.assertTrue(proc.err.count("Submitting 1 tests") == 2)
+
+        build = (
+            self.squad.group("my_group").project("my_project").build("next-20201021")
+        )
+        self.assertIsNotNone(build)
+
+        for arch in ["arm64", "x86"]:
+            environment = (
+                self.squad.group("my_group").project("my_project").environment(arch)
+            )
+            self.assertIsNotNone(environment, "environment %s does not exist" % (arch))
+
+        suite = self.squad.group("my_group").project("my_project").suite("build")
+        self.assertIsNotNone(suite)
+
+        test = first(self.squad.tests(name="gcc-9-defconfig-b9979cfa"))
+        self.assertEqual("build/gcc-9-defconfig-b9979cfa", test.name)
+        self.assertEqual("pass", test.status)
+
+        test = first(self.squad.tests(name="gcc-9-defconfig-5b09568e"))
+        self.assertEqual("build/gcc-9-defconfig-5b09568e", test.name)
+        self.assertEqual("fail", test.status)
+
+    def test_submit_tuxbuild_buildset(self):
+        proc = self.submit_tuxbuild("tests/data/submit/tuxbuild/buildset.json")
+        self.assertTrue(proc.ok, msg=proc.out)
+        self.assertIn("Submitting 3 tests", proc.err)
+
+        build = (
+            self.squad.group("my_group").project("my_project").build("next-20201030")
+        )
+        self.assertIsNotNone(build)
+
+        environment = (
+            self.squad.group("my_group").project("my_project").environment("x86")
+        )
+        self.assertIsNotNone(environment)
+
+        suite = self.squad.group("my_group").project("my_project").suite("build")
+        self.assertIsNotNone(suite)
+
+        test = first(self.squad.tests(name="gcc-8-allnoconfig"))
+        self.assertEqual("build/gcc-8-allnoconfig", test.name)
+        self.assertEqual("pass", test.status)
+
+        test = first(self.squad.tests(name="gcc-8-tinyconfig"))
+        self.assertEqual("build/gcc-8-tinyconfig", test.name)
+        self.assertEqual("pass", test.status)
+
+        test = first(self.squad.tests(name="gcc-8-x86_64_defconfig"))
+        self.assertEqual("build/gcc-8-x86_64_defconfig", test.name)
+        self.assertEqual("pass", test.status)
+
+    def test_submit_tuxbuild_empty(self):
+        proc = self.submit_tuxbuild("")
+        self.assertFalse(proc.ok, msg=proc.err)
+        self.assertIn("No such file or directory: ''", proc.err)
+
+    def test_submit_tuxbuild_malformed(self):
+        proc = self.submit_tuxbuild("tests/data/submit/tuxbuild/malformed.json")
+        self.assertFalse(proc.ok, msg=proc.err)
+        self.assertIn("Failed to load json", proc.err)
+
+    def test_submit_tuxbuild_missing(self):
+        proc = self.submit_tuxbuild("tests/data/submit/tuxbuild/missing.json")
+        self.assertFalse(proc.ok)
+        self.assertIn(
+            "No such file or directory: 'tests/data/submit/tuxbuild/missing.json'",
+            proc.err,
+        )
+
+    def test_submit_tuxbuild_empty_build_status(self):
+        proc = self.submit_tuxbuild(
+            "tests/data/submit/tuxbuild/empty_build_status.json"
+        )
+        self.assertFalse(proc.ok, msg=proc.err)
+        self.assertIn(
+            "Failed to validate tuxbuild data: '' is not one of ['fail', 'pass']",
+            proc.err,
+        )
+        self.assertIn(
+            "Failed validating 'enum' in schema['properties']['build_status']", proc.err
+        )
+
+    def test_submit_tuxbuild_malformed_build_status(self):
+        proc = self.submit_tuxbuild(
+            "tests/data/submit/tuxbuild/malformed_build_status.json"
+        )
+        self.assertFalse(proc.ok, msg=proc.err)
+        self.assertIn(
+            "Failed to validate tuxbuild data: {'build': 'pass'} is not of type 'string'",
+            proc.err,
+        )
+        self.assertIn(
+            "Failed validating 'type' in schema['properties']['build_status']", proc.err
+        )
+
+    def test_submit_tuxbuild_missing_build_status(self):
+        proc = self.submit_tuxbuild(
+            "tests/data/submit/tuxbuild/missing_build_status.json"
+        )
+        self.assertFalse(proc.ok, msg=proc.err)
+        self.assertIn(
+            "Failed to validate tuxbuild data: 'build_status' is a required property",
+            proc.err,
+        )
+
+    def test_submit_tuxbuild_empty_kconfig(self):
+        proc = self.submit_tuxbuild("tests/data/submit/tuxbuild/empty_kconfig.json")
+        self.assertFalse(proc.ok, msg=proc.err)
+        self.assertIn("Failed to validate tuxbuild data: [] is too short", proc.err)
+        self.assertIn(
+            "Failed validating 'minItems' in schema['properties']['kconfig']", proc.err
+        )
+
+    def test_submit_tuxbuild_malformed_kconfig(self):
+        proc = self.submit_tuxbuild("tests/data/submit/tuxbuild/malformed_kconfig.json")
+        self.assertFalse(proc.ok, msg=proc.err)
+        self.assertIn(
+            "Failed to validate tuxbuild data: {'CONFIG_ARM64_MODULE_PLTS': 'y'} is not of type 'string'",
+            proc.err,
+        )
+        self.assertIn(
+            "Failed validating 'type' in schema['properties']['kconfig']['items'][0]",
+            proc.err,
+        )
+
+    def test_submit_tuxbuild_missing_kconfig(self):
+        proc = self.submit_tuxbuild("tests/data/submit/tuxbuild/missing_kconfig.json")
+        self.assertFalse(proc.ok, msg=proc.err)
+        self.assertIn(
+            "Failed to validate tuxbuild data: 'kconfig' is a required property",
+            proc.err,
+        )