diff options
-rw-r--r-- | tempest-pull/.gitignore | 3 | ||||
-rw-r--r-- | tempest-pull/app/__init__.py (renamed from tempest-pull/src/__init__.py) | 0 | ||||
-rw-r--r-- | tempest-pull/app/app.py | 44 | ||||
-rw-r--r-- | tempest-pull/app/bundle.py (renamed from tempest-pull/src/bundle.py) | 0 | ||||
-rw-r--r-- | tempest-pull/app/bundlestore.py | 48 | ||||
-rw-r--r-- | tempest-pull/app/lava.py (renamed from tempest-pull/src/lava.py) | 0 | ||||
-rw-r--r-- | tempest-pull/app/neo4j.py | 69 | ||||
-rw-r--r-- | tempest-pull/app/subunitresults.py (renamed from tempest-pull/src/subunitresults.py) | 0 | ||||
-rw-r--r-- | tempest-pull/app/tempest-lava-pull.py (renamed from tempest-pull/src/tempest-lava-pull.py) | 0 | ||||
-rw-r--r-- | tempest-pull/app/util.py (renamed from tempest-pull/src/util.py) | 0 | ||||
-rw-r--r-- | tempest-pull/config.py | 46 | ||||
-rw-r--r-- | tempest-pull/config.sample | 4 | ||||
-rwxr-xr-x | tempest-pull/run | 3 |
13 files changed, 211 insertions, 6 deletions
diff --git a/tempest-pull/.gitignore b/tempest-pull/.gitignore new file mode 100644 index 0000000..e7cc346 --- /dev/null +++ b/tempest-pull/.gitignore @@ -0,0 +1,3 @@ +venv/** +**/*.pyc + diff --git a/tempest-pull/src/__init__.py b/tempest-pull/app/__init__.py index e69de29..e69de29 100644 --- a/tempest-pull/src/__init__.py +++ b/tempest-pull/app/__init__.py diff --git a/tempest-pull/app/app.py b/tempest-pull/app/app.py new file mode 100644 index 0000000..19ce855 --- /dev/null +++ b/tempest-pull/app/app.py @@ -0,0 +1,44 @@ +import gc +import os +import string + +from neo4j import Neo4JDatabase +from lava import LAVADashboard +from bundlestore import BundleStore +from bundle import Bundle + +def run(config, print_config): + if print_config: + print config + + lava = LAVADashboard(config["LAVA_XMLRPC_ENDPOINT"], config["BUNDLE_STREAM_NAME"]) + lava.connect() + + store = BundleStore(config["BUNDLES_ROOT"]) + store_sha1_list = store.bundle_list() + + bundle_sha1_list, bundle_list = lava.server_bundle_list() + for entry in bundle_list: + sha1 = entry["content_sha1"] + if sha1 in store_sha1_list: + print "[%s] skipping, already processed" % sha1 + continue + + print "------------------------------------------------------------------------------------------------------" + + print "downloading new entry:" + print entry + + bundle = lava.retrieve_bundle(entry) + + print "[%s]:" % sha1 + + metadata = bundle.expand(config["WHITELIST"], config["LOGS_ROOT"]) + + database = Neo4JDatabase(config["NEO4J_ENDPOINT"]) + database.store_bundle(bundle) + + store.write_bundle_receipt(bundle) + del bundle + gc.collect() + diff --git a/tempest-pull/src/bundle.py b/tempest-pull/app/bundle.py index 6a79004..6a79004 100644 --- a/tempest-pull/src/bundle.py +++ b/tempest-pull/app/bundle.py diff --git a/tempest-pull/app/bundlestore.py b/tempest-pull/app/bundlestore.py new file mode 100644 index 0000000..d70e98d --- /dev/null +++ b/tempest-pull/app/bundlestore.py @@ -0,0 +1,48 @@ +import base64 +import datetime +import fnmatch +import json +import os +import string +import subprocess + +from util import * +from bundle import Bundle + +SUBUNIT_RESULTS_FILE = "results.subunit" +ALL_TESTS_FILE = "all-tests.txt" + +class BundleStore(object): + def __init__(self, root_path): + self.root_path = root_path + + def bundle_list(self): + dirs = [] + root = os.path.expanduser(self.root_path) + for subdir in os.listdir(root): + if os.path.isdir(os.path.join(root, subdir)): + dirs.append(subdir) + return dirs + + def is_bundle_present(self, bundle): + if not isinstance(bundle, Bundle): + raise Exception("argument is not a Bundle") + bundles = self.bundle_list() + return bundle.sha1 in bundles + + def write_bundle_receipt(self, bundle, include_data=False): + if not isinstance(bundle, Bundle): + raise Exception("argument is not a Bundle") + root = os.path.expanduser(self.root_path) + bundle_root = os.path.join(root, bundle.sha1) + create_dir(bundle_root) + + full_meta_path = os.path.join(bundle_root, "metadata") + with open(full_meta_path, "w") as f: + f.write(json.dumps(bundle.entry_data, default=json_serial)) + if include_data: + full_data_path = os.path.join(bundle_root, "data") + with open(full_data_path, "w") as f: + f.write(json.dumps(bundle.data, default=json_serial)) + + diff --git a/tempest-pull/src/lava.py b/tempest-pull/app/lava.py index 5be75cc..5be75cc 100644 --- a/tempest-pull/src/lava.py +++ b/tempest-pull/app/lava.py diff --git a/tempest-pull/app/neo4j.py b/tempest-pull/app/neo4j.py new file mode 100644 index 0000000..243b0d5 --- /dev/null +++ b/tempest-pull/app/neo4j.py @@ -0,0 +1,69 @@ +import os +import string + +from py2neo import Graph, Node, Relationship +from bundle import Bundle + +class Neo4JDatabase(object): + def __init__(self, endpoint): + self.endpoint = endpoint + + def store_bundle(self, bundle): + if not isinstance(bundle, Bundle): + raise Exception("argument is not a Bundle") + + graph = Graph(self.endpoint) + + print "creating graph for bundle" + + # find (or create) the OS version + distro node + os_version = bundle.metadata["lava_job_attributes"]["os-version"] + os_distro = bundle.metadata["lava_job_attributes"]["os-distro"] + os_name = "%s/%s" % (os_distro, os_version) + OS_node = graph.find_one("OS", "name", os_name) + if not OS_node: + OS_node = Node("OS", name=os_name, distro=os_distro, version=os_version) + graph.create(OS_node) + + # find (or create) the devstack branch node + devstack_branch = bundle.metadata["lava_job_attributes"]["devstack-branch"] + Branch_node = graph.find_one("Devstack", "name", devstack_branch) + if not Branch_node: + Branch_node = Node("Devstack", name=devstack_branch) + graph.create(Branch_node) + + # create the main tempest run node and associate with the OS and Branch + TempestRun_node = Node("TempestRun", \ + date = bundle.metadata["date_uploaded"], \ + lava_job = bundle.metadata["lava_job_id"], \ + sha1 = bundle.metadata["bundle_sha1"]) + if bundle.all_tests: + TempestRun_node.properties["all_tests"] = len(bundle.all_tests) + if bundle.tests_run: + TempestRun_node.properties["tests_run"] = len(bundle.tests_run) + if bundle.failing_tests: + TempestRun_node.properties["failing_tests"] = len(bundle.failing_tests) + if bundle.skipped_tests: + TempestRun_node.properties["skipped_tests"] = len(bundle.skipped_tests) + if bundle.passing_tests: + TempestRun_node.properties["passing_tests"] = len(bundle.passing_tests) + OS_relationship = Relationship(TempestRun_node, "ON", OS_node) + Branch_relationship = Relationship(TempestRun_node, "USING", Branch_node) + graph.create(TempestRun_node, OS_relationship, Branch_relationship) + + # create all of the tests and relate them back to the tempest node + for test_set in [bundle.failing_tests, bundle.passing_tests, bundle.skipped_tests]: + if test_set: + print "adding tests" + for test in test_set: + Test_node = Node("Test", test["status"], \ + name=test["test"], \ + status=test["status"], \ + start_time=test["start_time"], \ + stop_time=test["stop_time"], \ + test_class=test["class"]) + Test_relationship = Relationship(TempestRun_node, \ + "HAS_TEST", Test_node, status=test["status"]) + graph.create(Test_node, Test_relationship) + + diff --git a/tempest-pull/src/subunitresults.py b/tempest-pull/app/subunitresults.py index e2caf7e..e2caf7e 100644 --- a/tempest-pull/src/subunitresults.py +++ b/tempest-pull/app/subunitresults.py diff --git a/tempest-pull/src/tempest-lava-pull.py b/tempest-pull/app/tempest-lava-pull.py index aa74c50..aa74c50 100644 --- a/tempest-pull/src/tempest-lava-pull.py +++ b/tempest-pull/app/tempest-lava-pull.py diff --git a/tempest-pull/src/util.py b/tempest-pull/app/util.py index b9b3582..b9b3582 100644 --- a/tempest-pull/src/util.py +++ b/tempest-pull/app/util.py diff --git a/tempest-pull/config.py b/tempest-pull/config.py new file mode 100644 index 0000000..32a8bb5 --- /dev/null +++ b/tempest-pull/config.py @@ -0,0 +1,46 @@ +config = dict( + + +# this is the URL of the LAVA instance to retrieve from +LAVA_XMLRPC_ENDPOINT = "https://openstack.validation.linaro.org/RPC2", + +# this is the URL of the Neo4J instance +NEO4J_ENDPOINT = "http://neo4j:linaro@localhost:7474/db/data/", + +# this is the name of the bundle stream to retrieve from +BUNDLE_STREAM_NAME = "/public/team/openstack/tempest-ci/", + +# this is the root location where the raw bundle JSON will be downloaded +# and stored. Each bundle will be stored as a single .txt file inside +# a directory named with the bundle UUID. Any bundle that already has a +# directory present will be skipped on subsequent runs. +BUNDLES_ROOT = "~/lava_pull/bundles/", #"/srv/tempest/bundles/" + +# this is the root location where the bundle's extracted log data and +# other files will be written +LOGS_ROOT = "~/lava_pull/logs/", #"/srv/tempest/logs/" + +# these are the important files that are to be brought to the top-level of +# the output directory +WHITELIST = { + 'devstack' : [ + { 'src' : "stdout.log", 'new-name' : "stdout-devstack.log" } + ], + 'tempest-summary' : [ + { 'src' : "tempest-summary.txt" } + ], + 'tempest-run' : [ + { 'src' : "*.log.gz" }, + { 'src' : "*.txt.gz" }, + { 'src' : "apache2/*" }, + { 'src' : "libvirt/*" }, + { 'src' : "all-tests.txt" }, + { 'src' : "results.subunit" }, + { 'src' : "stack.sh.summary.gz" }, + { 'src' : "stdout.log", 'new-name' : "stdout-tempest.log" }, + { 'src' : "tempest_conf.txt" } + ] +} + + +) diff --git a/tempest-pull/config.sample b/tempest-pull/config.sample deleted file mode 100644 index 4b579a3..0000000 --- a/tempest-pull/config.sample +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -export LAVA_PULL_BUNDLEROOT=~/lava_pull/bundles/ -export LAVA_PULL_LOGROOT=~/lava_pull/logs/ - diff --git a/tempest-pull/run b/tempest-pull/run index 5d5386c..12d14dc 100755 --- a/tempest-pull/run +++ b/tempest-pull/run @@ -1,6 +1,5 @@ #!/bin/bash -source config source venv/bin/activate -python src/tempest-lava-pull.py +python -c "import config; from app import app; app.run(config.config, False)" deactivate |