lots of reorg

13 files changed, 211 insertions, 6 deletions

diff --git a/tempest-pull/.gitignore b/tempest-pull/.gitignore
new file mode 100644
index 0000000..e7cc346
--- /dev/null
+++ b/tempest-pull/.gitignore
@@ -0,0 +1,3 @@
+venv/**
+**/*.pyc
+
diff --git a/tempest-pull/src/__init__.py b/tempest-pull/app/__init__.py
index e69de29..e69de29 100644
--- a/tempest-pull/src/__init__.py
+++ b/tempest-pull/app/__init__.py
diff --git a/tempest-pull/app/app.py b/tempest-pull/app/app.py
new file mode 100644
index 0000000..19ce855
--- /dev/null
+++ b/tempest-pull/app/app.py
@@ -0,0 +1,44 @@
+import gc
+import os
+import string
+
+from neo4j import Neo4JDatabase
+from lava import LAVADashboard
+from bundlestore import BundleStore
+from bundle import Bundle
+
+def run(config, print_config):
+   if print_config:
+      print config
+
+   lava = LAVADashboard(config["LAVA_XMLRPC_ENDPOINT"], config["BUNDLE_STREAM_NAME"])
+   lava.connect()
+
+   store = BundleStore(config["BUNDLES_ROOT"])
+   store_sha1_list = store.bundle_list()
+
+   bundle_sha1_list, bundle_list = lava.server_bundle_list()
+   for entry in bundle_list:
+      sha1 = entry["content_sha1"]
+      if sha1 in store_sha1_list:
+         print "[%s] skipping, already processed" % sha1
+         continue
+
+      print "------------------------------------------------------------------------------------------------------"
+
+      print "downloading new entry:"
+      print entry
+
+      bundle = lava.retrieve_bundle(entry)
+
+      print "[%s]:" % sha1
+      
+      metadata = bundle.expand(config["WHITELIST"], config["LOGS_ROOT"])
+
+      database = Neo4JDatabase(config["NEO4J_ENDPOINT"])
+      database.store_bundle(bundle)
+
+      store.write_bundle_receipt(bundle)
+      del bundle
+      gc.collect()
+
diff --git a/tempest-pull/src/bundle.py b/tempest-pull/app/bundle.py
index 6a79004..6a79004 100644
--- a/tempest-pull/src/bundle.py
+++ b/tempest-pull/app/bundle.py
diff --git a/tempest-pull/app/bundlestore.py b/tempest-pull/app/bundlestore.py
new file mode 100644
index 0000000..d70e98d
--- /dev/null
+++ b/tempest-pull/app/bundlestore.py
@@ -0,0 +1,48 @@
+import base64
+import datetime
+import fnmatch
+import json
+import os
+import string
+import subprocess
+
+from util import *
+from bundle import Bundle
+
+SUBUNIT_RESULTS_FILE = "results.subunit"
+ALL_TESTS_FILE = "all-tests.txt"
+
+class BundleStore(object):
+   def __init__(self, root_path):
+      self.root_path = root_path
+
+   def bundle_list(self):
+      dirs = []
+      root = os.path.expanduser(self.root_path)
+      for subdir in os.listdir(root):
+          if os.path.isdir(os.path.join(root, subdir)):
+             dirs.append(subdir)
+      return dirs
+
+   def is_bundle_present(self, bundle):
+      if not isinstance(bundle, Bundle):
+         raise Exception("argument is not a Bundle")
+      bundles = self.bundle_list()
+      return bundle.sha1 in bundles
+
+   def write_bundle_receipt(self, bundle, include_data=False):
+      if not isinstance(bundle, Bundle):
+         raise Exception("argument is not a Bundle")
+      root = os.path.expanduser(self.root_path)
+      bundle_root = os.path.join(root, bundle.sha1)
+      create_dir(bundle_root)
+
+      full_meta_path = os.path.join(bundle_root, "metadata")
+      with open(full_meta_path, "w") as f:
+            f.write(json.dumps(bundle.entry_data, default=json_serial))
+      if include_data:
+         full_data_path = os.path.join(bundle_root, "data")
+         with open(full_data_path, "w") as f:
+            f.write(json.dumps(bundle.data, default=json_serial))
+
+
diff --git a/tempest-pull/src/lava.py b/tempest-pull/app/lava.py
index 5be75cc..5be75cc 100644
--- a/tempest-pull/src/lava.py
+++ b/tempest-pull/app/lava.py
diff --git a/tempest-pull/app/neo4j.py b/tempest-pull/app/neo4j.py
new file mode 100644
index 0000000..243b0d5
--- /dev/null
+++ b/tempest-pull/app/neo4j.py
@@ -0,0 +1,69 @@
+import os
+import string
+
+from py2neo import Graph, Node, Relationship
+from bundle import Bundle
+
+class Neo4JDatabase(object):
+   def __init__(self, endpoint):
+      self.endpoint = endpoint
+
+   def store_bundle(self, bundle):
+      if not isinstance(bundle, Bundle):
+         raise Exception("argument is not a Bundle")
+
+      graph = Graph(self.endpoint)
+
+      print "creating graph for bundle"
+
+      # find (or create) the OS version + distro node
+      os_version = bundle.metadata["lava_job_attributes"]["os-version"]
+      os_distro = bundle.metadata["lava_job_attributes"]["os-distro"]
+      os_name = "%s/%s" % (os_distro, os_version)
+      OS_node = graph.find_one("OS", "name", os_name)
+      if not OS_node:
+         OS_node = Node("OS", name=os_name, distro=os_distro, version=os_version)
+         graph.create(OS_node)
+
+      # find (or create) the devstack branch  node
+      devstack_branch = bundle.metadata["lava_job_attributes"]["devstack-branch"]
+      Branch_node = graph.find_one("Devstack", "name", devstack_branch)
+      if not Branch_node:
+         Branch_node = Node("Devstack", name=devstack_branch)
+         graph.create(Branch_node)
+
+      # create the main tempest run node and associate with the OS and Branch
+      TempestRun_node = Node("TempestRun",          \
+         date = bundle.metadata["date_uploaded"],     \
+         lava_job = bundle.metadata["lava_job_id"],   \
+         sha1 = bundle.metadata["bundle_sha1"])
+      if bundle.all_tests:
+         TempestRun_node.properties["all_tests"] = len(bundle.all_tests)
+      if bundle.tests_run:
+         TempestRun_node.properties["tests_run"] = len(bundle.tests_run)
+      if bundle.failing_tests:
+         TempestRun_node.properties["failing_tests"] = len(bundle.failing_tests)
+      if bundle.skipped_tests:
+         TempestRun_node.properties["skipped_tests"] = len(bundle.skipped_tests)
+      if bundle.passing_tests:
+         TempestRun_node.properties["passing_tests"] = len(bundle.passing_tests)
+      OS_relationship = Relationship(TempestRun_node, "ON", OS_node)
+      Branch_relationship = Relationship(TempestRun_node, "USING", Branch_node)
+      graph.create(TempestRun_node, OS_relationship, Branch_relationship)
+
+      # create all of the tests and relate them back to the tempest node
+      for test_set in [bundle.failing_tests, bundle.passing_tests, bundle.skipped_tests]:
+         if test_set:
+            print "adding tests"
+            for test in test_set:
+               Test_node = Node("Test", test["status"],  \
+                  name=test["test"], \
+                  status=test["status"], \
+                  start_time=test["start_time"], \
+                  stop_time=test["stop_time"], \
+                  test_class=test["class"])
+               Test_relationship = Relationship(TempestRun_node, \
+                  "HAS_TEST", Test_node, status=test["status"])
+               graph.create(Test_node, Test_relationship)
+
+
diff --git a/tempest-pull/src/subunitresults.py b/tempest-pull/app/subunitresults.py
index e2caf7e..e2caf7e 100644
--- a/tempest-pull/src/subunitresults.py
+++ b/tempest-pull/app/subunitresults.py
diff --git a/tempest-pull/src/tempest-lava-pull.py b/tempest-pull/app/tempest-lava-pull.py
index aa74c50..aa74c50 100644
--- a/tempest-pull/src/tempest-lava-pull.py
+++ b/tempest-pull/app/tempest-lava-pull.py
diff --git a/tempest-pull/src/util.py b/tempest-pull/app/util.py
index b9b3582..b9b3582 100644
--- a/tempest-pull/src/util.py
+++ b/tempest-pull/app/util.py
diff --git a/tempest-pull/config.py b/tempest-pull/config.py
new file mode 100644
index 0000000..32a8bb5
--- /dev/null
+++ b/tempest-pull/config.py
@@ -0,0 +1,46 @@
+config = dict(
+
+
+# this is the URL of the LAVA instance to retrieve from
+LAVA_XMLRPC_ENDPOINT = "https://openstack.validation.linaro.org/RPC2",
+
+# this is the URL of the Neo4J instance
+NEO4J_ENDPOINT = "http://neo4j:linaro@localhost:7474/db/data/",
+
+# this is the name of the bundle stream to retrieve from
+BUNDLE_STREAM_NAME = "/public/team/openstack/tempest-ci/",
+
+# this is the root location where the raw bundle JSON will be downloaded
+# and stored.  Each bundle will be stored as a single .txt file inside
+# a directory named with the bundle UUID.  Any bundle that already has a
+# directory present will be skipped on subsequent runs.
+BUNDLES_ROOT = "~/lava_pull/bundles/",  #"/srv/tempest/bundles/"
+
+# this is the root location where the bundle's extracted log data and
+# other files will be written
+LOGS_ROOT = "~/lava_pull/logs/",  #"/srv/tempest/logs/"
+
+# these are the important files that are to be brought to the top-level of
+# the output directory
+WHITELIST = {
+   'devstack' : [
+       { 'src' : "stdout.log", 'new-name' : "stdout-devstack.log" }
+   ],
+   'tempest-summary' : [
+       { 'src' : "tempest-summary.txt" }
+   ],
+   'tempest-run' : [
+       { 'src' : "*.log.gz" },
+       { 'src' : "*.txt.gz" },
+       { 'src' : "apache2/*" },
+       { 'src' : "libvirt/*" },
+       { 'src' : "all-tests.txt" },
+       { 'src' : "results.subunit" },
+       { 'src' : "stack.sh.summary.gz" },
+       { 'src' : "stdout.log", 'new-name' : "stdout-tempest.log" },
+       { 'src' : "tempest_conf.txt" }
+   ]
+}
+
+
+)
diff --git a/tempest-pull/config.sample b/tempest-pull/config.sample
deleted file mode 100644
index 4b579a3..0000000
--- a/tempest-pull/config.sample
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash
-export LAVA_PULL_BUNDLEROOT=~/lava_pull/bundles/
-export LAVA_PULL_LOGROOT=~/lava_pull/logs/
-
diff --git a/tempest-pull/run b/tempest-pull/run
index 5d5386c..12d14dc 100755
--- a/tempest-pull/run
+++ b/tempest-pull/run
@@ -1,6 +1,5 @@
 #!/bin/bash
-source config
 source venv/bin/activate
-python src/tempest-lava-pull.py
+python -c "import config; from app import app; app.run(config.config, False)"
 deactivate