reorg

9 files changed, 159 insertions, 8 deletions

diff --git a/tempest-pull/config.sample b/tempest-pull/config.sample
new file mode 100644
index 0000000..4b579a3
--- /dev/null
+++ b/tempest-pull/config.sample
@@ -0,0 +1,4 @@
+#!/bin/bash
+export LAVA_PULL_BUNDLEROOT=~/lava_pull/bundles/
+export LAVA_PULL_LOGROOT=~/lava_pull/logs/
+
diff --git a/tempest-pull/run b/tempest-pull/run
index 11c5ca4..1cb7c6c 100755
--- a/tempest-pull/run
+++ b/tempest-pull/run
@@ -1,8 +1,6 @@
 #!/bin/bash
-
-export LAVA_PULL_BUNDLEROOT=~/lava_pull/bundles/
-export LAVA_PULL_LOGROOT=~/lava_pull/logs/
+source config
 source venv/bin/activate
-python tempest-pull.py
+python tempest-lava-pull.py
 deactivate
 
diff --git a/tempest-pull/__init__.py b/tempest-pull/src/__init__.py
index e69de29..e69de29 100644
--- a/tempest-pull/__init__.py
+++ b/tempest-pull/src/__init__.py
diff --git a/tempest-pull/bundle.py b/tempest-pull/src/bundle.py
index 6e786e4..6a79004 100644
--- a/tempest-pull/bundle.py
+++ b/tempest-pull/src/bundle.py
@@ -97,10 +97,10 @@ class Bundle(object):
       bundle_metadata['lava_job_attributes'] = attributes
 
       # create the names for the root directories
-      output_subdir = self.lava_job_id
-      if "os-distro" in attributes and "os-version" in attributes and "devstack-branch" in attributes:
-	 output_subdir = "%s,os=%s,osver=%s,branch=%s" %   \
-	   (self.lava_job_id, attributes["os-distro"], attributes["os-version"], attributes["devstack-branch"])
+      output_subdir = str(self.lava_job_id)
+      #if "os-distro" in attributes and "os-version" in attributes and "devstack-branch" in attributes:
+	# output_subdir = "%s,os=%s,osver=%s,branch=%s" %   \
+	#   (self.lava_job_id, attributes["os-distro"], attributes["os-version"], attributes["devstack-branch"])
       full_root_path = os.path.join(os.path.expanduser(output_root), output_subdir)
 
       whitelist_path = os.path.join(full_root_path, self.subdir_whitelist)
diff --git a/tempest-pull/lava.py b/tempest-pull/src/lava.py
index 5be75cc..5be75cc 100644
--- a/tempest-pull/lava.py
+++ b/tempest-pull/src/lava.py
diff --git a/tempest-pull/subunitresults.py b/tempest-pull/src/subunitresults.py
index e2caf7e..e2caf7e 100644
--- a/tempest-pull/subunitresults.py
+++ b/tempest-pull/src/subunitresults.py
diff --git a/tempest-pull/tempest-pull.py b/tempest-pull/src/tempest-lava-pull.py
index aa74c50..aa74c50 100644
--- a/tempest-pull/tempest-pull.py
+++ b/tempest-pull/src/tempest-lava-pull.py
diff --git a/tempest-pull/util.py b/tempest-pull/src/util.py
index b9b3582..b9b3582 100644
--- a/tempest-pull/util.py
+++ b/tempest-pull/src/util.py
diff --git a/tempest-pull/tempest-lava-pull.py b/tempest-pull/tempest-lava-pull.py
new file mode 100644
index 0000000..aa74c50
--- /dev/null
+++ b/tempest-pull/tempest-lava-pull.py
@@ -0,0 +1,149 @@
+import gc
+import os
+import string
+
+from py2neo import Graph, Node, Relationship
+from lava import LAVADashboard
+from bundle import BundleStore
+from bundle import Bundle
+
+# this is the URL of the LAVA instance to retrieve from
+LAVA_XMLRPC_ENDPOINT = "https://openstack.validation.linaro.org/RPC2"
+
+# this is the URL of the Neo4J instance
+NEO4J_ENDPOINT = "http://neo4j:linaro@localhost:7474/db/data/"
+
+# this is the name of the bundle stream to retrieve from
+BUNDLE_STREAM_NAME = "/public/team/openstack/tempest-ci/"
+
+# this is the root location where the raw bundle JSON will be downloaded
+# and stored.  Each bundle will be stored as a single .txt file inside
+# a directory named with the bundle UUID.  Any bundle that already has a
+# directory present will be skipped on subsequent runs.
+BUNDLES_ROOT = os.environ["LAVA_PULL_BUNDLEROOT"] #"/srv/tempest/bundles/"
+
+# this is the root location where the bundle's extracted log data and
+# other files will be written
+LOGS_ROOT = os.environ["LAVA_PULL_LOGROOT"] #"/srv/tempest/logs/"
+
+# these are the important files that are to be brought to the top-level of
+# the output directory
+WHITELIST = {
+   'devstack' : [
+       { 'src' : "stdout.log", 'new-name' : "stdout-devstack.log" }
+   ],
+   'tempest-summary' : [
+       { 'src' : "tempest-summary.txt" }
+   ],
+   'tempest-run' : [
+       { 'src' : "*.log.gz" },
+       { 'src' : "*.txt.gz" },
+       { 'src' : "apache2/*" },
+       { 'src' : "libvirt/*" },
+       { 'src' : "all-tests.txt" },
+       { 'src' : "results.subunit" },
+       { 'src' : "stack.sh.summary.gz" },
+       { 'src' : "stdout.log", 'new-name' : "stdout-tempest.log" },
+       { 'src' : "tempest_conf.txt" }
+   ]
+}
+
+class Neo4JDatabase(object):
+   def __init__(self, endpoint):
+      self.endpoint = endpoint
+
+   def store_bundle(self, bundle):
+      if not isinstance(bundle, Bundle):
+         raise Exception("argument is not a Bundle")
+
+      graph = Graph(self.endpoint)
+
+      print "creating graph for bundle"
+
+      # find (or create) the OS version + distro node
+      os_version = bundle.metadata["lava_job_attributes"]["os-version"]
+      os_distro = bundle.metadata["lava_job_attributes"]["os-distro"]
+      os_name = "%s/%s" % (os_distro, os_version)
+      OS_node = graph.find_one("OS", "name", os_name)
+      if not OS_node:
+         OS_node = Node("OS", name=os_name, distro=os_distro, version=os_version)
+         graph.create(OS_node)
+
+      # find (or create) the devstack branch  node
+      devstack_branch = bundle.metadata["lava_job_attributes"]["devstack-branch"]
+      Branch_node = graph.find_one("Devstack", "name", devstack_branch)
+      if not Branch_node:
+         Branch_node = Node("Devstack", name=devstack_branch)
+         graph.create(Branch_node)
+
+      # create the main tempest run node and associate with the OS and Branch
+      TempestRun_node = Node("TempestRun",          \
+         date = bundle.metadata["date_uploaded"],     \
+         lava_job = bundle.metadata["lava_job_id"],   \
+         sha1 = bundle.metadata["bundle_sha1"])
+      if bundle.all_tests:
+         TempestRun_node.properties["all_tests"] = len(bundle.all_tests)
+      if bundle.tests_run:
+         TempestRun_node.properties["tests_run"] = len(bundle.tests_run)
+      if bundle.failing_tests:
+         TempestRun_node.properties["failing_tests"] = len(bundle.failing_tests)
+      if bundle.skipped_tests:
+         TempestRun_node.properties["skipped_tests"] = len(bundle.skipped_tests)
+      if bundle.passing_tests:
+         TempestRun_node.properties["passing_tests"] = len(bundle.passing_tests)
+      OS_relationship = Relationship(TempestRun_node, "ON", OS_node)
+      Branch_relationship = Relationship(TempestRun_node, "USING", Branch_node)
+      graph.create(TempestRun_node, OS_relationship, Branch_relationship)
+
+      # create all of the tests and relate them back to the tempest node
+      for test_set in [bundle.failing_tests, bundle.passing_tests, bundle.skipped_tests]:
+         if test_set:
+            print "adding tests"
+            for test in test_set:
+               Test_node = Node("Test", test["status"],  \
+                  name=test["test"], \
+                  status=test["status"], \
+                  start_time=test["start_time"], \
+                  stop_time=test["stop_time"], \
+                  test_class=test["class"])
+               Test_relationship = Relationship(TempestRun_node, \
+                  "HAS_TEST", Test_node, status=test["status"])
+               graph.create(Test_node, Test_relationship)
+
+
+
+
+def main():
+   lava = LAVADashboard(LAVA_XMLRPC_ENDPOINT, BUNDLE_STREAM_NAME)
+   lava.connect()
+
+   store = BundleStore(BUNDLES_ROOT)
+   store_sha1_list = store.bundle_list()
+
+   bundle_sha1_list, bundle_list = lava.server_bundle_list()
+   for entry in bundle_list:
+      sha1 = entry["content_sha1"]
+      if sha1 in store_sha1_list:
+         print "[%s] skipping, already processed" % sha1
+         continue
+
+      print "------------------------------------------------------------------------------------------------------"
+
+      print "downloading new entry:"
+      print entry
+
+      bundle = lava.retrieve_bundle(entry)
+
+      print "[%s]:" % sha1
+      
+      metadata = bundle.expand(WHITELIST, LOGS_ROOT)
+
+      database = Neo4JDatabase(NEO4J_ENDPOINT)
+      database.store_bundle(bundle)
+
+      store.write_bundle_receipt(bundle)
+      del bundle
+      gc.collect()
+
+
+main()