From 10e31ddc25ac57dc427c204e648742b0be94cd51 Mon Sep 17 00:00:00 2001 From: Clark Laughlin Date: Tue, 12 May 2015 14:06:08 -0400 Subject: starting using virtualenv, new graph library --- tempest-pull/bundle.py | 12 ++----- tempest-pull/run | 6 +++- tempest-pull/setup | 5 +++ tempest-pull/tempest-pull.py | 75 +++++++++++++++++++++++--------------------- 4 files changed, 53 insertions(+), 45 deletions(-) mode change 100644 => 100755 tempest-pull/run create mode 100755 tempest-pull/setup diff --git a/tempest-pull/bundle.py b/tempest-pull/bundle.py index a489fa5..6e786e4 100644 --- a/tempest-pull/bundle.py +++ b/tempest-pull/bundle.py @@ -116,7 +116,7 @@ class Bundle(object): # loop through all of the tests in the bundle for test_run in self.data["test_runs"]: test_id = test_run["test_id"] - print "test [%s]:" % test_id + print "processing test [%s]" % test_id # create directories if necessary test_root_path = os.path.join(unimp_path, test_id) @@ -133,18 +133,13 @@ class Bundle(object): matching_whitelist_filter = None filename = attachment["pathname"] - print "- %s" % filename - - if filename in self.skip_files: - print " skipping!" - else: + if not filename in self.skip_files: if test_whitelist: # see if the file matches one of the whitelist patterns for filter in test_whitelist: pattern = filter["src"] if fnmatch.fnmatch(filename, pattern): matching_whitelist_filter = filter - print " whitelist: [%s] using pattern [%s]" % (filename, pattern) break # build the full path to the output file, assuming it @@ -181,8 +176,7 @@ class Bundle(object): json.dump(bundle_metadata, f) # touch the directory with the original creation date - print "setting date on [%s]" % full_root_path - print subprocess.check_output(["touch", "--date=%s" % self.upload_date, full_root_path]) + subprocess.check_output(["touch", "--date=%s" % self.upload_date, full_root_path]) self.metadata = bundle_metadata diff --git a/tempest-pull/run b/tempest-pull/run old mode 100644 new mode 100755 index a0da69d..11c5ca4 --- a/tempest-pull/run +++ b/tempest-pull/run @@ -1,4 +1,8 @@ +#!/bin/bash + export LAVA_PULL_BUNDLEROOT=~/lava_pull/bundles/ export LAVA_PULL_LOGROOT=~/lava_pull/logs/ - +source venv/bin/activate python tempest-pull.py +deactivate + diff --git a/tempest-pull/setup b/tempest-pull/setup new file mode 100755 index 0000000..cca7e75 --- /dev/null +++ b/tempest-pull/setup @@ -0,0 +1,5 @@ +#!/bin/bash +virtualenv venv +source venv/bin/activate +pip install py2neo + diff --git a/tempest-pull/tempest-pull.py b/tempest-pull/tempest-pull.py index 2b85ed9..c3fa1c5 100644 --- a/tempest-pull/tempest-pull.py +++ b/tempest-pull/tempest-pull.py @@ -2,8 +2,7 @@ import gc import os import string -from neo4jrestclient import client -from neo4jrestclient.client import GraphDatabase +from py2neo import Graph, Node, Relationship from lava import LAVADashboard from bundle import BundleStore from bundle import Bundle @@ -12,7 +11,7 @@ from bundle import Bundle LAVA_XMLRPC_ENDPOINT = "https://openstack.validation.linaro.org/RPC2" # this is the URL of the Neo4J instance -NEO4J_ENDPOINT = "http://localhost:7474" +NEO4J_ENDPOINT = "http://neo4j:linaro@localhost:7474/db/data/" # this is the name of the bundle stream to retrieve from BUNDLE_STREAM_NAME = "/public/team/openstack/tempest-ci/" @@ -56,48 +55,54 @@ class Neo4JDatabase(object): def store_bundle(self, bundle): if not isinstance(bundle, Bundle): raise Exception("argument is not a Bundle") - gdb = GraphDatabase(self.endpoint, username="neo4j", password="linaro") + + graph = Graph(self.endpoint) # find (or create) the OS version + distro node - OS = None - results = gdb.query("match(n:OS {name:\"%s\", distro:\"%s\"}) return n" % \ - (bundle.metadata["lava_job_attributes"]["os-version"], bundle.metadata["lava_job_attributes"]["os-distro"]), \ - returns=client.Node) - if len(results) == 1: - OS = results[0] - else: - OS = gdb.nodes.create( - name=bundle.metadata["lava_job_attributes"]["os-version"], \ - distro=bundle.metadata["lava_job_attributes"]["os-distro"]) - OS.labels.add("OS") + os_version = bundle.metadata["lava_job_attributes"]["os-version"] + os_distro = bundle.metadata["lava_job_attributes"]["os-distro"] + os_name = "%s/%s" % (os_distro, os_version) + OS_node = graph.find_one("OS", "name", os_name) + if not OS_node: + print "creating OS node" + OS_node = Node("OS", name=os_name, distro=os_distro, version=os_version) + graph.create(OS_node) # find (or create) the devstack branch node - Branch = None - results = gdb.query("match(n:Branch {name:\"%s\"}) return n" % \ - bundle.metadata["lava_job_attributes"]["devstack-branch"], \ - returns=client.Node) - if len(results) == 1: - Branch = results[0] - else: - Branch = gdb.nodes.create( - name=bundle.metadata["lava_job_attributes"]["devstack-branch"]) - Branch.labels.add("Branch") - - # create the main tempest run node - TempestRun = gdb.nodes.create( \ - date=bundle.metadata["date_uploaded"], \ - lava_job=bundle.metadata["lava_job_id"], \ - sha1=bundle.metadata["bundle_sha1"]) - TempestRun.labels.add("TempestRun") - TempestRun.relationships.create("ON", OS) - TempestRun.relationships.create("USING", Branch) + devstack_branch = bundle.metadata["lava_job_attributes"]["devstack-branch"] + Branch_node = graph.find_one("Devstack", "name", devstack_branch) + if not Branch_node: + print "creating Branch node" + Branch_node = Node("Devstack", name=devstack_branch) + graph.create(Branch_node) + + # create the main tempest run node and associate with the OS and Branch + TempestRun_node = Node("TempestRun", \ + date = bundle.metadata["date_uploaded"], \ + lava_job = bundle.metadata["lava_job_id"], \ + sha1 = bundle.metadata["bundle_sha1"]) + if bundle.all_tests: + TempestRun_node.properties["all_tests"] = len(bundle.all_tests) + if bundle.tests_run: + TempestRun_node.properties["tests_run"] = len(bundle.tests_run) + if bundle.failing_tests: + TempestRun_node.properties["failing_tests"] = len(bundle.failing_tests) + if bundle.skipped_tests: + TempestRun_node.properties["skipped_tests"] = len(bundle.skipped_tests) + if bundle.passing_tests: + TempestRun_node.properties["passing_tests"] = len(bundle.passing_tests) + OS_relationship = Relationship(TempestRun_node, "ON", OS_node) + Branch_relationship = Relationship(TempestRun_node, "USING", Branch_node) + graph.create(TempestRun_node, OS_relationship, Branch_relationship) + + return # create all of the tests and relate them back to the tempest node for test_set in [bundle.failing_tests, bundle.passing_tests, bundle.skipped_tests]: if test_set: for test in test_set: Test = gdb.nodes.create( \ - test=test["test"], \ + name=test["test"], \ status=test["status"], \ start_time=test["start_time"], \ stop_time=test["stop_time"], \ -- cgit v1.2.3