reorg, update web app to create charts based on queries and not hard-coded categorization

3 files changed, 134 insertions, 197 deletions

diff --git a/tempest-pull/app/neo4j.py b/tempest-pull/app/neo4j.py
index 517e171..2415d07 100644
--- a/tempest-pull/app/neo4j.py
+++ b/tempest-pull/app/neo4j.py
@@ -30,14 +30,14 @@ class Neo4JDatabase(object):
       # axis:
       # find (or create) the devstack branch node
       devstack_branch = bundle.metadata["lava_job_attributes"]["devstack-branch"]
-      Branch_node = graph.find_one("Devstack", "name", devstack_branch)
+      Branch_node = graph.find_one("Branch", "name", devstack_branch)
       if not Branch_node:
-         Branch_node = Node("Devstack", name=devstack_branch)
+         Branch_node = Node("Branch", name=devstack_branch)
          graph.create(Branch_node)
 
       # create the main tempest run node and associate with the OS and Branch
       
-      TempestRun_node = Node("TempestRun",                   \
+      TempestRun_node = Node("Run", "Tempest",                \
          date = bundle.metadata["date_uploaded"],            \
          epoch_time = bundle.metadata["timestamp_uploaded"],  \
          lava_job = bundle.metadata["lava_job_id"],          \
diff --git a/web-app/server.go b/web-app/server.go
index eedf86a..916be35 100644
--- a/web-app/server.go
+++ b/web-app/server.go
@@ -19,66 +19,62 @@ var (
 )
 
 
-func Results_AllDevstack(w http.ResponseWriter, r *http.Request) {
-        w.Header().Set("Content-Type", "application/json")
+func Results_AllCategorization(w http.ResponseWriter, r *http.Request) {
 
-        db, err := neoism.Connect(*neo4j_server)
+	unionres := struct {
+		Branch interface{}
+		OS interface{}
+	}{}
+
+	db, err := neoism.Connect(*neo4j_server)
         if err != nil {
                 log.Println("error connecting to database: ", err)
-        }
+        }       
+        
+	// branches
 
-        res := [] struct {
+        resBranches := [] struct {
                 Name string `json:"n.name"`
-        }{}
-
-        cq := neoism.CypherQuery{
-                Statement: `MATCH (n:Devstack) RETURN n.name`,
-                Result: &res,
-        }
-
-        err = db.Cypher(&cq)
+                Occurances int32 `json:"occurances"`
+        }{}     
+        
+        cq1 := neoism.CypherQuery{
+                Statement: `MATCH (n:Branch)-[:USING]-(x) RETURN n.name, count(x) as occurances ORDER BY occurances DESC`,
+                Result: &resBranches,
+        }       
+        
+        err = db.Cypher(&cq1)
         if err != nil {
                 log.Println("query error: ", err)
-        }
+        } 
 
-        //log.Printf("%#v", res)
+	// OSes
 
-        enc := json.NewEncoder(w)
-        enc.Encode(res)
-}
-
-
-func Results_AllOS(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Content-Type", "application/json")
-
-	db, err := neoism.Connect(*neo4j_server)
-	if err != nil {
-		log.Println("error connecting to database: ", err)
-	}
-
-	res := [] struct {
+        resOSes := [] struct {
                 Name string `json:"n.name"`
                 Version string `json:"n.version"`
                 Distro string `json:"n.distro"`
-        }{}
-
-	cq := neoism.CypherQuery{
-		Statement: `MATCH (n:OS) RETURN n.name, n.version, n.distro`,
-		Result: &res,
-	}
-
-	err = db.Cypher(&cq)
-	if err != nil {
-		log.Println("query error: ", err)
-	}
-
-        //log.Printf("%#v", res)
+                Occurances int32 `json:"occurances"`
+        }{}     
+        
+        cq2 := neoism.CypherQuery{
+                Statement: `MATCH (n:OS)-[:ON]-(x) RETURN n.name, n.version, n.distro, count(x) as occurances ORDER BY occurances DESC`,
+                Result: &resOSes,
+        }       
+        
+        err = db.Cypher(&cq2)
+        if err != nil {
+                log.Println("query error: ", err)
+        }
 
-	enc := json.NewEncoder(w)
-	enc.Encode(res)
+      	
+	unionres.Branch = resBranches
+	unionres.OS = resOSes
+        
+        enc := json.NewEncoder(w)
+        enc.Encode(unionres)
 }
 
-
 func Results_Tempest_Summary(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
 
@@ -90,15 +86,15 @@ func Results_Tempest_Summary(w http.ResponseWriter, r *http.Request) {
         res := [] struct {
                 Number_Of_Runs int32 `json:"Number_Of_Runs"`
                 OS_Name string `json:"OS_Name"`
-                Devstack_Branch string `json:"Devstack_Branch"`
+                Branch string `json:"Branch"`
                 Average_Pct_Failing float64 `json:"Average_Pct_Failing"`
 		Average_Pct_Passing float64 `json:"Average_Pct_Passing"`
 		Average_Pct_Skipped float64 `json:"Average_Pct_Skipped"`
         }{}
 
         cq := neoism.CypherQuery{
-                Statement: `MATCH (d:Devstack)<-[:USING]-(t:TempestRun)-[:ON]->(o:OS)
-				RETURN count(t) as Number_Of_Runs, o.name as OS_Name, d.name as Devstack_Branch,
+                Statement: `MATCH (b:Branch)<-[:USING]-(t:Tempest:Run)-[:ON]->(o:OS)
+				RETURN count(t) as Number_Of_Runs, o.name as OS_Name, b.name as Branch,
 				ROUND(avg(toFloat(t.failing_tests)/t.tests_run) * 100) as Average_Pct_Failing,
 				ROUND(avg(toFloat(t.passing_tests)/t.tests_run) * 100) as Average_Pct_Passing,
 				ROUND(avg(toFloat(t.skipped_tests)/t.tests_run) * 100) as Average_Pct_Skipped`,
@@ -155,17 +151,17 @@ func Results_Tempest_SinglePermutation(w http.ResponseWriter, r *http.Request) {
                 Skipped_tests int32 `json:"t.skipped_tests"`
 		OS_Distro string `json:"o.distro"`
 		OS_Version string `json:"o.version"`
-		Devstack_Branch string `json:"d.name"`
+		Branch string `json:"b.name"`
                 Epoch_time int64 `json:"t.epoch_time"`
         }{}
 
         cq := neoism.CypherQuery{
-                Statement: `MATCH (d:Devstack)<-[:USING]-(t:TempestRun)-[:ON]->(o:OS)
+                Statement: `MATCH (b:Branch)<-[:USING]-(t:Tempest:Run)-[:ON]->(o:OS)
 				WHERE o.distro = {osdistro} AND o.version = {osversion}
-                                AND d.name = {branch}
+                                AND b.name = {branch}
 				RETURN t.lava_job, t.date, t.sha1, t.all_tests,
 				t.passing_tests, t.failing_tests, t.tests_run, t.skipped_tests,
-				t.epoch_time, o.distro, o.version, d.name
+				t.epoch_time, o.distro, o.version, b.name
 				ORDER BY t.epoch_time DESC
                                 LIMIT {count}`,
                 Parameters: neoism.Props{"count": count, "osdistro": mux.Vars(r)["osdistro"], "osversion": mux.Vars(r)["osversion"], "branch": mux.Vars(r)["branch"]},
@@ -217,16 +213,16 @@ func Results_Tempest_LastN(w http.ResponseWriter, r *http.Request) {
                 Skipped_tests int32 `json:"t.skipped_tests"`
 		OS_Distro string `json:"o.distro"`
 		OS_Version string `json:"o.version"`
-		Devstack_Branch string `json:"d.name"`
+		Branch string `json:"b.name"`
                 Epoch_time int64 `json:"t.epoch_time"`
         }{}
 
         cq := neoism.CypherQuery{
-                Statement: `MATCH (d:Devstack)<-[:USING]-(t:TempestRun)-[:ON]->(o:OS)
+                Statement: `MATCH (b:Branch)<-[:USING]-(t:Tempest:Run)-[:ON]->(o:OS)
                                 RETURN t.lava_job, t.date, t.sha1, t.all_tests,
                                         t.passing_tests, t.failing_tests,
                                         t.tests_run, t.skipped_tests,
-                                        t.epoch_time, o.distro, o.version, d.name
+                                        t.epoch_time, o.distro, o.version, b.name
                                 ORDER BY t.epoch_time DESC
                                 LIMIT {count}`,
                 Parameters: neoism.Props{"count": count},
@@ -270,17 +266,17 @@ func Results_Tempest_Job_Summary(w http.ResponseWriter, r *http.Request) {
                 Skipped_tests int32 `json:"t.skipped_tests"`
                 OS_Distro string `json:"o.distro"`
                 OS_Version string `json:"o.version"`
-                Devstack_Branch string `json:"d.name"`
+                Branch string `json:"b.name"`
                 Epoch_time int64 `json:"t.epoch_time"`
         }{}
 
 	cq := neoism.CypherQuery{
-		Statement: `MATCH (d:Devstack)<-[:USING]-(t:TempestRun)-[:ON]->(o:OS)
+		Statement: `MATCH (b:Branch)<-[:USING]-(t:Tempest:Run)-[:ON]->(o:OS)
 				WHERE t.lava_job = {job}
                         	RETURN t.lava_job, t.date, t.sha1, t.all_tests,
                                 	t.passing_tests, t.failing_tests,
                                 	t.tests_run, t.skipped_tests,
-                                	t.epoch_time, o.distro, o.version, d.name`,
+                                	t.epoch_time, o.distro, o.version, b.name`,
 		Parameters: neoism.Props{"job": job},
 		Result: &res,
 	}
@@ -360,16 +356,16 @@ func Results_Tempest_OS(w http.ResponseWriter, r *http.Request) {
                 Skipped_tests int32 `json:"t.skipped_tests"`
 		OS_Distro string `json:"o.distro"`
 		OS_Version string `json:"o.version"`
-		Devstack_Branch string `json:"d.name"`
+		Branch string `json:"b.name"`
                 Epoch_time int64 `json:"t.epoch_time"`
         }{}
 
         cq := neoism.CypherQuery{
-                Statement: `MATCH (d:Devstack)<-[:USING]-(t:TempestRun)-[:ON]->(o:OS)
+                Statement: `MATCH (b:Branch)<-[:USING]-(t:Tempest:Run)-[:ON]->(o:OS)
 				WHERE o.distro = {osdistro} AND o.version = {osversion}
 				RETURN t.lava_job, t.date, t.sha1, t.all_tests,
 				t.passing_tests, t.failing_tests, t.tests_run, t.skipped_tests,
-				t.epoch_time, o.distro, o.version, d.name
+				t.epoch_time, o.distro, o.version, b.name
 				ORDER BY t.epoch_time DESC
                                 LIMIT {count}`,
                 Parameters: neoism.Props{"count": count, "osdistro": mux.Vars(r)["osdistro"], "osversion": mux.Vars(r)["osversion"]},
@@ -426,7 +422,7 @@ func Results_Tempest_Job_Failures(w http.ResponseWriter, r *http.Request) {
         }{}     
         
         cq := neoism.CypherQuery{
-                Statement: `MATCH (n:TempestRun)-[:HAS_TEST {status:"failure"}]-(t:Test)
+                Statement: `MATCH (n:Tempest:Run)-[:HAS_TEST {status:"failure"}]-(t:Test)
 				WHERE n.lava_job = {job}
 				RETURN t.name, t.status, t.start_time, t.stop_time, t.test_class`,
 		Parameters: neoism.Props{"job" : job},
@@ -468,7 +464,7 @@ func Results_Tempest_Job_Skipped(w http.ResponseWriter, r *http.Request) {
         }{}
 
         cq := neoism.CypherQuery{
-                Statement: `MATCH (n:TempestRun)-[:HAS_TEST {status:"skip"}]-(t:Test)
+                Statement: `MATCH (n:Tempest:Run)-[:HAS_TEST {status:"skip"}]-(t:Test)
                                 WHERE n.lava_job = {job}
                                 RETURN t.name, t.status, t.start_time, t.stop_time, t.test_class`,
                 Parameters: neoism.Props{"job" : job},
@@ -496,8 +492,7 @@ func main() {
         s := r.Methods("GET").PathPrefix("/results").Subrouter()	
 
 	// global data
-	s.HandleFunc("/devstack-branches", Results_AllDevstack)
-	s.HandleFunc("/operating-systems", Results_AllOS)
+	s.HandleFunc("/all-categorization", Results_AllCategorization)
 
 	// tempest
 	s.HandleFunc("/tempest", Results_Tempest_SinglePermutation).Queries("count", "{count:[0-9]+}", "branch", "{branch}", "osdistro", "{osdistro}", "osversion", "{osversion}")
@@ -506,8 +501,8 @@ func main() {
         s.HandleFunc("/tempest/jobs", Results_Tempest_AllJobIds)     // DONE
 	s.HandleFunc("/tempest/operating-system/{osdistro}/{osversion}", Results_Tempest_OS).Queries("count", "{count:[0-9]+}")
 	s.HandleFunc("/tempest/operating-system/{osdistro}/{osversion}", Results_Tempest_OS_Summary)
-        s.HandleFunc("/tempest/devstack-branch/{branch}", Results_Tempest_Branch).Queries("count", "{count:[0-9]+}")
-	s.HandleFunc("/tempest/devstack-branch/{branch}", Results_Tempest_Branch_Summary)
+        s.HandleFunc("/tempest/branch/{branch}", Results_Tempest_Branch).Queries("count", "{count:[0-9]+}")
+	s.HandleFunc("/tempest/branch/{branch}", Results_Tempest_Branch_Summary)
 	s.HandleFunc("/tempest/job/{job:[0-9]+}", Results_Tempest_Job_Summary)      // DONE
 	s.HandleFunc("/tempest/job/{job:[0-9]+}/failures", Results_Tempest_Job_Failures)    // DONE
 	s.HandleFunc("/tempest/job/{job:[0-9]+}/skipped", Results_Tempest_Job_Skipped)    // DONE	
diff --git a/web-app/static/index.html b/web-app/static/index.html
index 5d563db..9024e59 100644
--- a/web-app/static/index.html
+++ b/web-app/static/index.html
@@ -13,155 +13,99 @@
   </style>
 
 <script type="text/javascript">
-window.onload = function () {
 
-   // ubuntu/trusty
-   $.getJSON( "/results/tempest?count=60&branch=master&osdistro=ubuntu&osversion=trusty", function( data ) {
-
-      var failureDataPoints = [];
-      var skippedDataPoints = [];
-      var passingDataPoints = [];
-      
-      $.each(data, function (index,value) {
-         if (value['t.all_tests'] > 0) {
-            failureDataPoints.unshift({label: value['t.lava_job'], y: value['t.failing_tests']});
-            skippedDataPoints.unshift({label: value['t.lava_job'], y: value['t.skipped_tests']});
-            passingDataPoints.unshift({label: value['t.lava_job'], y: value['t.passing_tests']});
-         }
-       });
-
-         var chart = new CanvasJS.Chart("chartTrustyOverview",
-          {
-            title:{
-              text: "Devstack \"master\" on \"ubuntu/trusty\" (up to last 60 results)"
-            },
-            legend: {
-            cursor: "pointer",
-            itemclick: function (e) {
-                //console.log("legend click: " + e.dataPointIndex);
-                //console.log(e);
-                if (typeof (e.dataSeries.visible) === "undefined" || e.dataSeries.visible) {
-                    e.dataSeries.visible = false;
-                } else {
-                    e.dataSeries.visible = true;
-                }
-
-                e.chart.render();
-            }
-        },
-            axisY:{title: "# of Tests" },
-            axisX:{title: "Test Job", prefix: "Job " }, 
-            toolTip:{
-               content: function(e){
-                  var content;
-                  content = e.entries[0].dataSeries.name + " <strong>"+e.entries[0].dataPoint.y  ;
-                  return content;
-               }            
-            },
-            animationEnabled: true, zoomEnabled: true, 
-            data: [
-             {
-                name: "Skipped Tests", showInLegend: true,
-                type: "stackedColumn",
-                color: "rgba(54,158,173,.7)",
-                dataPoints: skippedDataPoints
-             },
-             {
-                name: "Failing Tests", showInLegend: true,
-                type: "stackedColumn",
-                color: "rgba(194,70,66,.7)",
-                dataPoints: failureDataPoints
-             },
-             {
-                name: "Passing Tests", showInLegend: true,
-                type: "stackedColumn",
-                color: "rgba(54,200,30,.7)",
-                dataPoints: passingDataPoints
-             }              
-            ]
-          });
-
-         chart.render();
+$(function() {
+   // get the set of categorization data
+   $.getJSON("/results/all-categorization", function(data) {
+      data['Branch'].forEach(function(value, index) {
+         var branch = value['n.name'];
+         data['OS'].forEach(function(value, index) {
+            var distro = value['n.distro'];
+            var version = value['n.version'];
+            var div_name = branch + ":" + distro + ":" + version;
+            
+            $("<div>", {
+               'id': div_name,
+               css: { "height": "450px", "width": "90%" }
+            }).appendTo($("#chartContainer"));
+            $("<br>").appendTo($("#chartContainer"));
+
+            create_chart(div_name, branch, distro, version, 60);
+         });
+      });
    });
+});
 
+function create_chart(div_name, branch, distro, version, max_results) {
+   var div_name = branch + ":" + distro + ":" + version;
 
-
-   // ubuntu/vivid
-   $.getJSON( "/results/tempest?count=60&branch=master&osdistro=ubuntu&osversion=vivid", function( data ) {
-
+   $.getJSON( "/results/tempest?count=" + max_results + "&branch=" + branch + "&osdistro=" + distro + "&osversion=" + version, function(data) {
       var failureDataPoints = [];
       var skippedDataPoints = [];
       var passingDataPoints = [];
       
-      $.each(data, function (index,value) {
+      $.each(data, function (index, value) {
          if (value['t.all_tests'] > 0) {
             failureDataPoints.unshift({label: value['t.lava_job'], y: value['t.failing_tests']});
             skippedDataPoints.unshift({label: value['t.lava_job'], y: value['t.skipped_tests']});
             passingDataPoints.unshift({label: value['t.lava_job'], y: value['t.passing_tests']});
          }
-       });
+      });
 
-         var chart = new CanvasJS.Chart("chartVividOverview",
-          {
-            title:{
-              text: "Devstack \"master\" on \"ubuntu/vivid\" (up to last 60 results)"
+      var chart = new CanvasJS.Chart(div_name,
+         {
+            title: {
+              text: "Branch \"" + branch + "\" on \"" + distro + "/" + version + "\" (up to last " + max_results + " results)"
             },
-legend: {
-            cursor: "pointer",
-            itemclick: function (e) {
-                //console.log("legend click: " + e.dataPointIndex);
-                //console.log(e);
-                if (typeof (e.dataSeries.visible) === "undefined" || e.dataSeries.visible) {
-                    e.dataSeries.visible = false;
-                } else {
-                    e.dataSeries.visible = true;
-                }
-
-                e.chart.render();
-            }
-        },
-            axisY:{title: "# of Tests" },
-            axisX:{title: "Test Job", prefix: "Job " },
-            toolTip:{
-               content: function(e){
+            legend: {
+               cursor: "pointer",
+               itemclick: function (e) {
+                  if (typeof (e.dataSeries.visible) === "undefined" || e.dataSeries.visible) {
+                     e.dataSeries.visible = false;
+                  } else {
+                     e.dataSeries.visible = true;
+                  }
+                  e.chart.render();
+               }
+            },
+            axisY: {title: "# of Tests" },
+            axisX: {title: "Test Job", prefix: "Job " }, 
+            toolTip: { 
+               content: function(e) {
                   var content;
-                  content = e.entries[0].dataSeries.name + " <strong>"+e.entries[0].dataPoint.y  ;
+                  content = e.entries[0].dataSeries.name + " <strong>"+ e.entries[0].dataPoint.y  ;
                   return content;
                }            
             },
-            animationEnabled: true, zoomEnabled: true, 
+            animationEnabled: true,
+            zoomEnabled: true, 
             data: [
-             {
-        
-          click: function(e){
-        alert(  "dataSeries Event => Type: "+ e.dataSeries.type+ ", dataPoint { x:" + e.dataPoint.x + ", y: "+ e.dataPoint.y + " }" );
-        },
-                name: "Skipped Tests", showInLegend: true,
-                type: "stackedColumn",
-                color: "rgba(54,158,173,.7)",
-                dataPoints: skippedDataPoints
-             },
-             {
-                name: "Failing Tests", showInLegend: true,
-                type: "stackedColumn",
-                color: "rgba(194,70,66,.7)",
-                dataPoints: failureDataPoints
-             },
-             {
-                name: "Passing Tests", showInLegend: true,
-                type: "stackedColumn",
-                color: "rgba(54,200,30,.7)",
-                dataPoints: passingDataPoints
-             }              
+               {
+                  name: "Skipped Tests", showInLegend: true,
+                  type: "stackedColumn",
+                  color: "rgba(54,158,173,.7)",
+                  dataPoints: skippedDataPoints
+               },
+               {
+                  name: "Failing Tests", showInLegend: true,
+                  type: "stackedColumn",
+                  color: "rgba(194,70,66,.7)",
+                  dataPoints: failureDataPoints
+               },
+               {
+                  name: "Passing Tests", showInLegend: true,
+                  type: "stackedColumn",
+                  color: "rgba(54,200,30,.7)",
+                  dataPoints: passingDataPoints
+               }              
             ]
-          });
+         });
 
          chart.render();
    });
+}
 
-
-  }
-  </script>
+</script>
 
 
 </head>
@@ -173,8 +117,6 @@ legend: {
 Linaro runs Openstack Tempest on arm64 on a nighty schedule.  For more information on the hardware and test methodology, see <a href="https://wiki.linaro.org/OpenStack/OpenstackTempestCI">https://wiki.linaro.org/OpenStack/OpenstackTempestCI</a>.
 </p>
 <p>In the charts below, you can click on the data series labels in the chart legends to show/hide the series.</p>
-<div id="chartTrustyOverview" style="height: 450px; width: 90%;"></div>
-<br/>
-<div id="chartVividOverview" style="height: 450px; width: 90%;"></div>
+<div id="chartContainer"></div>
 </body>
 </html>