Allow for multiple pages and use vector operations rather than peck though the tables

1 files changed, 120 insertions, 99 deletions

diff --git a/member_heatmaps.R b/member_heatmaps.R
index 58e2715..e7fea24 100644
--- a/member_heatmaps.R
+++ b/member_heatmaps.R
@@ -4,6 +4,8 @@ library(jsonlite)
 library(gsubfn)
 library(gplots)
 library(RColorBrewer)
+library(plyr)
+library(tibble)
 
 #functions
 companies <- function (names)
@@ -25,8 +27,12 @@ display_heatmap <- function(report, m, suffix) {
   png(file=report_name, title = report["name"])
   
   #display raw data
+
+  cat("\nGenerating", report_name,'\n')
   print(m)
-  cat("Generating", report_name,'\n')
+  
+  #calucaltions require a matix
+  m <- as.matrix(m)
   
   # following code limits the lowest and highest color to 5%, and 95% of your range, respectively
   quantile.range <- quantile(m, probs = seq(0, 1, 0.01))
@@ -58,7 +64,7 @@ watcher_heatmap <- function(report) {
   fields <- "&fields=watches,key,project"
   
   type <- "search?"
-  jql <- paste ("jql=project+in+(",report["team"],")+AND+issuetype+in+(Initiative,Epic)&maxResults=2000", sep = "")
+  jql <- paste ("jql=project+in+(",report["team"],")+AND+issuetype+in+(Initiative,Epic)&maxResults=1000", sep = "")
   
   url <- paste(baseurl, type, jql, fields, sep = "")
   cat (url,"\n")
@@ -97,7 +103,6 @@ watcher_heatmap <- function(report) {
   num_comp <- length(comp)
   m <- matrix(NA, nrow = num_comp, ncol = num_projects)
   colnames(m) <- projects
-  rownames(m) <- comp
   m[] <- 0L
   
   #from the website now get watchers names with another query for all keys found previously
@@ -140,68 +145,76 @@ sponsor_heatmap <- function (report) {
   fields <- "&fields=customfield_10101,key,project"
   type <- "search?"
   
-  jql <-
-    paste(
-      "jql=project+in+(",
-      report["team"],
-      ")+AND+issuetype+in+(Initiative,Epic)&maxResults=1000",
-      sep = ""
-    )
-  url <- paste(baseurl, type, jql, fields, sep = "")
-  cat (url,"\n")
-  
-  webpage <- curl (url, handle = h)
-  
-  #get data it into R
-  jira_raw <- fromJSON(webpage, simplifyDataFrame = TRUE)
+  page <- 0
+  #empty container for all data returned
+  jira_raw=data.frame()
+  
+  repeat {
+    jql <-
+      paste(
+        "jql=project+in+(",
+        report["team"],
+        ")+AND+issuetype+in+(Initiative,Epic)&maxResults=1000",
+        "&page=",
+        page,
+        sep = ""
+      )
+    url <- paste(baseurl, type, jql, fields, sep = "")
+    webpage <- curl (url, handle = h)
+    
+    #get data it into R
+    new_data <- fromJSON(webpage, simplifyDataFrame = TRUE)
+    new_data_issues <- data.frame(key = new_data$issues$key,
+                                  project_key = new_data$issues$fields$project$key)
+    new_data_issues$sponsor <-
+      sapply(new_data$issues$fields$customfield_10101,
+             function(x) unlist(x[2])  )
+    
+    jira_raw <- rbind(jira_raw, new_data_issues)
+    
+    #get following pages if there are any
+    page <- page + 1
+    
+    if (new_data$total < (new_data$maxResults * new_data$maxResults))
+      break
+  } 
+
+  rm(new_data)
+  rm(new_data_issues)
   #list of every issue
-  keys <- jira_raw$issues$key
+  keys <- jira_raw$key
   
   #x axis the number of projects
-  projects <-  unique(jira_raw$issues$fields$project$key)
+  projects <-  unique(jira_raw$project_key)
   num_projects <- length(projects)
   
   #y axis the companies or sponsors
-  sponsor <- jira_raw$issues$fields$customfield_10101
-  
-  names = list()
-  for (key in keys) {
-    s <- sponsor[[match (key, keys)]][[2]]
-    for (n in s) {
-      names[[length(names) + 1]] <- n
-    }
-  }
+  sponsor <- jira_raw$sponsor
   
   #create empty matrix
-  comp <- unique(names)
-  num_comp <- length(comp)
-  m <- matrix(NA, nrow = num_comp, ncol = num_projects)
-  colnames(m) <- projects
-  rownames(m) <- comp
-  m[] <- 0L
-  
+  m <- data.frame()
+  m$Company <- rownames(m)
+
   # count companies per project by looking at every key and update matrix
-  
-  for (key in keys) {
-    #y
-    mike <- match (key, keys)
-    team <- jira_raw$issues$fields$project$key[mike]
-    y <- match (team, projects)
+  for (project in unique(jira_raw$project_key)) {
+    team <- jira_raw[jira_raw$project_key == project,]
     
-    #x
-    s <- sponsor[[match (key, keys)]][[2]]
-    for (c in s)
-    {
-      x <- match (c, comp)
-      m[x, y] <- m[x, y] + 1
-    }
+    team <- unlist(team['sponsor'])
+    freq <- data.frame(table(unlist(team)))
+    freq <- rename (freq,c("Freq"=project,"Var1"="Company"))
+    m <- merge(m, freq, all = TRUE)
   }
   
-  #remove companies we don't like ;)
+  #add row names and remove the  colum that was the row names
+  rownames(m) <- m$Company
+  m$Company <- NULL
+
+  #remove companies we don't like ;) - Linaro adds no value to this analysis
   rows.to.delete <- c('Linaro')
-  for (d in rows.to.delete) {
-    m <- as.matrix(m[!grepl(d, rownames(m)),])
-  }
+  m <- m[!rownames(m) %in% rows.to.delete,]
+  
+  #fix the NA's they really mean 0
+  m[is.na(m)] <- 0
 
   display_heatmap(report, m, "_sp")
 
@@ -212,68 +225,76 @@ sponsor_heatmap_current <- function (report) {
   fields <- "&fields=customfield_10101,key,project"
   type <- "search?"
   
-  jql <-
-    paste(
-      "jql=project+in+(",
-      report["team"],
-      ")+AND+status!=Closed+AND+issuetype+in+(Initiative,Epic)&maxResults=1000",
-      sep = ""
-    )
-  url <- paste(baseurl, type, jql, fields, sep = "")
-  cat (url,"\n")
-  
-  webpage <- curl (url, handle = h)
+  page <- 0
+  #empty container for all data returned
+  jira_raw=data.frame()
+  
+  repeat {
+    jql <-
+      paste(
+        "jql=project+in+(",
+        report["team"],
+        ")+AND+status!=Closed+AND+issuetype+in+(Initiative,Epic)&maxResults=1000",
+        "&page=",
+        page,
+        sep = ""
+      )
+    url <- paste(baseurl, type, jql, fields, sep = "")
+    webpage <- curl (url, handle = h)
+    
+    #get data it into R
+    new_data <- fromJSON(webpage, simplifyDataFrame = TRUE)
+    new_data_issues <- data.frame(key = new_data$issues$key,
+                                  project_key = new_data$issues$fields$project$key)
+    new_data_issues$sponsor <-
+      sapply(new_data$issues$fields$customfield_10101,
+             function(x) unlist(x[2])  )
+    
+    jira_raw <- rbind(jira_raw, new_data_issues)
+    
+    #get following pages if there are any
+    page <- page + 1
+    
+    if (new_data$total < (new_data$maxResults * new_data$maxResults))
+      break
+  } 
   
-  #get data it into R
-  jira_raw <- fromJSON(webpage, simplifyDataFrame = TRUE)
+  rm(new_data)
+  rm(new_data_issues)
   #list of every issue
-  keys <- jira_raw$issues$key
+  keys <- jira_raw$key
   
   #x axis the number of projects
-  projects <-  unique(jira_raw$issues$fields$project$key)
+  projects <-  unique(jira_raw$project_key)
   num_projects <- length(projects)
   
   #y axis the companies or sponsors
-  sponsor <- jira_raw$issues$fields$customfield_10101
-  
-  names = list()
-  for (key in keys) {
-    s <- sponsor[[match (key, keys)]][[2]]
-    for (n in s) {
-      names[[length(names) + 1]] <- n
-    }
-  }
+  sponsor <- jira_raw$sponsor
   
   #create empty matrix
-  comp <- unique(names)
-  num_comp <- length(comp)
-  m <- matrix(NA, nrow = num_comp, ncol = num_projects)
-  colnames(m) <- projects
-  rownames(m) <- comp
-  m[] <- 0L
+  m <- data.frame()
+  m$Company <- rownames(m)
   
   # count companies per project by looking at every key and update matrix
-  
-  for (key in keys) {
-    #y
-    mike <- match (key, keys)
-    team <- jira_raw$issues$fields$project$key[mike]
-    y <- match (team, projects)
+  for (project in unique(jira_raw$project_key)) {
+    team <- jira_raw[jira_raw$project_key == project,]
     
-    #x
-    s <- sponsor[[match (key, keys)]][[2]]
-    for (c in s)
-    {
-      x <- match (c, comp)
-      m[x, y] <- m[x, y] + 1
-    }
+    team <- unlist(team['sponsor'])
+    freq <- data.frame(table(unlist(team)))
+    freq <- rename (freq,c("Freq"=project,"Var1"="Company"))
+    m <- merge(m, freq, all = TRUE)
   }
   
-  #remove companies we don't like ;)
+  #add row names and remove the  colum that was the row names
+  rownames(m) <- m$Company
+  m$Company <- NULL
+  
+  #remove companies we don't like ;) - Linaro adds no value to this analysis
   rows.to.delete <- c('Linaro')
-  for (d in rows.to.delete) {
-    m <- as.matrix(m[!grepl(d, rownames(m)),])
-  }
+  m <- m[!rownames(m) %in% rows.to.delete,]
+  
+  #fix the NA's they really mean 0
+  m[is.na(m)] <- 0
   
   display_heatmap(report, m, "_spc")