devapp: add per-release issue tracker dashboard

This imports https://swtch.com/tmp/dash.html and makes it render for any
release. Initially, the only graph is the # of open issues by milestone
over the course of the release.

The dashboard is not currently tracking label history, which is needed
to draw the second graph on that page.

Change-Id: I9bd031f8709701b304e18208ae3c972bdfe3b276
Reviewed-on: https://go-review.googlesource.com/30012
Reviewed-by: Brad Fitzpatrick <[email protected]>

Author: quentinmit

devapp/app.yaml

@@ -26,7 +26,7 @@ handlers:
   static_dir: static
   application_readable: true
   secure: always
-- url: /(|dash|release|cl|stats/raw|stats/svg)
+- url: /(|dash|release|cl|stats/raw|stats/svg|stats/release|stats/release/data.js)
   script: _go_app
   secure: always
 - url: /update.*

devapp/devapp.go

@@ -38,6 +38,8 @@ func init() {
 	// Defined in stats.go
 	http.HandleFunc("/stats/raw", rawHandler)
 	http.HandleFunc("/stats/svg", svgHandler)
+	http.Handle("/stats/release", ctxHandler(release))
+	http.Handle("/stats/release/data.js", ctxHandler(releaseData))
 	http.Handle("/update/stats", ctxHandler(updateStats))
 }
 
@@ -125,6 +127,7 @@ func update(ctx context.Context, w http.ResponseWriter, _ *http.Request) error {
 		if cls {
 			data.PrintCLs(&output)
 		} else {
+			fmt.Fprintf(&output, fmt.Sprintf(`<a href="/stats/release?cycle=%d">Go 1.%d Issue Stats Dashboard</a>`, data.GoReleaseCycle, data.GoReleaseCycle))
 			data.PrintIssues(&output)
 		}
 		var html bytes.Buffer

devapp/stats.go

@@ -7,12 +7,15 @@ package devapp
 import (
 	"encoding/json"
 	"fmt"
+	"html/template"
 	"image/color"
+	"io/ioutil"
 	"math"
 	"net/http"
 	"regexp"
 	"sort"
 	"strconv"
+	"strings"
 	"time"
 
 	"golang.org/x/build/godash"
@@ -69,6 +72,37 @@ func updateStats(ctx context.Context, w http.ResponseWriter, r *http.Request) er
 	return writeCache(ctx, "gzstats", stats)
 }
 
+func release(ctx context.Context, w http.ResponseWriter, req *http.Request) error {
+	req.ParseForm()
+
+	tmpl, err := ioutil.ReadFile("template/release.html")
+	if err != nil {
+		return err
+	}
+
+	t, err := template.New("main").Parse(string(tmpl))
+	if err != nil {
+		return err
+	}
+
+	cycle, _, err := argtoi(req, "cycle")
+	if err != nil {
+		return err
+	}
+	if cycle == 0 {
+		data, err := loadData(ctx)
+		if err != nil {
+			return err
+		}
+		cycle = data.GoReleaseCycle
+	}
+
+	if err := t.Execute(w, struct{ GoReleaseCycle int }{cycle}); err != nil {
+		return err
+	}
+	return nil
+}
+
 func rawHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := appengine.NewContext(r)
 
@@ -143,19 +177,23 @@ func (s countChangeSlice) Less(i, j int) bool { return s[i].t.Before(s[j].t) }
 // were open at each time. This produces columns called "Time" and
 // "Count".
 type openCount struct {
-	// ByRelease will add a Release column and provide counts per release.
-	ByRelease bool
+	// By is the column to group by; if "" all issues will be
+	// grouped together. Only "Release" and "Milestone" are
+	// supported.
+	By string
 }
 
 func (o openCount) F(input table.Grouping) table.Grouping {
 	return table.MapTables(input, func(_ table.GroupID, t *table.Table) *table.Table {
-		releases := make(map[string]countChangeSlice)
+		groups := make(map[string]countChangeSlice)
 		add := func(milestone string, t time.Time, count int) {
-			r := milestoneToRelease(milestone)
-			if r == "" {
-				r = milestone
+			if o.By == "Release" {
+				r := milestoneToRelease(milestone)
+				if r != "" {
+					milestone = r
+				}
 			}
-			releases[r] = append(releases[r], countChange{t, count})
+			groups[milestone] = append(groups[milestone], countChange{t, count})
 		}
 
 		created := t.MustColumn("Created").([]time.Time)
@@ -189,9 +227,9 @@ func (o openCount) F(input table.Grouping) table.Grouping {
 
 		var times []time.Time
 		var counts []int
-		if o.ByRelease {
+		if o.By != "" {
 			var names []string
-			for name, s := range releases {
+			for name, s := range groups {
 				sort.Sort(s)
 				sum := 0
 				for _, c := range s {
@@ -201,10 +239,10 @@ func (o openCount) F(input table.Grouping) table.Grouping {
 					counts = append(counts, sum)
 				}
 			}
-			nt.Add("Release", names)
+			nt.Add(o.By, names)
 		} else {
 			var all countChangeSlice
-			for _, s := range releases {
+			for _, s := range groups {
 				all = append(all, s...)
 			}
 			sort.Sort(all)
@@ -308,25 +346,34 @@ func plot(w http.ResponseWriter, req *http.Request, stats table.Grouping) error
 	}
 	switch pivot := req.Form.Get("pivot"); pivot {
 	case "opencount":
-		byRelease := req.Form.Get("group") == "release"
-		plot.Stat(openCount{ByRelease: byRelease})
-		if byRelease {
-			plot.GroupBy("Release")
+		o := openCount{}
+		switch by := req.Form.Get("group"); by {
+		case "release":
+			o.By = "Release"
+		case "milestone":
+			o.By = "Milestone"
+		case "":
+		default:
+			return fmt.Errorf("unknown group %q", by)
+		}
+		plot.Stat(o)
+		if o.By != "" {
+			plot.GroupBy(o.By)
 		}
 		plot.SortBy("Time")
 		lp := gg.LayerPaths{
 			X: "Time",
 			Y: "Count",
 		}
-		if byRelease {
-			lp.Color = "Release"
+		if o.By != "" {
+			lp.Color = o.By
 		}
 		plot.Add(gg.LayerSteps{LayerPaths: lp})
-		if byRelease {
+		if o.By != "" {
 			plot.Add(gg.LayerTooltips{
 				X:     "Time",
 				Y:     "Count",
-				Label: "Release",
+				Label: o.By,
 			})
 		}
 	case "":
@@ -456,3 +503,99 @@ func plot(w http.ResponseWriter, req *http.Request, stats table.Grouping) error
 	plot.WriteSVG(w, 1200, 600)
 	return nil
 }
+
+func releaseData(ctx context.Context, w http.ResponseWriter, req *http.Request) error {
+	req.ParseForm()
+
+	stats := &godash.Stats{}
+	if err := loadCache(ctx, "gzstats", stats); err != nil {
+		return err
+	}
+
+	cycle, _, err := argtoi(req, "cycle")
+	if err != nil {
+		return err
+	}
+	if cycle == 0 {
+		data, err := loadData(ctx)
+		if err != nil {
+			return err
+		}
+		cycle = data.GoReleaseCycle
+	}
+
+	prefix := fmt.Sprintf("Go1.%d", cycle)
+
+	w.Header().Set("Content-Type", "application/javascript")
+
+	g := gdstats.IssueStats(stats)
+	g = openCount{By: "Milestone"}.F(g)
+	g = table.Filter(g, func(m string) bool { return strings.HasPrefix(m, prefix) }, "Milestone")
+	g = table.SortBy(g, "Time")
+
+	// Dump data; remember that each row only affects one count, so we need to hold the counts from the previous row. Kind of like Pivot.
+	data := [][]interface{}{{"Date"}}
+	counts := make(map[string]int)
+	var (
+		maxt time.Time
+		maxc int
+	)
+	for _, gid := range g.Tables() {
+		// Find all the milestones that exist
+		ms := g.Table(gid).MustColumn("Milestone").([]string)
+		for _, m := range ms {
+			counts[m] = 0
+		}
+		// Find the peak of the graph
+		ts := g.Table(gid).MustColumn("Time").([]time.Time)
+		cs := g.Table(gid).MustColumn("Count").([]int)
+		for i, c := range cs {
+			if c > maxc {
+				maxc = c
+				maxt = ts[i]
+			}
+		}
+	}
+
+	// Only show the most recent 6 months of data.
+	start := maxt.Add(time.Duration(-6 * 30 * 24 * time.Hour))
+	g = table.Filter(g, func(t time.Time) bool { return t.After(start) }, "Time")
+
+	milestones := []string{prefix + "Early", prefix, prefix + "Maybe"}
+	for m := range counts {
+		switch m {
+		case prefix + "Early", prefix, prefix + "Maybe":
+		default:
+			milestones = append(milestones, m)
+		}
+	}
+	for _, m := range milestones {
+		data[0] = append(data[0], m)
+	}
+	for _, gid := range g.Tables() {
+		t := g.Table(gid)
+		time := t.MustColumn("Time").([]time.Time)
+		milestone := t.MustColumn("Milestone").([]string)
+		count := t.MustColumn("Count").([]int)
+		for i := range time {
+			counts[milestone[i]] = count[i]
+			row := []interface{}{time[i].UnixNano() / 1e6}
+			for _, m := range milestones {
+				row = append(row, counts[m])
+			}
+			data = append(data, row)
+		}
+	}
+	fmt.Fprintf(w, "var ReleaseData = ")
+	if err := json.NewEncoder(w).Encode(data); err != nil {
+		return err
+	}
+	fmt.Fprintf(w, ";\n")
+	fmt.Fprintf(w, `
+ReleaseData.map(function(row, i) {
+  if (i > 0) {
+    row[0] = new Date(row[0])
+  }
+});`)
+	return nil
+}

devapp/template/release.html

@@ -0,0 +1,69 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <script type="text/javascript" src="https://www.google.com/jsapi"></script>
+    <script type="text/javascript">
+      google.load("visualization", "1", {packages:["corechart"]});
+      google.setOnLoadCallback(drawCharts);
+      function drawCharts() {
+          var data = google.visualization.arrayToDataTable(ReleaseData);
+          var options = {
+            title: 'Go 1.{{.GoReleaseCycle}} Release Issues',
+            isStacked: true,
+            width: 1100, height: 450,
+            vAxis: {minValue: 0},
+            focusTarget: 'category',
+            series: [
+              // TODO: What if we change the set of labels? How to map these more intelligently?
+              {color: '#008'}, // Early
+              {color: '#44c'}, // Release
+              {color: '#ccc'}, // Maybe
+            ]
+          };
+          var chart = new google.visualization.AreaChart(document.getElementById('ReleaseDiv'));
+          chart.draw(data, options);
+
+          var data = google.visualization.arrayToDataTable(TriageData);
+          var options = {
+            title: 'Issue Progress',
+            isStacked: true,
+            width: 1100, height: 450,
+            vAxis: {minValue: 0},
+            focusTarget: 'category',
+            series: [
+              {color: '#c00'}, // Triage needed
+              {color: '#cc0', lineDashStyle: [4, 4]}, // NeedsInvestigation
+              {color: '#ee4', lineDashStyle: [4, 4]}, // NeedsInvestigation+Waiting
+              {color: '#ff8', lineDashStyle: [4, 4]}, // NeedsInvestigation+Blocked
+              {color: '#0a0', lineDashStyle: [4, 4]}, // NeedsDecision
+              {color: '#4d4', lineDashStyle: [4, 4]}, // NeedsDecision+Waiting
+              {color: '#8f8', lineDashStyle: [4, 4]}, // NeedsDecision+Blocked
+              {color: '#00c', lineDashStyle: [4, 4]}, // NeedsFix
+              {color: '#44e', lineDashStyle: [4, 4]}, // NeedsFix+Waiting
+              {color: '#88f', lineDashStyle: [4, 4]}, // NeedsFix+Blocked
+            ]
+          };
+          var chart = new google.visualization.AreaChart(document.getElementById('TriageDiv'));
+          chart.draw(data, options);
+      }
+      function myDate(s) {
+        return new Date(s)
+      }
+    </script>
+    <script type="text/javascript" src="/stats/release/data.js?cycle={{.GoReleaseCycle}}"></script>
+
+<style>
+body { font-family: sans-serif; }
+h1 { text-align: center; }
+</style>
+</head>
+
+<body>
+
+<h1>Go 1.{{.GoReleaseCycle}} Issue Tracker Dashboard</h1>
+
+<div id="ReleaseDiv"></div>
+
+<div id="TriageDiv"></div>
+
+</body>