_content: serve wiki pages on go.dev

The wiki pages have been copied to go.googlesource.com/wiki
and will be code reviewed using Gerrit, like other Go repos.
Unlike other Go repos, self-review will be permitted, and
there is no requirement for Googlers to be involved to make a change.
These relaxations are possible because the wiki has no production code.
The set of wiki +2'ers is a superset of the usual code +2'ers.

Fixes golang/go#61940.

Change-Id: I01823720091fbaa24e95e9c82abeaadba867a17c
Reviewed-on: https://go-review.googlesource.com/c/website/+/518297
Reviewed-by: Dmitri Shuralyov <[email protected]>
Reviewed-by: Dmitri Shuralyov <[email protected]>
LUCI-TryBot-Result: Go LUCI <[email protected]>

Author: rsc

_content/css/styles.css

@@ -5370,3 +5370,9 @@ a.downloadBox:hover .filename span {
     display: none;
   }
 }
+
+hr {
+  border: none;
+  border-top: 1px solid var(--color-border);
+  height: 1px;
+}

_content/site.tmpl

@@ -71,7 +71,7 @@
             <a href="{{if .children}}#{{else}}{{.url}}{{end}}" {{if .children}} class="js-desktop-menu-hover"{{end}} aria-label={{.name}} aria-describedby="dropdown-description">
               {{.name}} {{if .children}}<i class="material-icons" aria-hidden="true">arrow_drop_down</i>{{end}}
             </a>
-            <div class="screen-reader-only" id="dropdown-description" hidden> 
+            <div class="screen-reader-only" id="dropdown-description" hidden>
               Press Enter to activate/deactivate dropdown
             </div>
             {{- if .children}}
@@ -270,7 +270,7 @@
   <script async src="/js/copypaste.js"></script>
 </footer>
 <section class="Cookie-notice js-cookieNotice">
-  <div>go.dev uses cookies from Google to deliver and enhance the quality of its services and to 
+  <div>go.dev uses cookies from Google to deliver and enhance the quality of its services and to
   analyze traffic. <a target=_blank href="https://policies.google.com/technologies/cookies">Learn more.</a></div>
   <div><button class="go-Button">Okay</button></div>
 </section>
@@ -438,7 +438,7 @@
 {{end}}
 
 {{if .title}}
-<h1>{{.title}}</h1>
+<h1>{{if strings.HasPrefix .URL "/wiki/"}}Go Wiki: {{end}}{{.title}}</h1>
 {{else if eq .layout "error"}}
 <h1>Error</h1>
 {{else if eq .layout "dir"}}
@@ -471,6 +471,13 @@
 {{end}}
 </div>
 {{end}}
+
+{{if strings.HasPrefix .URL "/wiki/"}}
+<hr>
+<p>
+<i>This content is part of the <a href="/wiki/">Go Wiki</a>.</i>
+</p>
+{{end}}
 </article>
 
 {{end}}

_content/wiki/Comments.md

@@ -0,0 +1,50 @@
+---
+title: Comments
+---
+
+<!--
+This is just a placeholder page for enabling a test.
+In the deployed site it is overwritten with the content of go.googlesource.com/wiki.
+-->
+
+Every package should have a package comment. It should immediately precede the ` package ` statement in one of the files in the package. (It only needs to appear in one file.) It should begin with a single sentence that begins "Package _packagename_" and give a concise summary of the package functionality. This introductory sentence will be used in godoc's list of all packages.
+
+Subsequent sentences and/or paragraphs can give more details. Sentences should be properly punctuated.
+
+```go
+// Package superman implements methods for saving the world.
+//
+// Experience has shown that a small number of procedures can prove
+// helpful when attempting to save the world.
+package superman
+```
+
+Nearly every top-level type, const, var and func should have a comment. A comment for bar should be in the form "_bar_ floats on high o'er vales and hills.". The first letter of _bar_ should not be capitalized unless it's capitalized in the code.
+
+```go
+// enterOrbit causes Superman to fly into low Earth orbit, a position
+// that presents several possibilities for planet salvation.
+func enterOrbit() os.Error {
+  ...
+}
+```
+
+All text that you indent inside a comment, godoc will render as a pre-formatted block. This facilitates code samples.
+
+```go
+// fight can be used on any enemy and returns whether Superman won.
+//
+// Examples:
+//
+//  fight("a random potato")
+//  fight(LexLuthor{})
+//
+func fight(enemy interface{}) bool {
+	// This is testing proper escaping in the wiki.
+	for i := 0; i < 10; i++ {
+		println("fight!")
+	}
+}
+```
+
+

_content/wiki/default.tmpl

@@ -0,0 +1,3 @@
+{{define "layout"}}
+{{doclayout .}}
+{{end}}

_content/wiki/index.md

@@ -0,0 +1,5 @@
+---
+title: Index
+---
+
+The Wiki is still loading...

cmd/golangorg/cloudbuild.yaml

@@ -3,21 +3,36 @@
 # Do not run directly.
 
 steps:
+  # Clone go repo to _goroot.zip for use by uploaded app.
   - name: gcr.io/cloud-builders/git
     args: ["clone", "--branch=release-branch.go1.21", "--depth=1", "https://go.googlesource.com/go", "_gotmp"]
   - name: gcr.io/cloud-builders/git
     args: ["archive", "--format=zip", "--output=../_goroot.zip", "HEAD"]
     dir: _gotmp
   - name: golang
     args: ["rm", "-rf", "_gotmp"]
+  # Clone wiki repo into _content/wiki as initial wiki content.
+  # The server will replace this with the wiki repo when it can,
+  # but this provides a good fallback.
+  - name: gcr.io/cloud-builders/git
+    args: ["clone", "--depth=1", "https://go.googlesource.com/wiki", "_wikitmp"]
+  - name: golang
+    args: ["rm", "-rf", "_wikitmp/.git"]
+  - name: golang
+    args: ["cp", "-a", "_wikitmp/*", "_content/wiki"]
+  # Run tests.
   - name: golang
     args: ["go", "test", "./..."]
+  # Coordinate with other Cloud Build jobs to deploy only newest commit.
+  # May abort job here.
   - name: golang
     args: ["go", "run", "./cmd/locktrigger", "--project=$PROJECT_ID",
     "--build=$BUILD_ID", "--repo=https://go.googlesource.com/website"]
+  # Deploy site and redirect traffic (maybe; tests again in prod first).
   - name: gcr.io/cloud-builders/gcloud
     entrypoint: bash
     args: ["./go-app-deploy.sh", "cmd/golangorg/app.yaml"]
+  # Clean up stale versions.
   - name: golang
     args: ["go", "run", "./cmd/versionprune", "--dry_run=false", "--project=$PROJECT_ID", "--service=default"]
 

cmd/golangorg/server.go

@@ -59,7 +59,8 @@ var (
 
 	runningOnAppEngine = os.Getenv("PORT") != ""
 
-	tipFlag = flag.Bool("tip", runningOnAppEngine, "load git content for tip.golang.org")
+	tipFlag  = flag.Bool("tip", runningOnAppEngine, "load git content for tip.golang.org")
+	wikiFlag = flag.Bool("wiki", runningOnAppEngine, "load git content for go.dev/wiki")
 
 	googleAnalytics string
 )
@@ -157,14 +158,27 @@ func NewHandler(contentDir, goroot string) http.Handler {
 		gorootFS = os.DirFS(goroot)
 	}
 
+	// go.dev/wiki serves content from the very latest Git commit of the wiki repo.
+	// Start with the _content/wiki directory as placeholder until Git loads.
+	var wikiFS atomicFS
+	wikiDefault, err := fs.Sub(contentFS, "wiki")
+	if err != nil {
+		log.Fatalf("loading default wiki content: %v", err)
+	}
+	wikiFS.Set(wikiDefault)
+	if *wikiFlag {
+		go watchGit(&wikiFS, "https://go.googlesource.com/wiki")
+	}
+	contentFS = &mountFS{contentFS, "wiki", &wikiFS}
+
 	// tip.golang.org serves content from the very latest Git commit
 	// of the main Go repo, instead of the one the app is bundled with.
 	var tipGoroot atomicFS
 	if _, err := newSite(mux, "tip.golang.org", contentFS, &tipGoroot); err != nil {
 		log.Fatalf("loading tip site: %v", err)
 	}
 	if *tipFlag {
-		go watchTip(&tipGoroot)
+		go watchGit(&tipGoroot, "https://go.googlesource.com/go")
 	}
 
 	// beta.golang.org is an old name for tip.
@@ -280,32 +294,32 @@ func parseRFC3339(s string) (time.Time, error) {
 	return time.Parse(time.RFC3339, s)
 }
 
-// watchTip is a background goroutine that watches the main Go repo for updates.
-// When a new commit is available, watchTip downloads the new tree and calls
-// tipGoroot.Set to install the new file system.
-func watchTip(tipGoroot *atomicFS) {
+// watchGit is a background goroutine that watches a Git repo for updates.
+// When a new commit is available, watchGit downloads the new tree and calls
+// fsys.Set to install the new file system.
+func watchGit(fsys *atomicFS, repo string) {
 	for {
-		// watchTip1 runs until it panics (hopefully never).
+		// watchGit1 runs until it panics (hopefully never).
 		// If that happens, sleep 5 minutes and try again.
-		watchTip1(tipGoroot)
+		watchGit1(fsys, repo)
 		time.Sleep(5 * time.Minute)
 	}
 }
 
-// watchTip1 does the actual work of watchTip and recovers from panics.
-func watchTip1(tipGoroot *atomicFS) {
+// watchGit1 does the actual work of watchGit and recovers from panics.
+func watchGit1(afs *atomicFS, repo string) {
 	defer func() {
 		if e := recover(); e != nil {
-			log.Printf("watchTip panic: %v\n%s", e, debug.Stack())
+			log.Printf("watchGit %s panic: %v\n%s", repo, e, debug.Stack())
 		}
 	}()
 
 	var r *gitfs.Repo
 	for {
 		var err error
-		r, err = gitfs.NewRepo("https://go.googlesource.com/go")
+		r, err = gitfs.NewRepo(repo)
 		if err != nil {
-			log.Printf("tip: %v", err)
+			log.Printf("watchGit %s: %v", repo, err)
 			time.Sleep(1 * time.Minute)
 			continue
 		}
@@ -318,29 +332,29 @@ func watchTip1(tipGoroot *atomicFS) {
 		var err error
 		h, fsys, err = r.Clone("HEAD")
 		if err != nil {
-			log.Printf("tip: %v", err)
+			log.Printf("watchGit %s: %v", repo, err)
 			time.Sleep(1 * time.Minute)
 			continue
 		}
-		tipGoroot.Set(fsys)
+		afs.Set(fsys)
 		break
 	}
 
 	for {
 		time.Sleep(5 * time.Minute)
 		h2, err := r.Resolve("HEAD")
 		if err != nil {
-			log.Printf("tip: %v", err)
+			log.Printf("watchGit %s: %v", repo, err)
 			continue
 		}
 		if h2 != h {
 			fsys, err := r.CloneHash(h2)
 			if err != nil {
-				log.Printf("tip: %v", err)
+				log.Printf("watchGit %s: %v", repo, err)
 				time.Sleep(1 * time.Minute)
 				continue
 			}
-			tipGoroot.Set(fsys)
+			afs.Set(fsys)
 			h = h2
 		}
 	}
@@ -715,6 +729,7 @@ func (fsys fixSpecsFS) Open(name string) (fs.File, error) {
 // README.md, and SECURITY.md. The last is particularly problematic
 // when running locally on a Mac, because it can be opened as
 // security.md, which takes priority over _content/security.html.
+// Same for wiki.
 type hideRootMDFS struct {
 	fs fs.FS
 }
@@ -723,6 +738,10 @@ func (fsys hideRootMDFS) Open(name string) (fs.File, error) {
 	if !strings.Contains(name, "/") && strings.HasSuffix(name, ".md") {
 		return nil, errors.New(".md file not available")
 	}
+	switch name {
+	case "wiki/README.md", "wiki/CONTRIBUTING.md", "wiki/LICENSE", "wiki/PATENTS", "wiki/codereview.cfg":
+		return nil, errors.New("wiki meta file not available")
+	}
 	return fsys.fs.Open(name)
 }
 
@@ -795,6 +814,23 @@ func (a *atomicFS) Set(fsys fs.FS) {
 	a.v.Store(&fsys)
 }
 
+// A mountFS is a root FS with a second FS mounted at a specific location.
+type mountFS struct {
+	old fs.FS  // root file system
+	dir string // mount point
+	new fs.FS  // fs mounted on dir
+}
+
+func (m *mountFS) Open(name string) (fs.File, error) {
+	if name == m.dir {
+		return m.new.Open(".")
+	}
+	if strings.HasPrefix(name, m.dir) && len(name) > len(m.dir) && name[len(m.dir)] == '/' {
+		return m.new.Open(name[len(m.dir)+1:])
+	}
+	return m.old.Open(name)
+}
+
 // Open returns fsys.Open(name) where fsys is the file system passed to the most recent call to Set.
 // If there has been no call to Set, Open returns an error with text “no file system”.
 func (a *atomicFS) Open(name string) (fs.File, error) {

cmd/golangorg/server_test.go

@@ -114,6 +114,9 @@ func TestAll(t *testing.T) {
 
 	// Do not process these paths or path prefixes.
 	ignores := []string{
+		// Wiki is in a different repo; errors there should not block production push.
+		"/wiki/",
+
 		// Support files not meant to be served directly.
 		"/doc/articles/wiki/",
 		"/talks/2013/highperf/",

cmd/golangorg/testdata/godev.txt

@@ -109,3 +109,11 @@ redirect == /doc/security/
 
 GET https://go.dev/doc/security/
 body contains Security
+
+GET https://go.dev/wiki/
+body contains Go Wiki: Index
+body contains <i>This content is part of the <a href="/wiki/">Go Wiki</a>.</i>
+
+GET https://go.dev/wiki/Comments
+body contains Go Wiki: Comments
+body contains <i>This content is part of the <a href="/wiki/">Go Wiki</a>.</i>

cmd/golangorg/testdata/live.txt

@@ -62,3 +62,11 @@ header Content-Type == text/plain; charset=utf-8
 body !contains The Go Playground
 body !contains About the Playground
 body contains Hello, 世界
+
+GET https://go.dev/wiki/Comments
+body contains Go Wiki: Comments
+body contains <i>This content is part of the <a href="/wiki/">Go Wiki</a>.</i>
+
+GET https://go.dev/wiki/CommonMistakes
+body contains Go Wiki: Common Mistakes
+body contains <i>This content is part of the <a href="/wiki/">Go Wiki</a>.</i>

internal/redirect/redirect.go

@@ -111,8 +111,6 @@ var redirects = map[string]string{
 	"/doc/mem":  "/ref/mem",
 	"/doc/spec": "/ref/spec",
 
-	"/wiki": "https://github.com/golang/go/wiki",
-
 	"/doc/articles/c_go_cgo.html":                    "/blog/c-go-cgo",
 	"/doc/articles/concurrency_patterns.html":        "/blog/go-concurrency-patterns-timing-out-and",
 	"/doc/articles/defer_panic_recover.html":         "/blog/defer-panic-and-recover",
@@ -163,7 +161,6 @@ var newIssueRedirects = [...]string{
 var prefixHelpers = map[string]string{
 	"issue":  "https://github.com/golang/go/issues/",
 	"issues": "https://github.com/golang/go/issues/",
-	"wiki":   "https://github.com/golang/go/wiki/",
 }
 
 func Handler(target string) http.Handler {

internal/redirect/redirect_test.go

@@ -85,9 +85,6 @@ func TestRedirects(t *testing.T) {
 		"/issues/new/choose":        errorResult(404),
 		"/issues/1/2/3":             errorResult(404),
 
-		"/wiki/foo":  {302, "https://github.com/golang/go/wiki/foo"},
-		"/wiki/foo/": {302, "https://github.com/golang/go/wiki/foo/"},
-
 		"/design":              {301, "https://go.googlesource.com/proposal/+/master/design"},
 		"/design/":             {302, "/design"},
 		"/design/123-foo":      {302, "https://go.googlesource.com/proposal/+/master/design/123-foo.md"},

internal/web/render.go

@@ -127,6 +127,12 @@ func (site *Site) renderHTML(p Page, tmpl string, r *http.Request) ([]byte, erro
 		// Either the page explicitly requested templating, or it is markdown,
 		// which is treated as a template by default.
 		isTemplate, explicit := p["template"].(bool)
+
+		// The wiki is not templated by default.
+		if !explicit && strings.HasPrefix(file, "wiki/") {
+			isTemplate, explicit = false, true
+		}
+
 		tdata := data
 		if !explicit || isTemplate {
 			// Load content as a template.
@@ -193,6 +199,7 @@ func markdownToHTML(markdown string) (template.HTML, error) {
 				extension.WithLinkifyEmailRegexp(regexp.MustCompile(`[^\x00-\x{10FFFF}]`)), // impossible
 			),
 			extension.DefinitionList,
+			extension.NewTable(),
 		),
 	)
 	var buf bytes.Buffer