cmd/tip: start building website from x/website repository

As part of golang/go#29206, the golang.org website is being moved to
from x/tools to x/website repository. This change updates the tip
command to start deploying it from its new location.

The new website is using Go modules for reproducible builds. This
simplifies the code, because we no longer need to manually fetch
any dependent repositories (such as x/net).

Update instances of "godoc" (the binary that was previously used to
host the website) to "golangorg", the new binary name in x/website.

Fixes golang/go#30232
Updates golang/go#29206

Change-Id: I428eddf95a7ed6a43a25138ffeb919e79f42f9d2
Reviewed-on: https://go-review.googlesource.com/c/162908
Reviewed-by: Andrew Bonventre <andybons@golang.org>

cmd/tip/deployment-prod.yaml

@@ -22,7 +22,7 @@ spec:
         - name: TMPDIR
           value: /build
         - name: TIP_BUILDER
-          value: godoc
+          value: golangorg
         volumeMounts:
         - mountPath: /build
           name: cache-volume

cmd/tip/deployment-staging.yaml

@@ -22,7 +22,7 @@ spec:
         - name: TMPDIR
           value: /build
         - name: TIP_BUILDER
-          value: godoc
+          value: golangorg
         volumeMounts:
         - mountPath: /build
           name: cache-volume

cmd/tip/golangorg.go

@@ -0,0 +1,92 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by the Apache 2.0
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+)
+
+type golangorgBuilder struct{}
+
+func prefix8(s string) string {
+	if len(s) < 8 {
+		return s
+	}
+	return s[:8]
+}
+
+func (b golangorgBuilder) Signature(heads map[string]string) string {
+	return fmt.Sprintf("go=%v/website=%v", prefix8(heads["go"]), prefix8(heads["website"]))
+}
+
+func (b golangorgBuilder) Init(logger *log.Logger, dir, hostport string, heads map[string]string) (*exec.Cmd, error) {
+	goDir := filepath.Join(dir, "go")
+	websiteDir := filepath.Join(dir, "website")
+	logger.Printf("checking out go repo ...")
+	if err := checkout(repoURL+"go", heads["go"], goDir); err != nil {
+		return nil, fmt.Errorf("checkout of go: %v", err)
+	}
+	logger.Printf("checking out website repo ...")
+	if err := checkout(repoURL+"website", heads["website"], websiteDir); err != nil {
+		return nil, fmt.Errorf("checkout of website: %v", err)
+	}
+
+	var logWriter io.Writer = toLoggerWriter{logger}
+
+	make := exec.Command(filepath.Join(goDir, "src/make.bash"))
+	make.Dir = filepath.Join(goDir, "src")
+	make.Stdout = logWriter
+	make.Stderr = logWriter
+	logger.Printf("running make.bash in %s ...", make.Dir)
+	if err := make.Run(); err != nil {
+		return nil, fmt.Errorf("running make.bash: %v", err)
+	}
+
+	logger.Printf("installing golangorg ...")
+	goBin := filepath.Join(goDir, "bin/go")
+	binDir := filepath.Join(dir, "bin")
+	install := exec.Command(goBin, "install", "golang.org/x/website/cmd/golangorg")
+	install.Stdout = logWriter
+	install.Stderr = logWriter
+	install.Env = append(os.Environ(),
+		"GOROOT="+goDir,
+		"GO111MODULE=on",
+		"GOBIN="+binDir,
+	)
+	if err := install.Run(); err != nil {
+		return nil, fmt.Errorf("go install golang.org/x/website/cmd/golangorg: %v", err)
+	}
+
+	logger.Printf("starting golangorg ...")
+	golangorgBin := filepath.Join(binDir, "golangorg")
+	golangorg := exec.Command(golangorgBin, "-http="+hostport, "-index", "-index_interval=-1s", "-play")
+	golangorg.Env = append(os.Environ(), "GOROOT="+goDir)
+	golangorg.Stdout = logWriter
+	golangorg.Stderr = logWriter
+	if err := golangorg.Start(); err != nil {
+		return nil, fmt.Errorf("starting golangorg: %v", err)
+	}
+	return golangorg, nil
+}
+
+var indexingMsg = []byte("Indexing in progress: result may be inaccurate")
+
+func (b golangorgBuilder) HealthCheck(hostport string) error {
+	body, err := getOK(fmt.Sprintf("http://%v/search?q=FALLTHROUGH", hostport))
+	if err != nil {
+		return err
+	}
+	if bytes.Contains(body, indexingMsg) {
+		return errors.New("still indexing")
+	}
+	return nil
+}

cmd/tip/tip.go

@@ -44,8 +44,8 @@ func main() {
 	const k = "TIP_BUILDER"
 	var b Builder
 	switch os.Getenv(k) {
-	case "godoc":
-		b = godocBuilder{}
+	case "golangorg":
+		b = golangorgBuilder{}
 	case "talks":
 		b = talksBuilder{}
 	default:
@@ -75,14 +75,14 @@ func main() {
 }
 
 // Proxy implements the tip.golang.org server: a reverse-proxy
-// that builds and runs godoc instances showing the latest docs.
+// that builds and runs golangorg instances showing the latest docs.
 type Proxy struct {
 	builder Builder
 
 	mu       sync.Mutex // protects following fields
 	proxy    http.Handler
 	cur      string    // signature of gorepo+toolsrepo
-	cmd      *exec.Cmd // live godoc instance, or nil for none
+	cmd      *exec.Cmd // live golangorg instance, or nil for none
 	side     string
 	hostport string // host and port of the live instance
 	err      error
@@ -102,7 +102,7 @@ func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// Redirect the old beta.golang.org URL to tip.golang.org,
 	// just in case there are old links out there to
 	// beta.golang.org. (We used to run a "temporary" beta.golang.org
-	// GCE VM running godoc where "temporary" lasted two years.
+	// GCE VM running golangorg where "temporary" lasted two years.
 	// So it lasted so long, there are probably links to it out there.)
 	if r.Host == "beta.golang.org" {
 		u := *r.URL

cmd/tip/tip_test.go

@@ -10,7 +10,7 @@ import (
 )
 
 func TestTipRedirects(t *testing.T) {
-	mux := newServeMux(&Proxy{builder: &godocBuilder{}})
+	mux := newServeMux(&Proxy{builder: &golangorgBuilder{}})
 	req := httptest.NewRequest("GET", "http://example.com/foo?bar=baz", nil)
 	req.Header.Set("X-Forwarded-Proto", "http")
 	w := httptest.NewRecorder()