From da35c5f8393ceb4ddf77a333b424c3cc4799a70f Mon Sep 17 00:00:00 2001 From: Dmitri Shuralyov Date: Fri, 15 Feb 2019 13:46:30 -0500 Subject: [PATCH] cmd/tip: start building website from x/website repository As part of golang/go#29206, the golang.org website is being moved to from x/tools to x/website repository. This change updates the tip command to start deploying it from its new location. The new website is using Go modules for reproducible builds. This simplifies the code, because we no longer need to manually fetch any dependent repositories (such as x/net). Update instances of "godoc" (the binary that was previously used to host the website) to "golangorg", the new binary name in x/website. Fixes golang/go#30232 Updates golang/go#29206 Change-Id: I428eddf95a7ed6a43a25138ffeb919e79f42f9d2 Reviewed-on: https://go-review.googlesource.com/c/162908 Reviewed-by: Andrew Bonventre --- cmd/tip/deployment-prod.yaml | 2 +- cmd/tip/deployment-staging.yaml | 2 +- cmd/tip/godoc.go | 102 -------------------------------- cmd/tip/golangorg.go | 92 ++++++++++++++++++++++++++++ cmd/tip/tip.go | 10 ++-- cmd/tip/tip_test.go | 2 +- 6 files changed, 100 insertions(+), 110 deletions(-) delete mode 100644 cmd/tip/godoc.go create mode 100644 cmd/tip/golangorg.go diff --git a/cmd/tip/deployment-prod.yaml b/cmd/tip/deployment-prod.yaml index 8b30a973cd..9ac2503971 100644 --- a/cmd/tip/deployment-prod.yaml +++ b/cmd/tip/deployment-prod.yaml @@ -22,7 +22,7 @@ spec: - name: TMPDIR value: /build - name: TIP_BUILDER - value: godoc + value: golangorg volumeMounts: - mountPath: /build name: cache-volume diff --git a/cmd/tip/deployment-staging.yaml b/cmd/tip/deployment-staging.yaml index 16b454353e..f43dfa27ce 100644 --- a/cmd/tip/deployment-staging.yaml +++ b/cmd/tip/deployment-staging.yaml @@ -22,7 +22,7 @@ spec: - name: TMPDIR value: /build - name: TIP_BUILDER - value: godoc + value: golangorg volumeMounts: - mountPath: /build name: cache-volume diff --git a/cmd/tip/godoc.go b/cmd/tip/godoc.go deleted file mode 100644 index 13b891253d..0000000000 --- a/cmd/tip/godoc.go +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by the Apache 2.0 -// license that can be found in the LICENSE file. - -package main - -import ( - "bytes" - "errors" - "fmt" - "io" - "log" - "os" - "os/exec" - "path/filepath" -) - -type godocBuilder struct{} - -func prefix8(s string) string { - if len(s) < 8 { - return s - } - return s[:8] -} - -func (b godocBuilder) Signature(heads map[string]string) string { - // x/net is intentionally not a part of the signature, because - // at this time it does not contribute substantially to the deployed - // website, and so we don't want tip.golang.org redeployed whenever - // x/net changes. This will no longer matter when the Go website uses - // modules to pin its dependencies to specific versions. - return fmt.Sprintf("go=%v/tools=%v", prefix8(heads["go"]), prefix8(heads["tools"])) -} - -func (b godocBuilder) Init(logger *log.Logger, dir, hostport string, heads map[string]string) (*exec.Cmd, error) { - goDir := filepath.Join(dir, "go") - toolsDir := filepath.Join(dir, "gopath/src/golang.org/x/tools") - netDir := filepath.Join(dir, "gopath/src/golang.org/x/net") - logger.Printf("checking out go repo ...") - if err := checkout(repoURL+"go", heads["go"], goDir); err != nil { - return nil, fmt.Errorf("checkout of go: %v", err) - } - logger.Printf("checking out tools repo ...") - if err := checkout(repoURL+"tools", heads["tools"], toolsDir); err != nil { - return nil, fmt.Errorf("checkout of tools: %v", err) - } - logger.Printf("checking out net repo ...") - if err := checkout(repoURL+"net", heads["net"], netDir); err != nil { - return nil, fmt.Errorf("checkout of net: %v", err) - } - - var logWriter io.Writer = toLoggerWriter{logger} - - make := exec.Command(filepath.Join(goDir, "src/make.bash")) - make.Dir = filepath.Join(goDir, "src") - make.Stdout = logWriter - make.Stderr = logWriter - logger.Printf("running make.bash in %s ...", make.Dir) - if err := make.Run(); err != nil { - return nil, fmt.Errorf("running make.bash: %v", err) - } - - logger.Printf("installing godoc ...") - goBin := filepath.Join(goDir, "bin/go") - goPath := filepath.Join(dir, "gopath") - install := exec.Command(goBin, "install", "golang.org/x/tools/cmd/godoc") - install.Stdout = logWriter - install.Stderr = logWriter - install.Env = append(os.Environ(), - "GOROOT="+goDir, - "GOPATH="+goPath, - "GOROOT_BOOTSTRAP="+os.Getenv("GOROOT_BOOTSTRAP"), - ) - if err := install.Run(); err != nil { - return nil, fmt.Errorf("go install golang.org/x/tools/cmd/godoc: %v", err) - } - - logger.Printf("starting godoc ...") - godocBin := filepath.Join(goPath, "bin/godoc") - godoc := exec.Command(godocBin, "-http="+hostport, "-index", "-index_interval=-1s", "-play") - godoc.Env = append(os.Environ(), "GOROOT="+goDir) - godoc.Stdout = logWriter - godoc.Stderr = logWriter - if err := godoc.Start(); err != nil { - return nil, fmt.Errorf("starting godoc: %v", err) - } - return godoc, nil -} - -var indexingMsg = []byte("Indexing in progress: result may be inaccurate") - -func (b godocBuilder) HealthCheck(hostport string) error { - body, err := getOK(fmt.Sprintf("http://%v/search?q=FALLTHROUGH", hostport)) - if err != nil { - return err - } - if bytes.Contains(body, indexingMsg) { - return errors.New("still indexing") - } - return nil -} diff --git a/cmd/tip/golangorg.go b/cmd/tip/golangorg.go new file mode 100644 index 0000000000..e2fa4b524a --- /dev/null +++ b/cmd/tip/golangorg.go @@ -0,0 +1,92 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "errors" + "fmt" + "io" + "log" + "os" + "os/exec" + "path/filepath" +) + +type golangorgBuilder struct{} + +func prefix8(s string) string { + if len(s) < 8 { + return s + } + return s[:8] +} + +func (b golangorgBuilder) Signature(heads map[string]string) string { + return fmt.Sprintf("go=%v/website=%v", prefix8(heads["go"]), prefix8(heads["website"])) +} + +func (b golangorgBuilder) Init(logger *log.Logger, dir, hostport string, heads map[string]string) (*exec.Cmd, error) { + goDir := filepath.Join(dir, "go") + websiteDir := filepath.Join(dir, "website") + logger.Printf("checking out go repo ...") + if err := checkout(repoURL+"go", heads["go"], goDir); err != nil { + return nil, fmt.Errorf("checkout of go: %v", err) + } + logger.Printf("checking out website repo ...") + if err := checkout(repoURL+"website", heads["website"], websiteDir); err != nil { + return nil, fmt.Errorf("checkout of website: %v", err) + } + + var logWriter io.Writer = toLoggerWriter{logger} + + make := exec.Command(filepath.Join(goDir, "src/make.bash")) + make.Dir = filepath.Join(goDir, "src") + make.Stdout = logWriter + make.Stderr = logWriter + logger.Printf("running make.bash in %s ...", make.Dir) + if err := make.Run(); err != nil { + return nil, fmt.Errorf("running make.bash: %v", err) + } + + logger.Printf("installing golangorg ...") + goBin := filepath.Join(goDir, "bin/go") + binDir := filepath.Join(dir, "bin") + install := exec.Command(goBin, "install", "golang.org/x/website/cmd/golangorg") + install.Stdout = logWriter + install.Stderr = logWriter + install.Env = append(os.Environ(), + "GOROOT="+goDir, + "GO111MODULE=on", + "GOBIN="+binDir, + ) + if err := install.Run(); err != nil { + return nil, fmt.Errorf("go install golang.org/x/website/cmd/golangorg: %v", err) + } + + logger.Printf("starting golangorg ...") + golangorgBin := filepath.Join(binDir, "golangorg") + golangorg := exec.Command(golangorgBin, "-http="+hostport, "-index", "-index_interval=-1s", "-play") + golangorg.Env = append(os.Environ(), "GOROOT="+goDir) + golangorg.Stdout = logWriter + golangorg.Stderr = logWriter + if err := golangorg.Start(); err != nil { + return nil, fmt.Errorf("starting golangorg: %v", err) + } + return golangorg, nil +} + +var indexingMsg = []byte("Indexing in progress: result may be inaccurate") + +func (b golangorgBuilder) HealthCheck(hostport string) error { + body, err := getOK(fmt.Sprintf("http://%v/search?q=FALLTHROUGH", hostport)) + if err != nil { + return err + } + if bytes.Contains(body, indexingMsg) { + return errors.New("still indexing") + } + return nil +} diff --git a/cmd/tip/tip.go b/cmd/tip/tip.go index fdb149782a..b4cb080ef9 100644 --- a/cmd/tip/tip.go +++ b/cmd/tip/tip.go @@ -44,8 +44,8 @@ func main() { const k = "TIP_BUILDER" var b Builder switch os.Getenv(k) { - case "godoc": - b = godocBuilder{} + case "golangorg": + b = golangorgBuilder{} case "talks": b = talksBuilder{} default: @@ -75,14 +75,14 @@ func main() { } // Proxy implements the tip.golang.org server: a reverse-proxy -// that builds and runs godoc instances showing the latest docs. +// that builds and runs golangorg instances showing the latest docs. type Proxy struct { builder Builder mu sync.Mutex // protects following fields proxy http.Handler cur string // signature of gorepo+toolsrepo - cmd *exec.Cmd // live godoc instance, or nil for none + cmd *exec.Cmd // live golangorg instance, or nil for none side string hostport string // host and port of the live instance err error @@ -102,7 +102,7 @@ func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Redirect the old beta.golang.org URL to tip.golang.org, // just in case there are old links out there to // beta.golang.org. (We used to run a "temporary" beta.golang.org - // GCE VM running godoc where "temporary" lasted two years. + // GCE VM running golangorg where "temporary" lasted two years. // So it lasted so long, there are probably links to it out there.) if r.Host == "beta.golang.org" { u := *r.URL diff --git a/cmd/tip/tip_test.go b/cmd/tip/tip_test.go index 878954d571..e042f4b55b 100644 --- a/cmd/tip/tip_test.go +++ b/cmd/tip/tip_test.go @@ -10,7 +10,7 @@ import ( ) func TestTipRedirects(t *testing.T) { - mux := newServeMux(&Proxy{builder: &godocBuilder{}}) + mux := newServeMux(&Proxy{builder: &golangorgBuilder{}}) req := httptest.NewRequest("GET", "http://example.com/foo?bar=baz", nil) req.Header.Set("X-Forwarded-Proto", "http") w := httptest.NewRecorder()