diff --git a/internal/gapicgen/cmd/genbot/Dockerfile b/internal/gapicgen/cmd/genbot/Dockerfile index dfc8067cea1f..6ef6e7a2da12 100644 --- a/internal/gapicgen/cmd/genbot/Dockerfile +++ b/internal/gapicgen/cmd/genbot/Dockerfile @@ -41,6 +41,11 @@ RUN mkdir /root/.ssh && echo -e "Host github.com\n\tStrictHostKeyChecking no\n" RUN echo -e '#!/bin/bash\n\ set -ex\n\ + if [[ ${GENBOT_LOCAL_MODE} = "true" ]]; then\n\ + cd internal/gapicgen\n\ + go run cloud.google.com/go/internal/gapicgen/cmd/genbot\n\ + exit 0\n\ + fi\n\ go mod download \n\ go run cloud.google.com/go/internal/gapicgen/cmd/genbot \ ' >> /run.sh diff --git a/internal/gapicgen/cmd/genbot/README.md b/internal/gapicgen/cmd/genbot/README.md index 0328f86744a1..455fd5fb76a4 100644 --- a/internal/gapicgen/cmd/genbot/README.md +++ b/internal/gapicgen/cmd/genbot/README.md @@ -3,23 +3,38 @@ genbot is a binary for generating gapics and creating PRs with the results. It is intended to be used as a bot, though it can be run locally too. -## Getting certs - -### Github +## Prerequisites for running locally + +Note that only step one, listed below, is required if you plan to run the code +in docker. + +1. Clone this project: `git clone https://github.com/googleapis/google-cloud-go.git` +1. Install [protoc](https://github.com/protocolbuffers/protobuf/releases) +1. Install [Go](http://golang.org/dl) +1. Add `$GOPATH/bin` to `PATH` +1. Create a [GitHub access token](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line). +1. Install Go tools: + +```bash +go get \ + github.com/golang/protobuf/protoc-gen-go \ + golang.org/x/lint/golint \ + golang.org/x/tools/cmd/goimports \ + honnef.co/go/tools/cmd/staticcheck \ + github.com/googleapis/gapic-generator-go/cmd/protoc-gen-go_gapic +``` -For Github, you need to generate/supply a Personal Access Token. More -information on how that's done can be found here: -[creating a personal access token](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line). +## Generating code and PRs(bot mode) -## Running locally +### Run genbot locally Note: this may change your `~/.gitconfig`, `~/.gitcookies`, and use up non-trivial amounts of space on your computer. 1. Make sure you are on a non-Windows platform. If you are using windows continue on to the docker instructions. -2. Make sure you have all the tools installed listed in genlocal's README.md -3. Run: +1. Make sure you have all the tools installed listed in genlocal's README.md +1. Run: ```shell cd /path/to/internal/gapicgen @@ -30,7 +45,7 @@ go run cloud.google.com/go/internal/gapicgen/cmd/genbot \ --githubEmail=deklerk@google.com \ ``` -## Run with docker +### Run genbot with docker Note: this can be quite slow (~10m). @@ -41,12 +56,51 @@ Note: this may leave a lot of docker resources laying around. Use cd /path/to/internal/gapicgen/cmd/genbot docker build . -t genbot docker run -t --rm --privileged \ - -v `pwd`/../..:/gapicgen \ - -e GITHUB_ACCESS_TOKEN \ - -e GITHUB_USERNAME \ - -e GITHUB_NAME \ - -e GITHUB_EMAIL \ - genbot + -v `pwd`/../..:/gapicgen \ + -e GITHUB_ACCESS_TOKEN \ + -e GITHUB_USERNAME \ + -e GITHUB_NAME \ + -e GITHUB_EMAIL \ + genbot +``` + +## Generating code (local mode) + +Sometimes you may want to just generate gapic sources to test out +new changes, test the generation of a new library, test new generator tweaks, +run generators against googleapis-private, and various other local tasks. The +local mode in genbot allows you to do just that. + +### Run genbot(local mode) locally + +```shell +cd /path/to/internal/gapicgen +go run cloud.google.com/go/internal/gapicgen/cmd/genbot \ + -local \ + -only-gapics \ + -gocloud-dir=/path/to/google-cloud-go \ + -gapic=cloud.google.com/go/foo/apiv1 +``` + +### Run genbot(local mode) with docker + +```shell +cd /path/to/internal/gapicgen +docker build -t genbot -f cmd/genbot/Dockerfile . +docker run --rm \ + -v `pwd`/../..:/gapicgen \ + -e GENBOT_LOCAL_MODE=true \ + -e ONLY_GAPICS=true \ + -e GOCLOUD_DIR=/gapicgen \ + -e GAPIC_TO_GENERATE=cloud.google.com/go/foo/apiv1 \ + genbot \ +``` + +Note you can optionally mount in your Go module cache if you have Go installed. +This will speed up the build a bit: + +```shell +-v `go env GOMODCACHE`:/root/go/pkg/mod ``` ## FAQ diff --git a/internal/gapicgen/cmd/genbot/bot.go b/internal/gapicgen/cmd/genbot/bot.go new file mode 100644 index 000000000000..e5d8b8cbac77 --- /dev/null +++ b/internal/gapicgen/cmd/genbot/bot.go @@ -0,0 +1,83 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !windows + +package main + +import ( + "context" + "flag" + "fmt" + "log" + "time" +) + +func genBot(ctx context.Context, githubAccessToken, githubUsername, githubName, githubEmail string) error { + for k, v := range map[string]string{ + "githubAccessToken": githubAccessToken, + "githubUsername": githubUsername, + "githubName": githubName, + "githubEmail": githubEmail, + } { + if v == "" { + log.Printf("missing or empty value for required flag --%s\n", k) + flag.PrintDefaults() + } + } + + // Setup the client and git environment. + githubClient, err := NewGithubClient(ctx, githubUsername, githubName, githubEmail, githubAccessToken) + if err != nil { + return err + } + + // Check current regen status. + if pr, err := githubClient.GetRegenPR(ctx, "go-genproto", "open"); err != nil { + return err + } else if pr != nil { + return fmt.Errorf("There is already a re-generation in progress") + } + if pr, err := githubClient.GetRegenPR(ctx, "google-cloud-go", "open"); err != nil { + return err + } else if pr != nil { + if err := updateGocloudPR(ctx, githubClient, pr); err != nil { + return err + } + return nil + } + log.Println("checking if a pull request was already opened and merged today") + if pr, err := githubClient.GetRegenPR(ctx, "go-genproto", "closed"); err != nil { + return err + } else if pr != nil && hasCreatedPRToday(pr.Created) { + log.Println("skipping generation, already created and merged a go-genproto PR today") + return nil + } + if pr, err := githubClient.GetRegenPR(ctx, "google-cloud-go", "closed"); err != nil { + return err + } else if pr != nil && hasCreatedPRToday(pr.Created) { + log.Println("skipping generation, already created and merged a google-cloud-go PR today") + return nil + } + + return generate(ctx, githubClient) +} + +// hasCreatedPRToday checks if the created time of a PR is from today. +func hasCreatedPRToday(created time.Time) bool { + now := time.Now() + today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) + log.Printf("Times -- Now: %v\tToday: %v\tPR Created: %v", now, today, created) + return created.After(today) +} diff --git a/internal/gapicgen/cmd/genbot/generate.go b/internal/gapicgen/cmd/genbot/generate.go index b4b023ca1b85..154db21f4ca5 100644 --- a/internal/gapicgen/cmd/genbot/generate.go +++ b/internal/gapicgen/cmd/genbot/generate.go @@ -53,16 +53,16 @@ func generate(ctx context.Context, githubClient *GithubClient) error { grp, _ := errgroup.WithContext(ctx) grp.Go(func() error { - return gitClone("https://github.com/googleapis/googleapis", googleapisDir) + return gitDeepClone("https://github.com/googleapis/googleapis", googleapisDir) }) grp.Go(func() error { - return gitClone("https://github.com/googleapis/go-genproto", genprotoDir) + return gitDeepClone("https://github.com/googleapis/go-genproto", genprotoDir) }) grp.Go(func() error { - return gitClone("https://github.com/googleapis/google-cloud-go", gocloudDir) + return gitDeepClone("https://github.com/googleapis/google-cloud-go", gocloudDir) }) grp.Go(func() error { - return gitClone("https://github.com/protocolbuffers/protobuf", protoDir) + return gitDeepClone("https://github.com/protocolbuffers/protobuf", protoDir) }) if err := grp.Wait(); err != nil { log.Println(err) @@ -140,7 +140,7 @@ func generate(ctx context.Context, githubClient *GithubClient) error { } // gitClone clones a repository in the given directory. -func gitClone(repo, dir string) error { +func gitDeepClone(repo, dir string) error { log.Printf("cloning %s\n", repo) _, err := git.PlainClone(dir, false, &git.CloneOptions{ diff --git a/internal/gapicgen/cmd/genbot/github.go b/internal/gapicgen/cmd/genbot/github.go index c84e04d18330..24b296d5e6cc 100644 --- a/internal/gapicgen/cmd/genbot/github.go +++ b/internal/gapicgen/cmd/genbot/github.go @@ -235,7 +235,7 @@ git push origin $BRANCH_NAME // Can't assign the submitter of the PR as a reviewer. var reviewers []string for _, r := range githubReviewers { - if r != *githubUsername { + if r != gc.Username { reviewers = append(reviewers, r) } } diff --git a/internal/gapicgen/cmd/genbot/local.go b/internal/gapicgen/cmd/genbot/local.go new file mode 100644 index 000000000000..a45b5d6656e7 --- /dev/null +++ b/internal/gapicgen/cmd/genbot/local.go @@ -0,0 +1,104 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !windows + +package main + +import ( + "context" + "io/ioutil" + "log" + "os" + "path/filepath" + + "cloud.google.com/go/internal/gapicgen/generator" + "golang.org/x/sync/errgroup" + "gopkg.in/src-d/go-git.v4" +) + +type localConfig struct { + googleapisDir string + gocloudDir string + genprotoDir string + protoDir string + gapicToGenerate string + onlyGapics bool +} + +func genLocal(ctx context.Context, c localConfig) error { + log.Println("creating temp dir") + tmpDir, err := ioutil.TempDir("", "update-genproto") + if err != nil { + log.Fatal(err) + } + log.Printf("temp dir created at %s\n", tmpDir) + tmpGoogleapisDir := filepath.Join(tmpDir, "googleapis") + tmpGenprotoDir := filepath.Join(tmpDir, "genproto") + tmpGocloudDir := filepath.Join(tmpDir, "gocloud") + tmpProtoDir := filepath.Join(tmpDir, "proto") + + // Clone repositories if needed. + grp, _ := errgroup.WithContext(ctx) + gitShallowClone(grp, "https://github.com/googleapis/googleapis.git", c.googleapisDir, tmpGoogleapisDir) + gitShallowClone(grp, "https://github.com/googleapis/go-genproto", c.genprotoDir, tmpGenprotoDir) + gitShallowClone(grp, "https://github.com/googleapis/google-cloud-go", c.gocloudDir, tmpGocloudDir) + gitShallowClone(grp, "https://github.com/protocolbuffers/protobuf", c.protoDir, tmpProtoDir) + if err := grp.Wait(); err != nil { + log.Println(err) + } + + // Regen. + conf := &generator.Config{ + GoogleapisDir: deafultDir(tmpGoogleapisDir, c.googleapisDir), + GenprotoDir: deafultDir(tmpGenprotoDir, c.genprotoDir), + GapicDir: deafultDir(tmpGocloudDir, c.gocloudDir), + ProtoDir: deafultDir(tmpProtoDir, c.protoDir), + GapicToGenerate: c.gapicToGenerate, + OnlyGenerateGapic: c.onlyGapics, + LocalMode: true, + } + if _, err := generator.Generate(ctx, conf); err != nil { + log.Printf("Generator ran (and failed) in %s\n", tmpDir) + log.Fatal(err) + } + return nil +} + +// gitShallowClone clones a repository into the provided tmpDir if a dir has not +// been specified. +func gitShallowClone(eg *errgroup.Group, repo, dir, tmpDir string) { + if dir != "" { + return + } + eg.Go(func() error { + log.Printf("cloning %s\n", repo) + + _, err := git.PlainClone(tmpDir, false, &git.CloneOptions{ + URL: repo, + Progress: os.Stdout, + Depth: 1, + Tags: git.NoTags, + }) + return err + }) +} + +// deafultDir returns the default option if dir is not set. +func deafultDir(def, dir string) string { + if dir == "" { + return def + } + return dir +} diff --git a/internal/gapicgen/cmd/genbot/main.go b/internal/gapicgen/cmd/genbot/main.go index bf07d2f44495..aa598b77869a 100644 --- a/internal/gapicgen/cmd/genbot/main.go +++ b/internal/gapicgen/cmd/genbot/main.go @@ -21,44 +21,13 @@ package main import ( "context" "flag" - "fmt" "log" "os" - "time" + "strconv" "cloud.google.com/go/internal/gapicgen" ) -var ( - toolsNeeded = []string{"git", "go", "protoc"} - - githubAccessToken = flag.String("githubAccessToken", "", "Get an access token at https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line") - githubUsername = flag.String("githubUsername", "", "ex -githubUsername=jadekler") - githubName = flag.String("githubName", "", "ex -githubName=\"Jean de Klerk\"") - githubEmail = flag.String("githubEmail", "", "ex -githubEmail=deklerk@google.com") - - usage = func() { - fmt.Fprintln(os.Stderr, `genbot \ - -githubAccessToken=11223344556677889900aabbccddeeff11223344 \ - -githubUsername=jadekler \ - -githubEmail=deklerk@google.com \ - -githubName="Jean de Klerk" \ - --githubAccessToken - The access token to authenticate to github. Get this at https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line. - --githubUsername - The username to use in the github commit. - --githubName - The name to use in the github commit. - --githubEmail - The email to use in the github commit.`) - os.Exit(2) - } -) - type prStatus uint8 func (ps prStatus) Has(status prStatus) bool { return ps&status != 0 } @@ -71,83 +40,48 @@ const ( func main() { log.SetFlags(0) - - flag.Usage = usage - flag.Parse() - if *githubAccessToken == "" { - *githubAccessToken = os.Getenv("GITHUB_ACCESS_TOKEN") - } - if *githubUsername == "" { - *githubUsername = os.Getenv("GITHUB_USERNAME") - } - if *githubName == "" { - *githubName = os.Getenv("GITHUB_NAME") - } - if *githubEmail == "" { - *githubEmail = os.Getenv("GITHUB_EMAIL") - } - - for k, v := range map[string]string{ - "githubAccessToken": *githubAccessToken, - "githubUsername": *githubUsername, - "githubName": *githubName, - "githubEmail": *githubEmail, - } { - if v == "" { - log.Printf("missing or empty value for required flag --%s\n", k) - usage() - } + if err := gapicgen.VerifyAllToolsExist([]string{"git", "go", "protoc"}); err != nil { + log.Fatal(err) } - ctx := context.Background() - // Setup the client and git environment. - if err := gapicgen.VerifyAllToolsExist(toolsNeeded); err != nil { - log.Fatal(err) - } - githubClient, err := NewGithubClient(ctx, *githubUsername, *githubName, *githubEmail, *githubAccessToken) - if err != nil { - log.Fatal(err) - } + // General bot flags + githubAccessToken := flag.String("githubAccessToken", os.Getenv("GITHUB_ACCESS_TOKEN"), "The token used to open pull requests.") + githubUsername := flag.String("githubUsername", os.Getenv("GITHUB_USERNAME"), "The GitHub user name for the author.") + githubName := flag.String("githubName", os.Getenv("GITHUB_NAME"), "The name of the author for git commits.") + githubEmail := flag.String("githubEmail", os.Getenv("GITHUB_EMAIL"), "The email address of the author.") + localMode := flag.Bool("local", strToBool(os.Getenv("GENBOT_LOCAL_MODE")), "Enables generating sources locally. This mode will not open any pull requests.") + + // flags for local mode + googleapisDir := flag.String("googleapis-dir", os.Getenv("GOOGLEAPIS_DIR"), "Directory where sources of googleapis/googleapis resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.") + gocloudDir := flag.String("gocloud-dir", os.Getenv("GOCLOUD_DIR"), "Directory where sources of googleapis/google-cloud-go resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.") + genprotoDir := flag.String("genproto-dir", os.Getenv("GENPROTO_DIR"), "Directory where sources of googleapis/go-genproto resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.") + protoDir := flag.String("proto-dir", os.Getenv("PROTO_DIR"), "Directory where sources of google/protobuf resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.") + gapicToGenerate := flag.String("gapic", os.Getenv("GAPIC_TO_GENERATE"), `Specifies which gapic to generate. The value should be in the form of an import path (Ex: cloud.google.com/go/pubsub/apiv1). The default "" generates all gapics.`) + onlyGapics := flag.Bool("only-gapics", strToBool(os.Getenv("ONLY_GAPICS")), "Enabling stops regenerating genproto.") - // Check current regen status. - if pr, err := githubClient.GetRegenPR(ctx, "go-genproto", "open"); err != nil { - log.Fatal(err) - } else if pr != nil { - log.Println("There is already a re-generation in progress") - return - } - if pr, err := githubClient.GetRegenPR(ctx, "google-cloud-go", "open"); err != nil { - log.Fatal(err) - } else if pr != nil { - if err := updateGocloudPR(ctx, githubClient, pr); err != nil { + flag.Parse() + + if *localMode { + if err := genLocal(ctx, localConfig{ + googleapisDir: *googleapisDir, + gocloudDir: *gocloudDir, + genprotoDir: *genprotoDir, + protoDir: *protoDir, + gapicToGenerate: *gapicToGenerate, + onlyGapics: *onlyGapics, + }); err != nil { log.Fatal(err) } return } - log.Println("checking if a pull request was already opened and merged today") - if pr, err := githubClient.GetRegenPR(ctx, "go-genproto", "closed"); err != nil { - log.Fatal(err) - } else if pr != nil && hasCreatedPRToday(pr.Created) { - log.Println("skipping generation, already created and merged a go-genproto PR today") - return - } - if pr, err := githubClient.GetRegenPR(ctx, "google-cloud-go", "closed"); err != nil { - log.Fatal(err) - } else if pr != nil && hasCreatedPRToday(pr.Created) { - log.Println("skipping generation, already created and merged a google-cloud-go PR today") - return - } - - if err := generate(ctx, githubClient); err != nil { + if err := genBot(ctx, *githubAccessToken, *githubUsername, *githubName, *githubEmail); err != nil { log.Fatal(err) } } -// hasCreatedPRToday checks if the created time of a PR is from today. -func hasCreatedPRToday(created time.Time) bool { - now := time.Now() - today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) - log.Printf("Times -- Now: %v\tToday: %v\tPR Created: %v", now, today, created) - return created.After(today) +func strToBool(s string) bool { + // Treat error as false + b, _ := strconv.ParseBool(s) + return b } diff --git a/internal/gapicgen/cmd/genlocal/README.md b/internal/gapicgen/cmd/genlocal/README.md deleted file mode 100644 index 3532ee437abe..000000000000 --- a/internal/gapicgen/cmd/genlocal/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# genlocal - -genlocal is a binary for generating gapics locally. It may be used to test out -new changes, test the generation of a new library, test new generator tweaks, -run generators against googleapis-private, and various other local tasks. - -## Required tools - -*Note*: If you are on a Windows platform you will need to install these tools -in a linux docker container: [Install docker](https://www.docker.com/get-started) - -1. Install [protoc](https://github.com/protocolbuffers/protobuf/releases) -2. Install [Go](http://golang.org/dl) -3. Install python3, pip3 -4. Install virtualenv `pip3 install virtualenv` -5. Install Go tools: - -```bash -go get \ - github.com/golang/protobuf/protoc-gen-go \ - golang.org/x/lint/golint \ - golang.org/x/tools/cmd/goimports \ - honnef.co/go/tools/cmd/staticcheck \ - github.com/googleapis/gapic-generator-go/cmd/protoc-gen-go_gapic -``` - -## Running - -`git clone` this project if you don't already have it checked-out locally. - -```bash -cd /path/to/google-cloud-go/internal/gapicgen -go run cloud.google.com/go/internal/gapicgen/cmd/genlocal -``` diff --git a/internal/gapicgen/cmd/genlocal/main.go b/internal/gapicgen/cmd/genlocal/main.go deleted file mode 100644 index 0f004a0e8152..000000000000 --- a/internal/gapicgen/cmd/genlocal/main.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build !windows - -// genlocal is a binary for generating gapics locally. It may be used to test out -// new changes, test the generation of a new library, test new generator tweaks, -// run generators against googleapis-private, and various other local tasks. -package main - -import ( - "context" - "flag" - "io/ioutil" - "log" - "os" - "path/filepath" - "strings" - - "cloud.google.com/go/internal/gapicgen" - "cloud.google.com/go/internal/gapicgen/generator" - "golang.org/x/sync/errgroup" - "gopkg.in/src-d/go-git.v4" -) - -var ( - toolsNeeded = []string{"go", "protoc"} -) - -func main() { - log.SetFlags(0) - if err := gapicgen.VerifyAllToolsExist(toolsNeeded); err != nil { - log.Fatal(err) - } - - log.Println("creating temp dir") - tmpDir, err := ioutil.TempDir("", "update-genproto") - if err != nil { - log.Fatal(err) - } - log.Printf("temp dir created at %s\n", tmpDir) - - googleapisDir := flag.String("googleapis-dir", filepath.Join(tmpDir, "googleapis"), "Directory where sources of googleapis/googleapis resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.") - gocloudDir := flag.String("gocloud-dir", filepath.Join(tmpDir, "gocloud"), "Directory where sources of googleapis/google-cloud-go resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.") - genprotoDir := flag.String("genproto-dir", filepath.Join(tmpDir, "genproto"), "Directory where sources of googleapis/go-genproto resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.") - protoDir := flag.String("proto-dir", filepath.Join(tmpDir, "proto"), "Directory where sources of google/protobuf resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.") - gapicToGenerate := flag.String("gapic", "", `Specifies which gapic to generate. The value should be in the form of an import path (Ex: cloud.google.com/go/pubsub/apiv1). The default "" generates all gapics.`) - onlyGapics := flag.Bool("only-gapics", false, "Enabling stops regenerating genproto.") - verbose := flag.Bool("verbose", false, "Enables verbose logging.") - flag.Parse() - - ctx := context.Background() - - // Clone repositories if needed. - grp, _ := errgroup.WithContext(ctx) - gitClone(grp, "https://github.com/googleapis/googleapis.git", *googleapisDir, tmpDir) - gitClone(grp, "https://github.com/googleapis/go-genproto", *genprotoDir, tmpDir) - gitClone(grp, "https://github.com/googleapis/google-cloud-go", *gocloudDir, tmpDir) - gitClone(grp, "https://github.com/protocolbuffers/protobuf", *protoDir, tmpDir) - if err := grp.Wait(); err != nil { - log.Println(err) - } - - // Regen. - conf := &generator.Config{ - GoogleapisDir: *googleapisDir, - GenprotoDir: *genprotoDir, - GapicDir: *gocloudDir, - ProtoDir: *protoDir, - GapicToGenerate: *gapicToGenerate, - OnlyGenerateGapic: *onlyGapics, - } - changes, err := generator.Generate(ctx, conf) - if err != nil { - log.Printf("Generator ran (and failed) in %s\n", tmpDir) - log.Fatal(err) - } - - // Log results. - log.Println(genprotoDir) - log.Println(gocloudDir) - - if *verbose { - log.Println(generator.FormatChanges(changes, false)) - } -} - -// gitClone clones a repository in the given directory if dir is not in tmpDir. -func gitClone(eg *errgroup.Group, repo, dir, tmpDir string) { - if !strings.HasPrefix(dir, tmpDir) { - return - } - eg.Go(func() error { - log.Printf("cloning %s\n", repo) - - _, err := git.PlainClone(dir, false, &git.CloneOptions{ - URL: repo, - Progress: os.Stdout, - }) - return err - }) -} diff --git a/internal/gapicgen/generator/generator.go b/internal/gapicgen/generator/generator.go index f5b7f9707a27..c3831e4b0308 100644 --- a/internal/gapicgen/generator/generator.go +++ b/internal/gapicgen/generator/generator.go @@ -36,6 +36,7 @@ type Config struct { ProtoDir string GapicToGenerate string OnlyGenerateGapic bool + LocalMode bool } // Generate generates genproto and gapics. @@ -51,13 +52,16 @@ func Generate(ctx context.Context, conf *Config) ([]*ChangeInfo, error) { return nil, fmt.Errorf("error generating gapics (may need to check logs for more errors): %v", err) } - changes, err := gatherChanges(conf.GoogleapisDir, conf.GenprotoDir) - if err != nil { - return nil, fmt.Errorf("error gathering commit info") - } - - if err := recordGoogleapisHash(conf.GoogleapisDir, conf.GenprotoDir); err != nil { - return nil, err + var changes []*ChangeInfo + if !conf.LocalMode { + var err error + changes, err = gatherChanges(conf.GoogleapisDir, conf.GenprotoDir) + if err != nil { + return nil, fmt.Errorf("error gathering commit info") + } + if err := recordGoogleapisHash(conf.GoogleapisDir, conf.GenprotoDir); err != nil { + return nil, err + } } return changes, nil