bazel: add bazel mirror (#39072)

close #39081
pingcap · Jul 18, 2023 · 17cda10 · 17cda10
1 parent 2ff2d11
commit 17cda10
Show file tree

Hide file tree

Showing 12 changed files with 7,561 additions and 2,174 deletions.
diff --git a/BUILD.bazel b/BUILD.bazel
@@ -1,5 +1,12 @@
 load("@bazel_gazelle//:def.bzl", "gazelle")
 
+exports_files([
+    "DEPS.bzl",
+    "WORKSPACE",
+    "go.mod",
+    "go.sum",
+])
+
 # gazelle:proto disable_global
 # gazelle:prefix github.com/pingcap/tidb
 # gazelle:exclude tidb-binlog/proto/proto

diff --git a/DEPS.bzl b/DEPS.bzl
diff --git a/Makefile b/Makefile
@@ -433,6 +433,8 @@ generate_grafana_scripts:
 bazel_ci_prepare:
 	bazel $(BAZEL_GLOBAL_CONFIG) run $(BAZEL_CMD_CONFIG) //:gazelle
 	bazel $(BAZEL_GLOBAL_CONFIG) run $(BAZEL_CMD_CONFIG) //:gazelle -- update-repos -from_file=go.mod -to_macro DEPS.bzl%go_deps  -build_file_proto_mode=disable
+	bazel $(BAZEL_GLOBAL_CONFIG) run $(BAZEL_CMD_CONFIG)  //cmd/mirror:mirror -- --mirror> tmp.txt
+	mv tmp.txt DEPS.bzl
 	bazel $(BAZEL_GLOBAL_CONFIG) run $(BAZEL_CMD_CONFIG)  \
 		--run_under="cd $(CURDIR) && " \
 		 //tools/tazel:tazel
@@ -448,6 +450,10 @@ bazel_prepare:
 	bazel run \
 		--run_under="cd $(CURDIR) && " \
 		 //tools/tazel:tazel
+	$(eval $@TMP_OUT := $(shell mktemp -d -t tidbbzl.XXXXXX))
+	bazel run  //cmd/mirror -- --mirror> $($@TMP_OUT)/tmp.txt
+	cp $($@TMP_OUT)/tmp.txt DEPS.bzl
+	rm -rf $($@TMP_OUT)
 
 bazel_ci_prepare_rbe:
 	bazel run //:gazelle
@@ -473,7 +479,7 @@ bazel_coverage_test: check-bazel-prepare failpoint-enable bazel_ci_prepare
 		-- //... -//cmd/... -//tests/graceshutdown/... \
 		-//tests/globalkilltest/... -//tests/readonlytest/... -//br/pkg/task:task_test -//tests/realtikvtest/...
 
-bazel_build: bazel_ci_prepare
+bazel_build:
 	mkdir -p bin
 	bazel $(BAZEL_GLOBAL_CONFIG) build $(BAZEL_CMD_CONFIG) \
 		//... --//build:with_nogo_flag=true
@@ -555,3 +561,12 @@ docker:
 
 docker-test:
 	docker buildx build --platform linux/amd64,linux/arm64 --push -t "$(DOCKERPREFIX)tidb:latest" --build-arg 'GOPROXY=$(shell go env GOPROXY),' -f Dockerfile .
+
+bazel_mirror:
+	$(eval $@TMP_OUT := $(shell mktemp -d -t tidbbzl.XXXXXX))
+	bazel $(BAZEL_GLOBAL_CONFIG) run $(BAZEL_CMD_CONFIG)  //cmd/mirror:mirror -- --mirror> $($@TMP_OUT)/tmp.txt
+	cp $($@TMP_OUT)/tmp.txt DEPS.bzl
+	rm -rf $($@TMP_OUT)
+
+bazel_sync:
+	bazel $(BAZEL_GLOBAL_CONFIG) sync $(BAZEL_SYNC_CONFIG)
diff --git a/Makefile.common b/Makefile.common
@@ -122,4 +122,5 @@ TEST_COVERAGE_DIR := "test_coverage"
 ifneq ("$(CI)", "")
 	BAZEL_GLOBAL_CONFIG := --output_user_root=/home/jenkins/.tidb/tmp
 	BAZEL_CMD_CONFIG := --config=ci --repository_cache=/home/jenkins/.tidb/tmp
+	BAZEL_SYNC_CONFIG := --repository_cache=/home/jenkins/.tidb/tmp
 endif
diff --git a/build/nogo_config.json b/build/nogo_config.json
@@ -160,6 +160,7 @@
     "exclude_files": {
       "parser/parser.go": "parser/parser.go code",
       "external/": "no need to vet third party code",
+      ".cgo/": "no need to cgo code",
       ".*_generated\\.go$": "ignore generated code",
       ".*_/testmain\\.go$": "ignore code"
     }
@@ -278,6 +279,7 @@
       "/cgo/": "ignore cgo code",
       "/rules_go_work-*": "ignore generated code",
       "tools/check/ut.go": "ignore tools/check code",
+      "cmd/mirror": "ignore cmd/mirror code",
       "tools/check/xprog.go": "ignore tools/check code",
       "cmd/pluginpkg/pluginpkg.go": "ignore cmd/pluginpkg code",
       "tools/check/xprog.go:": "ignore tools/check code",
@@ -1155,6 +1157,7 @@
   "prealloc": {
     "exclude_files": {
       "parser/parser.go": "parser/parser.go code",
+      "cmd/mirror": "cmd/mirror code",
       "external/": "no need to vet third party code",
       ".*_generated\\.go$": "ignore generated code",
       "parser/yy_parser.go": "ignore generated code",
@@ -1164,6 +1167,7 @@
   "predeclared": {
     "exclude_files": {
       "external/": "no need to vet third party code",
+      "cmd/mirror": "no need to mirror",
       ".*_generated\\.go$": "ignore generated code",
       "parser/yy_parser.go": "ignore generated code",
       "parser/parser.go": "ignore generated code",

diff --git a/build/patches/io_etcd_go_etcd_api_v3.patch b/build/patches/io_etcd_go_etcd_api_v3.patch
@@ -6,7 +6,7 @@ diff -uprN old/io_etcd_go_etcd_api_v3/etcdserverpb/BUILD.bazel new/io_etcd_go_et
          "@com_github_gogo_protobuf//gogoproto",
          "@com_github_golang_protobuf//proto:go_default_library",
 -        "@go_googleapis//google/api:annotations_go_proto",
-+        "@org_golang_google_genproto_googleapis_api//annotations:annotations",
++        "@org_golang_google_genproto//googleapis/api/annotations:annotations",
          "@org_golang_google_grpc//:go_default_library",
          "@org_golang_google_grpc//codes",
          "@org_golang_google_grpc//status",

diff --git a/cmd/mirror/BUILD.bazel b/cmd/mirror/BUILD.bazel
@@ -0,0 +1,34 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "mirror_lib",
+    srcs = [
+        "mirror.go",
+        "skylarkutil.go",
+    ],
+    importpath = "github.com/pingcap/tidb/cmd/mirror",
+    visibility = ["//visibility:private"],
+    deps = [
+        "@com_github_google_skylark//syntax",
+        "@com_google_cloud_go_storage//:storage",
+        "@io_bazel_rules_go//go/tools/bazel:go_default_library",
+        "@org_golang_google_api//googleapi",
+        "@org_golang_x_exp//slices",
+        "@org_golang_x_sync//errgroup",
+    ],
+)
+
+go_binary(
+    name = "mirror",
+    data = [
+        "//:DEPS.bzl",
+        "//:go.mod",
+        "//:go.sum",
+        "//build/patches:all_patches",
+        "//parser:go.mod",
+        "//parser:go.sum",
+        "@go_sdk//:files",
+    ],
+    embed = [":mirror_lib"],
+    visibility = ["//visibility:public"],
+)