From 379665ce1e349833c5993335139e05cccc229e0f Mon Sep 17 00:00:00 2001 From: seph Date: Mon, 14 Dec 2020 15:37:47 -0500 Subject: [PATCH] Support architectures in the build/release side (#682) Update the build/release process to support creating darwin universal binaries. This requires building on an m1 machine, with go installed at the specified paths. To make this a bit smoother, it also updates a handful of the abstractions across our build tools. --- .circleci/config.yml | 18 ---- .github/workflows/go.yml | 2 +- Makefile | 173 +++++++++++++++++++++++++-------------- cmd/make/make.go | 63 ++++++++++---- pkg/make/builder.go | 66 +++++++++------ pkg/make/builder_test.go | 16 ++-- 6 files changed, 206 insertions(+), 132 deletions(-) delete mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 29ac9b1ef..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,18 +0,0 @@ -version: 2 -jobs: - build: - docker: - - image: golang:1.12 - working_directory: /tmp/launcher - steps: &steps - - checkout - - run: make deps - - run: make -j lint - - run: make -j {linux,windows}-xp-{launcher,extension,grpc-extension} - - run: make test - -workflows: - version: 2 - build: - jobs: - - build diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 850f37393..6254d487e 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -39,7 +39,7 @@ jobs: if: runner.os != 'Windows' - name: Build - run: make -j launcher extension grpc-extension table.ext + run: make -j launcher extension grpc.ext tables.ext package-builder - name: Test run: make test diff --git a/Makefile b/Makefile index d05ac77f7..953207efd 100644 --- a/Makefile +++ b/Makefile @@ -10,20 +10,17 @@ PATH := $(GOPATH)/bin:$(PATH) export GO111MODULE=on -# If on macOS, set the shell to bash explicitly +# If not windows, set the shell to bash explicitly ifneq ($(OS), Windows_NT) ifeq ($(shell uname), Darwin) SHELL := /bin/bash endif endif -fake-launcher: .pre-build - go run cmd/make/make.go -targets=launcher -linkstamp -fakedata - -rm build/darwin/launcher - mv build/launcher build/launcher-fake all: build -build: launcher extension +build: build_launcher build_extension + .pre-build: ${BUILD_DIR} ${BUILD_DIR}: @@ -33,62 +30,127 @@ else mkdir -p ${BUILD_DIR} endif -# Simple things, pointers into our build -launcher: .pre-build - go run cmd/make/make.go -targets=launcher -linkstamp +## +## Build +## -table.ext: .pre-build - go run cmd/make/make.go -targets=table-extension -linkstamp -table.ext-windows: .pre-build deps - go run cmd/make/make.go -targets=table-extension -linkstamp --os windows +build_%: TARGET = $(word 2, $(subst _, ,$@)) +build_%: OS = $(word 3, $(subst _, ,$@)) +build_%: OSARG = $(if $(OS), --os $(OS)) +build_%: ARCH = $(word 4, $(subst _, ,$@)) +build_%: ARCHARG = $(if $(ARCH), --arch $(ARCH)) +build_%: GOARG = $(if $(CROSSGOPATH), --go $(CROSSGOPATH)) +build_%: .pre-build + go run cmd/make/make.go -targets=$(TARGET) -linkstamp $(OSARG) $(ARCHARG) $(GOARG) + +fake_%: TARGET = $(word 2, $(subst _, ,$@)) +fake_%: OS = $(word 3, $(subst _, ,$@)) +fake_%: OSARG = $(if $(OS), --os $(OS)) +fake_%: ARCH = $(word 4, $(subst _, ,$@)) +fake_%: ARCHARG = $(if $(ARCH), --arch $(ARCH)) +fake_%: .pre-build + go run cmd/make/make.go -targets=$(TARGET) -linkstamp -fakedata $(OSARG) $(ARCHARG) + +# The lipo command will combine things into universal +# binaries. Because of the go path needs, there is little point in +# abstracting this further +lipo_%: build/darwin.amd64/% build/darwin.arm64/% + @mkdir -p build/darwin.universal + lipo -create $^ -output build/darwin.universal/$* + +# pointers, mostly for legacy reasons +launcher: build_launcher +tables.ext: build_tables.ext +extension: build_osquery-extension.ext +grpc.ext: build_grpc.ext +fake-launcher: fake_launcher -extension: .pre-build - go run cmd/make/make.go -targets=extension +## +## Cross Build targets +## -grpc-extension: .pre-build - go run cmd/make/make.go -targets=grpc-extension +RELEASE_TARGETS=launcher osquery-extension.ext package-builder +MANUAL_CROSS_OSES=darwin windows linux +ARM64_OSES=darwin +AMD64_OSES=darwin windows linux +# xp is a helper for quick cross platform builds, and sanity checking +# for breakage. humans only +xp: $(foreach target, $(RELEASE_TARGETS), $(foreach os, $(MANUAL_CROSS_OSES), build_$(target)_$(os))) -# Convenience tools -osqueryi-tables: table.ext - osqueryd -S --allow-unsafe --verbose --extension ./build/darwin/tables.ext -osqueryi-tables-linux: table.ext - osqueryd -S --allow-unsafe --verbose --extension ./build/linux/tables.ext -osqueryi-tables-windows: table.ext - osqueryd.exe -S --allow-unsafe --verbose --extension .\build\windows\tables.exe -sudo-osqueryi-tables: table.ext - sudo osqueryd -S --allow-unsafe --verbose --extension ./build/darwin/tables.ext -launchas-osqueryi-tables: table.ext - sudo launchctl asuser 0 osqueryd -S --allow-unsafe --verbose --extension ./build/darwin/tables.ext +# Actual release targets. Because of the m1 cgo cross stuff, this requires explicit go paths +rel-amd64: CROSSGOPATH = /Users/seph/go1.15.6.darwin-amd64/bin/go +rel-amd64: $(foreach target, $(RELEASE_TARGETS), $(foreach os, $(AMD64_OSES), build_$(target)_$(os)_amd64)) + +rel-arm64: CROSSGOPATH = /opt/homebrew/bin/go +rel-arm64: $(foreach target, $(RELEASE_TARGETS), $(foreach os, $(ARM64_OSES), build_$(target)_$(os)_arm64)) + +rel-lipo: $(foreach target, $(RELEASE_TARGETS), lipo_$(target)) + +## +## Release Process Stuff +## +RELEASE_VERSION = $(shell git describe --tags --always --dirty) +release: + @echo "Run 'make release-phase1' on the m1 machine" + @echo "Run 'make release-phase2' on a codesign machine" -xp: xp-launcher xp-extension xp-grpc-extension +release-phase1: + rm -rf build + $(MAKE) rel-amd64 rel-arm64 + $(MAKE) rel-lipo +# $(MAKE) codesign +# $(MAKE) binary-bundles -xp-%: darwin-xp-% windows-xp-% linux-xp-% - @true # make needs something here for the pattern rule +release-phase2: + rm -rf build + rsync -av 10.42.19.215:~/checkouts/kolide/launcher/build ./ +# $(MAKE) rel-amd64 rel-arm64 +# $(MAKE) rel-lipo + $(MAKE) codesign + $(MAKE) binary-bundles -darwin-xp-%: .pre-build deps - go run cmd/make/make.go -targets=$* -linkstamp -os=darwin -linux-xp-%: .pre-build deps - go run cmd/make/make.go -targets=$* -linkstamp -os=linux +# release: binary-bundle containers-push -windows-xp-%: .pre-build deps - go run cmd/make/make.go -targets=$* -linkstamp -os=windows +binary-bundles: + rm -rf build/binary-bundles + $(MAKE) $(foreach p, $(shell cd build && ls -d */ | tr -d /), build/binary-bundles/$(p)) +build/binary-bundles/%: + mkdir -p build/binary-bundles + mv build/$* build/$*_$(RELEASE_VERSION) + cd build && zip -r "binary-bundles/$*_$(RELEASE_VERSION)".zip $*_$(RELEASE_VERSION) + + +## +## Handy osqueryi command line +## + +osqueryi-tables: build_tables.ext + osqueryd -S --allow-unsafe --verbose --extension ./build/darwin/tables.ext +osqueryi-tables-linux: build_tables.ext + osqueryd -S --allow-unsafe --verbose --extension ./build/linux/tables.ext +osqueryi-tables-windows: build_tables.ext + osqueryd.exe -S --allow-unsafe --verbose --extension .\build\windows\tables.exe +sudo-osqueryi-tables: build_tables.ext + sudo osqueryd -S --allow-unsafe --verbose --extension ./build/darwin/tables.ext +launchas-osqueryi-tables: build_tables.ext + sudo launchctl asuser 0 osqueryd -S --allow-unsafe --verbose --extension ./build/darwin/tables.ext # `-o runtime` should be enough, however there was a catalina bug that # required we add `library`. This was fixed in 10.15.4. (from # macadmins slack) -codesign-darwin: xp - codesign --force -s "${CODESIGN_IDENTITY}" -v --options runtime,library --timestamp ./build/darwin/* +codesign-darwin: + codesign --force -s "${CODESIGN_IDENTITY}" -v --options runtime,library --timestamp ./build/darwin*/* notarize-darwin: codesign-darwin rm -f build/notarization-upload.zip - zip -r build/notarization-upload.zip ./build/darwin/* + zip -r build/notarization-upload.zip ./build/darwin* xcrun altool \ --username "${NOTARIZE_APPLE_ID}" \ --password @env:NOTARIZE_APP_PASSWD \ @@ -110,10 +172,15 @@ notarize-check-%: # Using the `osslsigncode` we can sign windows binaries from # non-windows platforms. codesign-windows: codesign-windows-launcher.exe codesign-windows-osquery-extension.exe -codesign-windows-%: xp +codesign-windows-%: P12 = ~/Documents/kolide-codesigning-2020.p12 +codesign-windows-%: @if [ -z "${AUTHENTICODE_PASSPHRASE}" ]; then echo "Missing AUTHENTICODE_PASSPHRASE"; exit 1; fi - osslsigncode -in build/windows/$* -out build/windows/$* -i https://kolide.com -h sha1 -t http://timestamp.verisign.com/scripts/timstamp.dll -pkcs12 ~/Documents/kolide-codesigning-2020.p12 -pass "${AUTHENTICODE_PASSPHRASE}" - osslsigncode -in build/windows/$* -out build/windows/$* -i https://kolide.com -h sha256 -nest -ts http://sha256timestamp.ws.symantec.com/sha256/timestamp -pkcs12 ~/Documents/kolide-codesigning-2020.p12 -pass "${AUTHENTICODE_PASSPHRASE}" + mv build/windows.amd64/$* build/windows.amd64/$*.tmp + osslsigncode sign -in build/windows.amd64/$*.tmp -out build/windows.amd64/$* -i https://kolide.com -h sha1 -t http://timestamp.verisign.com/scripts/timstamp.dll -pkcs12 $(P12) -pass "${AUTHENTICODE_PASSPHRASE}" + rm build/windows.amd64/$*.tmp + mv build/windows.amd64/$* build/windows.amd64/$*.tmp + osslsigncode sign -in build/windows.amd64/$*.tmp -out build/windows.amd64/$* -i https://kolide.com -h sha256 -nest -ts http://sha256timestamp.ws.symantec.com/sha256/timestamp -pkcs12 $(P12) -pass "${AUTHENTICODE_PASSPHRASE}" + rm build/windows.amd64/$*.tmp codesign: notarize-darwin codesign-windows @@ -170,7 +237,7 @@ lint-go-vet: go vet ./cmd/... ./pkg/... lint-go-nakedret: deps-go - nakedret ./... + nakedret ./pkg/... ./cmd/... # This is ugly. since go-fmt doesn't have a simple exit code, we use # some make trickery to handle failing if there;s output. @@ -180,24 +247,6 @@ fmt-fail/%: @echo fmt failure in: $* @false -## -## Release Process Stuff -## - -release: binary-bundle containers-push - -binary-bundle: VERSION = $(shell git describe --tags --always --dirty) -binary-bundle: codesign - rm -rf build/binary-bundle - $(MAKE) -j $(foreach p, darwin linux windows, build/binary-bundle/$(p)) - cd build/binary-bundle && zip -r "launcher_${VERSION}.zip" * - -build/binary-bundle/%: - mkdir -p $@ - cp build/$*/launcher* $@/ - cp build/$*/osquery-extension* $@/ - go run ./tools/download-osquery.go --platform=$* --output=$@/osqueryd - ## ## Docker Tooling ## diff --git a/cmd/make/make.go b/cmd/make/make.go index 953b2376c..8ccc00c13 100644 --- a/cmd/make/make.go +++ b/cmd/make/make.go @@ -3,6 +3,7 @@ package main import ( "context" "flag" + "os" "runtime" "strings" @@ -10,6 +11,7 @@ import ( "github.com/kolide/kit/logutil" "github.com/kolide/launcher/pkg/contexts/ctxlog" "github.com/kolide/launcher/pkg/make" + "github.com/peterbourgon/ff/v3" ) func main() { @@ -18,17 +20,30 @@ func main() { "install-tools", }, ",") + fs := flag.NewFlagSet("make", flag.ExitOnError) + var ( - flTargets = flag.String("targets", buildAll, "comma separated list of targets") - flDebug = flag.Bool("debug", false, "use a debug logger") - flBuildARCH = flag.String("arch", runtime.GOARCH, "Architecture to build for.") - flBuildOS = flag.String("os", runtime.GOOS, "Operating system to build for.") - flRace = flag.Bool("race", false, "Build race-detector version of binaries.") - flStatic = flag.Bool("static", false, "Build a static binary.") - flStampVersion = flag.Bool("linkstamp", false, "Add version info with ldflags.") - flFakeData = flag.Bool("fakedata", false, "Compile with build tags to falsify some data, like serial numbers") + flTargets = fs.String("targets", buildAll, "comma separated list of targets") + flDebug = fs.Bool("debug", false, "use a debug logger") + flBuildARCH = fs.String("arch", runtime.GOARCH, "Architecture to build for.") + flBuildOS = fs.String("os", runtime.GOOS, "Operating system to build for.") + flGoPath = fs.String("go", "", "Path for go binary. Will attempt auto detection") + flRace = fs.Bool("race", false, "Build race-detector version of binaries.") + flStatic = fs.Bool("static", false, "Build a static binary.") + flStampVersion = fs.Bool("linkstamp", false, "Add version info with ldflags.") + flFakeData = fs.Bool("fakedata", false, "Compile with build tags to falsify some data, like serial numbers") ) - flag.Parse() + + ffOpts := []ff.Option{ + ff.WithConfigFileFlag("config"), + ff.WithConfigFileParser(ff.PlainParser), + ff.WithEnvVarPrefix("MAKE"), + } + + if err := ff.Parse(fs, os.Args[1:], ffOpts...); err != nil { + logger := logutil.NewCLILogger(true) + logutil.Fatal(logger, "msg", "Error parsing flags", "err", err) + } logger := logutil.NewCLILogger(*flDebug) ctx := context.Background() @@ -51,6 +66,10 @@ func main() { opts = append(opts, make.WithFakeData()) } + if *flGoPath != "" { + opts = append(opts, make.WithGoPath(*flGoPath)) + } + b, err := make.New(opts...) if err != nil { logutil.Fatal(logger, "msg", "Failed to create builder", "err", err) @@ -58,15 +77,15 @@ func main() { } targetSet := map[string]func(context.Context) error{ - "deps-go": b.DepsGo, - "install-tools": b.InstallTools, - "generate-tuf": b.GenerateTUF, - "launcher": b.BuildCmd("./cmd/launcher", b.PlatformBinaryName("launcher")), - "extension": b.BuildCmd("./cmd/osquery-extension", b.PlatformExtensionName("osquery-extension")), - "table-extension": b.BuildCmd("./cmd/launcher.ext", b.PlatformExtensionName("tables")), - "grpc-extension": b.BuildCmd("./cmd/grpc.ext", b.PlatformExtensionName("grpc")), - "package-builder": b.BuildCmd("./cmd/package-builder", b.PlatformBinaryName("package-builder")), - "make": b.BuildCmd("./cmd/make", b.PlatformBinaryName("make")), + "deps-go": b.DepsGo, + "install-tools": b.InstallTools, + "generate-tuf": b.GenerateTUF, + "launcher": b.BuildCmd("./cmd/launcher", fakeName("launcher", *flFakeData)), + "osquery-extension.ext": b.BuildCmd("./cmd/osquery-extension", "osquery-extension.ext"), + "tables.ext": b.BuildCmd("./cmd/launcher.ext", "tables.ext"), + "grpc.ext": b.BuildCmd("./cmd/grpc.ext", "grpc.ext"), + "package-builder": b.BuildCmd("./cmd/package-builder", "package-builder"), + "make": b.BuildCmd("./cmd/make", "make"), } if t := strings.Split(*flTargets, ","); len(t) != 0 && t[0] != "" { @@ -82,3 +101,11 @@ func main() { } } } + +func fakeName(binName string, fake bool) string { + if !fake { + return binName + } + + return binName + "-fake" +} diff --git a/pkg/make/builder.go b/pkg/make/builder.go index fe9eb4df6..7ae4d6c99 100644 --- a/pkg/make/builder.go +++ b/pkg/make/builder.go @@ -43,10 +43,12 @@ type Builder struct { os string arch string goVer string + goPath string static bool race bool stampVersion bool fakedata bool + notStripped bool cmdEnv []string execCC func(context.Context, string, ...string) *exec.Cmd @@ -54,6 +56,12 @@ type Builder struct { type Option func(*Builder) +func WithGoPath(goPath string) Option { + return func(b *Builder) { + b.goPath = goPath + } +} + func WithOS(o string) Option { return func(b *Builder) { b.os = o @@ -72,6 +80,12 @@ func WithStatic() Option { } } +func WithOutStripped() Option { + return func(b *Builder) { + b.notStripped = true + } +} + func WithRace() Option { return func(b *Builder) { b.race = true @@ -92,9 +106,10 @@ func WithFakeData() Option { func New(opts ...Option) (*Builder, error) { b := Builder{ - os: runtime.GOOS, - arch: runtime.GOARCH, - goVer: strings.TrimPrefix(runtime.Version(), "go"), + os: runtime.GOOS, + arch: runtime.GOARCH, + goPath: "go", + goVer: strings.TrimPrefix(runtime.Version(), "go"), execCC: exec.CommandContext, } @@ -117,23 +132,17 @@ func New(opts ...Option) (*Builder, error) { return &b, nil } -// PlatformExtensionName is a helper to return the platform specific extension name. -func (b *Builder) PlatformExtensionName(input string) string { - input = filepath.Join("build", b.os, input) - if b.os == "windows" { - return input + ".exe" - } else { - return input + ".ext" - } -} - -// PlatformBinaryName is a helper to return the platform specific binary suffix. +// PlatformBinaryName is a helper to return the platform specific output path. func (b *Builder) PlatformBinaryName(input string) string { - input = filepath.Join("build", b.os, input) + // On windows, everything must end in .exe. Strip off the extension + // suffix, if present, and add .exe if b.os == "windows" { - return input + ".exe" + input = strings.TrimSuffix(input, ".ext") + ".exe" } - return input + + platformName := fmt.Sprintf("%s.%s", b.os, b.arch) + + return filepath.Join("build", platformName, input) } func (b *Builder) goVersionCompatible(logger log.Logger) error { @@ -386,9 +395,9 @@ func bootstrapFromNotary(notaryConfigDir, remoteServerURL, localRepo, gun string return nil } -func (b *Builder) BuildCmd(src, output string) func(context.Context) error { +func (b *Builder) BuildCmd(src, appName string) func(context.Context) error { return func(ctx context.Context) error { - _, appName := filepath.Split(output) + output := b.PlatformBinaryName(appName) ctx, span := trace.StartSpan(ctx, fmt.Sprintf("make.BuildCmd.%s", appName)) defer span.End() @@ -414,8 +423,13 @@ func (b *Builder) BuildCmd(src, output string) func(context.Context) error { var ldFlags []string if b.static { - ldFlags = append(ldFlags, "-w -d -linkmode internal") + ldFlags = append(ldFlags, "-d -linkmode internal") } + + if !b.notStripped { + ldFlags = append(ldFlags, "-w -s") + } + if b.stampVersion { v, err := b.getVersion(ctx) if err != nil { @@ -458,7 +472,7 @@ func (b *Builder) BuildCmd(src, output string) func(context.Context) error { } args := append(baseArgs, src) - cmd := b.execCC(ctx, "go", args...) + cmd := b.execCC(ctx, b.goPath, args...) cmd.Env = append(cmd.Env, b.cmdEnv...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr @@ -478,14 +492,16 @@ func (b *Builder) BuildCmd(src, output string) func(context.Context) error { // all the builds go to `build//binary`, but if the build OS is the same as the target OS, // we also want to hardlink the resulting binary at the root of `build/` for convenience. // ex: running ./build/launcher on macos instead of ./build/darwin/launcher - if b.os == runtime.GOOS { - platformPath := filepath.Join("build", appName) - if err := os.Remove(platformPath); err != nil && !os.IsNotExist(err) { + if b.os == runtime.GOOS && b.arch == runtime.GOARCH { + _, binName := filepath.Split(output) + symlinkTarget := filepath.Join("build", binName) + + if err := os.Remove(symlinkTarget); err != nil && !os.IsNotExist(err) { // log but don't fail. This could happen if for example ./build/launcher.exe is referenced by a running service. // if this becomes clearer, we can either return an error here, or go back to silently ignoring. level.Debug(ctxlog.FromContext(ctx)).Log("msg", "remove before hardlink failed", "err", err, "app_name", appName) } - return os.Link(output, platformPath) + return os.Link(output, symlinkTarget) } return nil } diff --git a/pkg/make/builder_test.go b/pkg/make/builder_test.go index 8edc98c22..3bf8e4652 100644 --- a/pkg/make/builder_test.go +++ b/pkg/make/builder_test.go @@ -36,26 +36,26 @@ func TestNamingHelpers(t *testing.T) { }{ { platform: "linux", - extensionOut: "build/linux/test.ext", - binaryOut: "build/linux/test", + extensionOut: "build/linux.amd64/test.ext", + binaryOut: "build/linux.amd64/test", }, { platform: "windows", - extensionOut: "build/windows/test.exe", - binaryOut: "build/windows/test.exe", + extensionOut: "build/windows.amd64/test.exe", + binaryOut: "build/windows.amd64/test.exe", }, { platform: "darwin", - extensionOut: "build/darwin/test.ext", - binaryOut: "build/darwin/test", + extensionOut: "build/darwin.amd64/test.ext", + binaryOut: "build/darwin.amd64/test", }, } for _, tt := range tests { t.Run("platform="+tt.platform, func(t *testing.T) { - b := Builder{os: tt.platform} + b := Builder{os: tt.platform, arch: "amd64"} require.Equal(t, filepath.Clean(tt.binaryOut), b.PlatformBinaryName("test")) - require.Equal(t, filepath.Clean(tt.extensionOut), b.PlatformExtensionName("test")) + require.Equal(t, filepath.Clean(tt.extensionOut), b.PlatformBinaryName("test.ext")) }) } }