Removing osquery-extension (kolide#838)

* removed osquery-extension references from main launcher code

* removing tests around osquery extension being removed

* removed all traces of osquery-extension, updated code in response to comments

* now passing list of extensions to osquery required_extension flag, removed osquery-extension.ext cmd

* added launcher flag for auto-loading extensions

Dockerfile

@@ -31,7 +31,7 @@ RUN cd launcher && git checkout "${gitver}"
 # Build!
 RUN cd launcher && make deps
 RUN cd launcher && make all
-RUN cd launcher && GO111MODULE=on go run cmd/make/make.go -targets=launcher,osquery-extension.ext -linkstamp $FAKE
+RUN cd launcher && GO111MODULE=on go run cmd/make/make.go -targets=launcher -linkstamp $FAKE
 
 # Install
 RUN mkdir -p /usr/local/kolide/bin/

Makefile

@@ -19,7 +19,7 @@ endif
 
 
 all: build
-build: build_launcher build_osquery-extension.ext
+build: build_launcher
 
 .pre-build: ${BUILD_DIR}
 
@@ -62,14 +62,13 @@ lipo_%: build/darwin.amd64/% build/darwin.arm64/%
 # pointers, mostly for legacy reasons
 launcher: build_launcher
 tables.ext: build_tables.ext
-extension: build_osquery-extension.ext
 grpc.ext: build_grpc.ext
 fake-launcher: fake_launcher
 
 ##
 ## GitHub Action Helpers
 ##
-GITHUB_TARGETS=launcher osquery-extension.ext grpc.ext tables.ext package-builder
+GITHUB_TARGETS=launcher grpc.ext tables.ext package-builder
 GITHUB_ARCHS=amd64 arm64
 # linux cross compiles aren't working. Disable for now
 github-build-no-cross: $(foreach t, $(GITHUB_TARGETS), build_$(t))
@@ -80,7 +79,7 @@ github-lipo: $(foreach t, $(GITHUB_TARGETS), lipo_$(t))
 ## Cross Build targets
 ##
 
-RELEASE_TARGETS=launcher osquery-extension.ext package-builder
+RELEASE_TARGETS=launcher package-builder
 MANUAL_CROSS_OSES=darwin windows linux
 ARM64_OSES=darwin
 AMD64_OSES=darwin windows linux
@@ -179,7 +178,7 @@ notarize-check-%:
 
 # Using the `osslsigncode` we can sign windows binaries from
 # non-windows platforms.
-codesign-windows: codesign-windows-launcher.exe  codesign-windows-osquery-extension.exe
+codesign-windows: codesign-windows-launcher.exe
 codesign-windows-%: P12 = ~/Documents/kolide-codesigning-2021-04.p12
 codesign-windows-%:
 	@if [ -z "${AUTHENTICODE_PASSPHRASE}" ]; then echo "Missing AUTHENTICODE_PASSPHRASE"; exit 1; fi

cmd/launcher/extension.go

@@ -207,6 +207,7 @@ func commonRunnerOptions(logger log.Logger, db *bbolt.DB, opts *launcher.Options
 		runtime.WithOsqueryVerbose(opts.OsqueryVerbose),
 		runtime.WithOsqueryFlags(opts.OsqueryFlags),
 		runtime.WithAugeasLensFunction(augeas.InstallLenses),
+		runtime.WithAutoloadedExtensions(opts.AutoloadedExtensions...),
 	}
 }
 

cmd/launcher/options.go

@@ -45,24 +45,25 @@ func parseOptions(args []string) (*launcher.Options, error) {
 
 	var (
 		// Primary options
-		flCertPins            = flagset.String("cert_pins", "", "Comma separated, hex encoded SHA256 hashes of pinned subject public key info")
-		flControl             = flagset.Bool("control", false, "Whether or not the control server is enabled (default: false)")
-		flControlServerURL    = flagset.String("control_hostname", "", "The hostname of the control server")
-		flEnrollSecret        = flagset.String("enroll_secret", "", "The enroll secret that is used in your environment")
-		flEnrollSecretPath    = flagset.String("enroll_secret_path", "", "Optionally, the path to your enrollment secret")
-		flInitialRunner       = flagset.Bool("with_initial_runner", false, "Run differential queries from config ahead of scheduled interval.")
-		flKolideServerURL     = flagset.String("hostname", "", "The hostname of the gRPC server")
-		flKolideHosted        = flagset.Bool("kolide_hosted", false, "Use Kolide SaaS settings for defaults")
-		flTransport           = flagset.String("transport", "grpc", "The transport protocol that should be used to communicate with remote (default: grpc)")
-		flLoggingInterval     = flagset.Duration("logging_interval", 60*time.Second, "The interval at which logs should be flushed to the server")
-		flOsquerydPath        = flagset.String("osqueryd_path", "", "Path to the osqueryd binary to use (Default: find osqueryd in $PATH)")
-		flRootDirectory       = flagset.String("root_directory", "", "The location of the local database, pidfiles, etc.")
-		flRootPEM             = flagset.String("root_pem", "", "Path to PEM file including root certificates to verify against")
-		flVersion             = flagset.Bool("version", false, "Print Launcher version and exit")
-		flLogMaxBytesPerBatch = flagset.Int("log_max_bytes_per_batch", 0, "Maximum size of a batch of logs. Recommend leaving unset, and launcher will determine")
-		flOsqueryFlags        arrayFlags // set below with flagset.Var
-		flCompactDbMaxTx      = flagset.Int64("compactdb-max-tx", 65536, "Maximum transaction size used when compacting the internal DB")
-		_                     = flagset.String("config", "", "config file to parse options from (optional)")
+		flAutoloadedExtensions arrayFlags
+		flCertPins             = flagset.String("cert_pins", "", "Comma separated, hex encoded SHA256 hashes of pinned subject public key info")
+		flControl              = flagset.Bool("control", false, "Whether or not the control server is enabled (default: false)")
+		flControlServerURL     = flagset.String("control_hostname", "", "The hostname of the control server")
+		flEnrollSecret         = flagset.String("enroll_secret", "", "The enroll secret that is used in your environment")
+		flEnrollSecretPath     = flagset.String("enroll_secret_path", "", "Optionally, the path to your enrollment secret")
+		flInitialRunner        = flagset.Bool("with_initial_runner", false, "Run differential queries from config ahead of scheduled interval.")
+		flKolideServerURL      = flagset.String("hostname", "", "The hostname of the gRPC server")
+		flKolideHosted         = flagset.Bool("kolide_hosted", false, "Use Kolide SaaS settings for defaults")
+		flTransport            = flagset.String("transport", "grpc", "The transport protocol that should be used to communicate with remote (default: grpc)")
+		flLoggingInterval      = flagset.Duration("logging_interval", 60*time.Second, "The interval at which logs should be flushed to the server")
+		flOsquerydPath         = flagset.String("osqueryd_path", "", "Path to the osqueryd binary to use (Default: find osqueryd in $PATH)")
+		flRootDirectory        = flagset.String("root_directory", "", "The location of the local database, pidfiles, etc.")
+		flRootPEM              = flagset.String("root_pem", "", "Path to PEM file including root certificates to verify against")
+		flVersion              = flagset.Bool("version", false, "Print Launcher version and exit")
+		flLogMaxBytesPerBatch  = flagset.Int("log_max_bytes_per_batch", 0, "Maximum size of a batch of logs. Recommend leaving unset, and launcher will determine")
+		flOsqueryFlags         arrayFlags // set below with flagset.Var
+		flCompactDbMaxTx       = flagset.Int64("compactdb-max-tx", 65536, "Maximum transaction size used when compacting the internal DB")
+		_                      = flagset.String("config", "", "config file to parse options from (optional)")
 
 		// osquery TLS endpoints
 		flOsqTlsConfig    = flagset.String("config_tls_endpoint", "", "Config endpoint for the osquery tls transport")
@@ -91,7 +92,9 @@ func parseOptions(args []string) (*launcher.Options, error) {
 		// deprecated options, kept for any kind of config file compatibility
 		_ = flagset.String("debug_log_file", "", "DEPRECATED")
 	)
+
 	flagset.Var(&flOsqueryFlags, "osquery_flag", "Flags to pass to osquery (possibly overriding Launcher defaults)")
+	flagset.Var(&flAutoloadedExtensions, "autoloaded_extension", "extension paths to autoload, filename without path may be used in same directory as launcher")
 
 	ffOpts := []ff.Option{
 		ff.WithConfigFileFlag("config"),
@@ -187,6 +190,7 @@ func parseOptions(args []string) (*launcher.Options, error) {
 		EnableInitialRunner:                *flInitialRunner,
 		EnrollSecret:                       *flEnrollSecret,
 		EnrollSecretPath:                   *flEnrollSecretPath,
+		AutoloadedExtensions:               flAutoloadedExtensions,
 		InsecureTLS:                        *flInsecureTLS,
 		InsecureTransport:                  *flInsecureTransport,
 		KolideHosted:                       *flKolideHosted,
@@ -209,6 +213,7 @@ func parseOptions(args []string) (*launcher.Options, error) {
 		Transport:                          *flTransport,
 		UpdateChannel:                      updateChannel,
 	}
+
 	return opts, nil
 }
 

cmd/launcher/options_test.go

@@ -93,12 +93,13 @@ func getArgsAndResponse() (map[string]string, *launcher.Options) {
 
 	// includes both `-` and `--` for variety.
 	args := map[string]string{
-		"-control":             "", // This is a bool, it's special cased in the test routines
-		"--hostname":           randomHostname,
-		"-autoupdate_interval": "48h",
-		"-logging_interval":    fmt.Sprintf("%ds", randomInt),
-		"-osqueryd_path":       windowsAddExe("/dev/null"),
-		"-transport":           "grpc",
+		"-control":              "", // This is a bool, it's special cased in the test routines
+		"--hostname":            randomHostname,
+		"-autoupdate_interval":  "48h",
+		"-logging_interval":     fmt.Sprintf("%ds", randomInt),
+		"-osqueryd_path":        windowsAddExe("/dev/null"),
+		"-transport":            "grpc",
+		"-autoloaded_extension": "some-extension.ext",
 	}
 
 	opts := &launcher.Options{
@@ -114,6 +115,7 @@ func getArgsAndResponse() (map[string]string, *launcher.Options) {
 		OsquerydPath:           windowsAddExe("/dev/null"),
 		Transport:              "grpc",
 		UpdateChannel:          "stable",
+		AutoloadedExtensions:   []string{"some-extension.ext"},
 	}
 
 	return args, opts

cmd/make/make.go

@@ -86,15 +86,14 @@ func main() {
 	}
 
 	targetSet := map[string]func(context.Context) error{
-		"deps-go":               make.New(opts...).DepsGo,
-		"install-tools":         make.New(opts...).InstallTools,
-		"generate-tuf":          make.New(opts...).GenerateTUF,
-		"launcher":              make.New(optsMaybeCgo...).BuildCmd("./cmd/launcher", fakeName("launcher", *flFakeData)),
-		"osquery-extension.ext": make.New(opts...).BuildCmd("./cmd/osquery-extension", "osquery-extension.ext"),
-		"tables.ext":            make.New(optsMaybeCgo...).BuildCmd("./cmd/launcher.ext", "tables.ext"),
-		"grpc.ext":              make.New(opts...).BuildCmd("./cmd/grpc.ext", "grpc.ext"),
-		"package-builder":       make.New(opts...).BuildCmd("./cmd/package-builder", "package-builder"),
-		"make":                  make.New(opts...).BuildCmd("./cmd/make", "make"),
+		"deps-go":         make.New(opts...).DepsGo,
+		"install-tools":   make.New(opts...).InstallTools,
+		"generate-tuf":    make.New(opts...).GenerateTUF,
+		"launcher":        make.New(optsMaybeCgo...).BuildCmd("./cmd/launcher", fakeName("launcher", *flFakeData)),
+		"tables.ext":      make.New(optsMaybeCgo...).BuildCmd("./cmd/launcher.ext", "tables.ext"),
+		"grpc.ext":        make.New(opts...).BuildCmd("./cmd/grpc.ext", "grpc.ext"),
+		"package-builder": make.New(opts...).BuildCmd("./cmd/package-builder", "package-builder"),
+		"make":            make.New(opts...).BuildCmd("./cmd/make", "make"),
 	}
 
 	if t := strings.Split(*flTargets, ","); len(t) != 0 && t[0] != "" {

docs/launcher.md

@@ -207,10 +207,8 @@ Each zip will contain the following files:
 ```
 |-- darwin
 |   |-- launcher
-|   |-- osquery-extension.ext
 |   `-- osqueryd
 `-- linux
     |-- launcher
-    |-- osquery-extension.ext
     `-- osqueryd
 ```

docs/package-builder.md

@@ -2,8 +2,8 @@
 
 ## Background & Requirements
 
-Kolide launcher packages are a collection of binaries (`osqueryd`,
-`launcher`, and `osquery-extension.ext`), configuration, and init
+Kolide launcher packages are a collection of binaries (`osqueryd` and
+`launcher`), configuration, and init
 scripts. This repository contains `package-builder`, a tool to produce
 these packages.
 
@@ -108,8 +108,7 @@ You can now use `package-builder` to make packages with those:
    --hostname=grpc.launcher.example.com:443 \
    --enroll_secret=foobar123 \
    --osquery_version stable \
-   --launcher_version ./build/darwin/launcher \
-   --extension_version ./build/darwin/osquery-extension.ext
+   --launcher_version ./build/darwin/launcher
 ```
 
 If you'd like to customize the keys that are used to sign the

pkg/launcher/options.go

@@ -8,6 +8,9 @@ import (
 
 // Options is the set of options that may be configured for Launcher.
 type Options struct {
+	// AutoloadedExtensions to load with osquery, expected to be in same
+	// directory as launcher binary.
+	AutoloadedExtensions []string
 	// KolideServerURL is the URL of the management server to connect to.
 	KolideServerURL string
 	// KolideHosted true if using Kolide SaaS settings