From 6a410b53e6425d105eb98a22255ccee561491725 Mon Sep 17 00:00:00 2001 From: James Pickett Date: Tue, 28 Jun 2022 16:05:36 +0200 Subject: [PATCH] Removing osquery-extension (#838) * removed osquery-extension references from main launcher code * removing tests around osquery extension being removed * removed all traces of osquery-extension, updated code in response to comments * now passing list of extensions to osquery required_extension flag, removed osquery-extension.ext cmd * added launcher flag for auto-loading extensions --- Dockerfile | 2 +- Makefile | 9 +- cmd/launcher/extension.go | 1 + cmd/launcher/options.go | 41 +++-- cmd/launcher/options_test.go | 14 +- cmd/make/make.go | 17 +- cmd/osquery-extension/osquery-extension.go | 59 ------ docs/launcher.md | 2 - docs/package-builder.md | 7 +- pkg/launcher/options.go | 3 + pkg/osquery/runtime/osqueryinstance.go | 103 ++++++++--- pkg/osquery/runtime/runner.go | 19 +- pkg/osquery/runtime/runtime_helpers.go | 2 - .../runtime/runtime_helpers_windows.go | 2 - pkg/osquery/runtime/runtime_test.go | 168 ++++++------------ pkg/packaging/fetch.go | 1 - pkg/packaging/packaging.go | 6 +- 17 files changed, 200 insertions(+), 256 deletions(-) delete mode 100644 cmd/osquery-extension/osquery-extension.go diff --git a/Dockerfile b/Dockerfile index 88292ae6b..fff12429d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,7 @@ RUN cd launcher && git checkout "${gitver}" # Build! RUN cd launcher && make deps RUN cd launcher && make all -RUN cd launcher && GO111MODULE=on go run cmd/make/make.go -targets=launcher,osquery-extension.ext -linkstamp $FAKE +RUN cd launcher && GO111MODULE=on go run cmd/make/make.go -targets=launcher -linkstamp $FAKE # Install RUN mkdir -p /usr/local/kolide/bin/ diff --git a/Makefile b/Makefile index 653ef1a89..7b39cbb8b 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ endif all: build -build: build_launcher build_osquery-extension.ext +build: build_launcher .pre-build: ${BUILD_DIR} @@ -62,14 +62,13 @@ lipo_%: build/darwin.amd64/% build/darwin.arm64/% # pointers, mostly for legacy reasons launcher: build_launcher tables.ext: build_tables.ext -extension: build_osquery-extension.ext grpc.ext: build_grpc.ext fake-launcher: fake_launcher ## ## GitHub Action Helpers ## -GITHUB_TARGETS=launcher osquery-extension.ext grpc.ext tables.ext package-builder +GITHUB_TARGETS=launcher grpc.ext tables.ext package-builder GITHUB_ARCHS=amd64 arm64 # linux cross compiles aren't working. Disable for now github-build-no-cross: $(foreach t, $(GITHUB_TARGETS), build_$(t)) @@ -80,7 +79,7 @@ github-lipo: $(foreach t, $(GITHUB_TARGETS), lipo_$(t)) ## Cross Build targets ## -RELEASE_TARGETS=launcher osquery-extension.ext package-builder +RELEASE_TARGETS=launcher package-builder MANUAL_CROSS_OSES=darwin windows linux ARM64_OSES=darwin AMD64_OSES=darwin windows linux @@ -179,7 +178,7 @@ notarize-check-%: # Using the `osslsigncode` we can sign windows binaries from # non-windows platforms. -codesign-windows: codesign-windows-launcher.exe codesign-windows-osquery-extension.exe +codesign-windows: codesign-windows-launcher.exe codesign-windows-%: P12 = ~/Documents/kolide-codesigning-2021-04.p12 codesign-windows-%: @if [ -z "${AUTHENTICODE_PASSPHRASE}" ]; then echo "Missing AUTHENTICODE_PASSPHRASE"; exit 1; fi diff --git a/cmd/launcher/extension.go b/cmd/launcher/extension.go index 266933a1f..f98deaf78 100644 --- a/cmd/launcher/extension.go +++ b/cmd/launcher/extension.go @@ -207,6 +207,7 @@ func commonRunnerOptions(logger log.Logger, db *bbolt.DB, opts *launcher.Options runtime.WithOsqueryVerbose(opts.OsqueryVerbose), runtime.WithOsqueryFlags(opts.OsqueryFlags), runtime.WithAugeasLensFunction(augeas.InstallLenses), + runtime.WithAutoloadedExtensions(opts.AutoloadedExtensions...), } } diff --git a/cmd/launcher/options.go b/cmd/launcher/options.go index 2557c167f..91323035b 100644 --- a/cmd/launcher/options.go +++ b/cmd/launcher/options.go @@ -45,24 +45,25 @@ func parseOptions(args []string) (*launcher.Options, error) { var ( // Primary options - flCertPins = flagset.String("cert_pins", "", "Comma separated, hex encoded SHA256 hashes of pinned subject public key info") - flControl = flagset.Bool("control", false, "Whether or not the control server is enabled (default: false)") - flControlServerURL = flagset.String("control_hostname", "", "The hostname of the control server") - flEnrollSecret = flagset.String("enroll_secret", "", "The enroll secret that is used in your environment") - flEnrollSecretPath = flagset.String("enroll_secret_path", "", "Optionally, the path to your enrollment secret") - flInitialRunner = flagset.Bool("with_initial_runner", false, "Run differential queries from config ahead of scheduled interval.") - flKolideServerURL = flagset.String("hostname", "", "The hostname of the gRPC server") - flKolideHosted = flagset.Bool("kolide_hosted", false, "Use Kolide SaaS settings for defaults") - flTransport = flagset.String("transport", "grpc", "The transport protocol that should be used to communicate with remote (default: grpc)") - flLoggingInterval = flagset.Duration("logging_interval", 60*time.Second, "The interval at which logs should be flushed to the server") - flOsquerydPath = flagset.String("osqueryd_path", "", "Path to the osqueryd binary to use (Default: find osqueryd in $PATH)") - flRootDirectory = flagset.String("root_directory", "", "The location of the local database, pidfiles, etc.") - flRootPEM = flagset.String("root_pem", "", "Path to PEM file including root certificates to verify against") - flVersion = flagset.Bool("version", false, "Print Launcher version and exit") - flLogMaxBytesPerBatch = flagset.Int("log_max_bytes_per_batch", 0, "Maximum size of a batch of logs. Recommend leaving unset, and launcher will determine") - flOsqueryFlags arrayFlags // set below with flagset.Var - flCompactDbMaxTx = flagset.Int64("compactdb-max-tx", 65536, "Maximum transaction size used when compacting the internal DB") - _ = flagset.String("config", "", "config file to parse options from (optional)") + flAutoloadedExtensions arrayFlags + flCertPins = flagset.String("cert_pins", "", "Comma separated, hex encoded SHA256 hashes of pinned subject public key info") + flControl = flagset.Bool("control", false, "Whether or not the control server is enabled (default: false)") + flControlServerURL = flagset.String("control_hostname", "", "The hostname of the control server") + flEnrollSecret = flagset.String("enroll_secret", "", "The enroll secret that is used in your environment") + flEnrollSecretPath = flagset.String("enroll_secret_path", "", "Optionally, the path to your enrollment secret") + flInitialRunner = flagset.Bool("with_initial_runner", false, "Run differential queries from config ahead of scheduled interval.") + flKolideServerURL = flagset.String("hostname", "", "The hostname of the gRPC server") + flKolideHosted = flagset.Bool("kolide_hosted", false, "Use Kolide SaaS settings for defaults") + flTransport = flagset.String("transport", "grpc", "The transport protocol that should be used to communicate with remote (default: grpc)") + flLoggingInterval = flagset.Duration("logging_interval", 60*time.Second, "The interval at which logs should be flushed to the server") + flOsquerydPath = flagset.String("osqueryd_path", "", "Path to the osqueryd binary to use (Default: find osqueryd in $PATH)") + flRootDirectory = flagset.String("root_directory", "", "The location of the local database, pidfiles, etc.") + flRootPEM = flagset.String("root_pem", "", "Path to PEM file including root certificates to verify against") + flVersion = flagset.Bool("version", false, "Print Launcher version and exit") + flLogMaxBytesPerBatch = flagset.Int("log_max_bytes_per_batch", 0, "Maximum size of a batch of logs. Recommend leaving unset, and launcher will determine") + flOsqueryFlags arrayFlags // set below with flagset.Var + flCompactDbMaxTx = flagset.Int64("compactdb-max-tx", 65536, "Maximum transaction size used when compacting the internal DB") + _ = flagset.String("config", "", "config file to parse options from (optional)") // osquery TLS endpoints flOsqTlsConfig = flagset.String("config_tls_endpoint", "", "Config endpoint for the osquery tls transport") @@ -91,7 +92,9 @@ func parseOptions(args []string) (*launcher.Options, error) { // deprecated options, kept for any kind of config file compatibility _ = flagset.String("debug_log_file", "", "DEPRECATED") ) + flagset.Var(&flOsqueryFlags, "osquery_flag", "Flags to pass to osquery (possibly overriding Launcher defaults)") + flagset.Var(&flAutoloadedExtensions, "autoloaded_extension", "extension paths to autoload, filename without path may be used in same directory as launcher") ffOpts := []ff.Option{ ff.WithConfigFileFlag("config"), @@ -187,6 +190,7 @@ func parseOptions(args []string) (*launcher.Options, error) { EnableInitialRunner: *flInitialRunner, EnrollSecret: *flEnrollSecret, EnrollSecretPath: *flEnrollSecretPath, + AutoloadedExtensions: flAutoloadedExtensions, InsecureTLS: *flInsecureTLS, InsecureTransport: *flInsecureTransport, KolideHosted: *flKolideHosted, @@ -209,6 +213,7 @@ func parseOptions(args []string) (*launcher.Options, error) { Transport: *flTransport, UpdateChannel: updateChannel, } + return opts, nil } diff --git a/cmd/launcher/options_test.go b/cmd/launcher/options_test.go index 3f6336a72..e6c8b1e93 100644 --- a/cmd/launcher/options_test.go +++ b/cmd/launcher/options_test.go @@ -93,12 +93,13 @@ func getArgsAndResponse() (map[string]string, *launcher.Options) { // includes both `-` and `--` for variety. args := map[string]string{ - "-control": "", // This is a bool, it's special cased in the test routines - "--hostname": randomHostname, - "-autoupdate_interval": "48h", - "-logging_interval": fmt.Sprintf("%ds", randomInt), - "-osqueryd_path": windowsAddExe("/dev/null"), - "-transport": "grpc", + "-control": "", // This is a bool, it's special cased in the test routines + "--hostname": randomHostname, + "-autoupdate_interval": "48h", + "-logging_interval": fmt.Sprintf("%ds", randomInt), + "-osqueryd_path": windowsAddExe("/dev/null"), + "-transport": "grpc", + "-autoloaded_extension": "some-extension.ext", } opts := &launcher.Options{ @@ -114,6 +115,7 @@ func getArgsAndResponse() (map[string]string, *launcher.Options) { OsquerydPath: windowsAddExe("/dev/null"), Transport: "grpc", UpdateChannel: "stable", + AutoloadedExtensions: []string{"some-extension.ext"}, } return args, opts diff --git a/cmd/make/make.go b/cmd/make/make.go index 256aea9a0..39e660f39 100644 --- a/cmd/make/make.go +++ b/cmd/make/make.go @@ -86,15 +86,14 @@ func main() { } targetSet := map[string]func(context.Context) error{ - "deps-go": make.New(opts...).DepsGo, - "install-tools": make.New(opts...).InstallTools, - "generate-tuf": make.New(opts...).GenerateTUF, - "launcher": make.New(optsMaybeCgo...).BuildCmd("./cmd/launcher", fakeName("launcher", *flFakeData)), - "osquery-extension.ext": make.New(opts...).BuildCmd("./cmd/osquery-extension", "osquery-extension.ext"), - "tables.ext": make.New(optsMaybeCgo...).BuildCmd("./cmd/launcher.ext", "tables.ext"), - "grpc.ext": make.New(opts...).BuildCmd("./cmd/grpc.ext", "grpc.ext"), - "package-builder": make.New(opts...).BuildCmd("./cmd/package-builder", "package-builder"), - "make": make.New(opts...).BuildCmd("./cmd/make", "make"), + "deps-go": make.New(opts...).DepsGo, + "install-tools": make.New(opts...).InstallTools, + "generate-tuf": make.New(opts...).GenerateTUF, + "launcher": make.New(optsMaybeCgo...).BuildCmd("./cmd/launcher", fakeName("launcher", *flFakeData)), + "tables.ext": make.New(optsMaybeCgo...).BuildCmd("./cmd/launcher.ext", "tables.ext"), + "grpc.ext": make.New(opts...).BuildCmd("./cmd/grpc.ext", "grpc.ext"), + "package-builder": make.New(opts...).BuildCmd("./cmd/package-builder", "package-builder"), + "make": make.New(opts...).BuildCmd("./cmd/make", "make"), } if t := strings.Split(*flTargets, ","); len(t) != 0 && t[0] != "" { diff --git a/cmd/osquery-extension/osquery-extension.go b/cmd/osquery-extension/osquery-extension.go deleted file mode 100644 index 2fed6524d..000000000 --- a/cmd/osquery-extension/osquery-extension.go +++ /dev/null @@ -1,59 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "os" - "os/signal" - "time" - - "github.com/kolide/kit/version" -) - -func main() { - var ( - _ = flag.Bool("verbose", false, "") - _ = flag.Int("interval", 0, "") - _ = flag.Int("timeout", 2, "") - _ = flag.String("socket", "", "") - flVersion = flag.Bool("version", false, "Print version and exit") - ) - flag.Parse() - - if *flVersion { - version.PrintFull() - os.Exit(0) - } - - fmt.Fprintf(os.Stderr, "%+v", os.Args) - - go monitorForParent() - - sig := make(chan os.Signal, 1) - signal.Notify(sig, os.Interrupt) - <-sig -} - -// continuously monitor for ppid and exit if osqueryd is no longer the parent process. -// because osqueryd is always the process starting the extension, when osqueryd is killed this process should also be cleaned up. -// sometimes the termination is not clean, causing this process to remain running, which sometimes prevents osqueryd from properly restarting. -// https://github.com/kolide/launcher/issues/341 -func monitorForParent() { - ticker := time.NewTicker(30 * time.Second) - defer ticker.Stop() - - f := func() { - ppid := os.Getppid() - if ppid <= 1 { - fmt.Println("extension process no longer owned by osqueryd, quitting") - os.Exit(1) - } - } - - f() - - select { - case <-ticker.C: - f() - } -} diff --git a/docs/launcher.md b/docs/launcher.md index 6333b2e40..f7a6683f3 100644 --- a/docs/launcher.md +++ b/docs/launcher.md @@ -207,10 +207,8 @@ Each zip will contain the following files: ``` |-- darwin | |-- launcher -| |-- osquery-extension.ext | `-- osqueryd `-- linux |-- launcher - |-- osquery-extension.ext `-- osqueryd ``` diff --git a/docs/package-builder.md b/docs/package-builder.md index 5a5f89de9..a8f443f24 100644 --- a/docs/package-builder.md +++ b/docs/package-builder.md @@ -2,8 +2,8 @@ ## Background & Requirements -Kolide launcher packages are a collection of binaries (`osqueryd`, -`launcher`, and `osquery-extension.ext`), configuration, and init +Kolide launcher packages are a collection of binaries (`osqueryd` and +`launcher`), configuration, and init scripts. This repository contains `package-builder`, a tool to produce these packages. @@ -108,8 +108,7 @@ You can now use `package-builder` to make packages with those: --hostname=grpc.launcher.example.com:443 \ --enroll_secret=foobar123 \ --osquery_version stable \ - --launcher_version ./build/darwin/launcher \ - --extension_version ./build/darwin/osquery-extension.ext + --launcher_version ./build/darwin/launcher ``` If you'd like to customize the keys that are used to sign the diff --git a/pkg/launcher/options.go b/pkg/launcher/options.go index 144d87365..c389aeefd 100644 --- a/pkg/launcher/options.go +++ b/pkg/launcher/options.go @@ -8,6 +8,9 @@ import ( // Options is the set of options that may be configured for Launcher. type Options struct { + // AutoloadedExtensions to load with osquery, expected to be in same + // directory as launcher binary. + AutoloadedExtensions []string // KolideServerURL is the URL of the management server to connect to. KolideServerURL string // KolideHosted true if using Kolide SaaS settings diff --git a/pkg/osquery/runtime/osqueryinstance.go b/pkg/osquery/runtime/osqueryinstance.go index 88d7c39da..96b02cd5c 100644 --- a/pkg/osquery/runtime/osqueryinstance.go +++ b/pkg/osquery/runtime/osqueryinstance.go @@ -10,6 +10,7 @@ import ( "os/exec" "path/filepath" "runtime" + "strings" "sync" "time" @@ -198,6 +199,14 @@ func WithAugeasLensFunction(f func(dir string) error) OsqueryInstanceOption { } } +// WithAutoloadedExtensions defines a list of extensions to load +// via the osquery autoloading. +func WithAutoloadedExtensions(extensions ...string) OsqueryInstanceOption { + return func(i *OsqueryInstance) { + i.opts.autoloadedExtensions = append(i.opts.autoloadedExtensions, extensions...) + } +} + // OsqueryInstance is the type which represents a currently running instance // of osqueryd. type OsqueryInstance struct { @@ -282,6 +291,7 @@ type osqueryOptions struct { configPluginFlag string distributedPluginFlag string extensionPlugins []osquery.OsqueryPlugin + autoloadedExtensions []string extensionSocketPath string enrollSecretPath string loggerPluginFlag string @@ -300,6 +310,22 @@ type osqueryOptions struct { verbose bool } +func (o osqueryOptions) uniqueExtensions() []string { + extensionsMap := make(map[string]bool) + uniqueExtensions := make([]string, 0) + + for _, extension := range append([]string{o.loggerPluginFlag, o.configPluginFlag, o.distributedPluginFlag}, o.autoloadedExtensions...) { + if _, ok := extensionsMap[extension]; ok { + continue + } + + extensionsMap[extension] = true + uniqueExtensions = append(uniqueExtensions, extension) + } + + return uniqueExtensions +} + func newInstance() *OsqueryInstance { i := &OsqueryInstance{} @@ -318,8 +344,8 @@ type osqueryFilePaths struct { augeasPath string databasePath string extensionAutoloadPath string - extensionPath string extensionSocketPath string + extensionPaths []string pidfilePath string } @@ -328,41 +354,67 @@ type osqueryFilePaths struct { // In return, a structure of paths is returned that can be used to launch an // osqueryd instance. An error may be returned if the supplied parameters are // unacceptable. -func calculateOsqueryPaths(rootDir, extensionSocketPath string) (*osqueryFilePaths, error) { +func calculateOsqueryPaths(opts osqueryOptions) (*osqueryFilePaths, error) { + + // Determine the path to the extension socket + extensionSocketPath := opts.extensionSocketPath + if extensionSocketPath == "" { + extensionSocketPath = socketPath(opts.rootDirectory) + } + + extensionAutoloadPath := filepath.Join(opts.rootDirectory, "osquery.autoload") + + osqueryFilePaths := &osqueryFilePaths{ + pidfilePath: filepath.Join(opts.rootDirectory, "osquery.pid"), + databasePath: filepath.Join(opts.rootDirectory, "osquery.db"), + augeasPath: filepath.Join(opts.rootDirectory, "augeas-lenses"), + extensionSocketPath: extensionSocketPath, + extensionAutoloadPath: extensionAutoloadPath, + extensionPaths: make([]string, len(opts.autoloadedExtensions)), + } + + osqueryAutoloadFile, err := os.Create(extensionAutoloadPath) + if err != nil { + return nil, errors.Wrap(err, "creating autoload file") + } + defer osqueryAutoloadFile.Close() + + if len(opts.autoloadedExtensions) == 0 { + return osqueryFilePaths, nil + } + // Determine the path to the extension exPath, err := os.Executable() if err != nil { return nil, errors.Wrap(err, "finding path of launcher executable") } - extensionPath := filepath.Join(autoupdate.FindBaseDir(exPath), extensionName) - if _, err := os.Stat(extensionPath); err != nil { - if os.IsNotExist(err) { - return nil, errors.Wrapf(err, "extension path does not exist: %s", extensionPath) - } else { - return nil, errors.Wrapf(err, "could not stat extension path") + for index, extension := range opts.autoloadedExtensions { + // first see if we just got a file name and check to see if it exists in the executable directory + extensionPath := filepath.Join(autoupdate.FindBaseDir(exPath), extension) + + if _, err := os.Stat(extensionPath); err != nil { + // if we got an error, try the raw flag + extensionPath = extension + + if _, err := os.Stat(extensionPath); err != nil { + if os.IsNotExist(err) { + return nil, errors.Wrapf(err, "extension path does not exist: %s", extension) + } else { + return nil, errors.Wrapf(err, "could not stat extension path") + } + } } - } - // Determine the path to the extension socket - if extensionSocketPath == "" { - extensionSocketPath = socketPath(rootDir) - } + osqueryFilePaths.extensionPaths[index] = extensionPath - // Write the autoload file - extensionAutoloadPath := filepath.Join(rootDir, "osquery.autoload") - if err := ioutil.WriteFile(extensionAutoloadPath, []byte(extensionPath), 0644); err != nil { - return nil, errors.Wrap(err, "could not write osquery extension autoload file") + _, err := osqueryAutoloadFile.WriteString(fmt.Sprintf("%s\n", extensionPath)) + if err != nil { + return nil, errors.Wrapf(err, "writing to autoload file") + } } - return &osqueryFilePaths{ - pidfilePath: filepath.Join(rootDir, "osquery.pid"), - databasePath: filepath.Join(rootDir, "osquery.db"), - augeasPath: filepath.Join(rootDir, "augeas-lenses"), - extensionPath: extensionPath, - extensionAutoloadPath: extensionAutoloadPath, - extensionSocketPath: extensionSocketPath, - }, nil + return osqueryFilePaths, nil } // createOsquerydCommand uses osqueryOptions to return an *exec.Cmd @@ -457,6 +509,7 @@ func (opts *osqueryOptions) createOsquerydCommand(osquerydBinary string, paths * "--disable_extensions=false", "--extensions_timeout=20", fmt.Sprintf("--config_plugin=%s", opts.configPluginFlag), + fmt.Sprintf("--extensions_require=%s", strings.Join(opts.uniqueExtensions(), ",")), ) // On darwin, run osquery using a magic macOS variable to ensure we diff --git a/pkg/osquery/runtime/runner.go b/pkg/osquery/runtime/runner.go index 2b172533c..0da92e27d 100644 --- a/pkg/osquery/runtime/runner.go +++ b/pkg/osquery/runtime/runner.go @@ -194,18 +194,21 @@ func (r *Runner) launchOsqueryInstance() error { // Based on the root directory, calculate the file names of all of the // required osquery artifact files. - paths, err := calculateOsqueryPaths(o.opts.rootDirectory, o.opts.extensionSocketPath) + paths, err := calculateOsqueryPaths(o.opts) if err != nil { return errors.Wrap(err, "could not calculate osquery file paths") } - // The extensions file should be owned by the process's UID or by root. - // Osquery will refuse to load the extension otherwise. - if err := ensureProperPermissions(o, paths.extensionPath); err != nil { - level.Info(o.logger).Log( - "msg", "unable to ensure proper permissions on extension path", - "err", err, - ) + for _, path := range paths.extensionPaths { + // The extensions files should be owned by the process's UID or by root. + // Osquery will refuse to load the extension otherwise. + if err := ensureProperPermissions(o, path); err != nil { + level.Info(o.logger).Log( + "msg", "unable to ensure proper permissions on extension path", + "path", path, + "err", err, + ) + } } // Populate augeas lenses, if requested diff --git a/pkg/osquery/runtime/runtime_helpers.go b/pkg/osquery/runtime/runtime_helpers.go index 8a683b873..1fc7f2d9f 100644 --- a/pkg/osquery/runtime/runtime_helpers.go +++ b/pkg/osquery/runtime/runtime_helpers.go @@ -13,8 +13,6 @@ import ( "github.com/pkg/errors" ) -const extensionName = `osquery-extension.ext` - func setpgid() *syscall.SysProcAttr { return &syscall.SysProcAttr{Setpgid: true} } diff --git a/pkg/osquery/runtime/runtime_helpers_windows.go b/pkg/osquery/runtime/runtime_helpers_windows.go index eb0c0d968..03b44fcfe 100644 --- a/pkg/osquery/runtime/runtime_helpers_windows.go +++ b/pkg/osquery/runtime/runtime_helpers_windows.go @@ -12,8 +12,6 @@ import ( "github.com/pkg/errors" ) -const extensionName = `osquery-extension.exe` - func setpgid() *syscall.SysProcAttr { // TODO: on unix we set the process group id and then // terminate that process group. diff --git a/pkg/osquery/runtime/runtime_test.go b/pkg/osquery/runtime/runtime_test.go index 07f1c7cbe..b435c202b 100644 --- a/pkg/osquery/runtime/runtime_test.go +++ b/pkg/osquery/runtime/runtime_test.go @@ -4,16 +4,11 @@ package runtime import ( - "bufio" - "bytes" "context" "fmt" - "io" "os" - "os/exec" "path/filepath" "runtime" - "strconv" "strings" "syscall" "testing" @@ -23,7 +18,6 @@ import ( "github.com/kolide/kit/testutil" "github.com/kolide/launcher/pkg/osquery/runtime/history" "github.com/kolide/launcher/pkg/packaging" - ps "github.com/mitchellh/go-ps" osquery "github.com/osquery/osquery-go" "github.com/pkg/errors" "github.com/stretchr/testify/assert" @@ -61,18 +55,6 @@ func TestMain(m *testing.M) { os.Exit(1) } - // Build the osquerty extension once - binDir, err := getBinDir() - if err != nil { - fmt.Printf("Failed to get binDir: %v\n", err) - os.Exit(1) - } - - if err := buildOsqueryExtensionInBinDir(binDir); err != nil { - fmt.Printf("Failed to build osquery extension: %v\n", err) - os.Exit(1) - } - // Run the tests! retCode := m.Run() os.Exit(retCode) @@ -94,16 +76,64 @@ func TestCalculateOsqueryPaths(t *testing.T) { binDir, err := getBinDir() require.NoError(t, err) - paths, err := calculateOsqueryPaths(binDir, "") + paths, err := calculateOsqueryPaths(osqueryOptions{ + rootDirectory: binDir, + }) + + require.NoError(t, err) + + // ensure that all of our resulting artifact files are in the rootDir that we + // dictated + require.Equal(t, binDir, filepath.Dir(paths.pidfilePath)) + require.Equal(t, binDir, filepath.Dir(paths.databasePath)) + require.Equal(t, binDir, filepath.Dir(paths.extensionSocketPath)) + require.Equal(t, binDir, filepath.Dir(paths.extensionAutoloadPath)) +} + +func TestCalculateOsqueryPathsWithAutoloadedExtensions(t *testing.T) { + t.Parallel() + binDir, err := getBinDir() + require.NoError(t, err) + + extensionPaths := make([]string, 0) + + for _, extension := range []string{"extensionInExecDir1", "extensionInExecDir2"} { + // create file at each extension path + extensionPath := filepath.Join(binDir, extension) + require.NoError(t, os.WriteFile(extensionPath, []byte("{}"), 0644)) + extensionPaths = append(extensionPaths, extensionPath) + } + + nonExecDir := t.TempDir() + for _, extension := range []string{"extensionNotInExecDir1", "extensionNotInExecDir2"} { + // create file at each extension path + extensionPath := filepath.Join(nonExecDir, extension) + require.NoError(t, os.WriteFile(extensionPath, []byte("{}"), 0644)) + extensionPaths = append(extensionPaths, extensionPath) + } + + paths, err := calculateOsqueryPaths(osqueryOptions{ + rootDirectory: binDir, + autoloadedExtensions: []string{"extensionInExecDir1", "extensionInExecDir2", filepath.Join(nonExecDir, "extensionNotInExecDir1"), filepath.Join(nonExecDir, "extensionNotInExecDir2")}, + }) + require.NoError(t, err) // ensure that all of our resulting artifact files are in the rootDir that we // dictated require.Equal(t, binDir, filepath.Dir(paths.pidfilePath)) require.Equal(t, binDir, filepath.Dir(paths.databasePath)) - require.Equal(t, binDir, filepath.Dir(paths.extensionPath)) require.Equal(t, binDir, filepath.Dir(paths.extensionSocketPath)) require.Equal(t, binDir, filepath.Dir(paths.extensionAutoloadPath)) + + osqueryAutoloadFilePath := filepath.Join(binDir, "osquery.autoload") + // read each line of the autoload file into a string array + bytes, err := os.ReadFile(osqueryAutoloadFilePath) + require.NoError(t, err) + autoloadFileLines := strings.Split(string(bytes), "\n") + + // add empty string to extensions path array so it matches the last line of autoloaded file + assert.ElementsMatch(t, append(extensionPaths, ""), autoloadFileLines) } func TestCreateOsqueryCommand(t *testing.T) { @@ -140,12 +170,15 @@ func TestCreateOsqueryCommandWithFlags(t *testing.T) { &osqueryFilePaths{}, ) require.NoError(t, err) + + // count of flags that cannot be overridden with this option + const nonOverridableFlagsCount = 8 + // Ensure that the provided flags were placed last (so that they can override) assert.Equal( t, []string{"--verbose=false", "--windows_event_channels=foo,bar"}, - // 7 is for the 7 flags that cannot be overridden with this option. - cmd.Args[len(cmd.Args)-2-7:len(cmd.Args)-7], + cmd.Args[len(cmd.Args)-2-nonOverridableFlagsCount:len(cmd.Args)-nonOverridableFlagsCount], ) } @@ -172,33 +205,6 @@ func downloadOsqueryInBinDir(binDirectory string) error { return nil } -// buildOsqueryExtensionInBinDir compiles the osquery extension and places it -// on disk in the same directory as the currently running executable (as -// expected when running an osquery process) -func buildOsqueryExtensionInBinDir(rootDirectory string) error { - goBinary, err := exec.LookPath("go") - if err != nil { - return err - } - - _, myFilename, _, _ := runtime.Caller(1) - launcherSrcDir := filepath.Join(filepath.Dir(myFilename), "..", "..", "..") - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - cmd := exec.CommandContext( - ctx, - goBinary, - "build", - "-o", - filepath.Join(rootDirectory, "osquery-extension.ext"), - filepath.Join(launcherSrcDir, "cmd/osquery-extension/osquery-extension.go"), - ) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - return cmd.Run() -} - func TestBadBinaryPath(t *testing.T) { t.Parallel() rootDirectory, rmRootDirectory, err := osqueryTempDir() @@ -260,7 +266,7 @@ func TestSimplePath(t *testing.T) { func TestRestart(t *testing.T) { t.Parallel() - runner, _, teardown := setupOsqueryInstanceForTests(t) + runner, teardown := setupOsqueryInstanceForTests(t) defer teardown() previousStats := runner.instance.stats @@ -335,7 +341,7 @@ func TestExtensionIsCleanedUp(t *testing.T) { t.Skip("https://github.com/kolide/launcher/issues/478") t.Parallel() - runner, extensionPid, teardown := setupOsqueryInstanceForTests(t) + runner, teardown := setupOsqueryInstanceForTests(t) defer teardown() osqueryPID := runner.instance.cmd.Process.Pid @@ -344,10 +350,7 @@ func TestExtensionIsCleanedUp(t *testing.T) { require.NoError(t, err) require.Equal(t, pgid, osqueryPID, "pgid must be set") - extensionProcess, err := ps.FindProcess(extensionPid) require.NoError(t, err) - // process name seems truncated - require.True(t, strings.HasPrefix(extensionProcess.Executable(), "osquery-ext")) // kill the current osquery process but not the extension err = syscall.Kill(osqueryPID, syscall.SIGKILL) @@ -366,24 +369,6 @@ func TestExtensionIsCleanedUp(t *testing.T) { // Ensure we've waited at least 32s <-timer1.C - - // check that the extension process is no longer running. Because - // this may be subject to PID reuse as a false positive, we have two - // test patterns. If we got an err, then the process is probably - // gone, and we test one way. If err==nil, check for PID - // reuse. go-ps will panic if you look for a missing process, so - // there is still some window for errors. - extpgid, err := syscall.Getpgid(extensionPid) - if err != nil { - require.EqualError(t, err, "no such process") - require.Equal(t, extpgid, -1) - } else { - extensionProcess, err := ps.FindProcess(extensionPid) - require.NoError(t, err) - require.False(t, strings.HasPrefix(extensionProcess.Executable(), "osquery-ext"), "old extension pid, still running. And named like osquery-extension") - require.NotEqual(t, osqueryPID, extpgid) - } - } func TestExtensionSocketPath(t *testing.T) { @@ -417,7 +402,7 @@ func TestExtensionSocketPath(t *testing.T) { } // sets up an osquery instance with a running extension to be used in tests. -func setupOsqueryInstanceForTests(t *testing.T) (runner *Runner, extensionPid int, teardown func()) { +func setupOsqueryInstanceForTests(t *testing.T) (runner *Runner, teardown func()) { rootDirectory, rmRootDirectory, err := osqueryTempDir() require.NoError(t, err) @@ -434,44 +419,9 @@ func setupOsqueryInstanceForTests(t *testing.T) (runner *Runner, extensionPid in require.NoError(t, err) require.Equal(t, pgid, osqueryPID) - extensionPid = getExtensionPid(t, rootDirectory, pgid) - teardown = func() { defer rmRootDirectory() require.NoError(t, runner.Shutdown()) } - return runner, extensionPid, teardown -} - -// get the osquery-extension.ext process' PID -func getExtensionPid(t *testing.T, rootDirectory string, pgid int) int { - out, err := exec.Command("ps", "xao", "pid,ppid,pgid,comm").CombinedOutput() - require.NoError(t, err) - - var extensionPid int - r := bufio.NewReader(bytes.NewReader(out)) - for { - line, _, err := r.ReadLine() - if err == io.EOF { - break - } - require.NoError(t, err) - - // some versions of the ps command truncate the comm field, using osquery-extensi - // instead of the full command name. - if bytes.Contains(line, []byte(`osquery-extensi`)) && - bytes.Contains(line, []byte(fmt.Sprintf("%d", pgid))) { - line = bytes.TrimSpace(line) - - cols := bytes.Split(line, []byte(" ")) - require.NotEqual(t, len(cols), 0) - - pid, err := strconv.Atoi(string(cols[0])) - require.NoError(t, err) - extensionPid = pid - } - } - - require.NotZero(t, extensionPid) - return extensionPid + return runner, teardown } diff --git a/pkg/packaging/fetch.go b/pkg/packaging/fetch.go index ae293e82f..26f9f9d81 100644 --- a/pkg/packaging/fetch.go +++ b/pkg/packaging/fetch.go @@ -96,7 +96,6 @@ func FetchBinary(ctx context.Context, localCacheDir, name, binaryName, version s return "", errors.Wrap(err, "couldn't untar download") } - // TODO / FIXME this fails for osquery-extension, since the binary name is inconsistent. if _, err := os.Stat(localBinaryPath); err != nil { level.Debug(logger).Log( "msg", "Missing local binary", diff --git a/pkg/packaging/packaging.go b/pkg/packaging/packaging.go index 5bc8d3590..30e845149 100644 --- a/pkg/packaging/packaging.go +++ b/pkg/packaging/packaging.go @@ -227,7 +227,7 @@ func (p *PackageOptions) Build(ctx context.Context, packageWriter io.Writer, tar } // Install binaries into packageRoot - // TODO parallization, osquery-extension.ext + // TODO parallization // TODO windows file extensions if p.OsqueryDownloadVersionOverride == "" { @@ -245,10 +245,6 @@ func (p *PackageOptions) Build(ctx context.Context, packageWriter io.Writer, tar return errors.Wrapf(err, "fetching binary launcher") } - if err := p.getBinary(ctx, "osquery-extension", p.target.PlatformExtensionName("osquery-extension"), p.ExtensionVersion); err != nil { - return errors.Wrapf(err, "fetching binary launcher") - } - // Some darwin specific bits if p.target.Platform == Darwin { if err := p.renderNewSyslogConfig(ctx); err != nil {