Osquerybeat: add extension uptycs/kubequery

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "x-pack/osquerybeat/ext/kubequery"]
+	path = x-pack/osquerybeat/ext/kubequery
+	url = https://github.com/Uptycs/kubequery.git

dev-tools/mage/build.go

@@ -34,6 +34,7 @@ import (
 // "go build" is invoked.
 type BuildArgs struct {
 	Name        string // Name of binary. (On Windows '.exe' is appended.)
+	WorkDir     string // Switch to this working directory. Input/output paths will be relative.
 	InputFiles  []string
 	OutputDir   string
 	CGO         bool
@@ -182,8 +183,40 @@ func Build(params BuildArgs) error {
 		defer os.Remove(syso)
 	}
 
+	var err error
+
+	if params.WorkDir != "" {
+		log.Println("Switching working directory:", params.WorkDir)
+
+		var wd string
+		wd, err = os.Getwd()
+		if err != nil {
+			return errors.Errorf("could not fetch current working directory: %w", err)
+		}
+
+		// Cleanup: must switch back to earlier working directory.
+		defer func() {
+			cerr := os.Chdir(wd)
+			if cerr != nil {
+				cerr = errors.Errorf("could not reset current working directory: %w", cerr)
+				if err == nil {
+					err = cerr
+				} else {
+					// Chain error with cleanup error.
+					err = errors.Errorf("%w, %w", err, cerr)
+				}
+			}
+		}()
+		err = os.Chdir(params.WorkDir)
+	}
+
 	log.Println("Adding build environment vars:", env)
-	return sh.RunWith(env, "go", args...)
+	err = sh.RunWith(env, "go", args...)
+	if err != nil {
+		err = errors.Errorf("failed to run build command: %w", err)
+	}
+
+	return err
 }
 
 // MakeWindowsSysoFile generates a .syso file containing metadata about the

x-pack/osquerybeat/.gitignore

@@ -14,5 +14,8 @@
 /osquery-extension.ext
 /osquery-extension.exe
 
+/kubequery.ext
+/kubequery.exe
+
 # VSCode
 .vscode/

x-pack/osquerybeat/internal/osqd/osqueryd.go

@@ -140,7 +140,7 @@ func (q *OSQueryD) Check(ctx context.Context) error {
 
 	err = cmd.Wait()
 
-	return nil
+	return err
 }
 
 // Run executes osqueryd binary as a child process
@@ -252,19 +252,21 @@ func (q *OSQueryD) prepare(ctx context.Context) (func(), error) {
 		return cleanupFn, nil
 	}
 
-	// Prepare autoload osquery-extension
-	extensionPath := osqueryExtensionPath(q.binPath)
-	if _, err := os.Stat(extensionPath); err != nil {
-		if os.IsNotExist(err) {
-			return nil, errors.Wrapf(err, "extension path does not exist: %s", extensionPath)
-		} else {
-			return nil, errors.Wrapf(err, "failed to stat extension path")
+	// Prepare autoload osquery-extension.
+	extensionPaths := extensionPaths(q.binPath)
+	for _, path := range extensionPaths {
+		if _, err := os.Stat(path); err != nil {
+			if os.IsNotExist(err) {
+				return nil, errors.Wrapf(err, "extension path does not exist: %s", path)
+			} else {
+				return nil, errors.Wrapf(err, "failed to stat extension path")
+			}
 		}
 	}
 
-	// Write the autoload file
+	// Write the autoload file.
 	extensionAutoloadPath := q.resolveDataPath(osqueryAutoload)
-	err = prepareAutoloadFile(extensionAutoloadPath, extensionPath, q.log)
+	err = prepareAutoloadFile(extensionAutoloadPath, extensionPaths, q.log)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to prepare extensions autoload file")
 	}
@@ -288,7 +290,7 @@ func (q *OSQueryD) prepare(ctx context.Context) (func(), error) {
 	return func() {}, nil
 }
 
-func prepareAutoloadFile(extensionAutoloadPath, mandatoryExtensionPath string, log *logp.Logger) error {
+func prepareAutoloadFile(extensionAutoloadPath string, mandatoryExtensionPaths []string, log *logp.Logger) error {
 	ok, err := fileutil.FileExists(extensionAutoloadPath)
 	if err != nil {
 		return errors.Wrapf(err, "failed to check osquery.autoload file exists")
@@ -297,8 +299,8 @@ func prepareAutoloadFile(extensionAutoloadPath, mandatoryExtensionPath string, l
 	rewrite := false
 
 	if ok {
-		log.Debugf("Extensions autoload file %s exists, verify the first extension is ours", extensionAutoloadPath)
-		err = verifyAutoloadFile(extensionAutoloadPath, mandatoryExtensionPath)
+		log.Debugf("Extensions autoload file %s exists, verify the mandatory extensions on top of file", extensionAutoloadPath)
+		err = verifyAutoloadFile(extensionAutoloadPath, mandatoryExtensionPaths)
 		if err != nil {
 			log.Debugf("Extensions autoload file %v verification failed, err: %v, create a new one", extensionAutoloadPath, err)
 			rewrite = true
@@ -309,26 +311,29 @@ func prepareAutoloadFile(extensionAutoloadPath, mandatoryExtensionPath string, l
 	}
 
 	if rewrite {
-		if err := ioutil.WriteFile(extensionAutoloadPath, []byte(mandatoryExtensionPath), 0644); err != nil {
+		if err := ioutil.WriteFile(extensionAutoloadPath, []byte(strings.Join(mandatoryExtensionPaths, "\n")), 0644); err != nil {
 			return errors.Wrap(err, "failed write osquery extension autoload file")
 		}
 	}
 	return nil
 }
 
-func verifyAutoloadFile(extensionAutoloadPath, mandatoryExtensionPath string) error {
+func verifyAutoloadFile(extensionAutoloadPath string, mandatoryExtensionPaths []string) error {
 	f, err := os.Open(extensionAutoloadPath)
 	if err != nil {
 		return err
 	}
 	defer f.Close()
+
 	scanner := bufio.NewScanner(f)
-	for i := 0; scanner.Scan(); i++ {
+	i := 0
+
+	for ; scanner.Scan(); i++ {
 		line := scanner.Text()
-		if i == 0 {
-			// Check that the first line is the mandatory extension
-			if line != mandatoryExtensionPath {
-				return errors.New("extentsions autoload file is missing mandatory extension in the first line of the file")
+		if i < len(mandatoryExtensionPaths) {
+			// Check that nth line of the file is the nth mandatory extension.
+			if line != mandatoryExtensionPaths[i] {
+				return errors.Errorf("extensions autoload file is missing mandatory extension %s on line %d", mandatoryExtensionPaths[i], i)
 			}
 		}
 
@@ -339,6 +344,10 @@ func verifyAutoloadFile(extensionAutoloadPath, mandatoryExtensionPath string) er
 		}
 	}
 
+	if i < len(mandatoryExtensionPaths) {
+		return errors.Errorf("expected %d mandatory extensions, but found only %d", len(mandatoryExtensionPaths), i)
+	}
+
 	return scanner.Err()
 }
 
@@ -416,8 +425,13 @@ func osquerydPath(dir string) string {
 	return filepath.Join(dir, osquerydFilename())
 }
 
-func osqueryExtensionPath(dir string) string {
-	return filepath.Join(dir, extensionName)
+func extensionPaths(dir string) []string {
+	p := make([]string, 0, len(extensionNames))
+	for i := range p {
+		p[i] = filepath.Join(dir, extensionNames[i])
+	}
+
+	return p
 }
 
 func (q *OSQueryD) resolveDataPath(filename string) string {
@@ -427,7 +441,7 @@ func (q *OSQueryD) resolveDataPath(filename string) string {
 func (q *OSQueryD) logOSQueryOutput(ctx context.Context, r io.ReadCloser) error {
 	log := q.log.With("ctx", "osqueryd output")
 
-	buf := make([]byte, 2048, 2048)
+	buf := make([]byte, 2048)
 LOOP:
 	for {
 		n, err := r.Read(buf[:])

x-pack/osquerybeat/internal/osqd/osqueryd_test.go

@@ -10,6 +10,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/elastic/beats/v7/libbeat/common"
@@ -60,8 +61,10 @@ func TestNew(t *testing.T) {
 func TestVerifyAutoloadFileMissing(t *testing.T) {
 	dir := uuid.Must(uuid.NewV4()).String()
 	extensionAutoloadPath := filepath.Join(dir, osqueryAutoload)
-	mandatoryExtensionPath := filepath.Join(dir, extensionName)
-	err := verifyAutoloadFile(extensionAutoloadPath, mandatoryExtensionPath)
+
+	mandatoryExtensionPaths := extensionPaths(dir)
+
+	err := verifyAutoloadFile(extensionAutoloadPath, mandatoryExtensionPaths)
 	if !errors.Is(err, os.ErrNotExist) {
 		t.Fatalf("expected error: %v, got: %v", os.ErrNotExist, err)
 	}
@@ -77,12 +80,16 @@ func TestPrepareAutoloadFile(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer os.RemoveAll(dir)
-	mandatoryExtensionPath := filepath.Join(dir, extensionName)
+
+	mandatoryExtensionPaths := extensionPaths(dir)
+	extensionsPathsBytes := []byte(strings.Join(mandatoryExtensionPaths, "\n"))
 
 	// Write fake extension file for testing
-	err = ioutil.WriteFile(mandatoryExtensionPath, nil, 0644)
-	if err != nil {
-		t.Fatal(err)
+	for _, path := range mandatoryExtensionPaths {
+		err = ioutil.WriteFile(path, nil, 0644)
+		if err != nil {
+			t.Fatal(err)
+		}
 	}
 
 	randomContent := func(sz int) []byte {
@@ -103,15 +110,15 @@ func TestPrepareAutoloadFile(t *testing.T) {
 		},
 		{
 			Name:        "File with mandatory extension",
-			FileContent: []byte(mandatoryExtensionPath),
+			FileContent: extensionsPathsBytes,
 		},
 		{
 			Name:        "Missing mandatory extension, should restore the file",
 			FileContent: []byte(filepath.Join(dir, "foobar.ext")),
 		},
 		{
 			Name:        "User extension path doesn't exists",
-			FileContent: []byte(mandatoryExtensionPath + "\n" + filepath.Join(dir, "foobar.ext")),
+			FileContent: append(extensionsPathsBytes, []byte("\n"+filepath.Join(dir, "foobar.ext"))...),
 		},
 		{
 			Name:        "Random garbage",
@@ -137,7 +144,7 @@ func TestPrepareAutoloadFile(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			err = prepareAutoloadFile(extensionAutoloadPath, mandatoryExtensionPath, validLogger)
+			err = prepareAutoloadFile(extensionAutoloadPath, mandatoryExtensionPaths, validLogger)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -151,9 +158,9 @@ func TestPrepareAutoloadFile(t *testing.T) {
 			scanner := bufio.NewScanner(f)
 			for i := 0; scanner.Scan(); i++ {
 				line := scanner.Text()
-				if i == 0 {
-					if line != mandatoryExtensionPath {
-						t.Fatalf("expected the fist line of the file to be: %v , got: %v", mandatoryExtensionPath, line)
+				if i < 0 {
+					if line != mandatoryExtensionPaths[i] {
+						t.Fatalf("expected line %d of the file to be: %v , got: %v", i, mandatoryExtensionPaths[i], line)
 					}
 				}
 				// Check that it is a valid path to the file on the disk

x-pack/osquerybeat/internal/osqd/osqueryd_unix.go

@@ -17,8 +17,11 @@ import (
 	"github.com/pkg/errors"
 )
 
-const (
-	extensionName = "osquery-extension.ext"
+var (
+	extensionNames = []string{
+		"osquery-extension.ext",
+		"kubequery.ext",
+	}
 )
 
 func CreateSocketPath() (string, func(), error) {

x-pack/osquerybeat/internal/osqd/osqueryd_windows.go

@@ -15,8 +15,11 @@ import (
 	"github.com/gofrs/uuid"
 )
 
-const (
-	extensionName = "osquery-extension.exe"
+var (
+	extensionNames = []string{
+		"osquery-extension.exe",
+		"kubequery.exe",
+	}
 )
 
 func CreateSocketPath() (string, func(), error) {

x-pack/osquerybeat/magefile.go

@@ -55,12 +55,29 @@ func Build() error {
 		return err
 	}
 
-	// Rename osquery-extension to osquery-extension.ext on non windows platforms
+	params.WorkDir = "./ext/kubequery"
+	params.InputFiles = []string{"./cmd/kubequery"}
+	params.OutputDir = "bin"
+	params.Name = "kubequery"
+	params.CGO = false
+	err = devtools.Build(params)
+	if err != nil {
+		return err
+	}
+
+	err = os.Rename("./ext/kubequery/bin/kubequery", "kubequery")
+
+	// Append .ext to extensions on non windows platforms.
 	if runtime.GOOS != "windows" {
 		err = os.Rename("osquery-extension", "osquery-extension.ext")
 		if err != nil {
 			return err
 		}
+
+		err = os.Rename("kubequery", "kubequery.ext")
+		if err != nil {
+			return err
+		}
 	}
 
 	return nil