ship logs to http endpoint

cmd/launcher/launcher.go

@@ -39,6 +39,8 @@ import (
 	"github.com/kolide/launcher/pkg/debug"
 	"github.com/kolide/launcher/pkg/launcher"
 	"github.com/kolide/launcher/pkg/log/checkpoint"
+	"github.com/kolide/launcher/pkg/log/logshipper"
+	"github.com/kolide/launcher/pkg/log/teelogger"
 	"github.com/kolide/launcher/pkg/osquery"
 	osqueryInstanceHistory "github.com/kolide/launcher/pkg/osquery/runtime/history"
 	"github.com/kolide/launcher/pkg/service"
@@ -144,6 +146,14 @@ func runLauncher(ctx context.Context, cancel func(), opts *launcher.Options) err
 	flagController := flags.NewFlagController(logger, stores[storage.AgentFlagsStore], fcOpts...)
 	k := knapsack.New(stores, flagController, db)
 
+	// Need to set up the log shipper so that we can get the logger early
+	// and pass it to the various systems.
+	var logShipper *logshipper.LogShipper
+	if k.ControlServerURL() != "" {
+		logShipper = logshipper.New(k, logger)
+		logger = teelogger.New(logger, logShipper)
+	}
+
 	// construct the appropriate http client based on security settings
 	httpClient := http.DefaultClient
 	if k.InsecureTLS() {
@@ -305,6 +315,14 @@ func runLauncher(ctx context.Context, cancel func(), opts *launcher.Options) err
 			runGroup.Add(exp.Execute, exp.Interrupt)
 			controlService.RegisterSubscriber(authTokensSubsystemName, exp)
 		}
+
+		// begin log shipping and subsribe to token updates
+		// nil check incase it failed to create for some reason
+		if logShipper != nil {
+			runGroup.Add(logShipper.Run, logShipper.Stop)
+			controlService.RegisterSubscriber(authTokensSubsystemName, logShipper)
+			controlService.RegisterSubscriber(agentFlagsSubsystemName, logShipper)
+		}
 	}
 
 	runEECode := k.ControlServerURL() != "" || k.IAmBreakingEELicense()

cmd/launcher/options.go

@@ -84,8 +84,9 @@ func parseOptions(subcommandName string, args []string) (*launcher.Options, erro
 		flConfigFilePath         = flagset.String("config", defaultConfigFilePath, "config file to parse options from (optional)")
 		flExportTraces           = flagset.Bool("export_traces", false, "Whether to export traces")
 		flTraceSamplingRate      = flagset.Float64("trace_sampling_rate", 0.0, "What fraction of traces should be sampled")
-		flIngestServerURL        = flagset.String("ingest_url", "", "Where to export traces and logs")
-		flDisableIngestTLS       = flagset.Bool("disable_ingest_tls", false, "Disable TLS for observability ingest server communication")
+		flLogIngestServerURL     = flagset.String("log_ingest_url", "", "Where to export logs")
+		flTraceIngestServerURL   = flagset.String("trace_ingest_url", "", "Where to export traces")
+		flDisableIngestTLS       = flagset.Bool("disable_trace_ingest_tls", false, "Disable TLS for observability ingest server communication")
 
 		// osquery TLS endpoints
 		flOsqTlsConfig    = flagset.String("config_tls_endpoint", "", "Config endpoint for the osquery tls transport")
@@ -248,8 +249,9 @@ func parseOptions(subcommandName string, args []string) (*launcher.Options, erro
 		EnrollSecret:                       *flEnrollSecret,
 		EnrollSecretPath:                   *flEnrollSecretPath,
 		ExportTraces:                       *flExportTraces,
-		ObservabilityIngestServerURL:       *flIngestServerURL,
-		DisableObservabilityIngestTLS:      *flDisableIngestTLS,
+		LogIngestServerURL:                 *flLogIngestServerURL,
+		TraceIngestServerURL:               *flTraceIngestServerURL,
+		DisableTraceIngestTLS:              *flDisableIngestTLS,
 		AutoloadedExtensions:               flAutoloadedExtensions,
 		IAmBreakingEELicense:               *flIAmBreakingEELicense,
 		InsecureTLS:                        *flInsecureTLS,

cmd/launcher/options_test.go

@@ -185,27 +185,27 @@ func getArgsAndResponse() (map[string]string, *launcher.Options) {
 	}
 
 	opts := &launcher.Options{
-		AutoupdateInitialDelay:        1 * time.Hour,
-		AutoupdateInterval:            48 * time.Hour,
-		CompactDbMaxTx:                int64(65536),
-		Control:                       false,
-		ControlServerURL:              "",
-		ControlRequestInterval:        60 * time.Second,
-		ExportTraces:                  false,
-		TraceSamplingRate:             0.0,
-		ObservabilityIngestServerURL:  "",
-		DisableObservabilityIngestTLS: false,
-		KolideServerURL:               randomHostname,
-		LoggingInterval:               time.Duration(randomInt) * time.Second,
-		MirrorServerURL:               "https://dl.kolide.co",
-		NotaryPrefix:                  "kolide",
-		NotaryServerURL:               "https://notary.kolide.co",
-		TufServerURL:                  "https://tuf.kolide.com",
-		OsquerydPath:                  windowsAddExe("/dev/null"),
-		Transport:                     "grpc",
-		UpdateChannel:                 "stable",
-		AutoloadedExtensions:          []string{"some-extension.ext"},
-		DelayStart:                    0 * time.Second,
+		AutoupdateInitialDelay: 1 * time.Hour,
+		AutoupdateInterval:     48 * time.Hour,
+		CompactDbMaxTx:         int64(65536),
+		Control:                false,
+		ControlServerURL:       "",
+		ControlRequestInterval: 60 * time.Second,
+		ExportTraces:           false,
+		TraceSamplingRate:      0.0,
+		LogIngestServerURL:     "",
+		DisableTraceIngestTLS:  false,
+		KolideServerURL:        randomHostname,
+		LoggingInterval:        time.Duration(randomInt) * time.Second,
+		MirrorServerURL:        "https://dl.kolide.co",
+		NotaryPrefix:           "kolide",
+		NotaryServerURL:        "https://notary.kolide.co",
+		TufServerURL:           "https://tuf.kolide.com",
+		OsquerydPath:           windowsAddExe("/dev/null"),
+		Transport:              "grpc",
+		UpdateChannel:          "stable",
+		AutoloadedExtensions:   []string{"some-extension.ext"},
+		DelayStart:             0 * time.Second,
 	}
 
 	return args, opts

pkg/agent/flags/flag_controller.go

@@ -465,20 +465,29 @@ func (fc *FlagController) TraceSamplingRate() float64 {
 	).get(fc.getControlServerValue(keys.TraceSamplingRate))
 }
 
-func (fc *FlagController) SetObservabilityIngestServerURL(url string) error {
-	return fc.setControlServerValue(keys.ObservabilityIngestServerURL, []byte(url))
+func (fc *FlagController) SetLogIngestServerURL(url string) error {
+	return fc.setControlServerValue(keys.LogIngestServerURL, []byte(url))
 }
-func (fc *FlagController) ObservabilityIngestServerURL() string {
+func (fc *FlagController) LogIngestServerURL() string {
 	return NewStringFlagValue(
-		WithDefaultString(fc.cmdLineOpts.ObservabilityIngestServerURL),
-	).get(fc.getControlServerValue(keys.ObservabilityIngestServerURL))
+		WithDefaultString(fc.cmdLineOpts.LogIngestServerURL),
+	).get(fc.getControlServerValue(keys.LogIngestServerURL))
 }
 
-func (fc *FlagController) SetDisableObservabilityIngestTLS(enabled bool) error {
-	return fc.setControlServerValue(keys.DisableObservabilityIngestTLS, boolToBytes(enabled))
+func (fc *FlagController) SetTraceIngestServerURL(url string) error {
+	return fc.setControlServerValue(keys.TraceIngestServerURL, []byte(url))
 }
-func (fc *FlagController) DisableObservabilityIngestTLS() bool {
+func (fc *FlagController) TraceIngestServerURL() string {
+	return NewStringFlagValue(
+		WithDefaultString(fc.cmdLineOpts.TraceIngestServerURL),
+	).get(fc.getControlServerValue(keys.TraceIngestServerURL))
+}
+
+func (fc *FlagController) SetDisableTraceIngestTLS(enabled bool) error {
+	return fc.setControlServerValue(keys.DisableTraceIngestTLS, boolToBytes(enabled))
+}
+func (fc *FlagController) DisableTraceIngestTLS() bool {
 	return NewBoolFlagValue(
-		WithDefaultBool(fc.cmdLineOpts.DisableObservabilityIngestTLS),
-	).get(fc.getControlServerValue(keys.DisableObservabilityIngestTLS))
+		WithDefaultBool(fc.cmdLineOpts.DisableTraceIngestTLS),
+	).get(fc.getControlServerValue(keys.DisableTraceIngestTLS))
 }

pkg/agent/flags/keys/keys.go

@@ -11,41 +11,42 @@ type FlagKey string
 // 4. Implement tests for any new APIs, sanitizers, limits, overrides.
 // 5. Update mocks -- in pkg/agent/types, run `mockery --name Knapsack` and `mockery --name Flags`.
 const (
-	KolideServerURL               FlagKey = "hostname"
-	KolideHosted                  FlagKey = "kolide_hosted"
-	Transport                     FlagKey = "transport"
-	LoggingInterval               FlagKey = "logging_interval"
-	OsquerydPath                  FlagKey = "osqueryd_path"
-	RootDirectory                 FlagKey = "root_directory"
-	RootPEM                       FlagKey = "root_pem"
-	DesktopEnabled                FlagKey = "desktop_enabled_v1"
-	DesktopUpdateInterval         FlagKey = "desktop_update_interval"
-	DesktopMenuRefreshInterval    FlagKey = "desktop_menu_refresh_interval"
-	DebugServerData               FlagKey = "debug_server_data"
-	ForceControlSubsystems        FlagKey = "force_control_subsystems"
-	ControlServerURL              FlagKey = "control_server_url"
-	ControlRequestInterval        FlagKey = "control_request_interval"
-	DisableControlTLS             FlagKey = "disable_control_tls"
-	InsecureControlTLS            FlagKey = "insecure_control_tls"
-	InsecureTLS                   FlagKey = "insecure_tls"
-	InsecureTransportTLS          FlagKey = "insecure_transport"
-	IAmBreakingEELicense          FlagKey = "i-am-breaking-ee-license"
-	Debug                         FlagKey = "debug"
-	DebugLogFile                  FlagKey = "debug_log_file"
-	OsqueryVerbose                FlagKey = "osquery_verbose"
-	Autoupdate                    FlagKey = "autoupdate"
-	NotaryServerURL               FlagKey = "notary_url"
-	TufServerURL                  FlagKey = "tuf_url"
-	MirrorServerURL               FlagKey = "mirror_url"
-	AutoupdateInterval            FlagKey = "autoupdate_interval"
-	UpdateChannel                 FlagKey = "update_channel"
-	NotaryPrefix                  FlagKey = "notary_prefix"
-	AutoupdateInitialDelay        FlagKey = "autoupdater_initial_delay"
-	UpdateDirectory               FlagKey = "update_directory"
-	ExportTraces                  FlagKey = "export_traces"
-	TraceSamplingRate             FlagKey = "trace_sampling_rate"
-	ObservabilityIngestServerURL  FlagKey = "ingest_url"
-	DisableObservabilityIngestTLS FlagKey = "disable_ingest_tls"
+	KolideServerURL            FlagKey = "hostname"
+	KolideHosted               FlagKey = "kolide_hosted"
+	Transport                  FlagKey = "transport"
+	LoggingInterval            FlagKey = "logging_interval"
+	OsquerydPath               FlagKey = "osqueryd_path"
+	RootDirectory              FlagKey = "root_directory"
+	RootPEM                    FlagKey = "root_pem"
+	DesktopEnabled             FlagKey = "desktop_enabled_v1"
+	DesktopUpdateInterval      FlagKey = "desktop_update_interval"
+	DesktopMenuRefreshInterval FlagKey = "desktop_menu_refresh_interval"
+	DebugServerData            FlagKey = "debug_server_data"
+	ForceControlSubsystems     FlagKey = "force_control_subsystems"
+	ControlServerURL           FlagKey = "control_server_url"
+	ControlRequestInterval     FlagKey = "control_request_interval"
+	DisableControlTLS          FlagKey = "disable_control_tls"
+	InsecureControlTLS         FlagKey = "insecure_control_tls"
+	InsecureTLS                FlagKey = "insecure_tls"
+	InsecureTransportTLS       FlagKey = "insecure_transport"
+	IAmBreakingEELicense       FlagKey = "i-am-breaking-ee-license"
+	Debug                      FlagKey = "debug"
+	DebugLogFile               FlagKey = "debug_log_file"
+	OsqueryVerbose             FlagKey = "osquery_verbose"
+	Autoupdate                 FlagKey = "autoupdate"
+	NotaryServerURL            FlagKey = "notary_url"
+	TufServerURL               FlagKey = "tuf_url"
+	MirrorServerURL            FlagKey = "mirror_url"
+	AutoupdateInterval         FlagKey = "autoupdate_interval"
+	UpdateChannel              FlagKey = "update_channel"
+	NotaryPrefix               FlagKey = "notary_prefix"
+	AutoupdateInitialDelay     FlagKey = "autoupdater_initial_delay"
+	UpdateDirectory            FlagKey = "update_directory"
+	ExportTraces               FlagKey = "export_traces"
+	TraceSamplingRate          FlagKey = "trace_sampling_rate"
+	LogIngestServerURL         FlagKey = "log_ingest_url"
+	TraceIngestServerURL       FlagKey = "trace_ingest_url"
+	DisableTraceIngestTLS      FlagKey = "disable_trace_ingest_tls"
 )
 
 func (key FlagKey) String() string {

pkg/agent/knapsack/knapsack.go

@@ -362,16 +362,23 @@ func (k *knapsack) TraceSamplingRate() float64 {
 	return k.flags.TraceSamplingRate()
 }
 
-func (k *knapsack) SetObservabilityIngestServerURL(url string) error {
-	return k.flags.SetObservabilityIngestServerURL(url)
+func (k *knapsack) SetTraceIngestServerURL(url string) error {
+	return k.flags.SetTraceIngestServerURL(url)
 }
-func (k *knapsack) ObservabilityIngestServerURL() string {
-	return k.flags.ObservabilityIngestServerURL()
+func (k *knapsack) TraceIngestServerURL() string {
+	return k.flags.TraceIngestServerURL()
 }
 
-func (k *knapsack) SetDisableObservabilityIngestTLS(enabled bool) error {
-	return k.flags.SetDisableObservabilityIngestTLS(enabled)
+func (k *knapsack) SetDisableTraceIngestTLS(enabled bool) error {
+	return k.flags.SetDisableTraceIngestTLS(enabled)
 }
-func (k *knapsack) DisableObservabilityIngestTLS() bool {
-	return k.flags.DisableObservabilityIngestTLS()
+func (k *knapsack) DisableTraceIngestTLS() bool {
+	return k.flags.DisableTraceIngestTLS()
+}
+
+func (k *knapsack) SetLogIngestServerURL(url string) error {
+	return k.flags.SetLogIngestServerURL(url)
+}
+func (k *knapsack) LogIngestServerURL() string {
+	return k.flags.LogIngestServerURL()
 }

pkg/agent/storage/keys.go

@@ -0,0 +1,5 @@
+package storage
+
+var (
+	ObservabilityIngestAuthTokenKey = []byte("observability_ingest_auth_token")
+)

pkg/agent/types/flags.go

@@ -177,11 +177,15 @@ type Flags interface {
 	SetTraceSamplingRate(rate float64) error
 	TraceSamplingRate() float64
 
-	// ObservabilityIngestServerURL is the URL of the ingest server for logs and traces
-	SetObservabilityIngestServerURL(url string) error
-	ObservabilityIngestServerURL() string
+	// LogIngestServerURL is the URL of the ingest server for logs
+	SetLogIngestServerURL(url string) error
+	LogIngestServerURL() string
 
-	// DisableObservabilityIngestTLS disables TLS for observability ingest server communication
-	SetDisableObservabilityIngestTLS(enabled bool) error
-	DisableObservabilityIngestTLS() bool
+	// TraceIngestServerURL is the URL of the ingest server for traces
+	SetTraceIngestServerURL(url string) error
+	TraceIngestServerURL() string
+
+	// DisableTraceIngestTLS disables TLS for observability ingest server communication
+	SetDisableTraceIngestTLS(enabled bool) error
+	DisableTraceIngestTLS() bool
 }