Merge branch 'main' into journalctl-for-journald-input

elastic · Aug 6, 2024 · b350bf2 · b350bf2
2 parents 2decbe6 + 99c1138
commit b350bf2
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 71 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -167,6 +167,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Fix handling of deeply nested numeric values in HTTP Endpoint CEL programs. {pull}40115[40115]
 - Prevent panic in CEL and salesforce inputs when github.com/hashicorp/go-retryablehttp exceeds maximum retries. {pull}40144[40144]
 - Fix bug in CEL input rate limit logic. {issue}40106[40106] {pull}40270[40270]
+- Relax requirements in Okta entity analytics provider user and device profile data shape. {pull}40359[40359]
 - Fix filestream not correctly tracking the offset of a file when using the `include_message` parsser. {pull}39873[39873] {issue}39653[39653]
 - Fix a number of crashes in the journald input. {pull}40061[40061]
 

diff --git a/x-pack/filebeat/input/entityanalytics/provider/okta/internal/okta/okta.go b/x-pack/filebeat/input/entityanalytics/provider/okta/internal/okta/okta.go
@@ -41,49 +41,12 @@ type User struct {
 	PasswordChanged       *time.Time     `json:"passwordChanged,omitempty"`
 	Type                  map[string]any `json:"type"`
 	TransitioningToStatus *string        `json:"transitioningToStatus,omitempty"`
-	Profile               Profile        `json:"profile"`
+	Profile               map[string]any `json:"profile"`
 	Credentials           *Credentials   `json:"credentials,omitempty"`
 	Links                 HAL            `json:"_links,omitempty"` // See https://developer.okta.com/docs/reference/api/users/#links-object for details.
 	Embedded              HAL            `json:"_embedded,omitempty"`
 }
 
-// Profile is an Okta user's profile.
-//
-// See https://developer.okta.com/docs/reference/api/users/#profile-object for details.
-type Profile struct {
-	Login             string  `json:"login"`
-	Email             string  `json:"email"`
-	SecondEmail       *string `json:"secondEmail,omitempty"`
-	FirstName         *string `json:"firstName,omitempty"`
-	LastName          *string `json:"lastName,omitempty"`
-	MiddleName        *string `json:"middleName,omitempty"`
-	HonorificPrefix   *string `json:"honorificPrefix,omitempty"`
-	HonorificSuffix   *string `json:"honorificSuffix,omitempty"`
-	Title             *string `json:"title,omitempty"`
-	DisplayName       *string `json:"displayName,omitempty"`
-	NickName          *string `json:"nickName,omitempty"`
-	ProfileUrl        *string `json:"profileUrl,omitempty"`
-	PrimaryPhone      *string `json:"primaryPhone,omitempty"`
-	MobilePhone       *string `json:"mobilePhone,omitempty"`
-	StreetAddress     *string `json:"streetAddress,omitempty"`
-	City              *string `json:"city,omitempty"`
-	State             *string `json:"state,omitempty"`
-	ZipCode           *string `json:"zipCode,omitempty"`
-	CountryCode       *string `json:"countryCode,omitempty"`
-	PostalAddress     *string `json:"postalAddress,omitempty"`
-	PreferredLanguage *string `json:"preferredLanguage,omitempty"`
-	Locale            *string `json:"locale,omitempty"`
-	Timezone          *string `json:"timezone,omitempty"`
-	UserType          *string `json:"userType,omitempty"`
-	EmployeeNumber    *string `json:"employeeNumber,omitempty"`
-	CostCenter        *string `json:"costCenter,omitempty"`
-	Organization      *string `json:"organization,omitempty"`
-	Division          *string `json:"division,omitempty"`
-	Department        *string `json:"department,omitempty"`
-	ManagerId         *string `json:"managerId,omitempty"`
-	Manager           *string `json:"manager,omitempty"`
-}
-
 // Credentials is a redacted Okta user's credential details. Only the credential provider is retained.
 //
 // See https://developer.okta.com/docs/reference/api/users/#credentials-object for details.
@@ -116,7 +79,7 @@ type Device struct {
 	Created             time.Time         `json:"created"`
 	ID                  string            `json:"id"`
 	LastUpdated         time.Time         `json:"lastUpdated"`
-	Profile             DeviceProfile     `json:"profile"`
+	Profile             map[string]any    `json:"profile"`
 	ResourceAlternateID string            `json:"resourceAlternateID"`
 	ResourceDisplayName DeviceDisplayName `json:"resourceDisplayName"`
 	ResourceID          string            `json:"resourceID"`
@@ -130,27 +93,6 @@ type Device struct {
 	Users []User `json:"users,omitempty"`
 }
 
-// DeviceProfile is an Okta device's hardware and security profile.
-//
-// See https://developer.okta.com/docs/api/openapi/okta-management/management/tag/Device/#tag/Device/operation/listDevices for details
-type DeviceProfile struct {
-	DiskEncryptionType    *string `json:"diskEncryptionType,omitempty"`
-	DisplayName           string  `json:"displayName"`
-	IMEI                  *string `json:"imei,omitempty"`
-	IntegrityJailBreak    *bool   `json:"integrityJailBreak,omitempty"`
-	Manufacturer          *string `json:"manufacturer,omitempty"`
-	MEID                  *string `json:"meid,omitempty"`
-	Model                 *string `json:"model,omitempty"`
-	OSVersion             *string `json:"osVersion,omitempty"`
-	Platform              string  `json:"platform"`
-	Registered            bool    `json:"registered"`
-	SecureHardwarePresent *bool   `json:"secureHardwarePresent,omitempty"`
-	SerialNumber          *string `json:"serialNumber,omitempty"`
-	SID                   *string `json:"sid,omitempty"`
-	TPMPublicKeyHash      *string `json:"tpmPublicKeyHash,omitempty"`
-	UDID                  *string `json:"udid,omitempty"`
-}
-
 // DeviceDisplayName is an Okta device's annotated display name.
 //
 // See https://developer.okta.com/docs/api/openapi/okta-management/management/tag/Device/#tag/Device/operation/listDevices for details

diff --git a/x-pack/filebeat/input/entityanalytics/provider/okta/internal/okta/okta_test.go b/x-pack/filebeat/input/entityanalytics/provider/okta/internal/okta/okta_test.go
@@ -117,22 +117,23 @@ func Test(t *testing.T) {
 			})
 
 			t.Run("user", func(t *testing.T) {
-				if me.Profile.Login == "" {
+				login, _ := me.Profile["login"].(string)
+				if login == "" {
 					b, _ := json.Marshal(me)
 					t.Skipf("cannot run user test without profile.login field set: %s", b)
 				}
 
 				query := make(url.Values)
 				query.Set("limit", "200")
-				users, _, err := GetUserDetails(context.Background(), http.DefaultClient, host, key, me.Profile.Login, query, omit, limiter, window, logger)
+				users, _, err := GetUserDetails(context.Background(), http.DefaultClient, host, key, login, query, omit, limiter, window, logger)
 				if err != nil {
 					t.Fatalf("unexpected error: %v", err)
 				}
 				if len(users) != 1 {
 					t.Fatalf("unexpected len(users): got:%d want:1", len(users))
 				}
 				if !cmp.Equal(me, users[0]) {
-					t.Errorf("unexpected result:\n-'me'\n+'%s'\n%s", me.Profile.Login, cmp.Diff(me, users[0]))
+					t.Errorf("unexpected result:\n-'me'\n+'%s'\n%s", login, cmp.Diff(me, users[0]))
 				}
 			})
 
@@ -218,7 +219,7 @@ var localTests = []struct {
 	{
 		// Test case constructed from API-returned value with details anonymised.
 		name: "users",
-		msg:  `[{"id":"userid","status":"STATUS","created":"2023-05-14T13:37:20.000Z","activated":null,"statusChanged":"2023-05-15T01:50:30.000Z","lastLogin":"2023-05-15T01:59:20.000Z","lastUpdated":"2023-05-15T01:50:32.000Z","passwordChanged":"2023-05-15T01:50:32.000Z","type":{"id":"typeid"},"profile":{"firstName":"name","lastName":"surname","mobilePhone":null,"secondEmail":null,"login":"name.surname@example.com","email":"name.surname@example.com"},"credentials":{"password":{"value":"secret"},"emails":[{"value":"name.surname@example.com","status":"VERIFIED","type":"PRIMARY"}],"provider":{"type":"OKTA","name":"OKTA"}},"_links":{"self":{"href":"https://localhost/api/v1/users/userid"}}}]`,
+		msg:  `[{"id":"userid","status":"STATUS","created":"2023-05-14T13:37:20.000Z","activated":null,"statusChanged":"2023-05-15T01:50:30.000Z","lastLogin":"2023-05-15T01:59:20.000Z","lastUpdated":"2023-05-15T01:50:32.000Z","passwordChanged":"2023-05-15T01:50:32.000Z","recovery_question":{"question":"Who's a major player in the cowboy scene?","answer":"Annie Oakley"},"type":{"id":"typeid"},"profile":{"firstName":"name","lastName":"surname","mobilePhone":null,"secondEmail":null,"login":"name.surname@example.com","email":"name.surname@example.com"},"credentials":{"password":{"value":"secret"},"emails":[{"value":"name.surname@example.com","status":"VERIFIED","type":"PRIMARY"}],"provider":{"type":"OKTA","name":"OKTA"}},"_links":{"self":{"href":"https://localhost/api/v1/users/userid"}}}]`,
 		fn: func(ctx context.Context, cli *http.Client, host, key, user string, query url.Values, lim *rate.Limiter, window time.Duration, log *logp.Logger) (any, http.Header, error) {
 			return GetUserDetails(context.Background(), cli, host, key, user, query, OmitNone, lim, window, log)
 		},
@@ -236,7 +237,7 @@ var localTests = []struct {
 	{
 		// Test case constructed from API-returned value with details anonymised.
 		name: "devices_users",
-		msg:  `[{"created":"2023-08-07T21:48:27.000Z","managementStatus":"NOT_MANAGED","user":{"id":"userid","status":"STATUS","created":"2023-05-14T13:37:20.000Z","activated":null,"statusChanged":"2023-05-15T01:50:30.000Z","lastLogin":"2023-05-15T01:59:20.000Z","lastUpdated":"2023-05-15T01:50:32.000Z","passwordChanged":"2023-05-15T01:50:32.000Z","type":{"id":"typeid"},"profile":{"firstName":"name","lastName":"surname","mobilePhone":null,"secondEmail":null,"login":"name.surname@example.com","email":"name.surname@example.com"},"credentials":{"password":{"value":"secret"},"emails":[{"value":"name.surname@example.com","status":"VERIFIED","type":"PRIMARY"}],"provider":{"type":"OKTA","name":"OKTA"}},"_links":{"self":{"href":"https://localhost/api/v1/users/userid"}}}}]`,
+		msg:  `[{"created":"2023-08-07T21:48:27.000Z","managementStatus":"NOT_MANAGED","user":{"id":"userid","status":"STATUS","created":"2023-05-14T13:37:20.000Z","activated":null,"statusChanged":"2023-05-15T01:50:30.000Z","lastLogin":"2023-05-15T01:59:20.000Z","lastUpdated":"2023-05-15T01:50:32.000Z","passwordChanged":"2023-05-15T01:50:32.000Z","type":{"id":"typeid"},"profile":{"firstName":"name","lastName":"surname","mobilePhone":null,"secondEmail":null,"login":"name.surname@example.com","email":"name.surname@example.com"},"credentials":{"password":{"value":"secret"},"recovery_question":{"question":"Who's a major player in the cowboy scene?","answer":"Annie Oakley"},"emails":[{"value":"name.surname@example.com","status":"VERIFIED","type":"PRIMARY"}],"provider":{"type":"OKTA","name":"OKTA"}},"_links":{"self":{"href":"https://localhost/api/v1/users/userid"}}}}]`,
 		id:   "devid",
 		fn: func(ctx context.Context, cli *http.Client, host, key, device string, query url.Values, lim *rate.Limiter, window time.Duration, log *logp.Logger) (any, http.Header, error) {
 			return GetDeviceUsers(context.Background(), cli, host, key, device, query, OmitNone, lim, window, log)

diff --git a/x-pack/filebeat/input/entityanalytics/provider/okta/statestore_test.go b/x-pack/filebeat/input/entityanalytics/provider/okta/statestore_test.go
@@ -104,14 +104,15 @@ func TestStateStore(t *testing.T) {
 					Type: map[string]interface{}{
 						"id": "typeid",
 					},
-					Profile: okta.Profile{
-						Login:     "name.surname@example.com",
-						Email:     "name.surname@example.com",
-						FirstName: ptr("name"),
-						LastName:  ptr("surname"),
+					Profile: map[string]interface{}{
+						"login":     "name.surname@example.com",
+						"email":     "name.surname@example.com",
+						"firstName": "name",
+						"lastName":  "surname",
 					},
 					Credentials: &okta.Credentials{
-						Password: &struct{}{}, // Had a password: not retained.
+						Password:         &struct{}{}, // Had a password: not retained.
+						RecoveryQuestion: &struct{}{}, // Had a question: not retained.
 						Provider: okta.Provider{
 							Type: "OKTA",
 							Name: ptr("OKTA"),