[Security_Solution][Endpoint] Resolver leverage ancestry array for qu…

…eries (elastic#69264) * Adding alerts route * Adding related alerts generator changes, tests, and script updates * Fixing missed parameter * Aligning the AlertEvent and ResolverEvent definition * Fixing type errors * Fixing import error * Adding ancestry functionality in generator * Creating some tests for ancestry field * Making progress on the ancestry * Fixing the ancestry verification * Fixing existing tests * Removing unused code and fixing test * Adding more comments * Fixing endgame queries Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
jonathan-buttner · Jun 19, 2020 · 212dbd9 · 212dbd9
1 parent aaa1b43
commit 212dbd9
Show file tree

Hide file tree

Showing 6 changed files with 248 additions and 89 deletions.
diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts
@@ -10,6 +10,7 @@ import {
   TreeNode,
   RelatedEventCategory,
   ECSCategory,
+  ANCESTRY_LIMIT,
 } from './generate_data';
 
 interface Node {
@@ -113,6 +114,7 @@ describe('data generator', () => {
         relatedEvents: 0,
         relatedAlerts: 0,
       });
+      tree.ancestry.delete(tree.origin.id);
     });
 
     it('creates an alert for the origin node but no other nodes', () => {
@@ -159,6 +161,30 @@ describe('data generator', () => {
       return (inRelated || inRelatedAlerts || inLifecycle) && event.process.entity_id === node.id;
     };
 
+    const verifyAncestry = (event: Event, genTree: Tree) => {
+      if (event.process.Ext.ancestry.length > 0) {
+        expect(event.process.parent?.entity_id).toBe(event.process.Ext.ancestry[0]);
+      }
+      for (let i = 0; i < event.process.Ext.ancestry.length; i++) {
+        const ancestor = event.process.Ext.ancestry[i];
+        const parent = genTree.children.get(ancestor) || genTree.ancestry.get(ancestor);
+        expect(ancestor).toBe(parent?.lifecycle[0].process.entity_id);
+
+        // the next ancestor should be the grandparent
+        if (i + 1 < event.process.Ext.ancestry.length) {
+          const grandparent = event.process.Ext.ancestry[i + 1];
+          expect(grandparent).toBe(parent?.lifecycle[0].process.parent?.entity_id);
+        }
+      }
+    };
+
+    it('has ancestry array defined', () => {
+      expect(tree.origin.lifecycle[0].process.Ext.ancestry.length).toBe(ANCESTRY_LIMIT);
+      for (const event of tree.allEvents) {
+        verifyAncestry(event, tree);
+      }
+    });
+
     it('has the right related events for each node', () => {
       const checkRelatedEvents = (node: TreeNode) => {
         expect(node.relatedEvents.length).toEqual(4);
@@ -185,8 +211,6 @@ describe('data generator', () => {
       for (const node of tree.children.values()) {
         checkRelatedEvents(node);
       }
-
-      checkRelatedEvents(tree.origin);
     });
 
     it('has the right number of related alerts for each node', () => {
@@ -202,7 +226,8 @@ describe('data generator', () => {
     });
 
     it('has the right number of ancestors', () => {
-      expect(tree.ancestry.size).toEqual(ancestors);
+      // +1 for the origin node
+      expect(tree.ancestry.size).toEqual(ancestors + 1);
     });
 
     it('has the right number of total children', () => {
@@ -239,10 +264,7 @@ describe('data generator', () => {
         const children = tree.children.get(event.process.entity_id);
         if (children) {
           expect(eventInNode(event, children)).toBeTruthy();
-          return;
         }
-
-        expect(eventInNode(event, tree.origin)).toBeTruthy();
       });
     });
 
@@ -260,8 +282,6 @@ describe('data generator', () => {
         total += nodeEventCount(node);
       }
 
-      total += nodeEventCount(tree.origin);
-
       expect(tree.allEvents.length).toEqual(total);
     });
   });

diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts
@@ -18,6 +18,16 @@ import {
 import { factory as policyFactory } from './models/policy_config';
 
 export type Event = AlertEvent | EndpointEvent;
+/**
+ * This value indicates the limit for the size of the ancestry array. The endpoint currently saves up to 20 values
+ * in its messages. To simulate a limit on the array size I'm using 2 here so that we can't rely on there being a large
+ * number like 20. The ancestry array contains entity_ids for the ancestors of a particular process.
+ *
+ * The array has a special format. The entity_ids towards the beginning of the array are closer ancestors and the
+ * values towards the end of the array are more distant ancestors (grandparents). Therefore
+ * ancestry_array[0] == process.parent.entity_id and ancestry_array[1] == process.parent.parent.entity_id
+ */
+export const ANCESTRY_LIMIT: number = 2;
 
 interface EventOptions {
   timestamp?: number;
@@ -26,6 +36,7 @@ interface EventOptions {
   eventType?: string;
   eventCategory?: string | string[];
   processName?: string;
+  ancestry?: string[];
   pid?: number;
   parentPid?: number;
   extensions?: object;
@@ -352,7 +363,8 @@ export class EndpointDocGenerator {
   public generateAlert(
     ts = new Date().getTime(),
     entityID = this.randomString(10),
-    parentEntityID?: string
+    parentEntityID?: string,
+    ancestryArray: string[] = []
   ): AlertEvent {
     return {
       ...this.commonInfo,
@@ -412,6 +424,9 @@ export class EndpointDocGenerator {
           sha256: 'fake sha256',
         },
         Ext: {
+          // simulate a finite ancestry array size, the endpoint limits the ancestry array to 20 entries we'll use
+          // 2 so that the backend can handle that case
+          ancestry: ancestryArray.slice(0, ANCESTRY_LIMIT),
           code_signature: [
             {
               trusted: false,
@@ -532,6 +547,9 @@ export class EndpointDocGenerator {
             }
           : undefined,
         name: processName,
+        // simulate a finite ancestry array size, the endpoint limits the ancestry array to 20 entries we'll use
+        // 2 so that the backend can handle that case
+        Ext: { ancestry: options.ancestry?.slice(0, ANCESTRY_LIMIT) || [] },
       },
       user: {
         domain: this.randomString(10),
@@ -589,9 +607,6 @@ export class EndpointDocGenerator {
       throw Error(`could not find origin while building tree: ${alert.process.entity_id}`);
     }
 
-    // remove the origin node from the ancestry array
-    ancestryNodes.delete(alert.process.entity_id);
-
     const children = Array.from(
       this.descendantsTreeGenerator(
         alert,
@@ -715,7 +730,7 @@ export class EndpointDocGenerator {
       }
     };
 
-    // generate related alerts for rootW
+    // generate related alerts for root
     const processDuration: number = 6 * 3600;
     if (this.randomN(100) < pctWithRelated) {
       addRelatedEvents(ancestor, processDuration, events);
@@ -740,6 +755,8 @@ export class EndpointDocGenerator {
       ancestor = this.generateEvent({
         timestamp,
         parentEntityID: ancestor.process.entity_id,
+        // add the parent to the ancestry array
+        ancestry: [ancestor.process.entity_id, ...ancestor.process.Ext.ancestry],
         parentPid: ancestor.process.pid,
         pid: this.randomN(5000),
       });
@@ -755,6 +772,7 @@ export class EndpointDocGenerator {
             parentEntityID: ancestor.process.parent?.entity_id,
             eventCategory: 'process',
             eventType: 'end',
+            ancestry: ancestor.process.Ext.ancestry,
           })
         );
       }
@@ -773,7 +791,12 @@ export class EndpointDocGenerator {
       }
     }
     events.push(
-      this.generateAlert(timestamp, ancestor.process.entity_id, ancestor.process.parent?.entity_id)
+      this.generateAlert(
+        timestamp,
+        ancestor.process.entity_id,
+        ancestor.process.parent?.entity_id,
+        ancestor.process.Ext.ancestry
+      )
     );
     return events;
   }
@@ -828,6 +851,10 @@ export class EndpointDocGenerator {
       const child = this.generateEvent({
         timestamp,
         parentEntityID: currentState.event.process.entity_id,
+        ancestry: [
+          currentState.event.process.entity_id,
+          ...currentState.event.process.Ext.ancestry,
+        ],
       });
 
       maxChildren = this.randomN(maxChildrenPerNode + 1);
@@ -849,6 +876,7 @@ export class EndpointDocGenerator {
           parentEntityID: child.process.parent?.entity_id,
           eventCategory: 'process',
           eventType: 'end',
+          ancestry: child.process.Ext.ancestry,
         });
       }
       if (this.randomN(100) < percentNodesWithRelated) {
@@ -893,6 +921,7 @@ export class EndpointDocGenerator {
           parentEntityID: node.process.parent?.entity_id,
           eventCategory: eventInfo.category,
           eventType: eventInfo.creationType,
+          ancestry: node.process.Ext.ancestry,
         });
       }
     }
@@ -911,7 +940,12 @@ export class EndpointDocGenerator {
   ) {
     for (let i = 0; i < relatedAlerts; i++) {
       const ts = node['@timestamp'] + this.randomN(alertCreationTime) * 1000;
-      yield this.generateAlert(ts, node.process.entity_id, node.process.parent?.entity_id);
+      yield this.generateAlert(
+        ts,
+        node.process.entity_id,
+        node.process.parent?.entity_id,
+        node.process.Ext.ancestry
+      );
     }
   }
 

diff --git a/x-pack/plugins/security_solution/common/endpoint/models/event.ts b/x-pack/plugins/security_solution/common/endpoint/models/event.ts
@@ -53,6 +53,13 @@ export function parentEntityId(event: ResolverEvent): string | undefined {
   return event.process.parent?.entity_id;
 }
 
+export function ancestryArray(event: ResolverEvent): string[] | undefined {
+  if (isLegacyEvent(event)) {
+    return undefined;
+  }
+  return event.process.Ext.ancestry;
+}
+
 /**
  * @param event The event to get the category for
  */

diff --git a/x-pack/plugins/security_solution/common/endpoint/types.ts b/x-pack/plugins/security_solution/common/endpoint/types.ts
@@ -300,6 +300,12 @@ export interface AlertEvent {
     thread?: ThreadFields[];
     uptime: number;
     Ext: {
+      /*
+       * The array has a special format. The entity_ids towards the beginning of the array are closer ancestors and the
+       * values towards the end of the array are more distant ancestors (grandparents). Therefore
+       * ancestry_array[0] == process.parent.entity_id and ancestry_array[1] == process.parent.parent.entity_id
+       */
+      ancestry: string[];
       code_signature: Array<{
         subject_name: string;
         trusted: boolean;
@@ -469,6 +475,14 @@ export interface EndpointEvent {
       name?: string;
       pid?: number;
     };
+    /*
+     * The array has a special format. The entity_ids towards the beginning of the array are closer ancestors and the
+     * values towards the end of the array are more distant ancestors (grandparents). Therefore
+     * ancestry_array[0] == process.parent.entity_id and ancestry_array[1] == process.parent.parent.entity_id
+     */
+    Ext: {
+      ancestry: string[];
+    };
   };
   user?: {
     domain?: string;