adding finding route in backend (elastic#61)

* adding finding a route in backend

x-pack/plugins/cloud_security_posture/common/constants.ts

@@ -8,3 +8,7 @@
 export const CSP_KUBEBEAT_INDEX_PATTERN = 'kubebeat*';
 export const CSP_KUBEBEAT_INDEX_NAME = 'findings';
 export const STATS_ROUTH_PATH = '/api/csp/stats';
+export const FINDINGS_ROUTH_PATH = '/api/csp/finding';
+export const AGENT_LOGS_INDEX = 'kubebeat*';
+export const RULE_PASSED = `passed`;
+export const RULE_FAILED = `failed`;

x-pack/plugins/cloud_security_posture/server/routes/findings/findings.test.ts

@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mocks';
+
+import { getLatestCycleIds } from './get_latest_cycle_ids';
+
+const mockEsClient = elasticsearchClientMock.createClusterClient().asScoped().asInternalUser;
+
+afterEach(() => {
+  mockEsClient.search.mockClear();
+  mockEsClient.count.mockClear();
+});
+
+describe('get latest cycle ids', () => {
+  it('expect for empty response from client and get undefined', async () => {
+    const response = await getLatestCycleIds(mockEsClient);
+    expect(response).toEqual(undefined);
+  });
+
+  it('expect to find empty bucket', async () => {
+    mockEsClient.search.mockResolvedValueOnce(
+      // @ts-expect-error @elastic/elasticsearch Aggregate only allows unknown values
+      elasticsearchClientMock.createSuccessTransportRequestPromise({
+        aggregations: {
+          group: {
+            buckets: [{}],
+          },
+        },
+      })
+    );
+    const response = await getLatestCycleIds(mockEsClient);
+    expect(response).toEqual(undefined);
+  });
+
+  it('expect to find 1 cycle id', async () => {
+    mockEsClient.search.mockResolvedValueOnce(
+      // @ts-expect-error @elastic/elasticsearch Aggregate only allows unknown values
+      elasticsearchClientMock.createSuccessTransportRequestPromise({
+        aggregations: {
+          group: {
+            buckets: [
+              { group_docs: { hits: { hits: [{ fields: { 'run_id.keyword': ['randomId1'] } }] } } },
+            ],
+          },
+        },
+      })
+    );
+    const response = await getLatestCycleIds(mockEsClient);
+    expect(response).toEqual(expect.arrayContaining(['randomId1']));
+  });
+
+  it('expect to find mutiple cycle ids', async () => {
+    mockEsClient.search.mockResolvedValueOnce(
+      // @ts-expect-error @elastic/elasticsearch Aggregate only allows unknown values
+      elasticsearchClientMock.createSuccessTransportRequestPromise({
+        aggregations: {
+          group: {
+            buckets: [
+              { group_docs: { hits: { hits: [{ fields: { 'run_id.keyword': ['randomId1'] } }] } } },
+              { group_docs: { hits: { hits: [{ fields: { 'run_id.keyword': ['randomId2'] } }] } } },
+              { group_docs: { hits: { hits: [{ fields: { 'run_id.keyword': ['randomId3'] } }] } } },
+            ],
+          },
+        },
+      })
+    );
+    const response = await getLatestCycleIds(mockEsClient);
+    expect(response).toEqual(expect.arrayContaining(['randomId1', 'randomId2', 'randomId3']));
+  });
+});

x-pack/plugins/cloud_security_posture/server/routes/findings/findings.ts

@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { SearchRequest, QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
+
+import { schema as rt, TypeOf } from '@kbn/config-schema';
+import type { ElasticsearchClient } from 'src/core/server';
+import type { IRouter } from 'src/core/server';
+import { getLatestCycleIds } from './get_latest_cycle_ids';
+import { CSP_KUBEBEAT_INDEX_PATTERN, FINDINGS_ROUTH_PATH } from '../../../common/constants';
+export const DEFAULT_FINDINGS_PER_PAGE = 20;
+type FindingsQuerySchema = TypeOf<typeof schema>;
+
+const buildQueryFilter = async (
+  esClient: ElasticsearchClient,
+  queryParams: FindingsQuerySchema
+): Promise<QueryDslQueryContainer> => {
+  if (queryParams.latest_cycle) {
+    const latestCycleIds = await getLatestCycleIds(esClient);
+    if (!!latestCycleIds) {
+      const filter = latestCycleIds.map((latestCycleId) => ({
+        term: { 'run_id.keyword': latestCycleId },
+      }));
+
+      return {
+        bool: { filter },
+      };
+    }
+  }
+  return {
+    match_all: {},
+  };
+};
+
+const getFindingsEsQuery = async (
+  esClient: ElasticsearchClient,
+  queryParams: FindingsQuerySchema
+): Promise<SearchRequest> => {
+  const query = await buildQueryFilter(esClient, queryParams);
+  return {
+    index: CSP_KUBEBEAT_INDEX_PATTERN,
+    query,
+    size: queryParams.per_page,
+    from:
+      queryParams.page <= 1
+        ? 0
+        : queryParams.page * queryParams.per_page - queryParams.per_page + 1,
+  };
+};
+
+export const defineFindingsIndexRoute = (router: IRouter): void =>
+  router.get(
+    {
+      path: FINDINGS_ROUTH_PATH,
+      validate: { query: schema },
+    },
+    async (context, request, response) => {
+      try {
+        const esClient = context.core.elasticsearch.client.asCurrentUser;
+        const { query } = request;
+        const esQuery = await getFindingsEsQuery(esClient, query);
+        const findings = await esClient.search(esQuery);
+        const hits = findings.body.hits.hits;
+        return response.ok({ body: hits });
+      } catch (err) {
+        return response.customError({ body: { message: err }, statusCode: 500 }); // TODO: research error handling
+      }
+    }
+  );
+
+const schema = rt.object({
+  latest_cycle: rt.maybe(rt.boolean()),
+  page: rt.number({ defaultValue: 1, min: 0 }), // TODO: research for pagintaion best practice
+  per_page: rt.number({ defaultValue: DEFAULT_FINDINGS_PER_PAGE, min: 0 }),
+});

x-pack/plugins/cloud_security_posture/server/routes/findings/get_latest_cycle_ids.ts

@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { AggregationsFiltersAggregate, SearchRequest } from '@elastic/elasticsearch/lib/api/types';
+import type { ElasticsearchClient } from 'src/core/server';
+import { AGENT_LOGS_INDEX } from '../../../common/constants';
+
+const getAgentLogsEsQuery = (): SearchRequest => ({
+  index: AGENT_LOGS_INDEX,
+  size: 0,
+  //   query: {
+  //     bool: {
+  //       filter: [
+  //         { term: { 'event_status.keyword': 'end' } }, // TODO: commment out when updateing agent to dend logs
+  //       ],
+  //     },
+  //   },
+  aggs: {
+    group: {
+      terms: { field: 'agent.id.keyword' },
+      aggs: {
+        group_docs: {
+          top_hits: {
+            size: 1,
+            sort: [{ '@timestamp': { order: 'desc' } }],
+          },
+        },
+      },
+    },
+  },
+  fields: ['run_id.keyword', 'agent.id.keyword'],
+  _source: false,
+});
+
+const getCycleId = (v: any): string => v.group_docs.hits.hits?.[0]?.fields['run_id.keyword'][0];
+
+export const getLatestCycleIds = async (
+  esClient: ElasticsearchClient
+): Promise<string[] | undefined> => {
+  try {
+    const agentLogs = await esClient.search(getAgentLogsEsQuery());
+    const aggregations = agentLogs.body.aggregations;
+    if (!aggregations) {
+      return;
+    }
+    const buckets = (aggregations.group as Record<string, AggregationsFiltersAggregate>).buckets;
+    if (!Array.isArray(buckets)) {
+      return;
+    }
+    return buckets.map(getCycleId);
+  } catch (err) {
+    // TODO: return meaningful error message
+    return;
+  }
+};

x-pack/plugins/cloud_security_posture/server/routes/index.ts

@@ -6,8 +6,10 @@
  */
 
 import { defineGetStatsRoute } from './stats/stats';
+import { defineFindingsIndexRoute as defineGetFindingsIndexRoute } from './findings/findings';
 import type { IRouter } from '../../../../../src/core/server';
 
 export function defineRoutes(router: IRouter) {
   defineGetStatsRoute(router);
+  defineGetFindingsIndexRoute(router);
 }

x-pack/plugins/cloud_security_posture/server/routes/stats/stats.ts

@@ -18,7 +18,7 @@ import {
   getBenchmarksQuery,
   getLatestFindingQuery,
 } from './stats_queries';
-import { STATS_ROUTH_PATH } from '../../../common/constants';
+import { STATS_ROUTH_PATH, RULE_PASSED, RULE_FAILED } from '../../../common/constants';
 interface LastCycle {
   run_id: string;
 }
@@ -57,8 +57,8 @@ export const getAllFindingsStats = async (
   cycleId: string
 ): Promise<BenchmarkStats> => {
   const findings = await esClient.count(getFindingsEsQuery(cycleId));
-  const passedFindings = await esClient.count(getFindingsEsQuery(cycleId, 'passed'));
-  const failedFindings = await esClient.count(getFindingsEsQuery(cycleId, 'failed'));
+  const passedFindings = await esClient.count(getFindingsEsQuery(cycleId, RULE_PASSED));
+  const failedFindings = await esClient.count(getFindingsEsQuery(cycleId, RULE_FAILED));
 
   const totalFindings = findings.body.count;
   const totalPassed = passedFindings.body.count;
@@ -83,10 +83,10 @@ export const getBenchmarksStats = async (
   for (const benchmark of benchmarks) {
     const benchmarkFindings = await esClient.count(getFindingsEsQuery(benchmark, cycleId));
     const benchmarkPassedFindings = await esClient.count(
-      getFindingsEsQuery(cycleId, 'passed', benchmark)
+      getFindingsEsQuery(cycleId, RULE_PASSED, benchmark)
     );
     const benchmarkFailedFindings = await esClient.count(
-      getFindingsEsQuery(cycleId, 'failed', benchmark)
+      getFindingsEsQuery(cycleId, RULE_FAILED, benchmark)
     );
     const totalFindings = benchmarkFindings.body.count;
     const totalPassed = benchmarkPassedFindings.body.count;
@@ -109,7 +109,7 @@ export const getResourcesEvaluation = async (
   cycleId: string
 ): Promise<EvaluationStats[]> => {
   const failedEvaluationsPerResourceResult = await esClient.search(
-    getResourcesEvaluationEsQuery(cycleId, 'failed', 5)
+    getResourcesEvaluationEsQuery(cycleId, RULE_FAILED, 5)
   );
 
   const failedResourcesGroup = failedEvaluationsPerResourceResult.body.aggregations
@@ -124,15 +124,15 @@ export const getResourcesEvaluation = async (
   });
 
   const passedEvaluationsPerResourceResult = await esClient.search(
-    getResourcesEvaluationEsQuery(cycleId, 'passed', 5, topFailedResources)
+    getResourcesEvaluationEsQuery(cycleId, RULE_PASSED, 5, topFailedResources)
   );
   const passedResourcesGroup = passedEvaluationsPerResourceResult.body.aggregations
     ?.group as AggregationsTermsAggregate<GroupFilename>;
   const passedEvaluationPerResorces = passedResourcesGroup.buckets.map((e) => {
     return {
       resource: e.key,
       value: e.doc_count,
-      evaluation: 'passed',
+      evaluation: RULE_PASSED,
     } as const;
   });