[Security Solution][Endpoint] Update to new manifest format (without …

…compression) (#70752)

* Stateless exception list translation with improved runtime checks

* use flatMap and reduce to simplify logic

* Update to new manifest format

* Fix test fixture SO data type

* Fix another test fixture data type

* Fix sha256 reference in artifact_client

x-pack/plugins/security_solution/common/endpoint/schema/common.ts

@@ -6,6 +6,15 @@
 
 import * as t from 'io-ts';
 
+export const compressionAlgorithm = t.keyof({
+  none: null,
+  zlib: null,
+});
+
+export const encryptionAlgorithm = t.keyof({
+  none: null,
+});
+
 export const identifier = t.string;
 
 export const manifestVersion = t.string;
@@ -15,8 +24,8 @@ export const manifestSchemaVersion = t.keyof({
 });
 export type ManifestSchemaVersion = t.TypeOf<typeof manifestSchemaVersion>;
 
+export const relativeUrl = t.string;
+
 export const sha256 = t.string;
 
 export const size = t.number;
-
-export const url = t.string;

x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts

@@ -5,13 +5,26 @@
  */
 
 import * as t from 'io-ts';
-import { identifier, manifestSchemaVersion, manifestVersion, sha256, size, url } from './common';
+import {
+  compressionAlgorithm,
+  encryptionAlgorithm,
+  identifier,
+  manifestSchemaVersion,
+  manifestVersion,
+  relativeUrl,
+  sha256,
+  size,
+} from './common';
 
 export const manifestEntrySchema = t.exact(
   t.type({
-    url,
-    sha256,
-    size,
+    relative_url: relativeUrl,
+    precompress_sha256: sha256,
+    precompress_size: size,
+    postcompress_sha256: sha256,
+    postcompress_size: size,
+    compression_algorithm: compressionAlgorithm,
+    encryption_algorithm: encryptionAlgorithm,
   })
 );
 

x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts

@@ -57,6 +57,8 @@ export const getPackageConfigCreateCallback = (
     try {
       return updatedPackageConfig;
     } finally {
+      // TODO: confirm creation of package config
+      // then commit.
       await manifestManager.commit(wrappedManifest);
     }
   };

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts

@@ -6,12 +6,12 @@
 
 export const ArtifactConstants = {
   GLOBAL_ALLOWLIST_NAME: 'endpoint-exceptionlist',
-  SAVED_OBJECT_TYPE: 'endpoint:exceptions-artifact',
+  SAVED_OBJECT_TYPE: 'endpoint:user-artifact',
   SUPPORTED_OPERATING_SYSTEMS: ['linux', 'macos', 'windows'],
   SCHEMA_VERSION: '1.0.0',
 };
 
 export const ManifestConstants = {
-  SAVED_OBJECT_TYPE: 'endpoint:exceptions-manifest',
+  SAVED_OBJECT_TYPE: 'endpoint:user-artifact-manifest',
   SCHEMA_VERSION: '1.0.0',
 };

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts

@@ -7,22 +7,20 @@
 import { createHash } from 'crypto';
 import { validate } from '../../../../common/validate';
 
-import {
-  Entry,
-  EntryNested,
-  EntryMatch,
-  EntryMatchAny,
-} from '../../../../../lists/common/schemas/types/entries';
+import { Entry, EntryNested } from '../../../../../lists/common/schemas/types/entries';
 import { FoundExceptionListItemSchema } from '../../../../../lists/common/schemas/response/found_exception_list_item_schema';
 import { ExceptionListClient } from '../../../../../lists/server';
 import {
   InternalArtifactSchema,
   TranslatedEntry,
-  TranslatedEntryMatch,
-  TranslatedEntryMatchAny,
-  TranslatedEntryNested,
   WrappedTranslatedExceptionList,
   wrappedExceptionList,
+  TranslatedEntryNestedEntry,
+  translatedEntryNestedEntry,
+  translatedEntry as translatedEntryType,
+  TranslatedEntryMatcher,
+  translatedEntryMatchMatcher,
+  translatedEntryMatchAnyMatcher,
 } from '../../schemas';
 import { ArtifactConstants } from './common';
 
@@ -36,11 +34,14 @@ export async function buildArtifact(
 
   return {
     identifier: `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`,
-    sha256,
-    encoding: 'application/json',
+    compressionAlgorithm: 'none',
+    encryptionAlgorithm: 'none',
+    decompressedSha256: sha256,
+    compressedSha256: sha256,
+    decompressedSize: exceptionsBuffer.byteLength,
+    compressedSize: exceptionsBuffer.byteLength,
     created: Date.now(),
     body: exceptionsBuffer.toString('base64'),
-    size: exceptionsBuffer.byteLength,
   };
 }
 
@@ -92,66 +93,80 @@ export function translateToEndpointExceptions(
   exc: FoundExceptionListItemSchema,
   schemaVersion: string
 ): TranslatedEntry[] {
-  const translatedList: TranslatedEntry[] = [];
-
   if (schemaVersion === '1.0.0') {
-    exc.data.forEach((list) => {
-      list.entries.forEach((entry) => {
-        const tEntry = translateEntry(schemaVersion, entry);
-        if (tEntry !== undefined) {
-          translatedList.push(tEntry);
+    return exc.data
+      .flatMap((list) => {
+        return list.entries;
+      })
+      .reduce((entries: TranslatedEntry[], entry) => {
+        const translatedEntry = translateEntry(schemaVersion, entry);
+        if (translatedEntry !== undefined && translatedEntryType.is(translatedEntry)) {
+          entries.push(translatedEntry);
         }
-      });
-    });
+        return entries;
+      }, []);
   } else {
     throw new Error('unsupported schemaVersion');
   }
-  return translatedList;
+}
+
+function getMatcherFunction(field: string, matchAny?: boolean): TranslatedEntryMatcher {
+  return matchAny
+    ? field.endsWith('.text')
+      ? 'exact_caseless_any'
+      : 'exact_cased_any'
+    : field.endsWith('.text')
+    ? 'exact_caseless'
+    : 'exact_cased';
+}
+
+function normalizeFieldName(field: string): string {
+  return field.endsWith('.text') ? field.substring(0, field.length - 5) : field;
 }
 
 function translateEntry(
   schemaVersion: string,
   entry: Entry | EntryNested
 ): TranslatedEntry | undefined {
-  let translatedEntry;
   switch (entry.type) {
     case 'nested': {
-      const e = (entry as unknown) as EntryNested;
-      const nestedEntries: TranslatedEntry[] = [];
-      for (const nestedEntry of e.entries) {
-        const translation = translateEntry(schemaVersion, nestedEntry);
-        if (translation !== undefined) {
-          nestedEntries.push(translation);
-        }
-      }
-      translatedEntry = {
+      const nestedEntries = entry.entries.reduce(
+        (entries: TranslatedEntryNestedEntry[], nestedEntry) => {
+          const translatedEntry = translateEntry(schemaVersion, nestedEntry);
+          if (nestedEntry !== undefined && translatedEntryNestedEntry.is(translatedEntry)) {
+            entries.push(translatedEntry);
+          }
+          return entries;
+        },
+        []
+      );
+      return {
         entries: nestedEntries,
-        field: e.field,
+        field: entry.field,
         type: 'nested',
-      } as TranslatedEntryNested;
-      break;
+      };
     }
     case 'match': {
-      const e = (entry as unknown) as EntryMatch;
-      translatedEntry = {
-        field: e.field.endsWith('.text') ? e.field.substring(0, e.field.length - 5) : e.field,
-        operator: e.operator,
-        type: e.field.endsWith('.text') ? 'exact_caseless' : 'exact_cased',
-        value: e.value,
-      } as TranslatedEntryMatch;
-      break;
+      const matcher = getMatcherFunction(entry.field);
+      return translatedEntryMatchMatcher.is(matcher)
+        ? {
+            field: normalizeFieldName(entry.field),
+            operator: entry.operator,
+            type: matcher,
+            value: entry.value,
+          }
+        : undefined;
+    }
+    case 'match_any': {
+      const matcher = getMatcherFunction(entry.field, true);
+      return translatedEntryMatchAnyMatcher.is(matcher)
+        ? {
+            field: normalizeFieldName(entry.field),
+            operator: entry.operator,
+            type: matcher,
+            value: entry.value,
+          }
+        : undefined;
     }
-    case 'match_any':
-      {
-        const e = (entry as unknown) as EntryMatchAny;
-        translatedEntry = {
-          field: e.field.endsWith('.text') ? e.field.substring(0, e.field.length - 5) : e.field,
-          operator: e.operator,
-          type: e.field.endsWith('.text') ? 'exact_caseless_any' : 'exact_cased_any',
-          value: e.value,
-        } as TranslatedEntryMatchAny;
-      }
-      break;
   }
-  return translatedEntry || undefined;
 }

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts

@@ -55,21 +55,33 @@ describe('manifest', () => {
       expect(manifest1.toEndpointFormat()).toStrictEqual({
         artifacts: {
           'endpoint-exceptionlist-linux-1.0.0': {
-            sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
-            size: 268,
-            url:
+            compression_algorithm: 'none',
+            encryption_algorithm: 'none',
+            precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
+            postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
+            precompress_size: 268,
+            postcompress_size: 268,
+            relative_url:
               '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
           },
           'endpoint-exceptionlist-macos-1.0.0': {
-            sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
-            size: 268,
-            url:
+            compression_algorithm: 'none',
+            encryption_algorithm: 'none',
+            precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
+            postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
+            precompress_size: 268,
+            postcompress_size: 268,
+            relative_url:
               '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
           },
           'endpoint-exceptionlist-windows-1.0.0': {
-            sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
-            size: 268,
-            url:
+            compression_algorithm: 'none',
+            encryption_algorithm: 'none',
+            precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
+            postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
+            precompress_size: 268,
+            postcompress_size: 268,
+            relative_url:
               '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
           },
         },
@@ -107,7 +119,7 @@ describe('manifest', () => {
 
     test('Manifest returns data for given artifact', async () => {
       const artifact = artifacts[0];
-      const returned = manifest1.getArtifact(`${artifact.identifier}-${artifact.sha256}`);
+      const returned = manifest1.getArtifact(`${artifact.identifier}-${artifact.compressedSha256}`);
       expect(returned).toEqual(artifact);
     });
 

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts

@@ -33,13 +33,17 @@ describe('manifest_entry', () => {
     });
 
     test('Correct sha256 is returned', () => {
-      expect(manifestEntry.getSha256()).toEqual(
+      expect(manifestEntry.getCompressedSha256()).toEqual(
+        '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c'
+      );
+      expect(manifestEntry.getDecompressedSha256()).toEqual(
         '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c'
       );
     });
 
     test('Correct size is returned', () => {
-      expect(manifestEntry.getSize()).toEqual(268);
+      expect(manifestEntry.getCompressedSize()).toEqual(268);
+      expect(manifestEntry.getDecompressedSize()).toEqual(268);
     });
 
     test('Correct url is returned', () => {
@@ -54,9 +58,13 @@ describe('manifest_entry', () => {
 
     test('Correct record is returned', () => {
       expect(manifestEntry.getRecord()).toEqual({
-        sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
-        size: 268,
-        url:
+        compression_algorithm: 'none',
+        encryption_algorithm: 'none',
+        precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
+        postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
+        precompress_size: 268,
+        postcompress_size: 268,
+        relative_url:
           '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
       });
     });

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts

@@ -15,23 +15,31 @@ export class ManifestEntry {
   }
 
   public getDocId(): string {
-    return `${this.getIdentifier()}-${this.getSha256()}`;
+    return `${this.getIdentifier()}-${this.getCompressedSha256()}`;
   }
 
   public getIdentifier(): string {
     return this.artifact.identifier;
   }
 
-  public getSha256(): string {
-    return this.artifact.sha256;
+  public getCompressedSha256(): string {
+    return this.artifact.compressedSha256;
   }
 
-  public getSize(): number {
-    return this.artifact.size;
+  public getDecompressedSha256(): string {
+    return this.artifact.decompressedSha256;
+  }
+
+  public getCompressedSize(): number {
+    return this.artifact.compressedSize;
+  }
+
+  public getDecompressedSize(): number {
+    return this.artifact.decompressedSize;
   }
 
   public getUrl(): string {
-    return `/api/endpoint/artifacts/download/${this.getIdentifier()}/${this.getSha256()}`;
+    return `/api/endpoint/artifacts/download/${this.getIdentifier()}/${this.getCompressedSha256()}`;
   }
 
   public getArtifact(): InternalArtifactSchema {
@@ -40,9 +48,13 @@ export class ManifestEntry {
 
   public getRecord(): ManifestEntrySchema {
     return {
-      sha256: this.getSha256(),
-      size: this.getSize(),
-      url: this.getUrl(),
+      compression_algorithm: 'none',
+      encryption_algorithm: 'none',
+      precompress_sha256: this.getDecompressedSha256(),
+      precompress_size: this.getDecompressedSize(),
+      postcompress_sha256: this.getCompressedSha256(),
+      postcompress_size: this.getCompressedSize(),
+      relative_url: this.getUrl(),
     };
   }
 }