Update to new manifest format

elastic · Jul 4, 2020 · e9f74b4 · e9f74b4
1 parent 9e43633
commit e9f74b4
Show file tree

Hide file tree

Showing 12 changed files with 142 additions and 57 deletions.
diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/common.ts b/x-pack/plugins/security_solution/common/endpoint/schema/common.ts
@@ -6,6 +6,15 @@
 
 import * as t from 'io-ts';
 
+export const compressionAlgorithm = t.keyof({
+  none: null,
+  zlib: null,
+});
+
+export const encryptionAlgorithm = t.keyof({
+  none: null,
+});
+
 export const identifier = t.string;
 
 export const manifestVersion = t.string;
@@ -15,8 +24,8 @@ export const manifestSchemaVersion = t.keyof({
 });
 export type ManifestSchemaVersion = t.TypeOf<typeof manifestSchemaVersion>;
 
+export const relativeUrl = t.string;
+
 export const sha256 = t.string;
 
 export const size = t.number;
-
-export const url = t.string;
diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts b/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts
@@ -5,13 +5,26 @@
  */
 
 import * as t from 'io-ts';
-import { identifier, manifestSchemaVersion, manifestVersion, sha256, size, url } from './common';
+import {
+  compressionAlgorithm,
+  encryptionAlgorithm,
+  identifier,
+  manifestSchemaVersion,
+  manifestVersion,
+  relativeUrl,
+  sha256,
+  size,
+} from './common';
 
 export const manifestEntrySchema = t.exact(
   t.type({
-    url,
-    sha256,
-    size,
+    relative_url: relativeUrl,
+    precompress_sha256: sha256,
+    precompress_size: size,
+    postcompress_sha256: sha256,
+    postcompress_size: size,
+    compression_algorithm: compressionAlgorithm,
+    encryption_algorithm: encryptionAlgorithm,
   })
 );
 

diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts
@@ -6,12 +6,12 @@
 
 export const ArtifactConstants = {
   GLOBAL_ALLOWLIST_NAME: 'endpoint-exceptionlist',
-  SAVED_OBJECT_TYPE: 'endpoint:exceptions-artifact',
+  SAVED_OBJECT_TYPE: 'endpoint:user-artifact',
   SUPPORTED_OPERATING_SYSTEMS: ['linux', 'macos', 'windows'],
   SCHEMA_VERSION: '1.0.0',
 };
 
 export const ManifestConstants = {
-  SAVED_OBJECT_TYPE: 'endpoint:exceptions-manifest',
+  SAVED_OBJECT_TYPE: 'endpoint:user-artifact-manifest',
   SCHEMA_VERSION: '1.0.0',
 };
diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts
@@ -34,11 +34,14 @@ export async function buildArtifact(
 
   return {
     identifier: `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`,
-    sha256,
-    encoding: 'application/json',
+    compressionAlgorithm: 'none',
+    encryptionAlgorithm: 'none',
+    decompressedSha256: sha256,
+    compressedSha256: sha256,
+    decompressedSize: exceptionsBuffer.byteLength,
+    compressedSize: exceptionsBuffer.byteLength,
     created: Date.now(),
     body: exceptionsBuffer.toString('base64'),
-    size: exceptionsBuffer.byteLength,
   };
 }
 

diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts
@@ -55,21 +55,33 @@ describe('manifest', () => {
       expect(manifest1.toEndpointFormat()).toStrictEqual({
         artifacts: {
           'endpoint-exceptionlist-linux-1.0.0': {
-            sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
-            size: 268,
-            url:
+            compression_algorithm: 'none',
+            encryption_algorithm: 'none',
+            precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
+            postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
+            precompress_size: 268,
+            postcompress_size: 268,
+            relative_url:
               '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
           },
           'endpoint-exceptionlist-macos-1.0.0': {
-            sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
-            size: 268,
-            url:
+            compression_algorithm: 'none',
+            encryption_algorithm: 'none',
+            precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
+            postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
+            precompress_size: 268,
+            postcompress_size: 268,
+            relative_url:
               '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
           },
           'endpoint-exceptionlist-windows-1.0.0': {
-            sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
-            size: 268,
-            url:
+            compression_algorithm: 'none',
+            encryption_algorithm: 'none',
+            precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
+            postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
+            precompress_size: 268,
+            postcompress_size: 268,
+            relative_url:
               '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
           },
         },
@@ -107,7 +119,7 @@ describe('manifest', () => {
 
     test('Manifest returns data for given artifact', async () => {
       const artifact = artifacts[0];
-      const returned = manifest1.getArtifact(`${artifact.identifier}-${artifact.sha256}`);
+      const returned = manifest1.getArtifact(`${artifact.identifier}-${artifact.compressedSha256}`);
       expect(returned).toEqual(artifact);
     });
 

diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts
@@ -33,13 +33,17 @@ describe('manifest_entry', () => {
     });
 
     test('Correct sha256 is returned', () => {
-      expect(manifestEntry.getSha256()).toEqual(
+      expect(manifestEntry.getCompressedSha256()).toEqual(
+        '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c'
+      );
+      expect(manifestEntry.getDecompressedSha256()).toEqual(
         '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c'
       );
     });
 
     test('Correct size is returned', () => {
-      expect(manifestEntry.getSize()).toEqual(268);
+      expect(manifestEntry.getCompressedSize()).toEqual(268);
+      expect(manifestEntry.getDecompressedSize()).toEqual(268);
     });
 
     test('Correct url is returned', () => {
@@ -54,9 +58,13 @@ describe('manifest_entry', () => {
 
     test('Correct record is returned', () => {
       expect(manifestEntry.getRecord()).toEqual({
-        sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
-        size: 268,
-        url:
+        compression_algorithm: 'none',
+        encryption_algorithm: 'none',
+        precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
+        postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
+        precompress_size: 268,
+        postcompress_size: 268,
+        relative_url:
           '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
       });
     });

diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts
@@ -15,23 +15,31 @@ export class ManifestEntry {
   }
 
   public getDocId(): string {
-    return `${this.getIdentifier()}-${this.getSha256()}`;
+    return `${this.getIdentifier()}-${this.getCompressedSha256()}`;
   }
 
   public getIdentifier(): string {
     return this.artifact.identifier;
   }
 
-  public getSha256(): string {
-    return this.artifact.sha256;
+  public getCompressedSha256(): string {
+    return this.artifact.compressedSha256;
   }
 
-  public getSize(): number {
-    return this.artifact.size;
+  public getDecompressedSha256(): string {
+    return this.artifact.decompressedSha256;
+  }
+
+  public getCompressedSize(): number {
+    return this.artifact.compressedSize;
+  }
+
+  public getDecompressedSize(): number {
+    return this.artifact.decompressedSize;
   }
 
   public getUrl(): string {
-    return `/api/endpoint/artifacts/download/${this.getIdentifier()}/${this.getSha256()}`;
+    return `/api/endpoint/artifacts/download/${this.getIdentifier()}/${this.getCompressedSha256()}`;
   }
 
   public getArtifact(): InternalArtifactSchema {
@@ -40,9 +48,13 @@ export class ManifestEntry {
 
   public getRecord(): ManifestEntrySchema {
     return {
-      sha256: this.getSha256(),
-      size: this.getSize(),
-      url: this.getUrl(),
+      compression_algorithm: 'none',
+      encryption_algorithm: 'none',
+      precompress_sha256: this.getDecompressedSha256(),
+      precompress_size: this.getDecompressedSize(),
+      postcompress_sha256: this.getCompressedSha256(),
+      postcompress_size: this.getCompressedSize(),
+      relative_url: this.getUrl(),
     };
   }
 }
diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts
@@ -16,11 +16,27 @@ export const exceptionsArtifactSavedObjectMappings: SavedObjectsType['mappings']
     identifier: {
       type: 'keyword',
     },
-    sha256: {
+    compressionAlgorithm: {
       type: 'keyword',
+      index: false,
+    },
+    encryptionAlgorithm: {
+      type: 'keyword',
+      index: false,
     },
-    encoding: {
+    compressedSha256: {
       type: 'keyword',
+    },
+    compressedSize: {
+      type: 'long',
+      index: false,
+    },
+    decompressedSha256: {
+      type: 'keyword',
+      index: false,
+    },
+    decompressedSize: {
+      type: 'long',
       index: false,
     },
     created: {
@@ -31,10 +47,6 @@ export const exceptionsArtifactSavedObjectMappings: SavedObjectsType['mappings']
       type: 'binary',
       index: false,
     },
-    size: {
-      type: 'long',
-      index: false,
-    },
   },
 };
 

diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts
@@ -14,7 +14,7 @@ import { EndpointAppContext } from '../../types';
 
 export const ManifestTaskConstants = {
   TIMEOUT: '1m',
-  TYPE: 'securitySolution:endpoint:exceptions-packager',
+  TYPE: 'endpoint:user-artifact-packager',
   VERSION: '1.0.0',
 };
 

diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts
@@ -5,17 +5,26 @@
  */
 
 import * as t from 'io-ts';
-import { identifier, sha256, size } from '../../../../common/endpoint/schema/common';
-import { body, created, encoding } from './common';
+import {
+  compressionAlgorithm,
+  encryptionAlgorithm,
+  identifier,
+  sha256,
+  size,
+} from '../../../../common/endpoint/schema/common';
+import { body, created } from './common';
 
 export const internalArtifactSchema = t.exact(
   t.type({
     identifier,
-    sha256,
-    encoding,
+    compressionAlgorithm,
+    encryptionAlgorithm,
+    decompressedSha256: sha256,
+    decompressedSize: size,
+    compressedSha256: sha256,
+    compressedSize: size,
     created,
     body,
-    size,
   })
 );
 

diff --git a/...ity_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts b/...ity_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts
@@ -38,9 +38,13 @@ describe('manifest_manager', () => {
         schema_version: '1.0.0',
         artifacts: {
           [artifact.identifier]: {
-            sha256: artifact.sha256,
-            size: artifact.size,
-            url: `/api/endpoint/artifacts/download/${artifact.identifier}/${artifact.sha256}`,
+            compression_algorithm: 'none',
+            encryption_algorithm: 'none',
+            precompress_sha256: artifact.decompressedSha256,
+            postcompress_sha256: artifact.compressedSha256,
+            precompress_size: artifact.decompressedSize,
+            postcompress_size: artifact.compressedSize,
+            relative_url: `/api/endpoint/artifacts/download/${artifact.identifier}/${artifact.compressedSha256}`,
           },
         },
       });

diff --git a/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json
@@ -1,20 +1,23 @@
 {
   "type": "doc",
   "value": {
-    "id": "endpoint:exceptions-artifact:endpoint-exceptionlist-linux-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d",
+    "id": "endpoint:user-artifact:endpoint-exceptionlist-linux-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d",
     "index": ".kibana",
     "source": {
       "references": [
       ],
       "endpoint:exceptions-artifact": {
         "body": "eyJleGNlcHRpb25zX2xpc3QiOltdfQ==",
         "created": 1593016187465,
-        "encoding": "application/json",
+        "compressionAlgorithm": "none",
+        "encryptionAlgorithm": "none",
         "identifier": "endpoint-exceptionlist-linux-1.0.0",
-        "sha256": "a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d",
-        "size": 22
+        "compressedSha256": "a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d",
+        "compressedSize": 22,
+        "decompressedSha256": "a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d",
+        "decompressedSize": 22
       },
-      "type": "endpoint:exceptions-artifact",
+      "type": "endpoint:user-artifact",
       "updated_at": "2020-06-24T16:29:47.584Z"
     }
   }
@@ -23,7 +26,7 @@
 {
   "type": "doc",
   "value": {
-    "id": "endpoint:exceptions-manifest:endpoint-manifest-1.0.0",
+    "id": "endpoint:user-artifact-manifest:endpoint-manifest-1.0.0",
     "index": ".kibana",
     "source": {
       "references": [
@@ -36,7 +39,7 @@
           "endpoint-exceptionlist-windows-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d"
         ]
       },
-      "type": "endpoint:exceptions-manifest",
+      "type": "endpoint:user-artifact-manifest",
       "updated_at": "2020-06-26T15:01:39.704Z"
     }
   }