[eventLog] prevent log writing when initialization fails

resolves elastic#68309

Previously, if the initialization of the elasticsearch resources failed
during initialization, the event logger would still try to write events.
Which is somewhat of a catastrophic failure, as typically the logger would
try writing to the alias name, but no alias exists, so a new index would
be created with the name of the alias.  Making it impossible to initialize
successfully later until that index was deleted.

The core initialization calls already returned success indicators, so this
PR just responds to those and prevents the logger from writing to the index
if intialization failed.

x-pack/plugins/event_log/server/es/context.test.ts

@@ -7,9 +7,8 @@
 import { createEsContext } from './context';
 import { LegacyClusterClient, Logger } from '../../../../../src/core/server';
 import { elasticsearchServiceMock, loggingSystemMock } from '../../../../../src/core/server/mocks';
-jest.mock('../lib/../../../../package.json', () => ({
-  version: '1.2.3',
-}));
+jest.mock('../lib/../../../../package.json', () => ({ version: '1.2.3' }));
+jest.mock('./init');
 type EsClusterClient = Pick<jest.Mocked<LegacyClusterClient>, 'callAsInternalUser' | 'asScoped'>;
 
 let logger: Logger;
@@ -92,4 +91,16 @@ describe('createEsContext', () => {
     );
     expect(doesIndexTemplateExist).toBeTruthy();
   });
+
+  test('should handled failed initialization', async () => {
+    jest.requireMock('./init').initializeEs.mockResolvedValue(false);
+    const context = createEsContext({
+      logger,
+      clusterClientPromise: Promise.resolve(clusterClient),
+      indexNameRoot: 'test2',
+    });
+    context.initialize();
+    const success = await context.waitTillReady();
+    expect(success).toBe(false);
+  });
 });

x-pack/plugins/event_log/server/es/context.ts

@@ -64,21 +64,23 @@ class EsContextImpl implements EsContext {
 
     setImmediate(async () => {
       try {
-        await this._initialize();
-        this.logger.debug('readySignal.signal(true)');
-        this.readySignal.signal(true);
+        const success = await this._initialize();
+        this.logger.debug(`readySignal.signal(${success})`);
+        this.readySignal.signal(success);
       } catch (err) {
         this.logger.debug('readySignal.signal(false)');
         this.readySignal.signal(false);
       }
     });
   }
 
+  // waits till the ES initialization is done, returns true if it was successful,
+  // false if it was not successful
   async waitTillReady(): Promise<boolean> {
     return await this.readySignal.wait();
   }
 
-  private async _initialize() {
-    await initializeEs(this);
+  private async _initialize(): Promise<boolean> {
+    return await initializeEs(this);
   }
 }

x-pack/plugins/event_log/server/event_logger.test.ts

@@ -17,7 +17,7 @@ const KIBANA_SERVER_UUID = '424-24-2424';
 
 describe('EventLogger', () => {
   let systemLogger: ReturnType<typeof loggingSystemMock.createLogger>;
-  let esContext: EsContext;
+  let esContext: jest.Mocked<EsContext>;
   let service: IEventLogService;
   let eventLogger: IEventLogger;
 
@@ -179,6 +179,32 @@ describe('EventLogger', () => {
     message = await waitForLogMessage(systemLogger);
     expect(message).toMatch(/invalid rel property.*ZZZ-primary.*/);
   });
+
+  test('handles failed but not thrown initialization', async () => {
+    esContext.waitTillReady.mockResolvedValueOnce(false);
+
+    service.registerProviderActions('test-provider', ['test-action-1']);
+    eventLogger = service.getLogger({
+      event: { provider: 'test-provider', action: 'test-action-1' },
+    });
+
+    eventLogger.logEvent({});
+    delay(3000); // sleep a bit since event logging is async
+    expect(esContext.esAdapter.indexDocument).not.toHaveBeenCalled();
+  });
+
+  test('handles failed thrown initialization', async () => {
+    esContext.waitTillReady.mockRejectedValueOnce(new Error('oof'));
+
+    service.registerProviderActions('test-provider', ['test-action-1']);
+    eventLogger = service.getLogger({
+      event: { provider: 'test-provider', action: 'test-action-1' },
+    });
+
+    eventLogger.logEvent({});
+    delay(3000); // sleep a bit since event logging is async
+    expect(esContext.esAdapter.indexDocument).not.toHaveBeenCalled();
+  });
 });
 
 // return the next logged event; throw if not an event

x-pack/plugins/event_log/server/event_logger.ts

@@ -183,7 +183,12 @@ function indexEventDoc(esContext: EsContext, doc: Doc): void {
 // whew, the thing that actually writes the event log document!
 async function indexLogEventDoc(esContext: EsContext, doc: unknown) {
   esContext.logger.debug(`writing to event log: ${JSON.stringify(doc)}`);
-  await esContext.waitTillReady();
+  const success = await esContext.waitTillReady();
+  if (!success) {
+    esContext.logger.debug(`event log did not initialize correctly, event not written`);
+    return;
+  }
+
   await esContext.esAdapter.indexDocument(doc);
   esContext.logger.debug(`writing to event log complete`);
 }