[Discover] Use fields API to retrieve fields

src/plugins/discover/common/index.ts

@@ -28,3 +28,4 @@ export const CONTEXT_DEFAULT_SIZE_SETTING = 'context:defaultSize';
 export const CONTEXT_STEP_SETTING = 'context:step';
 export const CONTEXT_TIE_BREAKER_FIELDS_SETTING = 'context:tieBreakerFields';
 export const MODIFY_COLUMNS_ON_SWITCH = 'discover:modifyColumnsOnSwitch';
+export const SEARCH_FIELDS_FROM_SOURCE = 'discover:searchFieldsFromSource';

src/plugins/discover/public/application/angular/context/api/_stubs.js

@@ -47,6 +47,7 @@ export function createSearchSourceStub(hits, timeField) {
 
   searchSourceStub.setParent = sinon.spy(() => searchSourceStub);
   searchSourceStub.setField = sinon.spy(() => searchSourceStub);
+  searchSourceStub.removeField = sinon.spy(() => searchSourceStub);
 
   searchSourceStub.getField = sinon.spy((key) => {
     const previousSetCall = searchSourceStub.setField.withArgs(key).lastCall;

src/plugins/discover/public/application/angular/context/api/anchor.js

@@ -20,7 +20,7 @@
 import _ from 'lodash';
 import { i18n } from '@kbn/i18n';
 
-export function fetchAnchorProvider(indexPatterns, searchSource) {
+export function fetchAnchorProvider(indexPatterns, searchSource, useNewFieldsApi = false) {
   return async function fetchAnchor(indexPatternId, anchorId, sort) {
     const indexPattern = await indexPatterns.get(indexPatternId);
     searchSource
@@ -41,7 +41,10 @@ export function fetchAnchorProvider(indexPatterns, searchSource) {
         language: 'lucene',
       })
       .setField('sort', sort);
-
+    if (useNewFieldsApi) {
+      searchSource.removeField('fieldsFromSource');
+      searchSource.setField('fields', ['*']);
+    }
     const response = await searchSource.fetch();
 
     if (_.get(response, ['hits', 'total'], 0) < 1) {

src/plugins/discover/public/application/angular/context/api/anchor.test.js

@@ -144,4 +144,29 @@ describe('context app', function () {
       });
     });
   });
+
+  describe('useNewFields API', () => {
+    let fetchAnchor;
+    let searchSourceStub;
+
+    beforeEach(() => {
+      searchSourceStub = createSearchSourceStub([{ _id: 'hit1' }]);
+      fetchAnchor = fetchAnchorProvider(createIndexPatternsStub(), searchSourceStub, true);
+    });
+
+    it('should request fields if useNewFieldsApi set', function () {
+      searchSourceStub._stubHits = [{ property1: 'value1' }, { property2: 'value2' }];
+
+      return fetchAnchor('INDEX_PATTERN_ID', 'id', [
+        { '@timestamp': 'desc' },
+        { _doc: 'desc' },
+      ]).then(() => {
+        const setFieldsSpy = searchSourceStub.setField.withArgs('fields');
+        const removeFieldsSpy = searchSourceStub.removeField.withArgs('fieldsFromSource');
+        expect(setFieldsSpy.calledOnce).toBe(true);
+        expect(removeFieldsSpy.calledOnce).toBe(true);
+        expect(setFieldsSpy.firstCall.args[1]).toEqual(['*']);
+      });
+    });
+  });
 });

src/plugins/discover/public/application/angular/context/api/context.predecessors.test.js

@@ -227,4 +227,81 @@ describe('context app', function () {
       });
     });
   });
+
+  describe('function fetchPredecessors with useNewFieldsApi set', function () {
+    let fetchPredecessors;
+    let mockSearchSource;
+
+    beforeEach(() => {
+      mockSearchSource = createContextSearchSourceStub([], '@timestamp', MS_PER_DAY * 8);
+
+      setServices({
+        data: {
+          search: {
+            searchSource: {
+              create: jest.fn().mockImplementation(() => mockSearchSource),
+            },
+          },
+        },
+      });
+
+      fetchPredecessors = (
+        indexPatternId,
+        timeField,
+        sortDir,
+        timeValIso,
+        timeValNr,
+        tieBreakerField,
+        tieBreakerValue,
+        size
+      ) => {
+        const anchor = {
+          _source: {
+            [timeField]: timeValIso,
+          },
+          sort: [timeValNr, tieBreakerValue],
+        };
+
+        return fetchContextProvider(createIndexPatternsStub(), true).fetchSurroundingDocs(
+          'predecessors',
+          indexPatternId,
+          anchor,
+          timeField,
+          tieBreakerField,
+          sortDir,
+          size,
+          []
+        );
+      };
+    });
+
+    it('should perform exactly one query when enough hits are returned', function () {
+      mockSearchSource._stubHits = [
+        mockSearchSource._createStubHit(MS_PER_DAY * 3000 + 2),
+        mockSearchSource._createStubHit(MS_PER_DAY * 3000 + 1),
+        mockSearchSource._createStubHit(MS_PER_DAY * 3000),
+        mockSearchSource._createStubHit(MS_PER_DAY * 2000),
+        mockSearchSource._createStubHit(MS_PER_DAY * 1000),
+      ];
+
+      return fetchPredecessors(
+        'INDEX_PATTERN_ID',
+        '@timestamp',
+        'desc',
+        ANCHOR_TIMESTAMP_3000,
+        MS_PER_DAY * 3000,
+        '_doc',
+        0,
+        3,
+        []
+      ).then((hits) => {
+        const setFieldsSpy = mockSearchSource.setField.withArgs('fields');
+        const removeFieldsSpy = mockSearchSource.removeField.withArgs('fieldsFromSource');
+        expect(mockSearchSource.fetch.calledOnce).toBe(true);
+        expect(removeFieldsSpy.calledOnce).toBe(true);
+        expect(setFieldsSpy.calledOnce).toBe(true);
+        expect(hits).toEqual(mockSearchSource._stubHits.slice(0, 3));
+      });
+    });
+  });
 });

src/plugins/discover/public/application/angular/context/api/context.successors.test.js

@@ -231,4 +231,81 @@ describe('context app', function () {
       });
     });
   });
+
+  describe('function fetchSuccessors with useNewFieldsApi set', function () {
+    let fetchSuccessors;
+    let mockSearchSource;
+
+    beforeEach(() => {
+      mockSearchSource = createContextSearchSourceStub([], '@timestamp');
+
+      setServices({
+        data: {
+          search: {
+            searchSource: {
+              create: jest.fn().mockImplementation(() => mockSearchSource),
+            },
+          },
+        },
+      });
+
+      fetchSuccessors = (
+        indexPatternId,
+        timeField,
+        sortDir,
+        timeValIso,
+        timeValNr,
+        tieBreakerField,
+        tieBreakerValue,
+        size
+      ) => {
+        const anchor = {
+          _source: {
+            [timeField]: timeValIso,
+          },
+          sort: [timeValNr, tieBreakerValue],
+        };
+
+        return fetchContextProvider(createIndexPatternsStub(), true).fetchSurroundingDocs(
+          'successors',
+          indexPatternId,
+          anchor,
+          timeField,
+          tieBreakerField,
+          sortDir,
+          size,
+          []
+        );
+      };
+    });
+
+    it('should perform exactly one query when enough hits are returned', function () {
+      mockSearchSource._stubHits = [
+        mockSearchSource._createStubHit(MS_PER_DAY * 5000),
+        mockSearchSource._createStubHit(MS_PER_DAY * 4000),
+        mockSearchSource._createStubHit(MS_PER_DAY * 3000),
+        mockSearchSource._createStubHit(MS_PER_DAY * 3000 - 1),
+        mockSearchSource._createStubHit(MS_PER_DAY * 3000 - 2),
+      ];
+
+      return fetchSuccessors(
+        'INDEX_PATTERN_ID',
+        '@timestamp',
+        'desc',
+        ANCHOR_TIMESTAMP_3000,
+        MS_PER_DAY * 3000,
+        '_doc',
+        0,
+        3,
+        []
+      ).then((hits) => {
+        expect(mockSearchSource.fetch.calledOnce).toBe(true);
+        expect(hits).toEqual(mockSearchSource._stubHits.slice(-3));
+        const setFieldsSpy = mockSearchSource.setField.withArgs('fields');
+        const removeFieldsSpy = mockSearchSource.removeField.withArgs('fieldsFromSource');
+        expect(removeFieldsSpy.calledOnce).toBe(true);
+        expect(setFieldsSpy.calledOnce).toBe(true);
+      });
+    });
+  });
 });

src/plugins/discover/public/application/angular/context/api/context.ts

@@ -40,7 +40,7 @@ const DAY_MILLIS = 24 * 60 * 60 * 1000;
 // look from 1 day up to 10000 days into the past and future
 const LOOKUP_OFFSETS = [0, 1, 7, 30, 365, 10000].map((days) => days * DAY_MILLIS);
 
-function fetchContextProvider(indexPatterns: IndexPatternsContract) {
+function fetchContextProvider(indexPatterns: IndexPatternsContract, useNewFieldsApi?: boolean) {
   return {
     fetchSurroundingDocs,
   };
@@ -116,6 +116,10 @@ function fetchContextProvider(indexPatterns: IndexPatternsContract) {
     const { data } = getServices();
 
     const searchSource = await data.search.searchSource.create();
+    if (useNewFieldsApi) {
+      searchSource.removeField('fieldsFromSource');
+      searchSource.setField('fields', ['*']);
+    }
     return searchSource
       .setParent(undefined)
       .setField('index', indexPattern)

src/plugins/discover/public/application/angular/context/query/actions.js

@@ -27,11 +27,17 @@ import { fetchContextProvider } from '../api/context';
 import { getQueryParameterActions } from '../query_parameters';
 import { FAILURE_REASONS, LOADING_STATUS } from './index';
 import { MarkdownSimple } from '../../../../../../kibana_react/public';
+import { SEARCH_FIELDS_FROM_SOURCE } from '../../../../../common';
 
 export function QueryActionsProvider(Promise) {
-  const { filterManager, indexPatterns, data } = getServices();
-  const fetchAnchor = fetchAnchorProvider(indexPatterns, data.search.searchSource.createEmpty());
-  const { fetchSurroundingDocs } = fetchContextProvider(indexPatterns);
+  const { filterManager, indexPatterns, data, uiSettings } = getServices();
+  const useNewFieldsApi = !uiSettings.get(SEARCH_FIELDS_FROM_SOURCE);
+  const fetchAnchor = fetchAnchorProvider(
+    indexPatterns,
+    data.search.searchSource.createEmpty(),
+    useNewFieldsApi
+  );
+  const { fetchSurroundingDocs } = fetchContextProvider(indexPatterns, useNewFieldsApi);
   const { setPredecessorCount, setQueryParameters, setSuccessorCount } = getQueryParameterActions(
     filterManager,
     indexPatterns

src/plugins/discover/public/application/angular/context_app.html

@@ -29,4 +29,5 @@
   successor-available="contextApp.state.rows.successors.length"
   successor-status="contextApp.state.loadingStatus.successors.status"
   on-change-successor-count="contextApp.actions.fetchGivenSuccessorRows"
+  use-new-fields-api="contextApp.state.useNewFieldsApi"
 ></context-app-legacy>

src/plugins/discover/public/application/angular/context_app.js

@@ -18,7 +18,11 @@
  */
 
 import _ from 'lodash';
-import { CONTEXT_STEP_SETTING, CONTEXT_TIE_BREAKER_FIELDS_SETTING } from '../../../common';
+import {
+  CONTEXT_STEP_SETTING,
+  CONTEXT_TIE_BREAKER_FIELDS_SETTING,
+  SEARCH_FIELDS_FROM_SOURCE,
+} from '../../../common';
 import { getAngularModule, getServices } from '../../kibana_services';
 import contextAppTemplate from './context_app.html';
 import './context/components/action_bar';
@@ -57,12 +61,14 @@ getAngularModule().directive('contextApp', function ContextApp() {
 
 function ContextAppController($scope, Private) {
   const { filterManager, indexPatterns, uiSettings } = getServices();
+  const useNewFieldsApi = !uiSettings.get(SEARCH_FIELDS_FROM_SOURCE);
   const queryParameterActions = getQueryParameterActions(filterManager, indexPatterns);
   const queryActions = Private(QueryActionsProvider);
   this.state = createInitialState(
     parseInt(uiSettings.get(CONTEXT_STEP_SETTING), 10),
     getFirstSortableField(this.indexPattern, uiSettings.get(CONTEXT_TIE_BREAKER_FIELDS_SETTING))
   );
+  this.state.useNewFieldsApi = useNewFieldsApi;
 
   this.actions = _.mapValues(
     {