[Discover] Use fields API to retrieve fields

src/plugins/data/common/constants.ts

@@ -31,6 +31,7 @@ export const UI_SETTINGS = {
   COURIER_CUSTOM_REQUEST_PREFERENCE: 'courier:customRequestPreference',
   COURIER_MAX_CONCURRENT_SHARD_REQUESTS: 'courier:maxConcurrentShardRequests',
   COURIER_BATCH_SEARCHES: 'courier:batchSearches',
+  SEARCH_FIELDS_FROM_SOURCE: 'search:fieldsFromSource',
   SEARCH_INCLUDE_FROZEN: 'search:includeFrozen',
   SEARCH_TIMEOUT: 'search:timeout',
   HISTOGRAM_BAR_TARGET: 'histogram:barTarget',

src/plugins/data/common/index_patterns/index_patterns/format_hit.ts

@@ -87,8 +87,10 @@ export function formatHitProvider(indexPattern: IndexPattern, defaultFormat: any
       partials = {};
       partialFormattedCache.set(hit, partials);
     }
-
-    const val = fieldName === '_source' ? hit._source : indexPattern.flattenHit(hit)[fieldName];
+    const unformattedFields = ['fields', '_source'];
+    const val = unformattedFields.includes(fieldName)
+      ? hit[fieldName]
+      : indexPattern.flattenHit(hit)[fieldName];
     return convert(hit, val, fieldName);
   };
 

src/plugins/data/common/search/search_source/search_source.ts

@@ -93,6 +93,7 @@ export const searchSourceRequiredUiSettings = [
   UI_SETTINGS.COURIER_IGNORE_FILTER_IF_FIELD_NOT_IN_INDEX,
   UI_SETTINGS.COURIER_MAX_CONCURRENT_SHARD_REQUESTS,
   UI_SETTINGS.COURIER_SET_REQUEST_PREFERENCE,
+  UI_SETTINGS.SEARCH_FIELDS_FROM_SOURCE,
   UI_SETTINGS.DOC_HIGHLIGHT,
   UI_SETTINGS.META_FIELDS,
   UI_SETTINGS.QUERY_ALLOW_LEADING_WILDCARDS,
@@ -404,8 +405,13 @@ export class SearchSource {
       case 'query':
         return addToRoot(key, (data[key] || []).concat(val));
       case 'fields':
-        const fields = uniq((data[key] || []).concat(val));
-        return addToRoot(key, fields);
+        if (getConfig(UI_SETTINGS.SEARCH_FIELDS_FROM_SOURCE)) {
+          // preserve legacy behavior
+          const fields = uniq((data[key] || []).concat(val));
+          return addToRoot(key, fields);
+        }
+        // use new Fields API
+        return addToBody('fields', val);
       case 'index':
       case 'type':
       case 'highlightAll':
@@ -451,8 +457,13 @@ export class SearchSource {
   }
 
   private flatten() {
+    const { getConfig } = this.dependencies;
     const searchRequest = this.mergeProps();
 
+    if (!getConfig(UI_SETTINGS.SEARCH_FIELDS_FROM_SOURCE)) {
+      return this.flattenWithFieldsApi(searchRequest);
+    }
+
     searchRequest.body = searchRequest.body || {};
     const { body, index, fields, query, filters, highlightAll } = searchRequest;
     searchRequest.indexType = this.getIndexType(index);
@@ -472,8 +483,6 @@ export class SearchSource {
       body._source = index.getSourceFiltering();
     }
 
-    const { getConfig } = this.dependencies;
-
     if (body._source) {
       // exclude source fields for this index pattern specified by the user
       const filter = fieldWildcardFilter(body._source.excludes, getConfig(UI_SETTINGS.META_FIELDS));
@@ -507,6 +516,76 @@ export class SearchSource {
     return searchRequest;
   }
 
+  private flattenWithFieldsApi(searchRequest: Record<string, any>) {
+    const { getConfig } = this.dependencies;
+
+    searchRequest.body = searchRequest.body || {};
+    const { body, index, query, filters, highlightAll } = searchRequest;
+    searchRequest.indexType = this.getIndexType(index);
+
+    const fieldListProvided = !!body.fields;
+
+    // get some special field types from the index pattern
+    const { storedFields, scriptFields, docvalueFields } = index
+      ? index.getComputedFields()
+      : {
+          storedFields: ['*'],
+          scriptFields: {},
+          docvalueFields: [],
+        };
+
+    // add docvalues to fields if no fields are explicitly provided
+    body.fields = fieldListProvided ? body.fields : docvalueFields;
+
+    // apply source filters from index pattern if specified by the user
+    if (index) {
+      const sourceFilters = index.getSourceFiltering();
+      if (!body.hasOwnProperty('_source')) {
+        body._source = sourceFilters;
+      }
+      const filter = fieldWildcardFilter(
+        sourceFilters.excludes,
+        getConfig(UI_SETTINGS.META_FIELDS)
+      );
+      body.fields = body.fields.filter((fld: any) =>
+        filter(typeof fld === 'string' ? fld : fld.field)
+      );
+    }
+
+    // add scripted fields
+    body.script_fields = pick(Object.assign(body.script_fields || {}, scriptFields), body.fields);
+    body.fields = body.fields.filter((fld: any) => {
+      const f = typeof fld === 'string' ? fld : fld.field;
+      return !Object.keys(body.script_fields).includes(f);
+    });
+
+    // request the remaining fields from stored_fields just in case, since the
+    // fields API does not handle stored fields
+    if (fieldListProvided) {
+      const bodyFieldNames = body.fields.map((field: any) =>
+        typeof field === 'string' ? field : field.field
+      );
+      const remainingFields = difference(bodyFieldNames, keys(body.script_fields));
+      body.stored_fields = remainingFields;
+
+      // remove _source, since everything's coming from fields API, scripted fields,
+      // or stored fields
+      body._source = false;
+    } else {
+      body.stored_fields = storedFields;
+    }
+
+    const esQueryConfigs = getEsQueryConfig({ get: getConfig });
+    body.query = buildEsQuery(index, query, filters, esQueryConfigs);
+
+    if (highlightAll && body.query) {
+      body.highlight = getHighlightRequest(body.query, getConfig(UI_SETTINGS.DOC_HIGHLIGHT));
+      delete searchRequest.highlightAll;
+    }
+
+    return searchRequest;
+  }
+
   /**
    * serializes search source fields (which can later be passed to {@link ISearchStartSearchSource})
    */

src/plugins/data/server/ui_settings.ts

@@ -285,6 +285,15 @@ export function getUiSettings(): Record<string, UiSettingsParams<unknown>> {
       category: ['search'],
       schema: schema.boolean(),
     },
+    [UI_SETTINGS.SEARCH_FIELDS_FROM_SOURCE]: {
+      name: 'Read fields from _source',
+      description: `Disable to retrieve fields via the new <a href="https://www.elastic.co/guide/en/elasticsearch/reference/master/search-fields.html#search-fields"
+        target="_blank" rel="noopener">Fields API</a> in the high-level search service. Keep this setting enabled
+        to continue retrieving fields directly from \`_source\`.`,
+      value: true,
+      category: ['search'],
+      schema: schema.boolean(),
+    },
     [UI_SETTINGS.SEARCH_INCLUDE_FROZEN]: {
       name: 'Search in frozen indices',
       description: `Will include <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/frozen-indices.html"

src/plugins/discover/public/application/angular/discover.js

@@ -82,6 +82,7 @@ import {
   DOC_HIDE_TIME_COLUMN_SETTING,
   MODIFY_COLUMNS_ON_SWITCH,
 } from '../../../common';
+import { UI_SETTINGS } from '../../../../data/common';
 
 const fetchStatuses = {
   UNINITIALIZED: 'uninitialized',
@@ -199,6 +200,14 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise
   const savedSearch = $route.current.locals.savedObjects.savedSearch;
   $scope.searchSource = savedSearch.searchSource;
   $scope.indexPattern = resolveIndexPatternLoading();
+  $scope.useNewFieldsApi = !$scope.searchSource.dependencies?.getConfig(
+    UI_SETTINGS.SEARCH_FIELDS_FROM_SOURCE
+  );
+  $scope.showUnmappedFields = false;
+  $scope.onShowUnmappedFieldsChange = (val) => {
+    $scope.showUnmappedFields = val;
+  };
+  const FIRST_N_COLUMNS_FROM_FIELDS_RESPONSE = 3;
   //used for functional testing
   $scope.fetchCounter = 0;
 
@@ -291,7 +300,8 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise
         nextIndexPattern,
         $scope.state.columns,
         $scope.state.sort,
-        config.get(MODIFY_COLUMNS_ON_SWITCH)
+        config.get(MODIFY_COLUMNS_ON_SWITCH),
+        $scope.useNewFieldsApi
       );
       await replaceUrlAppState(nextAppState);
       $route.reload();
@@ -594,15 +604,27 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise
     };
   };
 
+  function getDefaultColumns() {
+    const { columns } = savedSearch;
+    if ($scope.useNewFieldsApi) {
+      const indexOfSource = columns.indexOf('_source');
+      if (indexOfSource !== -1 && columns.splice(indexOfSource, 1).length > 0) {
+        return columns.splice(indexOfSource, 1).length > 0;
+      }
+      return [];
+    } else if (columns.length > 0) {
+      return columns;
+    }
+    return config.get(DEFAULT_COLUMNS_SETTING).slice();
+  }
+
   function getStateDefaults() {
     const query = $scope.searchSource.getField('query') || data.query.queryString.getDefaultQuery();
+    const columns = getDefaultColumns();
     return {
       query,
       sort: getSortArray(savedSearch.sort, $scope.indexPattern),
-      columns:
-        savedSearch.columns.length > 0
-          ? savedSearch.columns
-          : config.get(DEFAULT_COLUMNS_SETTING).slice(),
+      columns,
       index: $scope.indexPattern.id,
       interval: 'auto',
       filters: _.cloneDeep($scope.searchSource.getOwnField('filter')),
@@ -884,7 +906,18 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise
     }
 
     $scope.hits = resp.hits.total;
-    $scope.rows = resp.hits.hits;
+    $scope.rows = resp.hits.hits.map((hit) => {
+      if ($scope.useNewFieldsApi) {
+        const fields = {};
+        Object.keys(hit.fields)
+          .splice(0, FIRST_N_COLUMNS_FROM_FIELDS_RESPONSE)
+          .forEach((key) => {
+            fields[key] = hit.fields[key];
+          });
+        return { ...hit, ...fields };
+      }
+      return hit;
+    });
 
     // if we haven't counted yet, reset the counts
     const counts = ($scope.fieldCounts = $scope.fieldCounts || {});
@@ -896,6 +929,14 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise
       });
     });
 
+    if ($scope.useNewFieldsApi) {
+      // get some columns to display
+      if ($scope.state.columns.length === 0) {
+        const columns = ['fields'];
+        setAppState({ columns });
+      }
+    }
+
     $scope.fetchStatus = fetchStatuses.COMPLETE;
   }
 
@@ -958,7 +999,7 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise
   };
 
   $scope.updateDataSource = () => {
-    const { indexPattern, searchSource } = $scope;
+    const { indexPattern, searchSource, useNewFieldsApi } = $scope;
     searchSource
       .setField('index', $scope.indexPattern)
       .setField('size', $scope.opts.sampleSize)
@@ -972,6 +1013,10 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise
       )
       .setField('query', data.query.queryString.getQuery() || null)
       .setField('filter', filterManager.getFilters());
+    if (useNewFieldsApi) {
+      searchSource.setField('fields', ['*']);
+      searchSource.setField('source', false);
+    }
     return Promise.resolve();
   };
 
@@ -995,20 +1040,20 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise
   };
 
   $scope.addColumn = function addColumn(columnName) {
+    const { indexPattern } = $scope;
     if (uiCapabilities.discover.save) {
-      const { indexPattern } = $scope;
       popularizeField(indexPattern, columnName, indexPatterns);
     }
     const columns = columnActions.addColumn($scope.state.columns, columnName);
     setAppState({ columns });
   };
 
   $scope.removeColumn = function removeColumn(columnName) {
+    const { indexPattern, useNewFieldsApi } = $scope;
     if (uiCapabilities.discover.save) {
-      const { indexPattern } = $scope;
       popularizeField(indexPattern, columnName, indexPatterns);
     }
-    const columns = columnActions.removeColumn($scope.state.columns, columnName);
+    const columns = columnActions.removeColumn($scope.state.columns, columnName, useNewFieldsApi);
     // The state's sort property is an array of [sortByColumn,sortDirection]
     const sort = $scope.state.sort.length
       ? $scope.state.sort.filter((subArr) => subArr[0] !== columnName)

src/plugins/discover/public/application/angular/discover_legacy.html

@@ -31,6 +31,8 @@
     update-query="handleRefresh"
     update-saved-query-id="updateSavedQueryId"
     vis="vis"
+    use-new-fields-api="useNewFieldsApi"
+    on-show-unmapped-fields-change="onShowUnmappedFieldsChange"
   >
   </discover-legacy>
 </discover-app>

src/plugins/discover/public/application/angular/doc_table/actions/columns.ts

@@ -22,27 +22,30 @@
  * is empty, and removes _source if there are more than 1 columns given
  * @param columns
  */
-function buildColumns(columns: string[]) {
+function buildColumns(columns: string[], useNewFieldsApi: boolean) {
   if (columns.length > 1 && columns.indexOf('_source') !== -1) {
     return columns.filter((col) => col !== '_source');
   } else if (columns.length !== 0) {
     return columns;
   }
-  return ['_source'];
+  return useNewFieldsApi ? ['fields'] : ['_source'];
 }
 
 export function addColumn(columns: string[], columnName: string) {
   if (columns.includes(columnName)) {
     return columns;
   }
-  return buildColumns([...columns, columnName]);
+  return buildColumns([...columns, columnName], false);
 }
 
-export function removeColumn(columns: string[], columnName: string) {
+export function removeColumn(columns: string[], columnName: string, useNewFieldsApi: boolean) {
   if (!columns.includes(columnName)) {
     return columns;
   }
-  return buildColumns(columns.filter((col) => col !== columnName));
+  return buildColumns(
+    columns.filter((col) => col !== columnName),
+    useNewFieldsApi
+  );
 }
 
 export function moveColumn(columns: string[], columnName: string, newIndex: number) {

src/plugins/discover/public/application/angular/doc_table/components/table_header/helpers.tsx

@@ -69,7 +69,7 @@ export function getDisplayedColumns(
       name: column,
       displayName: isShortDots ? shortenDottedString(column) : column,
       isSortable: field && field.sortable ? true : false,
-      isRemoveable: column !== '_source' || columns.length > 1,
+      isRemoveable: (column !== '_source' && column !== 'fields') || columns.length > 1,
       colLeftIdx: idx - 1 < 0 ? -1 : idx - 1,
       colRightIdx: idx + 1 >= columns.length ? -1 : idx + 1,
     };

src/plugins/discover/public/application/components/create_discover_legacy_directive.ts

@@ -52,5 +52,7 @@ export function createDiscoverLegacyDirective(reactDirective: any) {
     ['updateQuery', { watchDepth: 'reference' }],
     ['updateSavedQueryId', { watchDepth: 'reference' }],
     ['vis', { watchDepth: 'reference' }],
+    ['useNewFieldsApi', { watchDepth: 'reference' }],
+    ['onShowUnmappedFieldsChange', { watchDepth: 'reference' }],
   ]);
 }

src/plugins/discover/public/application/components/discover_legacy.tsx

@@ -88,6 +88,8 @@ export interface DiscoverLegacyProps {
   updateQuery: (payload: { dateRange: TimeRange; query?: Query }, isUpdate?: boolean) => void;
   updateSavedQueryId: (savedQueryId?: string) => void;
   vis?: Vis;
+  useNewFieldsApi?: boolean;
+  onShowUnmappedFieldsChange: (value: boolean) => void;
 }
 
 export function DiscoverLegacy({
@@ -120,6 +122,8 @@ export function DiscoverLegacy({
   updateQuery,
   updateSavedQueryId,
   vis,
+  useNewFieldsApi,
+  onShowUnmappedFieldsChange,
 }: DiscoverLegacyProps) {
   const [isSidebarClosed, setIsSidebarClosed] = useState(false);
   const { TopNavMenu } = getServices().navigation.ui;
@@ -190,6 +194,8 @@ export function DiscoverLegacy({
                     onRemoveField={onRemoveColumn}
                     selectedIndexPattern={searchSource && searchSource.getField('index')}
                     setIndexPattern={setIndexPattern}
+                    useNewFieldsApi={useNewFieldsApi}
+                    onShowUnmappedFieldsChange={onShowUnmappedFieldsChange}
                   />
                 </div>
               )}