[Discover] Use fields API to retrieve fields

src/plugins/data/common/field_formats/constants/base_formatters.ts

@@ -33,6 +33,7 @@ import {
   StringFormat,
   TruncateFormat,
   UrlFormat,
+  FieldsFormat,
 } from '../converters';
 
 export const baseFormatters: FieldFormatInstanceType[] = [
@@ -49,4 +50,5 @@ export const baseFormatters: FieldFormatInstanceType[] = [
   StringFormat,
   TruncateFormat,
   UrlFormat,
+  FieldsFormat,
 ];

src/plugins/data/common/field_formats/converters/fields.ts

@@ -0,0 +1,49 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { keys } from 'lodash';
+import { doTemplate, SourceFormat } from './source';
+import { FIELD_FORMAT_IDS, HtmlContextTypeConvert } from '../types';
+import { KBN_FIELD_TYPES } from '../../kbn_field_types/types';
+import { shortenDottedString } from '../../utils';
+
+export class FieldsFormat extends SourceFormat {
+  static id = FIELD_FORMAT_IDS.FIELDS;
+  static title = 'fields';
+  static fieldType = KBN_FIELD_TYPES._SOURCE;
+
+  htmlConvert: HtmlContextTypeConvert = (value, options = {}) => {
+    const { hit, indexPattern } = options;
+
+    const highlights = (hit && hit.highlight) || {};
+    const formatted = indexPattern.formatHit(hit);
+    const highlightPairs: any[] = [];
+    const sourcePairs: any[] = [];
+    const isShortDots = false;
+
+    keys(formatted).forEach((key) => {
+      const pairs = highlights[key] ? highlightPairs : sourcePairs;
+      const newField = isShortDots ? shortenDottedString(key) : key;
+      const val = formatted[key];
+      pairs.push([newField, val]);
+    }, []);
+
+    return doTemplate({ defPairs: highlightPairs.concat(sourcePairs) });
+  };
+}

src/plugins/data/common/field_formats/converters/index.ts

@@ -30,3 +30,4 @@ export { ColorFormat } from './color';
 export { TruncateFormat } from './truncate';
 export { BoolFormat } from './boolean';
 export { StaticLookupFormat } from './static_lookup';
+export { FieldsFormat } from './fields';

src/plugins/data/common/field_formats/converters/source.ts

@@ -50,7 +50,7 @@ const templateHtml = `
       <%= ' ' %>
     <% }); %>
   </dl>`;
-const doTemplate = template(noWhiteSpace(templateHtml));
+export const doTemplate = template(noWhiteSpace(templateHtml));
 
 export class SourceFormat extends FieldFormat {
   static id = FIELD_FORMAT_IDS._SOURCE;

src/plugins/data/common/field_formats/index.ts

@@ -37,6 +37,7 @@ export {
   UrlFormat,
   StringFormat,
   TruncateFormat,
+  FieldsFormat,
 } from './converters';
 
 export { getHighlightRequest } from './utils';

src/plugins/data/common/field_formats/types.ts

@@ -67,6 +67,7 @@ export enum FIELD_FORMAT_IDS {
   STRING = 'string',
   TRUNCATE = 'truncate',
   URL = 'url',
+  FIELDS = 'fields',
 }
 
 export interface FieldFormatConfig {

src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts

@@ -21,12 +21,12 @@ import _, { each, reject } from 'lodash';
 import { FieldAttrs, FieldAttrSet } from '../..';
 import { DuplicateField } from '../../../../kibana_utils/common';
 
-import { ES_FIELD_TYPES, KBN_FIELD_TYPES, IIndexPattern, IFieldType } from '../../../common';
-import { IndexPatternField, IIndexPatternFieldList, fieldList } from '../fields';
+import { ES_FIELD_TYPES, IFieldType, IIndexPattern, KBN_FIELD_TYPES } from '../../../common';
+import { fieldList, IIndexPatternFieldList, IndexPatternField } from '../fields';
 import { formatHitProvider } from './format_hit';
 import { flattenHitWrapper } from './flatten_hit';
-import { FieldFormatsStartCommon, FieldFormat } from '../../field_formats';
-import { IndexPatternSpec, TypeMeta, SourceFilter, IndexPatternFieldMap } from '../types';
+import { FieldFormat, FieldFormatsStartCommon } from '../../field_formats';
+import { IndexPatternFieldMap, IndexPatternSpec, SourceFilter, TypeMeta } from '../types';
 import { SerializedFieldFormat } from '../../../../expressions/common';
 
 interface IndexPatternDeps {

src/plugins/discover/common/index.ts

@@ -28,3 +28,4 @@ export const CONTEXT_DEFAULT_SIZE_SETTING = 'context:defaultSize';
 export const CONTEXT_STEP_SETTING = 'context:step';
 export const CONTEXT_TIE_BREAKER_FIELDS_SETTING = 'context:tieBreakerFields';
 export const MODIFY_COLUMNS_ON_SWITCH = 'discover:modifyColumnsOnSwitch';
+export const SEARCH_FIELDS_FROM_SOURCE = 'discover:searchFieldsFromSource';

src/plugins/discover/public/application/angular/discover.js

@@ -64,6 +64,7 @@ import {
   DEFAULT_COLUMNS_SETTING,
   MODIFY_COLUMNS_ON_SWITCH,
   SAMPLE_SIZE_SETTING,
+  SEARCH_FIELDS_FROM_SOURCE,
   SEARCH_ON_PAGE_LOAD_SETTING,
 } from '../../../common';
 import { loadIndexPattern, resolveIndexPattern } from '../helpers/resolve_index_pattern';
@@ -192,6 +193,9 @@ function discoverController($element, $route, $scope, $timeout, Promise, uiCapab
     $scope.searchSource,
     toastNotifications
   );
+  $scope.useNewFieldsApi = !config.get(SEARCH_FIELDS_FROM_SOURCE);
+
+  const FIRST_N_COLUMNS_FROM_FIELDS_RESPONSE = 3;
   //used for functional testing
   $scope.fetchCounter = 0;
 
@@ -303,7 +307,8 @@ function discoverController($element, $route, $scope, $timeout, Promise, uiCapab
         nextIndexPattern,
         $scope.state.columns,
         $scope.state.sort,
-        config.get(MODIFY_COLUMNS_ON_SWITCH)
+        config.get(MODIFY_COLUMNS_ON_SWITCH),
+        $scope.useNewFieldsApi
       );
       await setAppState(nextAppState);
     }
@@ -408,15 +413,40 @@ function discoverController($element, $route, $scope, $timeout, Promise, uiCapab
 
   setBreadcrumbsTitle(savedSearch, chrome);
 
+  function removeElFromColumns(el, columns) {
+    const index = columns.indexOf(el);
+    if (index !== -1) {
+      columns.splice(index, 1);
+    }
+    return columns;
+  }
+
+  function removeSourceFromColumns(columns) {
+    return removeElFromColumns('_source', columns);
+  }
+
+  function removeFieldsFromColumns(columns) {
+    return removeElFromColumns('fields', columns);
+  }
+
+  function getDefaultColumns() {
+    const columns = [...savedSearch.columns];
+
+    if ($scope.useNewFieldsApi) {
+      return removeSourceFromColumns(columns);
+    } else if (columns.length > 0) {
+      return removeFieldsFromColumns(columns);
+    }
+    return config.get(DEFAULT_COLUMNS_SETTING).slice();
+  }
+
   function getStateDefaults() {
     const query = $scope.searchSource.getField('query') || data.query.queryString.getDefaultQuery();
+    const columns = getDefaultColumns();
     return {
       query,
       sort: getSortArray(savedSearch.sort, $scope.indexPattern),
-      columns:
-        savedSearch.columns.length > 0
-          ? savedSearch.columns
-          : config.get(DEFAULT_COLUMNS_SETTING).slice(),
+      columns,
       index: $scope.indexPattern.id,
       interval: 'auto',
       filters: _.cloneDeep($scope.searchSource.getOwnField('filter')),
@@ -659,7 +689,18 @@ function discoverController($element, $route, $scope, $timeout, Promise, uiCapab
     }
 
     $scope.hits = resp.hits.total;
-    $scope.rows = resp.hits.hits;
+    $scope.rows = resp.hits.hits.map((hit) => {
+      if ($scope.useNewFieldsApi) {
+        const fields = {};
+        Object.keys(hit.fields)
+          .splice(0, FIRST_N_COLUMNS_FROM_FIELDS_RESPONSE)
+          .forEach((key) => {
+            fields[key] = hit.fields[key];
+          });
+        return { ...hit };
+      }
+      return hit;
+    });
 
     $scope.fieldCounts = calcFieldCounts(
       $scope.fieldCounts || {},
@@ -721,10 +762,14 @@ function discoverController($element, $route, $scope, $timeout, Promise, uiCapab
   };
 
   $scope.updateDataSource = () => {
-    updateSearchSource($scope.searchSource, {
-      indexPattern: $scope.indexPattern,
+    const { indexPattern, searchSource, useNewFieldsApi } = $scope;
+    const { columns, sort } = $scope.state;
+    updateSearchSource(searchSource, {
+      indexPattern,
       services,
-      sort: $scope.state.sort,
+      sort,
+      columns,
+      useNewFieldsApi,
     });
     return Promise.resolve();
   };
@@ -752,20 +797,20 @@ function discoverController($element, $route, $scope, $timeout, Promise, uiCapab
   };
 
   $scope.addColumn = function addColumn(columnName) {
+    const { indexPattern, useNewFieldsApi } = $scope;
     if (uiCapabilities.discover.save) {
-      const { indexPattern } = $scope;
       popularizeField(indexPattern, columnName, indexPatterns);
     }
-    const columns = columnActions.addColumn($scope.state.columns, columnName);
+    const columns = columnActions.addColumn($scope.state.columns, columnName, useNewFieldsApi);
     setAppState({ columns });
   };
 
   $scope.removeColumn = function removeColumn(columnName) {
+    const { indexPattern, useNewFieldsApi } = $scope;
     if (uiCapabilities.discover.save) {
-      const { indexPattern } = $scope;
       popularizeField(indexPattern, columnName, indexPatterns);
     }
-    const columns = columnActions.removeColumn($scope.state.columns, columnName);
+    const columns = columnActions.removeColumn($scope.state.columns, columnName, useNewFieldsApi);
     // The state's sort property is an array of [sortByColumn,sortDirection]
     const sort = $scope.state.sort.length
       ? $scope.state.sort.filter((subArr) => subArr[0] !== columnName)

src/plugins/discover/public/application/angular/discover_legacy.html

@@ -30,6 +30,7 @@
     top-nav-menu="topNavMenu"
     update-query="handleRefresh"
     update-saved-query-id="updateSavedQueryId"
+    use-new-fields-api="useNewFieldsApi"
   >
   </discover-legacy>
 </discover-app>

src/plugins/discover/public/application/angular/doc_table/actions/columns.ts

@@ -21,28 +21,34 @@
  * Helper function to provide a fallback to a single _source column if the given array of columns
  * is empty, and removes _source if there are more than 1 columns given
  * @param columns
+ * @param useNewFieldsApi should a new fields API be used
  */
-function buildColumns(columns: string[]) {
+function buildColumns(columns: string[], useNewFieldsApi = false) {
   if (columns.length > 1 && columns.indexOf('_source') !== -1) {
     return columns.filter((col) => col !== '_source');
+  } else if (columns.length > 1 && useNewFieldsApi && columns.indexOf('fields') !== -1) {
+    return columns.filter((col) => col !== 'fields');
   } else if (columns.length !== 0) {
     return columns;
   }
-  return ['_source'];
+  return useNewFieldsApi ? ['fields'] : ['_source'];
 }
 
-export function addColumn(columns: string[], columnName: string) {
+export function addColumn(columns: string[], columnName: string, useNewFieldsApi?: boolean) {
   if (columns.includes(columnName)) {
     return columns;
   }
-  return buildColumns([...columns, columnName]);
+  return buildColumns([...columns, columnName], useNewFieldsApi);
 }
 
-export function removeColumn(columns: string[], columnName: string) {
+export function removeColumn(columns: string[], columnName: string, useNewFieldsApi?: boolean) {
   if (!columns.includes(columnName)) {
     return columns;
   }
-  return buildColumns(columns.filter((col) => col !== columnName));
+  return buildColumns(
+    columns.filter((col) => col !== columnName),
+    useNewFieldsApi
+  );
 }
 
 export function moveColumn(columns: string[], columnName: string, newIndex: number) {

src/plugins/discover/public/application/angular/doc_table/components/table_header/helpers.tsx

@@ -68,7 +68,7 @@ export function getDisplayedColumns(
       name: column,
       displayName: field ? field.displayName : column,
       isSortable: field && field.sortable ? true : false,
-      isRemoveable: column !== '_source' || columns.length > 1,
+      isRemoveable: (column !== '_source' && column !== 'fields') || columns.length > 1,
       colLeftIdx: idx - 1 < 0 ? -1 : idx - 1,
       colRightIdx: idx + 1 >= columns.length ? -1 : idx + 1,
     };

src/plugins/discover/public/application/angular/doc_table/components/table_row.ts

@@ -27,6 +27,7 @@ import cellTemplateHtml from '../components/table_row/cell.html';
 import truncateByHeightTemplateHtml from '../components/table_row/truncate_by_height.html';
 import { getServices } from '../../../../kibana_services';
 import { getContextUrl } from '../../../helpers/get_context_url';
+import { FieldsFormat } from '../../../../../../data/common';
 
 const TAGS_WITH_WS = />\s+</g;
 
@@ -145,7 +146,7 @@ export function createTableRowDirective($compile: ng.ICompileService) {
           newHtmls.push(
             cellTemplate({
               timefield: false,
-              sourcefield: column === '_source',
+              sourcefield: column === '_source' || column === 'fields',
               formatted: _displayField(row, column, true),
               filterable: isFilterable,
               column,
@@ -194,7 +195,13 @@ export function createTableRowDirective($compile: ng.ICompileService) {
        */
       function _displayField(row: any, fieldName: string, truncate = false) {
         const indexPattern = $scope.indexPattern;
-        const text = indexPattern.formatField(row, fieldName);
+        let text;
+        if (fieldName !== 'fields') {
+          text = indexPattern.formatField(row, fieldName);
+        } else {
+          const format = new FieldsFormat();
+          text = format.htmlConvert(row[fieldName], { hit: row, indexPattern });
+        }
 
         if (truncate && text.length > MIN_LINE_LENGTH) {
           return truncateByHeightTemplate({

src/plugins/discover/public/application/angular/doc_table/components/table_row/_cell.scss

@@ -1,9 +1,5 @@
-.kbnDocTableCell__dataField {
-  white-space: pre-wrap;
-}
-
 .kbnDocTableCell__toggleDetails {
-  padding: 4px 0 0 0!important;
+  padding: $euiSizeXS 0 0 0!important;
 }
 
 .kbnDocTableCell__filter {

src/plugins/discover/public/application/components/create_discover_legacy_directive.ts

@@ -51,5 +51,6 @@ export function createDiscoverLegacyDirective(reactDirective: any) {
     ['topNavMenu', { watchDepth: 'reference' }],
     ['updateQuery', { watchDepth: 'reference' }],
     ['updateSavedQueryId', { watchDepth: 'reference' }],
+    ['useNewFieldsApi', { watchDepth: 'reference' }],
   ]);
 }