[Monitoring] Use server side pagination for Logstash Pipelines page (#…

…46587) (#47153)

* Basic version working for cluster pipelines

* More support

* Refactoring

* Fixes

* Fix sorting issues

* Reduce the number of buckets too

* Fix tests

* This is actually not helping - it seems that the filter in the query doesn't work as expected - maybe related to the fact that are using nested fields

* Add more data for metric.debug

* Support sorting on throughput and node count

* Fix broken test

* Use getMetrics and support with numOfBuckets parameter

* Fix test for realz

* Fix logstash management pages by introducing a new api to just retrieve ids

* We need this to go back to 1000 but it doesn't affect the number of created buckets

* Fix issue with pagination data when filtering

* Fix sorting by id not working

* Make this a little more sturdy

x-pack/legacy/plugins/logstash/public/services/monitoring/monitoring_service.js

@@ -32,7 +32,7 @@ export class MonitoringService {
 
     return this.clusterService.loadCluster()
       .then(cluster => {
-        const url = `${this.basePath}/v1/clusters/${cluster.uuid}/logstash/pipelines`;
+        const url = `${this.basePath}/v1/clusters/${cluster.uuid}/logstash/pipeline_ids`;
         const now = moment.utc();
         const body = {
           timeRange: {
@@ -42,7 +42,7 @@ export class MonitoringService {
         };
         return this.$http.post(url, body);
       })
-      .then(response => response.data.pipelines.map(pipeline => PipelineListItem.fromUpstreamMonitoringJSON(pipeline)))
+      .then(response => response.data.map(pipeline => PipelineListItem.fromUpstreamMonitoringJSON(pipeline)))
       .catch(() => []);
   }
 }

x-pack/legacy/plugins/monitoring/public/components/logstash/pipeline_listing/pipeline_listing.js

@@ -11,7 +11,7 @@ import { EuiPage, EuiLink, EuiPageBody, EuiPageContent, EuiPanel, EuiSpacer, Eui
 import { formatMetric } from '../../../lib/format_number';
 import { ClusterStatus } from '../cluster_status';
 import { Sparkline } from 'plugins/monitoring/components/sparkline';
-import { EuiMonitoringTable } from '../../table';
+import { EuiMonitoringSSPTable } from '../../table';
 import { i18n } from '@kbn/i18n';
 
 export class PipelineListing extends Component {
@@ -137,6 +137,7 @@ export class PipelineListing extends Component {
       sorting,
       pagination,
       onTableChange,
+      fetchMoreData,
       upgradeMessage,
       className
     } = this.props;
@@ -151,31 +152,22 @@ export class PipelineListing extends Component {
           </EuiPanel>
           <EuiSpacer size="m" />
           <EuiPageContent>
-            <EuiMonitoringTable
+            <EuiMonitoringSSPTable
               className={className || 'logstashNodesTable'}
               rows={data}
               columns={columns}
-              sorting={{
-                ...sorting,
-                sort: {
-                  ...sorting.sort,
-                  field: 'id'
-                }
-              }}
+              sorting={sorting}
               message={upgradeMessage}
               pagination={pagination}
+              fetchMoreData={fetchMoreData}
               search={{
                 box: {
-                  incremental: true,
                   placeholder: i18n.translate('xpack.monitoring.logstash.filterPipelinesPlaceholder', {
                     defaultMessage: 'Filter Pipelines…'
                   })
                 },
               }}
               onTableChange={onTableChange}
-              executeQueryOptions={{
-                defaultFields: ['id']
-              }}
             />
           </EuiPageContent>
         </EuiPageBody>

x-pack/legacy/plugins/monitoring/public/components/table/eui_table_ssp.js

@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+import {
+  EuiBasicTable,
+  EuiSpacer,
+  EuiSearchBar
+} from '@elastic/eui';
+
+export function EuiMonitoringSSPTable({
+  rows: items,
+  search = {},
+  pagination,
+  columns: _columns,
+  onTableChange,
+  fetchMoreData,
+  ...props
+}) {
+  const [isLoading, setIsLoading] = React.useState(false);
+  const [queryText, setQueryText] = React.useState('');
+  const [page, setPage] = React.useState({
+    index: pagination.pageIndex,
+    size: pagination.pageSize
+  });
+  const [sort, setSort] = React.useState(props.sorting);
+
+  if (search.box && !search.box['data-test-subj']) {
+    search.box['data-test-subj'] = 'monitoringTableToolBar';
+  }
+
+  const columns = _columns.map(column => {
+    if (!column['data-test-subj']) {
+      column['data-test-subj'] = 'monitoringTableHasData';
+    }
+
+    if (!('sortable' in column)) {
+      column.sortable = true;
+    }
+
+    return column;
+  });
+
+  const onChange = async ({ page, sort }) => {
+    setPage(page);
+    setSort({ sort });
+    setIsLoading(true);
+    await fetchMoreData({ page, sort: { sort }, queryText });
+    setIsLoading(false);
+    onTableChange({ page, sort });
+  };
+
+  const onQueryChange = async ({ queryText }) => {
+    const newPage = { ...page, index: 0 };
+    setPage(newPage);
+    setQueryText(queryText);
+    setIsLoading(true);
+    await fetchMoreData({ page: newPage, sort, queryText });
+    setIsLoading(false);
+  };
+
+  return (
+    <div data-test-subj={`${props.className}Container`}>
+      <EuiSearchBar {...search} onChange={onQueryChange}/>
+      <EuiSpacer size="l"/>
+      <EuiBasicTable
+        {...props}
+        items={items}
+        pagination={pagination}
+        onChange={onChange}
+        loading={isLoading}
+        columns={columns}
+      />
+    </div>
+  );
+}

x-pack/legacy/plugins/monitoring/public/components/table/index.js

@@ -5,4 +5,5 @@
  */
 
 export { EuiMonitoringTable } from './eui_table';
+export { EuiMonitoringSSPTable } from './eui_table_ssp';
 export { tableStorageGetter, tableStorageSetter, euiTableStorageGetter, euiTableStorageSetter } from './storage';

x-pack/legacy/plugins/monitoring/public/views/alerts/index.js

@@ -66,6 +66,7 @@ uiRoutes.when('/alerts', {
         getPageData,
         $scope,
         $injector,
+        storageKey: 'alertsTable',
         reactNodeId: 'monitoringAlertsApp'
       });
 

x-pack/legacy/plugins/monitoring/public/views/base_controller.js

@@ -70,7 +70,8 @@ export class MonitoringViewBaseController {
     reactNodeId = null, // WIP: https://github.com/elastic/x-pack-kibana/issues/5198
     $scope,
     $injector,
-    options = {}
+    options = {},
+    fetchDataImmediately = true
   }) {
     const titleService = $injector.get('title');
     const $executor = $injector.get('$executor');
@@ -119,7 +120,7 @@ export class MonitoringViewBaseController {
         this.updateDataPromise = null;
       }
       const _api = apiUrlFn ? apiUrlFn() : api;
-      const promises = [_getPageData($injector, _api)];
+      const promises = [_getPageData($injector, _api, this.getPaginationRouteOptions())];
       const setupMode = getSetupModeState();
       if (setupMode.enabled) {
         promises.push(updateSetupModeData());
@@ -132,7 +133,7 @@ export class MonitoringViewBaseController {
         });
       });
     };
-    this.updateData();
+    fetchDataImmediately && this.updateData();
 
     $executor.register({
       execute: () => this.updateData()
@@ -175,4 +176,8 @@ export class MonitoringViewBaseController {
       render(component, document.getElementById(this.reactNodeId));
     }
   }
+
+  getPaginationRouteOptions() {
+    return {};
+  }
 }

x-pack/legacy/plugins/monitoring/public/views/base_eui_table_controller.js

@@ -8,6 +8,8 @@ import { MonitoringViewBaseController } from './';
 import { euiTableStorageGetter, euiTableStorageSetter } from 'plugins/monitoring/components/table';
 import { EUI_SORT_ASCENDING } from '../../common/constants';
 
+const PAGE_SIZE_OPTIONS = [5, 10, 20, 50];
+
 /**
  * Class to manage common instantiation behaviors in a view controller
  * And add persistent state to a table:
@@ -42,17 +44,22 @@ export class MonitoringViewBaseEuiTableController extends MonitoringViewBaseCont
     const setLocalStorageData = euiTableStorageSetter(storageKey);
     const { page, sort } = getLocalStorageData(storage);
 
-    this.pagination = page || {
+    this.pagination = {
+      pageSize: 20,
       initialPageSize: 20,
-      pageSizeOptions: [5, 10, 20, 50]
+      pageIndex: 0,
+      initialPageIndex: 0,
+      pageSizeOptions: PAGE_SIZE_OPTIONS
     };
 
-    this.sorting = sort || {
-      sort: {
-        field: 'name',
-        direction: EUI_SORT_ASCENDING
+    if (page) {
+      if (!PAGE_SIZE_OPTIONS.includes(page.size)) {
+        page.size = 20;
       }
-    };
+      this.setPagination(page);
+    }
+
+    this.setSorting(sort);
 
     this.onTableChange = ({ page, sort }) => {
       setLocalStorageData(storage, {
@@ -62,5 +69,59 @@ export class MonitoringViewBaseEuiTableController extends MonitoringViewBaseCont
         }
       });
     };
+
+    this.updateData();
+  }
+
+  setPagination(page) {
+    this.pagination = {
+      pageSize: page.size,
+      pageIndex: page.index,
+      pageSizeOptions: PAGE_SIZE_OPTIONS
+    };
+  }
+
+  setSorting(sort) {
+    this.sorting = sort || { sort: {} };
+
+    if (!this.sorting.sort.field) {
+      this.sorting.sort.field = 'name';
+    }
+    if (!this.sorting.sort.direction) {
+      this.sorting.sort.direction = EUI_SORT_ASCENDING;
+    }
+  }
+
+  setQueryText(queryText) {
+    this.queryText = queryText;
+  }
+
+  getPaginationRouteOptions() {
+    if (!this.pagination || !this.sorting) {
+      return {};
+    }
+
+    return {
+      pagination: {
+        size: this.pagination.pageSize,
+        index: this.pagination.pageIndex
+      },
+      ...this.sorting,
+      queryText: this.queryText,
+    };
+  }
+
+  getPaginationTableProps(pagination) {
+    return {
+      sorting: this.sorting,
+      pagination: pagination,
+      onTableChange: this.onTableChange,
+      fetchMoreData: async ({ page, sort, queryText }) => {
+        this.setPagination(page);
+        this.setSorting(sort);
+        this.setQueryText(queryText);
+        this.updateData();
+      }
+    };
   }
 }

x-pack/legacy/plugins/monitoring/public/views/logstash/node/pipelines/index.js

@@ -24,7 +24,7 @@ import { PipelineListing } from '../../../../components/logstash/pipeline_listin
 import { DetailStatus } from '../../../../components/logstash/detail_status';
 import { CODE_PATH_LOGSTASH } from '../../../../../common/constants';
 
-const getPageData = ($injector) => {
+const getPageData = ($injector, _api = undefined, routeOptions = {}) => {
   const $route = $injector.get('$route');
   const $http = $injector.get('$http');
   const globalState = $injector.get('globalState');
@@ -39,7 +39,8 @@ const getPageData = ($injector) => {
     timeRange: {
       min: timeBounds.min.toISOString(),
       max: timeBounds.max.toISOString()
-    }
+    },
+    ...routeOptions
   })
     .then(response => response.data)
     .catch((err) => {
@@ -70,7 +71,6 @@ uiRoutes
         const routeInit = Private(routeInitProvider);
         return routeInit({ codePaths: [CODE_PATH_LOGSTASH] });
       },
-      pageData: getPageData
     },
     controller: class extends MonitoringViewBaseEuiTableController {
       constructor($injector, $scope) {
@@ -82,9 +82,11 @@ uiRoutes
           getPageData,
           reactNodeId: 'monitoringLogstashNodePipelinesApp',
           $scope,
-          $injector
+          $injector,
+          fetchDataImmediately: false // We want to apply pagination before sending the first request
         });
 
+
         $scope.$watch(() => this.data, data => {
           if (!data || !data.nodeSummary) {
             return;
@@ -97,6 +99,11 @@ uiRoutes
             }
           }));
 
+          const pagination = {
+            ...this.pagination,
+            totalItemCount: data.totalPipelineCount
+          };
+
           this.renderReact(
             <I18nContext>
               <PipelineListing
@@ -106,9 +113,7 @@ uiRoutes
                 stats={data.nodeSummary}
                 statusComponent={DetailStatus}
                 data={data.pipelines}
-                sorting={this.sorting}
-                pagination={this.pagination}
-                onTableChange={this.onTableChange}
+                {...this.getPaginationTableProps(pagination)}
                 dateFormat={config.get('dateFormat')}
                 upgradeMessage={makeUpgradeMessage(data.nodeSummary.version, i18n)}
                 angular={{