Modify prometheus monitor

Signed-off-by: nkl199@yahoo.co.uk <nkl199@yahoo.co.uk>

packages/caliper-core/lib/common/config/Config.js

@@ -25,6 +25,10 @@ const keys = {
         PrometheusPush: {
             UserName: 'caliper-auth-prometheuspush-username',
             Password: 'caliper-auth-prometheuspush-password'
+        },
+        Prometheus: {
+            UserName: 'caliper-auth-prometheus-username',
+            Password: 'caliper-auth-prometheus-password'
         }
     },
     Bind: {

packages/caliper-core/lib/common/config/default.yaml

@@ -21,6 +21,12 @@ caliper:
             username:
             # password
             password:
+        # Prometheus Server
+        prometheus:
+            # username
+            username:
+            # password
+            password:
     # Settings related to the binding command
     bind:
         # The binding specification of the SUT in the <SUT type>:<SDK version> format

packages/caliper-core/lib/common/utils/constants.js

@@ -48,6 +48,7 @@ module.exports = {
         }
     },
     AuthComponents: {
-        PushGateway: 'PrometheusPush'
+        PushGateway: 'PrometheusPush',
+        Prometheus: 'Prometheus'
     }
 };

packages/caliper-core/lib/manager/monitors/monitor-prometheus.js

@@ -14,13 +14,16 @@
 
 'use strict';
 
-const Util = require('../../common/utils/caliper-utils.js');
-const ConfigUtil = require('../../common/config/config-util');
+const CaliperUtils = require('../../common/utils/caliper-utils');
 const ChartBuilder = require('../charts/chart-builder');
-const Logger = Util.getLogger('monitor-prometheus');
+const Constants = require('../../common/utils/constants');
+const ConfigUtil = require('../../common/config/config-util');
 const MonitorInterface = require('./monitor-interface');
 const PrometheusQueryClient = require('../../common/prometheus/prometheus-query-client');
 const PrometheusQueryHelper = require('../../common/prometheus/prometheus-query-helper');
+const Util = require('../../common/utils/caliper-utils.js');
+
+const Logger = Util.getLogger('monitor-prometheus');
 
 /**
  * Prometheus monitor implementation
@@ -34,21 +37,24 @@ class PrometheusMonitor extends MonitorInterface {
     constructor(resourceMonitorOptions) {
         super(resourceMonitorOptions);
         this.precision = ConfigUtil.get(ConfigUtil.keys.Report.Precision, 3);
-        this.prometheusQueryClient = new PrometheusQueryClient(this.options.url);
+
+        // Might be using basic auth
+        const url = CaliperUtils.augmentUrlWithBasicAuth(this.options.url, Constants.AuthComponents.Prometheus);
+        this.prometheusQueryClient = new PrometheusQueryClient(url);
         // User defined options for monitoring
         if (this.options.hasOwnProperty('metrics')) {
-            // Might have an ignore list
-            if (this.options.metrics.hasOwnProperty('ignore')) {
-                this.ignore = this.options.metrics.ignore;
+            // Might have an include list
+            if (this.options.metrics.hasOwnProperty('include')) {
+                this.include = this.options.metrics.include;
             } else {
-                Logger.info('No monitor metrics `ignore` option specified, will provide statistics on all items retrieved by queries');
+                Logger.info('No monitor metrics `include` option specified, will provide statistics on all items retrieved by queries');
             }
 
             // Might have user specified queries to run
-            if (this.options.metrics.hasOwnProperty('include')) {
-                this.include =  this.options.metrics.include;
+            if (this.options.metrics.hasOwnProperty('queries')) {
+                this.queries =  this.options.metrics.queries;
             } else {
-                Logger.info('No monitor metrics `include` options specified, unable to provide statistics on any resources');
+                Logger.info('No monitor metrics `queries` options specified, unable to provide statistics on any resources');
             }
         } else {
             Logger.info('No monitor `metrics` specified, will not provide statistics on any resources');
@@ -110,43 +116,41 @@ class PrometheusMonitor extends MonitorInterface {
     async getStatistics(testLabel) {
         this.endTime = Date.now()/1000;
 
-        if (this.include) {
+        if (this.queries) {
             const resourceStats = [];
             const chartArray = [];
-            for (const metricKey of Object.keys(this.include)) {
-                let newKey = true;
-                // Each metric is of the form
-                // Tag0: {
+            for (const queryObject of this.queries) {
+                // Each queryObject is of the form
+                // {
+                //     name: 'tag name',
                 //     query: 'the prometheus query to be made',
                 //     statistic: 'the action to be taken on returned metrics'
                 //     step: step size
                 // }
-                const params = this.include[metricKey];
-                const queryString = PrometheusQueryHelper.buildStringRangeQuery(params.query, this.startTime, this.endTime, params.step);
+                const queryString = PrometheusQueryHelper.buildStringRangeQuery(queryObject.query, this.startTime, this.endTime, queryObject.step);
                 const response = await this.prometheusQueryClient.getByEncodedUrl(queryString);
 
-                // Retrieve base mapped statistics and coerce into correct format
-                const resultMap = PrometheusQueryHelper.extractStatisticFromRange(response, params.statistic, params.label);
+                // Retrieve map of component names and corresponding values for the issued query
+                const componentNameValueMap = PrometheusQueryHelper.extractStatisticFromRange(response, queryObject.statistic, queryObject.label);
 
                 const metricArray = [];
-                for (const [key, value] of resultMap.entries()) {
-                    // Filter here
-                    if (this.ignore.includes(key)) {
-                        continue;
-                    } else {
+                let newQueryObjectIteration = true;
+                for (const [key, value] of componentNameValueMap.entries()) {
+                    // Filter here, based on a regex match
+                    if (this.includeStatistic(key)) {
                         // Build report table information
-                        const watchItemStat = newKey ? this.getResultColumnMapForQueryTag(params.query, metricKey) : this.getResultColumnMapForQueryTag('', '');
+                        const watchItemStat = newQueryObjectIteration ? this.getResultColumnMapForQueryTag(queryObject.query, queryObject.name) : this.getResultColumnMapForQueryTag('', '');
+                        newQueryObjectIteration = false;
                         watchItemStat.set('Name', key);
-                        const multiplier = params.multiplier ? params.multiplier : 1;
+                        const multiplier = queryObject.multiplier ? queryObject.multiplier : 1;
                         watchItemStat.set('Value', (value*multiplier).toPrecision(this.precision));
                         // Store
                         resourceStats.push(watchItemStat);
-                        newKey = false;
 
                         // Build separate charting information
                         const metricMap = new Map();
                         metricMap.set('Name',  watchItemStat.get('Name'));
-                        metricMap.set(metricKey, watchItemStat.get('Value'));
+                        metricMap.set(queryObject.name, watchItemStat.get('Value'));
                         metricArray.push(metricMap);
                     }
                 }
@@ -167,8 +171,25 @@ class PrometheusMonitor extends MonitorInterface {
 
             return { resourceStats, chartStats };
         } else {
-            Logger.debug('No include options specified for monitor - skipping action');
+            Logger.debug('No queries specified for monitor - skipping action');
+        }
+    }
+
+    /**
+     * Check if the passed key should be included
+     * @param {string} componentName the name to test for inclusion against the user supplied list of components to monitor
+     * @returns {boolean} boolean flag for inclusion
+     */
+    includeStatistic(componentName) {
+        let includeStat = false;
+        for (const includeItem of this.include) {
+            const regex = RegExp(includeItem);
+            if (regex.test(componentName)) {
+                includeStat = true;
+                continue;
+            }
         }
+        return includeStat;
     }
 
 }

packages/caliper-core/test/manager/monitors/monitor-prometheus.js

@@ -37,53 +37,44 @@ describe('Prometheus monitor implementation', () => {
     });
 
     // Test data
-    const ignore = {
+    const monitorOptions = {
         metrics : {
-            ignore: ['prometheus', 'pushgateway', 'cadvisor']
+            include: ['peer', 'pushgateway', 'dev.*'],
+            queries: [
+                {
+                    query: 'sum(rate(container_cpu_usage_seconds_total{name=~".+"}[$interval])) by (name) * 100',
+                    statistic: 'average'
+                },
+                {
+                    query: 'sum(rate(container_cpu_usage_seconds_total{name=~".+"}[$interval])) by (name) * 100',
+                    statistic: 'maximum'
+                }
+            ]
         }
     };
 
-    const includeOpts = {
-        Tag0: {
-            query: 'sum(rate(container_cpu_usage_seconds_total{name=~".+"}[$interval])) by (name) * 100',
-            statistic: 'average'
-        },
-        Tag1: {
-            query: 'sum(rate(container_cpu_usage_seconds_total{name=~".+"}[$interval])) by (name) * 100',
-            statistic: 'maximum'
-        },
-        Tag2: {
-            query: 'sum(container_memory_rss{name=~".+"}) by (name)',
-            statistic: 'average'
-        }
-    };
-
-    const include = {
-        metrics: {
-            include : includeOpts
-        }
-    };
+    const emptyOptions = {};
 
     describe('#constructor', () => {
 
-        it('should set ignore list if provided', () => {
-            const mon = new PrometheusMonitorRewire(ignore);
-            mon.ignore.should.be.an('array').that.deep.equals(['prometheus', 'pushgateway', 'cadvisor']);
+        it('should set include list if provided', () => {
+            const prometheusMonitor = new PrometheusMonitorRewire(monitorOptions);
+            prometheusMonitor.include.should.be.an('array').that.deep.equals(monitorOptions.metrics.include);
         });
 
-        it('should not set ignore list if missing', () => {
-            const mon = new PrometheusMonitorRewire(include);
-            should.not.exist(mon.ignore);
+        it('should not set include list if missing', () => {
+            const prometheusMonitor = new PrometheusMonitorRewire(emptyOptions);
+            should.not.exist(prometheusMonitor.include);
         });
 
-        it('should set include list if provided', () => {
-            const mon = new PrometheusMonitorRewire(include);
-            mon.include.should.be.an('object').that.deep.equals(includeOpts);
+        it('should set queries list if provided', () => {
+            const prometheusMonitor = new PrometheusMonitorRewire(monitorOptions);
+            prometheusMonitor.queries.should.be.an('array').that.deep.equals(monitorOptions.metrics.queries);
         });
 
-        it('should not set include list if missing', () => {
-            const mon = new PrometheusMonitorRewire(ignore);
-            should.not.exist(mon.include);
+        it('should not set queries list if missing', () => {
+            const prometheusMonitor = new PrometheusMonitorRewire(emptyOptions);
+            should.not.exist(prometheusMonitor.queries);
         });
     });
 
@@ -137,7 +128,7 @@ describe('Prometheus monitor implementation', () => {
 
         it('should return a map with keys that correspond to the passed `include` keys, with default entries populated', () => {
 
-            const mon = new PrometheusMonitorRewire(include);
+            const mon = new PrometheusMonitorRewire(monitorOptions);
             const map = mon.getResultColumnMapForQueryTag('query', 'MyTag');
 
             // Three keys
@@ -150,6 +141,23 @@ describe('Prometheus monitor implementation', () => {
         });
     });
 
+    describe('checking if a statistic should be included in the report', () => {
+
+
+        it('should identify user requested components', () => {
+
+            const mon = new PrometheusMonitorRewire(monitorOptions);
+            mon.includeStatistic('peer0.org0.example.com').should.equal(true);
+
+            mon.includeStatistic('pushgateway').should.equal(true);
+
+            mon.includeStatistic('dev-org0.example.com').should.equal(true);
+
+            mon.includeStatistic('penuin').should.equal(false);
+
+            mon.includeStatistic('orderer0.example.com').should.equal(false);
+        });
+    });
 
 
 });

packages/caliper-tests-integration/fabric_tests/phase4/benchconfig.yaml

@@ -59,19 +59,19 @@ monitors:
         interval: 5
         url: "http://localhost:9090"
         metrics:
-            ignore: [prometheus, pushGateway, cadvisor, grafana, node-exporter]
-            include:
-                Endorse Time (s):
-                    query: rate(endorser_propsal_duration_sum{chaincode="marbles:v0"}[1m])/rate(endorser_propsal_duration_count{chaincode="marbles:v0"}[1m])
-                    step: 1
-                    label: instance
-                    statistic: avg
-                Max Memory (MB):
-                    query: sum(container_memory_rss{name=~".+"}) by (name)
-                    step: 10
-                    label: name
-                    statistic: max
-                    multiplier: 0.000001
+            include: [peer*, dev*]
+            queries:
+            - name: Endorse Time (s)
+              query: rate(endorser_propsal_duration_sum{chaincode="marbles:v0"}[1m])/rate(endorser_propsal_duration_count{chaincode="marbles:v0"}[1m])
+              step: 1
+              label: instance
+              statistic: avg
+            - name: Max Memory (MB)
+              query: sum(container_memory_rss{name=~".+"}) by (name)
+              step: 10
+              label: name
+              statistic: max
+              multiplier: 0.000001
         charting:
             polar:
                 metrics: [Max Memory (MB)]

packages/caliper-tests-integration/fabric_tests/phase5/benchconfig.yaml

@@ -41,19 +41,19 @@ monitors:
         interval: 5
         url: "http://localhost:9090"
         metrics:
-            ignore: [prometheus, pushGateway, cadvisor, grafana, node-exporter]
-            include:
-                Endorse Time (s):
-                    query: rate(endorser_propsal_duration_sum{chaincode="marbles:v0"}[1m])/rate(endorser_propsal_duration_count{chaincode="marbles:v0"}[1m])
-                    step: 1
-                    label: instance
-                    statistic: avg
-                Max Memory (MB):
-                    query: sum(container_memory_rss{name=~".+"}) by (name)
-                    step: 10
-                    label: name
-                    statistic: max
-                    multiplier: 0.000001
+            include: [peer*, dev*]
+            queries:
+            - name: Endorse Time (s)
+              query: rate(endorser_propsal_duration_sum{chaincode="marbles:v0"}[1m])/rate(endorser_propsal_duration_count{chaincode="marbles:v0"}[1m])
+              step: 1
+              label: instance
+              statistic: avg
+            - name: Max Memory (MB)
+              query: sum(container_memory_rss{name=~".+"}) by (name)
+              step: 10
+              label: name
+              statistic: max
+              multiplier: 0.000001
         charting:
             polar:
                 metrics: [Max Memory (MB)]

packages/caliper-tests-integration/fabric_tests/phase6/benchconfig.yaml

@@ -54,19 +54,19 @@ monitors:
         interval: 5
         url: "http://localhost:9090"
         metrics:
-            ignore: [prometheus, pushGateway, cadvisor, grafana, node-exporter]
-            include:
-                Endorse Time (s):
-                    query: rate(endorser_propsal_duration_sum{chaincode="marbles:v0"}[1m])/rate(endorser_propsal_duration_count{chaincode="marbles:v0"}[1m])
-                    step: 1
-                    label: instance
-                    statistic: avg
-                Max Memory (MB):
-                    query: sum(container_memory_rss{name=~".+"}) by (name)
-                    step: 10
-                    label: name
-                    statistic: max
-                    multiplier: 0.000001
+            include: [peer*, dev*]
+            queries:
+            - name: Endorse Time (s)
+              query: rate(endorser_propsal_duration_sum{chaincode="marbles:v0"}[1m])/rate(endorser_propsal_duration_count{chaincode="marbles:v0"}[1m])
+              step: 1
+              label: instance
+              statistic: avg
+            - name: Max Memory (MB)
+              query: sum(container_memory_rss{name=~".+"}) by (name)
+              step: 10
+              label: name
+              statistic: max
+              multiplier: 0.000001
         charting:
             polar:
                 metrics: [Max Memory (MB)]