From ada19c3dfb422e413d3be029917be037aa21babb Mon Sep 17 00:00:00 2001 From: Dave Kelsey <25582377+davidkel@users.noreply.github.com> Date: Thu, 24 Mar 2022 17:57:43 +0000 Subject: [PATCH] Caliper terminates if prometheus is not available (#1288) This is due to the error event not being correctly captured when a request is made to prometheus Also added some extra code to output a warning and stop trying to do any more queries for the round. It won't stop it for all rounds but checks on every round. closes #1267 Signed-off-by: D Co-authored-by: D --- .../prometheus/prometheus-query-client.js | 9 +- .../manager/monitors/monitor-prometheus.js | 19 ++- .../manager/monitors/monitor-prometheus.js | 119 ++++++++++++++++-- 3 files changed, 124 insertions(+), 23 deletions(-) diff --git a/packages/caliper-core/lib/common/prometheus/prometheus-query-client.js b/packages/caliper-core/lib/common/prometheus/prometheus-query-client.js index 21c2617a9..433acb2c4 100644 --- a/packages/caliper-core/lib/common/prometheus/prometheus-query-client.js +++ b/packages/caliper-core/lib/common/prometheus/prometheus-query-client.js @@ -149,10 +149,11 @@ class PrometheusQueryClient { resolve(); } }); - res.on('error', err => { - Logger.error(err); - reject(err); - }); + }); + + req.on('error', err => { + Logger.error(err); + reject(err); }); req.end(); diff --git a/packages/caliper-core/lib/manager/monitors/monitor-prometheus.js b/packages/caliper-core/lib/manager/monitors/monitor-prometheus.js index 6f11c99df..0e690521a 100644 --- a/packages/caliper-core/lib/manager/monitors/monitor-prometheus.js +++ b/packages/caliper-core/lib/manager/monitors/monitor-prometheus.js @@ -65,7 +65,7 @@ class PrometheusMonitor extends MonitorInterface { * Retrieve the query client * @returns {PrometheusQueryClient} the query client */ - getQueryClient(){ + getQueryClient() { return this.prometheusQueryClient; } @@ -74,7 +74,7 @@ class PrometheusMonitor extends MonitorInterface { * @async */ async start() { - this.startTime = Date.now()/1000; + this.startTime = Date.now() / 1000; } /** @@ -114,7 +114,7 @@ class PrometheusMonitor extends MonitorInterface { * @async */ async getStatistics(testLabel) { - this.endTime = Date.now()/1000; + this.endTime = Date.now() / 1000; const resourceStats = []; const chartArray = []; @@ -131,7 +131,14 @@ class PrometheusMonitor extends MonitorInterface { // label: a matching label for the component of interest // } const queryString = PrometheusQueryHelper.buildStringRangeQuery(queryObject.query, this.startTime, this.endTime, queryObject.step); - const response = await this.prometheusQueryClient.getByEncodedUrl(queryString); + + let response; + try { + response = await this.prometheusQueryClient.getByEncodedUrl(queryString); + } catch (error) { + Logger.warn('Failed to connect to Prometheus, unable to perform queries'); + break; + } // Retrieve map of component names and corresponding values for the issued query const componentNameValueMap = PrometheusQueryHelper.extractStatisticFromRange(response, queryObject.statistic, queryObject.label); @@ -146,13 +153,13 @@ class PrometheusMonitor extends MonitorInterface { newQueryObjectIteration = false; watchItemStat.set('Name', key); const multiplier = queryObject.multiplier ? queryObject.multiplier : 1; - watchItemStat.set('Value', (value*multiplier).toPrecision(this.precision)); + watchItemStat.set('Value', (value * multiplier).toPrecision(this.precision)); // Store resourceStats.push(watchItemStat); // Build separate charting information const metricMap = new Map(); - metricMap.set('Name', watchItemStat.get('Name')); + metricMap.set('Name', watchItemStat.get('Name')); metricMap.set(queryObject.name, watchItemStat.get('Value')); metricArray.push(metricMap); } diff --git a/packages/caliper-core/test/manager/monitors/monitor-prometheus.js b/packages/caliper-core/test/manager/monitors/monitor-prometheus.js index 490c2c7a6..0d21de994 100644 --- a/packages/caliper-core/test/manager/monitors/monitor-prometheus.js +++ b/packages/caliper-core/test/manager/monitors/monitor-prometheus.js @@ -21,33 +21,58 @@ const chai = require('chai'); const should = chai.should(); const sinon = require('sinon'); +class FakeQueryClient { + static getByEncodedUrlCount = 0; + + static reset() { + FakeQueryClient.getByEncodedUrlCount = 0; + } + + static setGetByEncodedUrlResponse(ableToConnect, response) { + FakeQueryClient.ableToConnect = ableToConnect; + FakeQueryClient.response = response; + } + + async getByEncodedUrl() { + FakeQueryClient.getByEncodedUrlCount++; + if (!FakeQueryClient.ableToConnect) { + throw new Error('ECONNREFUSED'); + } + return FakeQueryClient.response; + } +} + describe('Prometheus monitor implementation', () => { - const fakeQueryClient = sinon.stub(); - PrometheusMonitorRewire.__set__('PrometheusQueryClient', fakeQueryClient); + PrometheusMonitorRewire.__set__('PrometheusQueryClient', FakeQueryClient); // Before/After let clock; - beforeEach( () => { + beforeEach(() => { clock = sinon.useFakeTimers(); }); - afterEach( () => { + afterEach(() => { clock.restore(); }); // Test data const monitorOptions = { metrics : { - include: ['peer', 'pushgateway', 'dev.*'], + url: 'http://localhost:9090', + include: ['peer', 'orderer', 'dev.*'], queries: [ { + name: 'avg cpu', + label: 'name', query: 'sum(rate(container_cpu_usage_seconds_total{name=~".+"}[$interval])) by (name) * 100', - statistic: 'average' + statistic: 'avg' }, { + name: 'max cpu', + label: 'name', query: 'sum(rate(container_cpu_usage_seconds_total{name=~".+"}[$interval])) by (name) * 100', - statistic: 'maximum' + statistic: 'max' } ] } @@ -78,7 +103,7 @@ describe('Prometheus monitor implementation', () => { }); }); - describe('#getQueryClient', ()=>{ + describe('#getQueryClient', () => { it('should return the internal Query Client', () => { const mon = new PrometheusMonitorRewire({}); @@ -92,7 +117,7 @@ describe('Prometheus monitor implementation', () => { it('should set the start time with the current time', () => { clock.tick(42); - const mon = new PrometheusMonitorRewire({push_url: '123'}); + const mon = new PrometheusMonitorRewire({ push_url: '123' }); mon.start(); mon.startTime.should.equal(0.042); }); @@ -102,7 +127,7 @@ describe('Prometheus monitor implementation', () => { describe('#stop', () => { it('should remove startTime if it exists', () => { clock.tick(42); - const mon = new PrometheusMonitorRewire({push_url: '123'}); + const mon = new PrometheusMonitorRewire({ push_url: '123' }); mon.start(); mon.startTime.should.equal(0.042); mon.stop(); @@ -114,7 +139,7 @@ describe('Prometheus monitor implementation', () => { it('should reset the start time', () => { clock.tick(42); - const mon = new PrometheusMonitorRewire({push_url: '123'}); + const mon = new PrometheusMonitorRewire({ push_url: '123' }); mon.start(); clock.tick(42); mon.restart(); @@ -149,15 +174,83 @@ describe('Prometheus monitor implementation', () => { const mon = new PrometheusMonitorRewire(monitorOptions); mon.includeStatistic('peer0.org0.example.com').should.equal(true); - mon.includeStatistic('pushgateway').should.equal(true); + mon.includeStatistic('pushgateway').should.equal(false); mon.includeStatistic('dev-org0.example.com').should.equal(true); mon.includeStatistic('penuin').should.equal(false); - mon.includeStatistic('orderer0.example.com').should.equal(false); + mon.includeStatistic('orderer0.example.com').should.equal(true); }); }); + describe('When getting statistics', () => { + const response = { + status: 'success', + data: { + resultType: 'matrix', + result: [ + { + metric: { + name: 'orderer.example.com' + }, + values: [ + [ + 1648125000.736, + '37318656' + ], + [ + 1648125010.736, + '37318656' + ], + [ + 1648125020.736, + '37318656' + ] + ] + }, + { + metric: { + name: 'peer0.org1.example.com' + }, + values: [ + [ + 1648125000.736, + '80855040' + ], + [ + 1648125010.736, + '80855040' + ], + [ + 1648125020.736, + '80855040' + ] + ] + } + ] + } + }; + + it('should stop processing further queries and return an empty set of results if it fails to connect to prometheus ', async () => { + const prometheusMonitor = new PrometheusMonitorRewire(monitorOptions); + FakeQueryClient.reset(); + FakeQueryClient.setGetByEncodedUrlResponse(false); + const res = await prometheusMonitor.getStatistics(); + FakeQueryClient.getByEncodedUrlCount.should.equal(1); + res.resourceStats.length.should.equal(0); + res.chartStats.length.should.equal(0); + }); + + it('should process all queries successfully if able to connect to prometheus', async () => { + const prometheusMonitor = new PrometheusMonitorRewire(monitorOptions); + FakeQueryClient.reset(); + FakeQueryClient.setGetByEncodedUrlResponse(true, response); + const res = await prometheusMonitor.getStatistics(); + FakeQueryClient.getByEncodedUrlCount.should.equal(2); + res.resourceStats.length.should.equal(4); + res.chartStats.length.should.equal(0); + }); + }); });