From 99217a89f90a9b4202bd1e81e0ec0100c5653091 Mon Sep 17 00:00:00 2001 From: "nkl199@yahoo.co.uk" Date: Tue, 29 Sep 2020 14:17:40 +0100 Subject: [PATCH] Make options robust and add unit tests to prom monitors Signed-off-by: nkl199@yahoo.co.uk --- .../caliper-core/lib/common/config/Config.js | 17 +- .../lib/common/config/default.yaml | 26 +- .../lib/manager/monitors/monitor-interface.js | 4 +- .../tx-observers/internal-tx-observer.js | 2 +- .../prometheus-push-tx-observer.js | 12 +- .../tx-observers/prometheus-tx-observer.js | 10 +- packages/caliper-core/package.json | 1 + .../prometheus-push-tx-observer.js | 227 ++++++++++++++++++ .../tx-observers/prometheus-tx-observer.js | 192 +++++++++++++++ 9 files changed, 466 insertions(+), 25 deletions(-) create mode 100644 packages/caliper-core/test/worker/tx-observers/prometheus-push-tx-observer.js create mode 100644 packages/caliper-core/test/worker/tx-observers/prometheus-tx-observer.js diff --git a/packages/caliper-core/lib/common/config/Config.js b/packages/caliper-core/lib/common/config/Config.js index 34025a93c..282e4d1f5 100644 --- a/packages/caliper-core/lib/common/config/Config.js +++ b/packages/caliper-core/lib/common/config/Config.js @@ -54,8 +54,18 @@ const keys = { } }, Monitor: { - DefaultInterval: 'caliper-monitor-default-interval', - PrometheusScrapePort: 'caliper-monitor-prometheus-scrape-port' + Interval: 'caliper-monitor-interval' + }, + Observer: { + Internal: { + Interval: 'caliper-observer-internal-interval' + }, + Prometheus: { + ScrapePort: 'caliper-observer-prometheus-scrapeport' + }, + PrometheusPush: { + Interval: 'caliper-observer-prometheuspush-interval' + } }, Workspace: 'caliper-workspace', ProjectConfig: 'caliper-projectconfig', @@ -111,9 +121,6 @@ const keys = { Communication: { Method: 'caliper-worker-communication-method', Address: 'caliper-worker-communication-address', - }, - Update: { - Interval: 'caliper-worker-update-interval' } }, Flow: { diff --git a/packages/caliper-core/lib/common/config/default.yaml b/packages/caliper-core/lib/common/config/default.yaml index 93b9b68c2..ad35942f6 100644 --- a/packages/caliper-core/lib/common/config/default.yaml +++ b/packages/caliper-core/lib/common/config/default.yaml @@ -72,12 +72,24 @@ caliper: enabled: true # Reporting frequency interval: 5000 - # Configurations related to caliper test monitors + # Configurations related to caliper resource monitors monitor: - # Default update interval - defaultinterval: 10000 - # Default scrape port for prometheus tx observer - prometheusscrapeport: 3000 + # Update interval + interval: 10000 + # Configurations related to caliper transaction observers + observer: + # Internal tx observer + internal: + # Default update interval + interval: 1000 + # Prometheus PushGateway tx observer + prometheuspush: + # Default update interval + interval: 10000 + # Prometheus tx observer + prometheus: + # Default scrape port for prometheus tx observer + scrapeport: 3000 # Configurations related to the logging mechanism logging: # Specifies the message structure through placeholders @@ -146,10 +158,6 @@ caliper: method: process # Address used for mqtt communications address: mqtt://localhost:1883 - # Worker update configuration - update: - # update interval for sending round statistics to the manager - interval: 1000 # Caliper flow options flow: # Skip options diff --git a/packages/caliper-core/lib/manager/monitors/monitor-interface.js b/packages/caliper-core/lib/manager/monitors/monitor-interface.js index 36690d129..34863b5e9 100644 --- a/packages/caliper-core/lib/manager/monitors/monitor-interface.js +++ b/packages/caliper-core/lib/manager/monitors/monitor-interface.js @@ -21,14 +21,14 @@ const ConfigUtil = require('../../common/config/config-util.js'); /** * Interface of resource consumption monitor */ -class MonitorInterface{ +class MonitorInterface { /** * Constructor * @param {JSON} resourceMonitorOptions Configuration options for the monitor */ constructor(resourceMonitorOptions) { this.options = resourceMonitorOptions; - this.interval = resourceMonitorOptions.interval ? resourceMonitorOptions.interval*1000 : ConfigUtil.get(ConfigUtil.keys.Monitor.DefaultInterval); + this.interval = resourceMonitorOptions.interval ? resourceMonitorOptions.interval*1000 : ConfigUtil.get(ConfigUtil.keys.Monitor.Interval); } /** diff --git a/packages/caliper-core/lib/worker/tx-observers/internal-tx-observer.js b/packages/caliper-core/lib/worker/tx-observers/internal-tx-observer.js index 6a5046bc2..1d6299938 100644 --- a/packages/caliper-core/lib/worker/tx-observers/internal-tx-observer.js +++ b/packages/caliper-core/lib/worker/tx-observers/internal-tx-observer.js @@ -33,7 +33,7 @@ class InternalTxObserver extends TxObserverInterface { */ constructor(messenger, managerUuid, workerIndex) { super(messenger, workerIndex); - this.updateInterval = ConfigUtil.get(ConfigUtil.keys.Worker.Update.Interval); + this.updateInterval = ConfigUtil.get(ConfigUtil.keys.Observer.Internal.Interval); this.intervalObject = undefined; this.messengerUUID = messenger.getUUID(); this.managerUuid = managerUuid; diff --git a/packages/caliper-core/lib/worker/tx-observers/prometheus-push-tx-observer.js b/packages/caliper-core/lib/worker/tx-observers/prometheus-push-tx-observer.js index ba32318f1..a0d29e7cd 100644 --- a/packages/caliper-core/lib/worker/tx-observers/prometheus-push-tx-observer.js +++ b/packages/caliper-core/lib/worker/tx-observers/prometheus-push-tx-observer.js @@ -36,14 +36,15 @@ class PrometheusPushTxObserver extends TxObserverInterface { */ constructor(options, messenger, workerIndex) { super(messenger, workerIndex); - this.pushInterval = options && options.pushInterval || ConfigUtil.get(ConfigUtil.keys.Monitor.DefaultInterval); + this.pushInterval = (options && options.pushInterval) ? options.pushInterval : ConfigUtil.get(ConfigUtil.keys.Observer.PrometheusPush.Interval); + this.processMetricCollectInterval = (options && options.processMetricCollectInterval) ? options.processMetricCollectInterval : undefined; this.intervalObject = undefined; // do not use global registry to avoid conflicts with other potential prom-based observers this.registry = new prometheusClient.Registry(); // automatically apply default internal and user supplied labels - this.defaultLabels = options.defaultLabels || {}; + this.defaultLabels = (options && options.defaultLabels) ? options.defaultLabels : {}; this.defaultLabels.workerIndex = this.workerIndex; this.defaultLabels.roundIndex = this.currentRound; this.defaultLabels.roundLabel = this.roundLabel; @@ -65,7 +66,7 @@ class PrometheusPushTxObserver extends TxObserverInterface { // configure buckets let buckets = prometheusClient.linearBuckets(0.1, 0.5, 10); // default - if (options.histogramBuckets) { + if (options && options.histogramBuckets) { if (options.histogramBuckets.explicit) { buckets = options.histogramBuckets.explicit; } else if (options.histogramBuckets.linear) { @@ -96,6 +97,11 @@ class PrometheusPushTxObserver extends TxObserverInterface { startGcStats(); } + if (!(options && options.pushUrl)) { + const msg = 'PushGateway transaction observer must be provided with a pushUrl within the passed options'; + Logger.error(msg); + throw new Error(msg); + } const url = CaliperUtils.augmentUrlWithBasicAuth(options.pushUrl, Constants.AuthComponents.PushGateway); this.prometheusPushGateway = new prometheusClient.Pushgateway(url, null, this.registry); } diff --git a/packages/caliper-core/lib/worker/tx-observers/prometheus-tx-observer.js b/packages/caliper-core/lib/worker/tx-observers/prometheus-tx-observer.js index 94129f712..6125fbf1e 100644 --- a/packages/caliper-core/lib/worker/tx-observers/prometheus-tx-observer.js +++ b/packages/caliper-core/lib/worker/tx-observers/prometheus-tx-observer.js @@ -37,13 +37,13 @@ class PrometheusTxObserver extends TxObserverInterface { */ constructor(options, messenger, workerIndex) { super(messenger, workerIndex); - this.metricPath = options.metricPath || '/metrics'; - this.scrapePort = Number(options.scrapePort) || ConfigUtil.get(ConfigUtil.keys.Monitor.PrometheusScrapePort); + this.metricPath = (options && options.metricPath) ? options.metricPath : '/metrics'; + this.scrapePort = (options && options.scrapePort) ? Number(options.scrapePort) : ConfigUtil.get(ConfigUtil.keys.Observer.Prometheus.ScrapePort); if (CaliperUtils.isForkedProcess()) { this.scrapePort += workerIndex; } - this.processMetricCollectInterval = options.processMetricCollectInterval; - this.defaultLabels = options.defaultLabels || {}; + this.processMetricCollectInterval = (options && options.processMetricCollectInterval) ? options.processMetricCollectInterval : undefined; + this.defaultLabels = (options && options.defaultLabels) ? options.defaultLabels : {}; Logger.debug(`Configuring Prometheus scrape server for worker ${workerIndex} on port ${this.scrapePort}, with metrics exposed on ${this.metricPath} endpoint`); @@ -72,7 +72,7 @@ class PrometheusTxObserver extends TxObserverInterface { // configure buckets let buckets = prometheusClient.linearBuckets(0.1, 0.5, 10); // default - if (options.histogramBuckets) { + if (options && options.histogramBuckets) { if (options.histogramBuckets.explicit) { buckets = options.histogramBuckets.explicit; } else if (options.histogramBuckets.linear) { diff --git a/packages/caliper-core/package.json b/packages/caliper-core/package.json index f9479bd34..eccf0660b 100644 --- a/packages/caliper-core/package.json +++ b/packages/caliper-core/package.json @@ -47,6 +47,7 @@ "chai": "^3.5.0", "eslint": "^5.16.0", "mocha": "3.4.2", + "mockery": "^2.1.0", "nyc": "11.1.0", "rewire": "^4.0.0", "sinon": "^7.3.2", diff --git a/packages/caliper-core/test/worker/tx-observers/prometheus-push-tx-observer.js b/packages/caliper-core/test/worker/tx-observers/prometheus-push-tx-observer.js new file mode 100644 index 000000000..d886bd906 --- /dev/null +++ b/packages/caliper-core/test/worker/tx-observers/prometheus-push-tx-observer.js @@ -0,0 +1,227 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +'use strict'; + +const chai = require('chai'); +const chaiAsPromised = require('chai-as-promised'); +chai.use(chaiAsPromised); +const should = chai.should(); +const mockery = require('mockery'); +const sinon = require('sinon'); + +/** + * simulate Util + */ +class Utils { + /** + * + * @param {*} path path + * @return {string} the fake path + */ + static resolvePath(path) { + return 'fake/path'; + } + + /** + * + * @return {boolean} the fake path + */ + static isForkedProcess() { + return false; + } + + /** + * + * @param {*} yaml res + * @return {string} the fake yaml + */ + static parseYaml(yaml) { + return 'yaml'; + } + + /** + * @returns {*} logger stub + */ + static getLogger() { + return { + debug: sinon.stub(), + error: sinon.stub() + }; + } + + /** + * @param {*} url url + * @returns {*} url + */ + static augmentUrlWithBasicAuth(url) { + return url; + } +} + +mockery.enable({ + warnOnReplace: false, + warnOnUnregistered: false +}); +mockery.registerMock('../../common/utils/caliper-utils', Utils); + + +describe('When using a PrometheusPushTxObserver', () => { + + // Require here to enable mocks to be established + const PrometheusPushTxObserver = require('../../../lib/worker/tx-observers/prometheus-push-tx-observer'); + + after(()=> { + mockery.deregisterAll(); + mockery.disable(); + }); + + it('should throw an error if no pushURL is specified within the options', async () => { + (() => { + PrometheusPushTxObserver.createTxObserver(undefined, undefined, 0); + }).should.throw('PushGateway transaction observer must be provided with a pushUrl within the passed options'); + }); + + it('should build from default values if no options are passed', async () => { + const options = { + pushUrl: 'http://my.url.com' + }; + const prometheusPushTxObserver = PrometheusPushTxObserver.createTxObserver(options, undefined, 0); + + // Assert expected default options + prometheusPushTxObserver.pushInterval.should.equal(10000); + prometheusPushTxObserver.defaultLabels.should.deep.equal({ + roundIndex: 0, + roundLabel: undefined, + workerIndex: 0 + }); + should.not.exist(prometheusPushTxObserver.processMetricCollectInterval); + prometheusPushTxObserver.defaultLabels.should.deep.equal({ + roundIndex: 0, + roundLabel: undefined, + workerIndex: 0 + }); + }); + + it('should build from the passed options if they exist', async () => { + const options = { + pushUrl: 'http://my.url.com', + pushInterval: 1234, + processMetricCollectInterval: 100, + defaultLabels: { + anotherLabel: 'anotherLabel' + } + }; + const prometheusPushTxObserver = PrometheusPushTxObserver.createTxObserver(options, undefined, 0); + + // Assert expected options + prometheusPushTxObserver.pushInterval.should.equal(1234); + prometheusPushTxObserver.processMetricCollectInterval.should.equal(100); + prometheusPushTxObserver.defaultLabels.should.deep.equal({ + roundIndex: 0, + roundLabel: undefined, + workerIndex: 0, + anotherLabel: 'anotherLabel' + }); + }); + + it('should update labels on activate to ensure statistics are scraped correctly', async () => { + const options = { + pushUrl: 'http://my.url.com' + }; + const prometheusPushTxObserver = PrometheusPushTxObserver.createTxObserver(options, undefined, 0); + prometheusPushTxObserver._sendUpdate = sinon.stub(); + await prometheusPushTxObserver.activate(2, 'myTestRound'); + + prometheusPushTxObserver.defaultLabels.should.deep.equal({ + roundIndex: 2, + roundLabel: 'myTestRound', + workerIndex: 0 + }); + }); + + it('should update transaction statistics during use', async () => { + const options = { + pushUrl: 'http://my.url.com' + }; + const prometheusPushTxObserver = PrometheusPushTxObserver.createTxObserver(options, undefined, 0); + prometheusPushTxObserver._sendUpdate = sinon.stub(); + await prometheusPushTxObserver.activate(2, 'myTestRound'); + prometheusPushTxObserver.txSubmitted(100); + prometheusPushTxObserver.txFinished({ + GetStatus: sinon.stub().returns('success'), + GetTimeFinal: sinon.stub().returns(101), + GetTimeCreate: sinon.stub().returns(10) + }); + + prometheusPushTxObserver.counterTxSubmitted.hashMap.should.deep.equal({ + '': { value: 100, labels: {} } + }); + prometheusPushTxObserver.counterTxFinished.hashMap.should.deep.equal({ + 'final_status:success': { + labels: { + 'final_status': 'success' + }, + value: 1 + } + }); + }); + + it('should reset all counters on deactivate so that statistics do not bleed into other rounds', async () => { + const options = { + pushUrl: 'http://my.url.com' + }; + const prometheusPushTxObserver = PrometheusPushTxObserver.createTxObserver(options, undefined, 0); + prometheusPushTxObserver._sendUpdate = sinon.stub(); + await prometheusPushTxObserver.activate(2, 'myTestRound'); + prometheusPushTxObserver.txSubmitted(100); + prometheusPushTxObserver.txFinished( + [{ + GetStatus: sinon.stub().returns('success'), + GetTimeFinal: sinon.stub().returns(101), + GetTimeCreate: sinon.stub().returns(10) + }, + { + GetStatus: sinon.stub().returns('success'), + GetTimeFinal: sinon.stub().returns(101), + GetTimeCreate: sinon.stub().returns(10) + }] + ); + + prometheusPushTxObserver.counterTxSubmitted.hashMap.should.deep.equal({ + '': { + labels: {}, + value: 100 + } + }); + prometheusPushTxObserver.counterTxFinished.hashMap.should.deep.equal({ + 'final_status:success': { + labels: { + 'final_status': 'success' + }, + value: 2 + } + }); + + await prometheusPushTxObserver.deactivate(); + + // Values should be zero, or empty (https://github.com/siimon/prom-client/blob/master/test/counterTest.js) + const txSubmitted = await prometheusPushTxObserver.counterTxSubmitted.get(); + txSubmitted.values[0].value.should.equal(0); + + const txFinished = await prometheusPushTxObserver.counterTxFinished.get(); + txFinished.values.should.deep.equal([]); + }); + +}); diff --git a/packages/caliper-core/test/worker/tx-observers/prometheus-tx-observer.js b/packages/caliper-core/test/worker/tx-observers/prometheus-tx-observer.js new file mode 100644 index 000000000..5a689f3ac --- /dev/null +++ b/packages/caliper-core/test/worker/tx-observers/prometheus-tx-observer.js @@ -0,0 +1,192 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +'use strict'; + +const chai = require('chai'); +const chaiAsPromised = require('chai-as-promised'); +chai.use(chaiAsPromised); +const should = chai.should(); +const mockery = require('mockery'); +const sinon = require('sinon'); + +/** + * simulate Util + */ +class Utils { + /** + * + * @param {*} path path + * @return {string} the fake path + */ + static resolvePath(path) { + return 'fake/path'; + } + + /** + * + * @return {boolean} the fake path + */ + static isForkedProcess() { + return false; + } + + /** + * + * @param {*} yaml res + * @return {string} the fake yaml + */ + static parseYaml(yaml) { + return 'yaml'; + } + + /** + * @returns {*} logger stub + */ + static getLogger() { + return { + debug: sinon.stub(), + error: sinon.stub() + }; + } +} + +mockery.enable({ + warnOnReplace: false, + warnOnUnregistered: false +}); +mockery.registerMock('../../common/utils/caliper-utils', Utils); + + +describe('When using a PrometheusTxObserver', () => { + + // Require here to enable mocks to be established + const PrometheusTxObserver = require('../../../lib/worker/tx-observers/prometheus-tx-observer'); + + after(()=> { + mockery.deregisterAll(); + mockery.disable(); + }); + + it('should build from default values if no options are passed', async () => { + const prometheusTxObserver = PrometheusTxObserver.createTxObserver(undefined, undefined, 0); + + // Assert expected default options + prometheusTxObserver.metricPath.should.equal('/metrics'); + prometheusTxObserver.scrapePort.should.equal(3000); + should.not.exist(prometheusTxObserver.processMetricCollectInterval); + prometheusTxObserver.defaultLabels.should.deep.equal({ + roundIndex: 0, + roundLabel: undefined, + workerIndex: 0 + }); + }); + + it('should build from the passed options if they exist', async () => { + const options = { + metricPath: '/newPath', + scrapePort: 1234, + processMetricCollectInterval: 100, + defaultLabels: { + anotherLabel: 'anotherLabel' + } + }; + const prometheusTxObserver = PrometheusTxObserver.createTxObserver(options, undefined, 0); + + // Assert expected options + prometheusTxObserver.metricPath.should.equal('/newPath'); + prometheusTxObserver.scrapePort.should.equal(1234); + prometheusTxObserver.processMetricCollectInterval.should.equal(100); + prometheusTxObserver.defaultLabels.should.deep.equal({ + roundIndex: 0, + roundLabel: undefined, + workerIndex: 0, + anotherLabel: 'anotherLabel' + }); + }); + + it('should update labels on activate to ensure statistics are scraped correctly', async () => { + const prometheusTxObserver = PrometheusTxObserver.createTxObserver(undefined, undefined, 0); + await prometheusTxObserver.activate(2, 'myTestRound'); + + prometheusTxObserver.defaultLabels.should.deep.equal({ + roundIndex: 2, + roundLabel: 'myTestRound', + workerIndex: 0 + }); + }); + + it('should update transaction statistics during use', async () => { + const prometheusTxObserver = PrometheusTxObserver.createTxObserver(undefined, undefined, 0); + await prometheusTxObserver.activate(2, 'myTestRound'); + prometheusTxObserver.txSubmitted(100); + prometheusTxObserver.txFinished({ + GetStatus: sinon.stub().returns('success'), + GetTimeFinal: sinon.stub().returns(101), + GetTimeCreate: sinon.stub().returns(10) + }); + + prometheusTxObserver.counterTxSubmitted.hashMap.should.deep.equal({ + '': { value: 100, labels: {} } + }); + prometheusTxObserver.counterTxFinished.hashMap.should.deep.equal({ + 'final_status:success': { + labels: { + 'final_status': 'success' + }, + value: 1 + } + }); + }); + + it('should reset all counters on deactivate so that statistics do not bleed into other rounds', async () => { + const prometheusTxObserver = PrometheusTxObserver.createTxObserver(undefined, undefined, 0); + await prometheusTxObserver.activate(2, 'myTestRound'); + prometheusTxObserver.txSubmitted(100); + prometheusTxObserver.txFinished( + [{ + GetStatus: sinon.stub().returns('success'), + GetTimeFinal: sinon.stub().returns(101), + GetTimeCreate: sinon.stub().returns(10) + }, + { + GetStatus: sinon.stub().returns('success'), + GetTimeFinal: sinon.stub().returns(101), + GetTimeCreate: sinon.stub().returns(10) + }] + ); + + prometheusTxObserver.counterTxSubmitted.hashMap.should.deep.equal({ + '': { value: 100, labels: {} } + }); + prometheusTxObserver.counterTxFinished.hashMap.should.deep.equal({ + 'final_status:success': { + labels: { + 'final_status': 'success' + }, + value: 2 + } + }); + + await prometheusTxObserver.deactivate(); + + // Values should be zero, or empty (https://github.com/siimon/prom-client/blob/master/test/counterTest.js) + const txSubmitted = await prometheusTxObserver.counterTxSubmitted.get(); + txSubmitted.values[0].value.should.equal(0); + + const txFinished = await prometheusTxObserver.counterTxFinished.get(); + txFinished.values.should.deep.equal([]); + }); + +});