Add include and exclude section in config

CHANGELOG.md

@@ -19,6 +19,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - `tnt_clock_delta` metric to compute clock difference on instances
 - set custom global labels in config and with `set_labels` function [#259](https://github.com/tarantool/metrics/issues/259)
+- allow to include and exclude default metrics in config and in `enable_default_metrics` function
+  [#222](https://github.com/tarantool/metrics/issues/222)
+
+### Deprecated
+- `enable_cartridge_metrics` function
 
 ## [0.9.0] - 2021-05-28
 ### Fixed

cartridge/roles/metrics.lua

@@ -46,11 +46,16 @@ local function set_labels(custom_labels)
     metrics_vars.custom_labels = custom_labels
 end
 
-local function check_config(_)
+local function check_config(config)
     checks({
         export = 'table',
         ['global-labels'] = '?table',
+        include = '?table',
+        exclude = '?table',
     })
+    if config.include and config.exclude then
+        error("don't use exclude and include sections together", 0)
+    end
 end
 
 local function delete_route(httpd, name)
@@ -144,7 +149,6 @@ end
 local function apply_config(conf)
     local metrics_conf = conf.metrics or {}
     metrics_conf.export = metrics_conf.export or {}
-    set_labels(metrics_conf['global-labels'])
     local paths = format_paths(metrics_conf.export)
     metrics_vars.config = table.copy(paths)
     for path, format in pairs(metrics_vars.default) do
@@ -153,6 +157,8 @@ local function apply_config(conf)
         end
     end
     apply_routes(paths)
+    metrics.enable_default_metrics(metrics_conf.include, metrics_conf.exclude)
+    set_labels(metrics_conf['global-labels'])
 end
 
 local function set_export(export)
@@ -186,7 +192,6 @@ end
 local function init()
     set_labels(metrics_vars.custom_labels)
     metrics.enable_default_metrics()
-    metrics.enable_cartridge_metrics()
     local current_paths = table.copy(metrics_vars.config)
     for path, format in pairs(metrics_vars.default) do
         if current_paths[path] == nil then

doc/monitoring/api_reference.rst

@@ -266,15 +266,33 @@ You can also set global labels by calling
 Metrics functions
 -------------------------------------------------------------------------------
 
-..  function:: enable_default_metrics()
+..  function:: enable_default_metrics([include, exclude])
 
-    Enables Tarantool metrics collections. See :ref:`metrics reference <metrics-reference>`
-    for details.
+    Enables Tarantool metrics collections.
 
-..  function:: enable_cartridge_metrics()
+    :param table include: Table containing names of default metrics which you need to enable.
 
-    Enables Cartridge metrics collections. See :ref:`metrics reference <metrics-cartridge>`
-    for details.
+    :param table exclude: Table containing names of default metrics which you need to exclude.
+
+    Default metrics names:
+
+    * "network"
+    * "operations"
+    * "system"
+    * "replicas"
+    * "info"
+    * "slab"
+    * "runtime"
+    * "memory"
+    * "spaces"
+    * "fibers"
+    * "cpu"
+    * "vinyl"
+    * "luajit"
+    * "cartridge_issues"
+    * "clock"
+
+    See :ref:`metrics reference <metrics-reference>` for details.
 
 ..  function:: metrics.set_global_labels(label_pairs)
 
@@ -301,6 +319,15 @@ Metrics functions
 
     Most common usage is for gauge metrics updates.
 
+..  function:: unregister_callback(callback)
+
+    Unregisters a function ``callback`` which will be called right before metrics
+    collection on plugin export.
+
+    :param function callback: Function which takes no parameters.
+
+    Most common usage is for unregister enabled callbacks.
+
 .. _collecting-http-statistics:
 
 -------------------------------------------------------------------------------

doc/monitoring/getting_started.rst

@@ -217,4 +217,36 @@ via configuration.
    ..  code-block:: lua
 
        local metrics = require('cartridge.roles.metrics')
-       metrics.set_labels({ ['my-custom-label'] = 'label-value'} )
+       metrics.set_labels({ ['my-custom-label'] = 'label-value' })
+
+#. To choose which default metrics are exported, you may use the following configuration.
+
+   When you add include section, only metrics from this section are exported:
+
+   ..  code-block:: yaml
+
+       metrics:
+         export:
+         - path: '/metrics'
+           format: 'json'
+         # export only vinyl, luajit and memory metrics:
+         include:
+         - vinyl
+         - luajit
+         - memory
+
+   When you add exclude section, metrics from this section are removed from default metrics list:
+
+   ..  code-block:: yaml
+
+       metrics:
+         export:
+         - path: '/metrics'
+           format: 'json'
+         # export all metrics except vinyl, luajit and memory:
+         exclude:
+         - vinyl
+         - luajit
+         - memory
+
+   You can see full list of default metrics in :ref:`API reference <metrics-functions>`.

metrics/cartridge/clock.lua

@@ -1,5 +1,11 @@
 local utils = require('metrics.utils')
-local membership = require('membership')
+local ok, membership = pcall(require, 'membership')
+
+if not ok then
+    return { update = function() end }
+end
+
+local collectors_list = {}
 
 -- from https://github.com/tarantool/cartridge/blob/cc607f5a6508449608f3953a3f93669e8c8c4ab0/cartridge/issues.lua#L375
 local function update_clock_metrics()
@@ -19,10 +25,11 @@ local function update_clock_metrics()
         end
     end
 
-    utils.set_gauge('clock_delta', 'Clock difference', min_delta * 1e-6, {delta = 'min'})
-    utils.set_gauge('clock_delta', 'Clock difference', max_delta * 1e-6, {delta = 'max'})
+    collectors_list.clock_delta = utils.set_gauge('clock_delta', 'Clock difference', min_delta * 1e-6, {delta = 'min'})
+    collectors_list.clock_delta = utils.set_gauge('clock_delta', 'Clock difference', max_delta * 1e-6, {delta = 'max'})
 end
 
 return {
-    update = update_clock_metrics
+    update = update_clock_metrics,
+    list = collectors_list,
 }

metrics/cartridge/issues.lua

@@ -1,6 +1,8 @@
-local utils = require('metrics.utils');
+local utils = require('metrics.utils')
 local fun = require('fun')
 
+local collectors_list = {}
+
 local function update_info_metrics()
     local list_on_instance = rawget(_G, '__cartridge_issues_list_on_instance')
 
@@ -14,11 +16,12 @@ local function update_info_metrics()
 
     for _, level in ipairs(levels) do
         local len = fun.iter(issues):filter(function(x) return x.level == level end):length()
-        utils.set_gauge('cartridge_issues', 'Tarantool Cartridge issues', len, {level = level})
+        collectors_list.cartridge_issues =
+            utils.set_gauge('cartridge_issues', 'Tarantool Cartridge issues', len, {level = level})
     end
-
 end
 
 return {
     update = update_info_metrics,
+    list = collectors_list,
 }

metrics/default_metrics/tarantool/cpu.lua

@@ -1,6 +1,8 @@
 local ffi = require('ffi')
 local utils = require('metrics.utils')
 
+local collectors_list = {}
+
 if not pcall(ffi.typeof, "struct timeval") then
     if ffi.os == 'OSX' then
         ffi.cdef[[
@@ -68,11 +70,12 @@ end
 local function update_info_metrics()
     local cpu_time = ss_get_rusage()
     if cpu_time then
-        utils.set_gauge('cpu_user_time', 'CPU user time usage', cpu_time.ru_utime)
-        utils.set_gauge('cpu_system_time', 'CPU system time usage', cpu_time.ru_stime)
+        collectors_list.cpu_user_time = utils.set_gauge('cpu_user_time', 'CPU user time usage', cpu_time.ru_utime)
+        collectors_list.cpu_system_time = utils.set_gauge('cpu_system_time', 'CPU system time usage', cpu_time.ru_stime)
     end
 end
 
 return {
     update = update_info_metrics,
+    list = collectors_list,
 }

metrics/default_metrics/tarantool/fibers.lua

@@ -1,6 +1,8 @@
 local fiber = require('fiber')
 local utils = require('metrics.utils')
 
+local collectors_list = {}
+
 local function update_fibers_metrics()
     local fibers_info = fiber.info()
     local fibers = 0
@@ -15,12 +17,13 @@ local function update_fibers_metrics()
         fused = fused + f.memory.used
     end
 
-    utils.set_gauge('fiber_count', 'Amount of fibers', fibers)
-    utils.set_gauge('fiber_csw', 'Fibers csw', csws)
-    utils.set_gauge('fiber_memalloc', 'Fibers memalloc', falloc)
-    utils.set_gauge('fiber_memused', 'Fibers memused', fused)
+    collectors_list.fiber_count = utils.set_gauge('fiber_count', 'Amount of fibers', fibers)
+    collectors_list.fiber_csw = utils.set_gauge('fiber_csw', 'Fibers csw', csws)
+    collectors_list.fiber_memalloc = utils.set_gauge('fiber_memalloc', 'Fibers memalloc', falloc)
+    collectors_list.fiber_memused = utils.set_gauge('fiber_memused', 'Fibers memused', fused)
 end
 
 return {
     update = update_fibers_metrics,
+    list = collectors_list,
 }

metrics/default_metrics/tarantool/info.lua

@@ -1,4 +1,6 @@
-local utils = require('metrics.utils');
+local utils = require('metrics.utils')
+
+local collectors_list = {}
 
 local function update_info_metrics()
     if not utils.box_is_configured() then
@@ -7,20 +9,23 @@ local function update_info_metrics()
 
     local info = box.info()
 
-    utils.set_gauge('info_lsn', 'Tarantool lsn', info.lsn)
-    utils.set_gauge('info_uptime', 'Tarantool uptime', info.uptime)
+    collectors_list.info_lsn = utils.set_gauge('info_lsn', 'Tarantool lsn', info.lsn)
+    collectors_list.info_uptime = utils.set_gauge('info_uptime', 'Tarantool uptime', info.uptime)
 
     for k, v in ipairs(info.vclock) do
         utils.set_gauge('info_vclock', 'VClock', v, {id = k})
     end
 
     for k, v in ipairs(info.replication) do
         if v.upstream ~= nil then
-            utils.set_gauge('replication_' .. k .. '_lag', 'Replication lag for instance ' .. k, v.upstream.lag)
+            local metric_name = 'replication_' .. k .. '_lag'
+            collectors_list[metric_name] =
+                utils.set_gauge(metric_name, 'Replication lag for instance ' .. k, v.upstream.lag)
         end
     end
 end
 
 return {
     update = update_info_metrics,
+    list = collectors_list,
 }

metrics/default_metrics/tarantool/init.lua

@@ -1,23 +1,63 @@
 local metrics = require('metrics')
 
 local default_metrics = {
-    require('metrics.default_metrics.tarantool.network'),
-    require('metrics.default_metrics.tarantool.operations'),
-    require('metrics.default_metrics.tarantool.system'),
-    require('metrics.default_metrics.tarantool.replicas'),
-    require('metrics.default_metrics.tarantool.info'),
-    require('metrics.default_metrics.tarantool.slab'),
-    require('metrics.default_metrics.tarantool.runtime'),
-    require('metrics.default_metrics.tarantool.memory'),
-    require('metrics.default_metrics.tarantool.spaces'),
-    require('metrics.default_metrics.tarantool.fibers'),
-    require('metrics.default_metrics.tarantool.cpu'),
-    require('metrics.tarantool.vinyl')
+    network             = require('metrics.default_metrics.tarantool.network'),
+    operations          = require('metrics.default_metrics.tarantool.operations'),
+    system              = require('metrics.default_metrics.tarantool.system'),
+    replicas            = require('metrics.default_metrics.tarantool.replicas'),
+    info                = require('metrics.default_metrics.tarantool.info'),
+    slab                = require('metrics.default_metrics.tarantool.slab'),
+    runtime             = require('metrics.default_metrics.tarantool.runtime'),
+    memory              = require('metrics.default_metrics.tarantool.memory'),
+    spaces              = require('metrics.default_metrics.tarantool.spaces'),
+    fibers              = require('metrics.default_metrics.tarantool.fibers'),
+    cpu                 = require('metrics.default_metrics.tarantool.cpu'),
+    vinyl               = require('metrics.tarantool.vinyl'),
+    luajit              = require('metrics.tarantool.luajit'),
+    cartridge_issues    = require('metrics.cartridge.issues'),
+    clock               = require('metrics.cartridge.clock'),
 }
 
-local function enable()
-    for _, metric in ipairs(default_metrics) do
-        metrics.register_callback(metric.update)
+local function delete_collectors(list)
+    for _, collector in pairs(list) do
+        metrics.registry:unregister(collector)
+    end
+end
+
+local function enable(include, exclude)
+    include = include or {}
+    exclude = exclude or {}
+    if next(include) ~= nil and next(exclude) ~= nil then
+        error('Only one of "exclude" or "include" should present')
+    end
+
+    local exclude_map = {}
+    for _, name in ipairs(exclude) do
+        exclude_map[name] = true
+    end
+    local include_map = {}
+    for _, name in ipairs(include) do
+        include_map[name] = true
+    end
+
+    for name, value in pairs(default_metrics) do
+        if next(include) ~= nil then
+            if include_map[name] ~= nil then
+                metrics.register_callback(value.update)
+            else
+                metrics.unregister_callback(value.update)
+                delete_collectors(value.list)
+            end
+        elseif next(exclude) ~= nil then
+            if exclude_map[name] ~= nil then
+                metrics.unregister_callback(value.update)
+                delete_collectors(value.list)
+            else
+                metrics.register_callback(value.update)
+            end
+        else
+            metrics.register_callback(value.update)
+        end
     end
 end
 

metrics/default_metrics/tarantool/memory.lua

@@ -1,5 +1,7 @@
 local utils = require('metrics.utils')
 
+local collectors_list = {}
+
 local function update_memory_metrics()
     if not utils.box_is_configured() then
         return
@@ -8,11 +10,13 @@ local function update_memory_metrics()
     if box.info.memory ~= nil then
         local i = box.info.memory()
         for k, v in pairs(i) do
-            utils.set_gauge('info_memory_' .. k, 'Memory' .. k, v)
+            local metric_name = 'info_memory_' .. k
+            collectors_list[metric_name] = utils.set_gauge(metric_name, 'Memory' .. k, v)
         end
     end
 end
 
 return {
-    update = update_memory_metrics
-}
+    update = update_memory_metrics,
+    list = collectors_list,
+}