diff --git a/README.md b/README.md index 6a4190173..9396b207d 100644 --- a/README.md +++ b/README.md @@ -753,6 +753,23 @@ returns. `count` is total requests count since instance start or stats restart. `latency` is average time of requests execution, `time` is the total time of requests execution. +`select` section additionally contains `details` collectors. +```lua +crud.stats('my_space').select.details +--- +- map_reduces: 4 + tuples_fetched: 10500 + tuples_lookup: 238000 +... +``` +`map_reduces` is the count of planned map reduces (including those not +executed successfully). `tuples_fetched` is the count of tuples fetched +from storages during execution, `tuples_lookup` is the count of tuples +looked up on storages while collecting responses for calls (including +scrolls for multibatch requests). Details data is updated as part of +the request process, so you may get new details before `select`/`pairs` +call is finished and observed with count, latency and time collectors. + Since `pairs` request behavior differs from any other crud request, its statistics collection also has specific behavior. Statistics (`select` section) are updated after `pairs` cycle is finished: you diff --git a/crud/select.lua b/crud/select.lua index a633f86b4..b0d1ef9bb 100644 --- a/crud/select.lua +++ b/crud/select.lua @@ -59,7 +59,7 @@ local function select_on_storage(space_name, index_id, conditions, opts) end -- execute select - local tuples, err = select_executor.execute(space, index, filter_func, { + local resp, err = select_executor.execute(space, index, filter_func, { scan_value = opts.scan_value, after_tuple = opts.after_tuple, tarantool_iter = opts.tarantool_iter, @@ -70,15 +70,20 @@ local function select_on_storage(space_name, index_id, conditions, opts) end local cursor - if #tuples < opts.limit or opts.limit == 0 then + if resp.tuples_fetched < opts.limit or opts.limit == 0 then cursor = {is_end = true} else - cursor = make_cursor(tuples) + cursor = make_cursor(resp.tuples) end + cursor.stats = { + tuples_lookup = resp.tuples_lookup, + tuples_fetched = resp.tuples_fetched, + } + -- getting tuples with user defined fields (if `fields` option is specified) -- and fields that are needed for comparison on router (primary key + scan key) - return cursor, schema.filter_tuples_fields(tuples, opts.field_names) + return cursor, schema.filter_tuples_fields(resp.tuples, opts.field_names) end function select_module.init() diff --git a/crud/select/compat/select.lua b/crud/select/compat/select.lua index 1984a87ab..a05e7a93c 100644 --- a/crud/select/compat/select.lua +++ b/crud/select/compat/select.lua @@ -8,6 +8,7 @@ local dev_checks = require('crud.common.dev_checks') local common = require('crud.select.compat.common') local schema = require('crud.common.schema') local sharding_metadata_module = require('crud.common.sharding.sharding_metadata') +local stats = require('crud.stats') local compare_conditions = require('crud.compare.conditions') local select_plan = require('crud.compare.plan') @@ -115,6 +116,8 @@ local function build_select_iterator(space_name, user_conditions, opts) if err ~= nil then return nil, err, true end + else + stats.update_map_reduces(space_name) end local tuples_limit = opts.first diff --git a/crud/select/compat/select_old.lua b/crud/select/compat/select_old.lua index 1cf887446..a4d79e858 100644 --- a/crud/select/compat/select_old.lua +++ b/crud/select/compat/select_old.lua @@ -9,6 +9,7 @@ local sharding = require('crud.common.sharding') local dev_checks = require('crud.common.dev_checks') local schema = require('crud.common.schema') local sharding_metadata_module = require('crud.common.sharding.sharding_metadata') +local stats = require('crud.stats') local compare_conditions = require('crud.compare.conditions') local select_plan = require('crud.compare.plan') @@ -59,6 +60,14 @@ local function select_iteration(space_name, plan, opts) local tuples = {} for replicaset_uuid, replicaset_results in pairs(results) do + -- Stats extracted with callback here and not passed + -- outside to wrapper because fetch for pairs can be + -- called even after pairs() return from generators. + local cursor = replicaset_results[1] + if cursor.stats ~= nil then + stats.update_fetch_stats(cursor.stats, space_name) + end + tuples[replicaset_uuid] = replicaset_results[2] end @@ -141,6 +150,8 @@ local function build_select_iterator(space_name, user_conditions, opts) if err ~= nil then return nil, err, true end + else + stats.update_map_reduces(space_name) end -- generate tuples comparator diff --git a/crud/select/executor.lua b/crud/select/executor.lua index 6d6f74837..10309be24 100644 --- a/crud/select/executor.lua +++ b/crud/select/executor.lua @@ -1,4 +1,5 @@ local errors = require('errors') +local fun = require('fun') local dev_checks = require('crud.common.dev_checks') local select_comparators = require('crud.compare.comparators') @@ -68,13 +69,12 @@ function executor.execute(space, index, filter_func, opts) opts = opts or {} + local resp = { tuples_fetched = 0, tuples_lookup = 0, tuples = {} } + if opts.limit == 0 then - return {} + return resp end - local tuples = {} - local tuples_count = 0 - local value = opts.scan_value if opts.after_tuple ~= nil then local new_value = generate_value(opts.after_tuple, opts.scan_value, index.parts, opts.tarantool_iter) @@ -84,7 +84,16 @@ function executor.execute(space, index, filter_func, opts) end local tuple - local gen = index:pairs(value, {iterator = opts.tarantool_iter}) + local raw_gen, param, state = index:pairs(value, {iterator = opts.tarantool_iter}) + local gen = fun.wrap(function(param, state) + local next_state, var = raw_gen(param, state) + + if var ~= nil then + resp.tuples_lookup = resp.tuples_lookup + 1 + end + + return next_state, var + end, param, state) if opts.after_tuple ~= nil then local err @@ -94,7 +103,7 @@ function executor.execute(space, index, filter_func, opts) end if tuple == nil then - return {} + return resp end end @@ -110,10 +119,10 @@ function executor.execute(space, index, filter_func, opts) local matched, early_exit = filter_func(tuple) if matched then - table.insert(tuples, tuple) - tuples_count = tuples_count + 1 + table.insert(resp.tuples, tuple) + resp.tuples_fetched = resp.tuples_fetched + 1 - if opts.limit ~= nil and tuples_count >= opts.limit then + if opts.limit ~= nil and resp.tuples_fetched >= opts.limit then break end elseif early_exit then @@ -123,7 +132,7 @@ function executor.execute(space, index, filter_func, opts) gen.state, tuple = gen(gen.param, gen.state) end - return tuples + return resp end return executor diff --git a/crud/select/merger.lua b/crud/select/merger.lua index fa443b849..e3c1bdf48 100644 --- a/crud/select/merger.lua +++ b/crud/select/merger.lua @@ -7,6 +7,7 @@ local compat = require('crud.common.compat') local merger_lib = compat.require('tuple.merger', 'merger') local Keydef = require('crud.compare.keydef') +local stats = require('crud.stats') local function bswap_u16(num) return bit.rshift(bit.bswap(tonumber(num)), 16) @@ -93,6 +94,7 @@ local function fetch_chunk(context, state) local replicaset = context.replicaset local vshard_call_name = context.vshard_call_name local timeout = context.timeout or call.DEFAULT_VSHARD_CALL_TIMEOUT + local space_name = context.space_name local future = state.future -- The source was entirely drained. @@ -109,6 +111,14 @@ local function fetch_chunk(context, state) -- Decode metainfo, leave data to be processed by the merger. local cursor = decode_metainfo(buf) + -- Extract stats info. + -- Stats extracted with callback here and not passed + -- outside to wrapper because fetch for pairs can be + -- called even after pairs() return from generators. + if cursor.stats ~= nil then + stats.update_fetch_stats(cursor.stats, space_name) + end + -- Check whether we need the next call. if cursor.is_end then local next_state = {} @@ -157,6 +167,7 @@ local function new(replicasets, space, index_id, func_name, func_args, opts) replicaset = replicaset, vshard_call_name = vshard_call_name, timeout = call_opts.timeout, + space_name = space.name, } local state = {future = future} local source = merger_lib.new_buffer_source(fetch_chunk, context, state) diff --git a/crud/stats/init.lua b/crud/stats/init.lua index 1e1a8f831..2714ee136 100644 --- a/crud/stats/init.lua +++ b/crud/stats/init.lua @@ -255,6 +255,62 @@ function stats.wrap(func, op, opts) end end +local storage_stats_schema = { tuples_fetched = 'number', tuples_lookup = 'number' } +--- Callback to collect storage tuples stats (select/pairs). +-- +-- @function update_fetch_stats +-- +-- @tab storage_stats +-- Statistics from select storage call. +-- +-- @number storage_stats.tuples_fetched +-- Count of tuples fetched during storage call. +-- +-- @number storage_stats.tuples_lookup +-- Count of tuples looked up on storages while collecting response. +-- +-- @string space_name +-- Name of space. +-- +-- @treturn boolean Returns `true`. +-- +function stats.update_fetch_stats(storage_stats, space_name) + dev_checks(storage_stats_schema, 'string') + + if not stats.is_enabled() then + return true + end + + registry.observe_fetch( + storage_stats.tuples_fetched, + storage_stats.tuples_lookup, + space_name + ) + + return true +end + +--- Callback to collect planned map reduces stats (select/pairs). +-- +-- @function update_map_reduces +-- +-- @string space_name +-- Name of space. +-- +-- @treturn boolean Returns `true`. +-- +function stats.update_map_reduces(space_name) + dev_checks('string') + + if not stats.is_enabled() then + return true + end + + registry.observe_map_reduces(1, space_name) + + return true +end + --- Table with CRUD operation lables. -- -- @tfield string INSERT diff --git a/crud/stats/local_registry.lua b/crud/stats/local_registry.lua index c5e125f1f..9626f75ba 100644 --- a/crud/stats/local_registry.lua +++ b/crud/stats/local_registry.lua @@ -4,6 +4,7 @@ local dev_checks = require('crud.common.dev_checks') local stash = require('crud.common.stash') +local op_module = require('crud.stats.operation') local registry_utils = require('crud.stats.registry_utils') local registry = {} @@ -98,4 +99,56 @@ function registry.observe(latency, space_name, op, status) return true end +--- Increase statistics of storage select/pairs calls +-- +-- @function observe_fetch +-- +-- @string space_name +-- Name of space. +-- +-- @number tuples_fetched +-- Count of tuples fetched during storage call. +-- +-- @number tuples_lookup +-- Count of tuples looked up on storages while collecting response. +-- +-- @treturn boolean Returns true. +-- +function registry.observe_fetch(tuples_fetched, tuples_lookup, space_name) + dev_checks('number', 'number', 'string') + + local op = op_module.SELECT + registry_utils.init_collectors_if_required(internal.registry.spaces, space_name, op) + local collectors = internal.registry.spaces[space_name][op].details + + collectors.tuples_fetched = collectors.tuples_fetched + tuples_fetched + collectors.tuples_lookup = collectors.tuples_lookup + tuples_lookup + + return true +end + +--- Increase statistics of planned map reduces during select/pairs +-- +-- @function observe_map_reduces +-- +-- @number count +-- Count of map reduces planned. +-- +-- @string space_name +-- Name of space. +-- +-- @treturn boolean Returns true. +-- +function registry.observe_map_reduces(count, space_name) + dev_checks('number', 'string') + + local op = op_module.SELECT + registry_utils.init_collectors_if_required(internal.registry.spaces, space_name, op) + local collectors = internal.registry.spaces[space_name][op].details + + collectors.map_reduces = collectors.map_reduces + count + + return true +end + return registry diff --git a/crud/stats/registry_utils.lua b/crud/stats/registry_utils.lua index 2c99f8a37..956544619 100644 --- a/crud/stats/registry_utils.lua +++ b/crud/stats/registry_utils.lua @@ -3,6 +3,7 @@ -- local dev_checks = require('crud.common.dev_checks') +local op_module = require('crud.stats.operation') local registry_utils = {} @@ -10,10 +11,17 @@ local registry_utils = {} -- -- @function build_collectors -- +-- @string op +-- Label of registry collectors. +-- Use `require('crud.stats').op` to pick one. +-- -- @treturn table Returns collectors for success and error requests. --- Collectors store 'count', 'latency' and 'time' values. +-- Collectors store 'count', 'latency' and 'time' values. Also +-- returns additional collectors for select operation. -- -function registry_utils.build_collectors() +function registry_utils.build_collectors(op) + dev_checks('string') + local collectors = { ok = { count = 0, @@ -27,6 +35,14 @@ function registry_utils.build_collectors() }, } + if op == op_module.SELECT then + collectors.details = { + tuples_fetched = 0, + tuples_lookup = 0, + map_reduces = 0, + } + end + return collectors end @@ -53,7 +69,7 @@ function registry_utils.init_collectors_if_required(spaces, space_name, op) local space_collectors = spaces[space_name] if space_collectors[op] == nil then - space_collectors[op] = registry_utils.build_collectors() + space_collectors[op] = registry_utils.build_collectors(op) end end diff --git a/doc/apidoc/index.html b/doc/apidoc/index.html new file mode 100644 index 000000000..fa6ccf6aa --- /dev/null +++ b/doc/apidoc/index.html @@ -0,0 +1,133 @@ + + + + + CRUD API reference + + + + +
+ +
+ +
+
+
+ + +
+ + + + + + + + + +
+ + + +

Modules

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
crud.bordersModule
crud.cfgModule for CRUD configuration.
crud.common.stashModule for preserving data between reloads.
crud.deleteModule
crud.getModule
crud.insertModule
crud.lenModule
crud.replaceModule
crud.selectModule
crud.statsCRUD statistics module.
crud.stats.local_registryInternal module used to store statistics.
crud.stats.metrics_registryInternal module used to store statistics in metrics registry.
crud.stats.registry_utilsInternal module used by statistics registries.
crud.truncateModule
crud.updateModule
crud.upsertModule
+ +
+
+
+ + diff --git a/doc/apidoc/ldoc.css b/doc/apidoc/ldoc.css new file mode 100644 index 000000000..83735aa26 --- /dev/null +++ b/doc/apidoc/ldoc.css @@ -0,0 +1,345 @@ +body { + font-size: 16px; + font-family: "Open Sans", sans-serif; + margin: 0; +} + +a:link { color: #fd4c4d; } +a:visited { color: #fd4c4d; } +a:hover { color: #fd4c4d; } + +h1 { font-size:26px; font-weight: normal; } +h2 { font-size:22px; font-weight: normal; } +h3 { font-size:18px; font-weight: normal; } +h4 { font-size:16px; font-weight: bold; } + +hr { + height: 1px; + background: #c1cce4; + border: 0px; + margin: 15px 0; +} + +#content code, tt { + font-family: monospace; + box-sizing: border-box; + background-color: #f9f2f4; + color: #c7254e; + border-radius: 6px; + font-size: 85%; + padding: 1px 0.4em; +} + +#content h1 > code { + background-color: #333333; + color: white; + padding: 1px 0.4em; +} + +.parameter { + font-family: monospace; + font-weight: bold; + color: #c7254e; +} + +.parameter-info { + color: #555555; +} + +.type { + font-style: italic +} + +.parameter-description { + display: inline-block; + padding-left: 15pt; + margin-bottom: 5px; +} + +.parameter-description > p { + margin-top: 0.3em; + margin-bottom: 0.5em; +} + +p.name { + font-family: monospace; +} + +#main { + position: relative; +} + +#navigation { + position: absolute; + color: white; + background-color: #262626; + border-bottom: 1px solid #d3dbec; + + height: 100%; + width: 18em; + vertical-align: top; + overflow: visible; +} + +#navigation a:link { + color: white; +} +#navigation a:visited { + color: white; +} +#navigation a:hover { + color: white; +} + +#navigation br { + display: none; +} + +#navigation h1 { + border-bottom: 1px solid #404040; + padding: 15px; + margin-top: 0px; + margin-bottom: 0px; +} + +#navigation h2 { + font-size: 18px; + border-bottom: 1px solid #404040; + padding-left: 15px; + padding-right: 15px; + padding-top: 10px; + padding-bottom: 10px; + margin-top: 30px; + margin-bottom: 0px; +} + +#content h1 { + background-color: black; + color: white; + padding: 15px; + margin: 0px; +} + +#content > p { + padding-left: 15px; +} + +#content h2 { + background-color: #f9f2f4; + padding: 15px; + padding-top: 15px; + padding-bottom: 15px; + margin-top: 0px; +} + +#content h2 a { + color: #fd4c4d; + text-decoration: none; +} + +#content h2 a:hover { + text-decoration: underline; +} + +#content h3 { + font-style: italic; + padding-top: 15px; + padding-bottom: 4px; + margin-right: 15px; + margin-left: 15px; + margin-bottom: 5px; + border-bottom: solid 1px #bcd; +} + +#content h4 { + margin-right: 15px; + margin-left: 15px; + border-bottom: solid 1px #bcd; +} + +pre { + background-color: #f9f2f4; + border-radius: 6px; + border: 1px solid #cccccc; + padding: 10px; + overflow: auto; + font-family: monospace; + font-size: 85%; +} + +#content ul pre.example { + margin-left: 0px; +} + +table.index { +/* border: 1px #00007f; */ +} +table.index td { text-align: left; vertical-align: top; } + +#navigation ul +{ + font-size:1em; + list-style-type: none; + margin: 1px 1px 10px 1px; +} + +#navigation li { + text-indent: -1em; + display: block; + margin: 3px 0px 0px 22px; +} + +#navigation li li a { + margin: 0px 3px 0px -1em; +} + +#content { + margin-left: 18em; +} + +#content .function dd > p { + margin-top: 0.3em; + margin-bottom: 0.5em; +} + +#content table { + padding-left: 15px; + padding-right: 15px; + background-color: white; +} + +#content p, #content table, #content ol, #content ul, #content dl { + max-width: 900px; +} + +table.module_list, table.function_list { + border-width: 1px; + border-style: solid; + border-color: #cccccc; + border-collapse: collapse; + margin: 15px; +} +table.module_list td, table.function_list td { + border-width: 1px; + padding-left: 10px; + padding-right: 10px; + padding-top: 5px; + padding-bottom: 5px; + border: solid 1px #cccccc; +} +table.module_list td.name, table.function_list td.name { + background-color: white; min-width: 200px; border-right-width: 0px; +} +table.module_list td.summary, table.function_list td.summary { + background-color: white; width: 100%; border-left-width: 0px; +} + +dl.function { + margin-right: 15px; + margin-left: 15px; + border-bottom: solid 1px #cccccc; + border-left: solid 1px #cccccc; + border-right: solid 1px #cccccc; + background-color: white; +} + +dl.function dt { + color: #c7254e; + font-family: monospace; + border-top: solid 1px #cccccc; + padding: 15px 15px 0.5em 15px; +} + +dl.function dd { + margin-left: 15px; + margin-right: 15px; + margin-top: 5px; + margin-bottom: 15px; +} + +#content dl.function dd h3 { + margin-top: 0px; + margin-left: 0px; + padding-left: 0px; + font-size: 16px; + color: #555555; + border-bottom: solid 1px #def; +} + +#content dl.function dd ul, #content dl.function dd ol { + padding: 0px; + padding-left: 15px; + list-style-type: none; +} + +/* Highlight XXX notes. */ +#content .xxx { + box-sizing: border-box; + background-color: yellow; + color: black; + border-radius: 6px; + border: #c7254e solid 1px; + padding: 0.1em 0.2em; +} + +ul.nowrap { + overflow:auto; + white-space:nowrap; +} + +.section-description { + padding-left: 15px; + padding-right: 15px; +} + +/* stop sublists from having initial vertical space */ +ul ul { margin-top: 0px; } +ol ul { margin-top: 0px; } +ol ol { margin-top: 0px; } +ul ol { margin-top: 0px; } + +/* make the target distinct; helps when we're navigating to a function */ +a:target + * { + background-color: #FF9; +} + + +/* styles for prettification of source */ +pre .comment { color: #bbccaa; } +pre .constant { color: #a8660d; } +pre .escape { color: #844631; } +pre .keyword { color: #ffc090; font-weight: bold; } +pre .library { color: #0e7c6b; } +pre .marker { color: #512b1e; background: #fedc56; font-weight: bold; } +pre .string { color: #8080ff; } +pre .number { color: #f8660d; } +pre .operator { color: #2239a8; font-weight: bold; } +pre .preprocessor, pre .prepro { color: #a33243; } +pre .global { color: #c040c0; } +pre .user-keyword { color: #800080; } +pre .prompt { color: #558817; } +pre .url { color: #272fc2; text-decoration: underline; } + +html body #content ul.markdown_list { + list-style-type: disc; +} + +#content table.markdown_table { + padding: 0px; + border-collapse: collapse; +} +.markdown_table td, th { + border: 1px solid #999999; + padding: 7px; +} +.markdown_table th { + background-color: #ccc; +} + +.markdown_table tr:nth-child(odd) td { + background-color: white; +} + +.markdown_table tr:nth-child(even) td { + background-color: #f9f2f4; +} diff --git a/doc/apidoc/modules/crud.borders.html b/doc/apidoc/modules/crud.borders.html new file mode 100644 index 000000000..4ca9866bb --- /dev/null +++ b/doc/apidoc/modules/crud.borders.html @@ -0,0 +1,210 @@ + + + + + CRUD API reference + + + + +
+ +
+ +
+
+
+ + +
+ + + + + + + + + +
+ +

Module crud.borders

+

Module

+

+
+

+ + +

Functions

+ + + + + + + + + +
min (string, string)Find the minimum value in the specified index
min (space_name[, index_name])Find the maximum value in the specified index
+ +
+
+ + +

Functions

+ +
+
+ + min (string, string) +
+
+

Find the minimum value in the specified index +
+

+ + +

Parameters:

+
    +
  • string +
    + +

    index_name + An index name (by default, primary index is used) +

    +     +
    • +
  • string +
    + +

    index_name + An index name (by default, primary index is used) +

    +     +
    • +
+ +

Returns:

+
    + + result +
+

Or

+
    +
  1. + nil + +
    +
    2. +
  2. + table + Error description
    3. +
+ + + + +
+
+ + min (space_name[, index_name]) +
+
+

Find the maximum value in the specified index +
+

+ + +

Parameters:

+
    +
  • space_name + + (string) + +
    + +

    + A space name +

    +     +
    • +
  • index_name + + (string, + + optional) + + +
    + +

    + An index name (by default, primary index is used) +

    +     +
    • +
+ +

Returns:

+
    + + result +
+

Or

+
    +
  1. + nil + +
    +
    2. +
  2. + table + Error description
    3. +
+ + + + +
+
+ + +
+
+
+ + diff --git a/doc/apidoc/modules/crud.cfg.html b/doc/apidoc/modules/crud.cfg.html new file mode 100644 index 000000000..23b741c66 --- /dev/null +++ b/doc/apidoc/modules/crud.cfg.html @@ -0,0 +1,197 @@ + + + + + CRUD API reference + + + + +
+ +
+ +
+
+
+ + +
+ + + + + + + + + +
+ +

Module crud.cfg

+

Module for CRUD configuration.

+

+
+

+ + +

Functions

+ + + + + +
__call (self[, opts])Configure CRUD module.
+ +
+
+ + +

Functions

+ +
+
+ + __call (self[, opts]) +
+
+

Configure CRUD module. +
+

+ + +

Parameters:

+
    +
  • self + + (tab) + +
    + +

    +
    +

    +     +
    • +
  • opts + + (tab, + + optional) + + + +
    + +

    +
    +

    +     +
      +
    • stats + + (bool, + + optional) + + +
      + +

      + Enable or disable statistics collect. + Statistics are observed only on router instances. +

      +       +
      • +
    • stats_driver + + (string, + + optional) + + +
      + +

      + 'local' or 'metrics'. + If 'local', stores statistics in local registry (some Lua tables) + and computes latency as overall average. 'metrics' requires + metrics >= 0.10.0 installed and stores statistics in + global metrics registry (integrated with exporters). + 'metrics' driver supports computing latency as 0.99 quantile with aging. + If 'metrics' driver is available, it is used by default, + otherwise 'local' is used. +

      +       +
      • +
    • stats_quantiles + + (bool, + + optional) + + +
      + +

      + Enable or disable statistics quantiles (only for metrics driver). + Quantiles computations increases performance overhead up to 10%. +

      +       +
      • +
    +
+ +

Returns:

+
    + + Copy of configuration table. +
+ + + + +
+
+ + +
+
+
+ + diff --git a/doc/apidoc/modules/crud.common.stash.html b/doc/apidoc/modules/crud.common.stash.html new file mode 100644 index 000000000..ae311ff53 --- /dev/null +++ b/doc/apidoc/modules/crud.common.stash.html @@ -0,0 +1,238 @@ + + + + + CRUD API reference + + + + +
+ +
+ +
+
+
+ + +
+ + + + + + + + + +
+ +

Module crud.common.stash

+

Module for preserving data between reloads.

+

+
+

+ + +

Functions

+ + + + + + + + + +
setup_cartridge_reload ()Setup Tarantool Cartridge reload.
get (name)Get a stash instance, initialize if needed.
+

Tables

+ + + + + +
nameAvailable stashes list.
+ +
+
+ + +

Functions

+ +
+
+ + setup_cartridge_reload () +
+
+

Setup Tarantool Cartridge reload. +
+

+ Call on Tarantool Cartridge roles that are expected + to use stashes. +

+ + + +

Returns:

+
    + + Returns +
+ + + + +
+
+ + get (name) +
+
+

Get a stash instance, initialize if needed. +
+

+ Stashes are persistent to package reload. + To use them with Cartridge roles reload, + call stash.setup_cartridge_reload in role. +

+ + +

Parameters:

+
    +
  • name + + (string) + +
    + +

    + Stash identifier. Use one from stash.name table. +

    +     +
    • +
+ +

Returns:

+
    + + table + A stash instance. +
+ + + + +
+
+

Tables

+ +
+
+ + name +
+
+

Available stashes list. +
+

+ + +

Fields:

+
    +
  • cfg + + (string) + +
    + +

    + Stash for CRUD module configuration. +

    +     +
    • +
  • stats_internal + + (string) + +
    + +

    + Stash for main stats module. +

    +     +
    • +
  • stats_local_registry + + (string) + +
    + +

    + Stash for local metrics registry. +

    +     +
    • +
  • stats_metrics_registry + + (string) + +
    + +

    + Stash for metrics rocks statistics registry. +

    +     +
    • +
+ + + + + +
+
+ + +
+
+
+ + diff --git a/doc/apidoc/modules/crud.delete.html b/doc/apidoc/modules/crud.delete.html new file mode 100644 index 000000000..ba46dee26 --- /dev/null +++ b/doc/apidoc/modules/crud.delete.html @@ -0,0 +1,149 @@ + + + + + CRUD API reference + + + + +
+ +
+ +
+
+
+ + +
+ + + + + + + + + +
+ +

Module crud.delete

+

Module

+

+
+

+ + +

Functions

+ + + + + +
delete (space_name, key)Deletes tuple from the specified space by key
+ +
+
+ + +

Functions

+ +
+
+ + delete (space_name, key) +
+
+

Deletes tuple from the specified space by key +
+

+ + +

Parameters:

+
    +
  • space_name + + (string) + +
    + +

    + A space name +

    +     +
    • +
  • key +
    + +

    + Primary key value +

    +     +
    • +
+ +

Returns:

+
    + + object +
+

Or

+
    +
  1. + nil + +
    +
    2. +
  2. + table + Error description
    3. +
+ + + + +
+
+ + +
+
+
+ + diff --git a/doc/apidoc/modules/crud.get.html b/doc/apidoc/modules/crud.get.html new file mode 100644 index 000000000..4e38492a0 --- /dev/null +++ b/doc/apidoc/modules/crud.get.html @@ -0,0 +1,149 @@ + + + + + CRUD API reference + + + + +
+ +
+ +
+
+
+ + +
+ + + + + + + + + +
+ +

Module crud.get

+

Module

+

+
+

+ + +

Functions

+ + + + + +
get (space_name, key)Get tuple from the specified space by key
+ +
+
+ + +

Functions

+ +
+
+ + get (space_name, key) +
+
+

Get tuple from the specified space by key +
+

+ + +

Parameters:

+
    +
  • space_name + + (string) + +
    + +

    + A space name +

    +     +
    • +
  • key +
    + +

    + Primary key value +

    +     +
    • +
+ +

Returns:

+
    + + object +
+

Or

+
    +
  1. + nil + +
    +
    2. +
  2. + table + Error description
    3. +
+ + + + +
+
+ + +
+
+
+ + diff --git a/doc/apidoc/modules/crud.insert.html b/doc/apidoc/modules/crud.insert.html new file mode 100644 index 000000000..e84cd6302 --- /dev/null +++ b/doc/apidoc/modules/crud.insert.html @@ -0,0 +1,80 @@ + + + + + CRUD API reference + + + + +
+ +
+ +
+
+
+ + +
+ + + + + + + + + +
+ +

Module crud.insert

+

Module

+

+
+

+ + + +
+
+ + + + +
+
+
+ + diff --git a/doc/apidoc/modules/crud.len.html b/doc/apidoc/modules/crud.len.html new file mode 100644 index 000000000..6c8fbec41 --- /dev/null +++ b/doc/apidoc/modules/crud.len.html @@ -0,0 +1,139 @@ + + + + + CRUD API reference + + + + +
+ +
+ +
+
+
+ + +
+ + + + + + + + + +
+ +

Module crud.len

+

Module

+

+
+

+ + +

Functions

+ + + + + +
len (string)Calculates the number of tuples in the space for memtx engine
+ +
+
+ + +

Functions

+ +
+
+ + len (string) +
+
+

Calculates the number of tuples in the space for memtx engine + Calculates the maximum approximate number of tuples in the space for vinyl engine +
+

+ + +

Parameters:

+
    +
  • string +
    + +

    |number space_name + A space name as well as numerical id +

    +     +
    • +
+ +

Returns:

+
    + + number +
+

Or

+
    +
  1. + nil + +
    +
    2. +
  2. + table + Error description
    3. +
+ + + + +
+
+ + +
+
+
+ + diff --git a/doc/apidoc/modules/crud.replace.html b/doc/apidoc/modules/crud.replace.html new file mode 100644 index 000000000..aa8c06954 --- /dev/null +++ b/doc/apidoc/modules/crud.replace.html @@ -0,0 +1,80 @@ + + + + + CRUD API reference + + + + +
+ +
+ +
+
+
+ + +
+ + + + + + + + + +
+ +

Module crud.replace

+

Module

+

+
+

+ + + +
+
+ + + + +
+
+
+ + diff --git a/doc/apidoc/modules/crud.select.html b/doc/apidoc/modules/crud.select.html new file mode 100644 index 000000000..a74d5f88d --- /dev/null +++ b/doc/apidoc/modules/crud.select.html @@ -0,0 +1,80 @@ + + + + + CRUD API reference + + + + +
+ +
+ +
+
+
+ + +
+ + + + + + + + + +
+ +

Module crud.select

+

Module

+

+
+

+ + + +
+
+ + + + +
+
+
+ + diff --git a/doc/apidoc/modules/crud.stats.html b/doc/apidoc/modules/crud.stats.html new file mode 100644 index 000000000..b4656186c --- /dev/null +++ b/doc/apidoc/modules/crud.stats.html @@ -0,0 +1,682 @@ + + + + + CRUD API reference + + + + +
+ +
+ +
+
+
+ + +
+ + + + + + + + + +
+ +

Module crud.stats

+

CRUD statistics module.

+

+
+

+ + +

Functions

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
is_enabled ()Check if statistics module was enabled.
get_default_driver ()Get default statistics driver name.
enable ([opts])Initializes statistics registry, enables callbacks and wrappers.
reset ()Resets statistics registry.
disable ()Destroys statistics registry and disable callbacks.
get ([space_name])Get statistics on CRUD operations.
wrap (func, op[, opts])Wrap CRUD operation call to collect statistics.
get_fetch_callback ()Returns callback to collect storage tuples stats (select/pairs).
update_map_reduces (space_name)Callback to collect planned map reduces stats (select/pairs).
+

Fields

+ + + + + + + + + +
opTable with CRUD operation lables.
internalStats module internal state (for debug/test).
+ +
+
+ + +

Functions

+ +
+
+ + is_enabled () +
+
+

Check if statistics module was enabled. +
+

+ + + +

Returns:

+
    + + boolean + Returns true or false. +
+ + + + +
+
+ + get_default_driver () +
+
+

Get default statistics driver name. +
+

+ + + +

Returns:

+
    + + string + metrics if supported, local if unsupported. +
+ + + + +
+
+ + enable ([opts]) +
+
+

Initializes statistics registry, enables callbacks and wrappers. +
+

+ If already enabled, do nothing. +

+ + +

Parameters:

+
    +
  • opts + + (tab, + + optional) + + + +
    + +

    +
    +

    +     +
      +
    • driver + + (string, + + optional) + + +
      + +

      + 'local' or 'metrics'. + If 'local', stores statistics in local registry (some Lua tables) + and computes latency as overall average. 'metrics' requires + metrics >= 0.9.0 installed and stores statistics in + global metrics registry (integrated with exporters). + 'metrics' driver supports computing latency as 0.99 quantile with aging. + If 'metrics' driver is available, it is used by default, + otherwise 'local' is used. +

      +       +
      • +
    • quantiles + + (bool, + default false + +
      + +

      + If 'metrics' driver used, you can enable + computing requests latency as 0.99 quantile with aging. + Performance overhead for enabling is near 10%. +

      +       +
      • +
    +
+ +

Returns:

+
    + + boolean + Returns true. +
+ + + + +
+
+ + reset () +
+
+

Resets statistics registry. +
+

+ After reset collectors are the same as right + after initial stats.enable(). +

+ + + +

Returns:

+
    + + boolean + Returns true. +
+ + + + +
+
+ + disable () +
+
+

Destroys statistics registry and disable callbacks. +
+

+ If already disabled, do nothing. +

+ + + +

Returns:

+
    + + boolean + Returns true. +
+ + + + +
+
+ + get ([space_name]) +
+
+

Get statistics on CRUD operations. +
+

+ + +

Parameters:

+
    +
  • space_name + + (string, + + optional) + + +
    + +

    + If specified, returns table with statistics + of operations on space, separated by operation type and + execution status. If there wasn't any requests of "op" type + for space, there won't be corresponding collectors. + If not specified, returns table with statistics + about all observed spaces. +

    +     +
    • +
+ +

Returns:

+
    + + table + Statistics on CRUD operations. + If statistics disabled, returns {}. + +
+ + + + +
+
+ + wrap (func, op[, opts]) +
+
+

Wrap CRUD operation call to collect statistics. +
+

+ Approach based on box.atomic(): + https://github.com/tarantool/tarantool/blob/b9f7204b5e0d10b443c6f198e9f7f04e0d16a867/src/box/lua/schema.lua#L369 +

+ + +

Parameters:

+
    +
  • func + + (func) + +
    + +

    + Function to wrap. First argument is expected to + be a space name string. If statistics enabled, + errors are caught and thrown again. +

    +     +
    • +
  • op + + (string) + +
    + +

    + Label of registry collectors. + Use require('crud.stats').op to pick one. +

    +     +
    • +
  • opts + + (tab, + + optional) + + + +
    + +

    +
    +

    +     +
      +
    • pairs + + (bool, + default false + +
      + +

      + If false, wraps only function passed as argument. + Second return value of wrapped function is treated + as error (nil, err case). + If true, also wraps gen() function returned by + call. Statistics observed on cycle end (last + element was fetched or error was thrown). If pairs + cycle was interrupted with break, statistics will + be collected when pairs objects are cleaned up with + Lua garbage collector. +

      +       +
      • +
    +
+ +

Returns:

+
    + + Wrapped function output. +
+ + + + +
+
+ + get_fetch_callback () +
+
+

Returns callback to collect storage tuples stats (select/pairs). +
+

+ + + +

Returns:

+
    + + function + update_fetch_stats function to collect tuples stats. +
+

Or

+
    + + function + Dummy function, if stats disabled. +
+ + + + +
+
+ + update_map_reduces (space_name) +
+
+

Callback to collect planned map reduces stats (select/pairs). +
+

+ + +

Parameters:

+
    +
  • space_name + + (string) + +
    + +

    + Name of space. +

    +     +
    • +
+ +

Returns:

+
    + + boolean + Returns true. +
+ + + + +
+
+

Fields

+ +
+
+ + op +
+
+

Table with CRUD operation lables. +
+

+ + +
    +
  • INSERT + + (string) + +
    + +

    + Identifies both insert and insert_object. +

    +     +
    • +
  • GET + + (string) + +
    + +

    +
    +

    +     +
    • +
  • REPLACE + + (string) + +
    + +

    + Identifies both replace and replace_object. +

    +     +
    • +
  • UPDATE + + (string) + +
    + +

    +
    +

    +     +
    • +
  • UPSERT + + (string) + +
    + +

    + Identifies both upsert and upsert_object. +

    +     +
    • +
  • DELETE + + (string) + +
    + +

    +
    +

    +     +
    • +
  • SELECT + + (string) + +
    + +

    + Identifies both pairs and select. +

    +     +
    • +
  • TRUNCATE + + (string) + +
    + +

    +
    +

    +     +
    • +
  • LEN + + (string) + +
    + +

    +
    +

    +     +
    • +
  • BORDERS + + (string) + +
    + +

    + Identifies both min and max. +

    +     +
    • +
+ + + + + +
+
+ + internal +
+
+

Stats module internal state (for debug/test). +
+

+ + +
    +
  • driver + + (string, + + optional) + + +
    + +

    Current statistics registry driver (if nil, stats disabled).

    +     +
    • +
  • quantiles + + (boolean, + + optional) + + +
    + +

    Is quantiles computed.

    +     +
    • +
+ + + + + +
+
+ + +
+
+
+ + diff --git a/doc/apidoc/modules/crud.stats.local_registry.html b/doc/apidoc/modules/crud.stats.local_registry.html new file mode 100644 index 000000000..07556ffe5 --- /dev/null +++ b/doc/apidoc/modules/crud.stats.local_registry.html @@ -0,0 +1,413 @@ + + + + + CRUD API reference + + + + +
+ +
+ +
+
+
+ + +
+ + + + + + + + + +
+ +

Module crud.stats.local_registry

+

Internal module used to store statistics.

+

+
+

+ + +

Functions

+ + + + + + + + + + + + + + + + + + + + + + + + + +
init (opts)Initialize local metrics registry.
destroy ()Destroy local metrics registry.
get ([space_name])Get copy of local metrics registry.
observe (space_name, latency, op, success)Increase requests count and update latency info.
observe_fetch (space_name, tuples_fetched, tuples_lookup)Increase statistics of storage select/pairs calls
observe_map_reduces (count, space_name)Increase statistics of planned map reduces during select/pairs
+ +
+
+ + +

Functions

+ +
+
+ + init (opts) +
+
+

Initialize local metrics registry. +
+

+ Registries are not meant to used explicitly + by users, init is not guaranteed to be idempotent. +

+ + +

Parameters:

+
    +
  • opts + + (tab) + + +
    + +

    +
    +

    +     +
      +
    • quantiles + + (bool) + +
      + +

      + Quantiles is not supported for local, only false is valid. +

      +       +
      • +
    +
+ +

Returns:

+
    + + boolean + Returns true. +
+ + + + +
+
+ + destroy () +
+
+

Destroy local metrics registry. +
+

+ Registries are not meant to used explicitly + by users, destroy is not guaranteed to be idempotent. +

+ + + +

Returns:

+
    + + boolean + Returns true. +
+ + + + +
+
+ + get ([space_name]) +
+
+

Get copy of local metrics registry. +
+

+ Registries are not meant to used explicitly + by users, get is not guaranteed to work without init. +

+ + +

Parameters:

+
    +
  • space_name + + (string, + + optional) + + +
    + +

    + If specified, returns table with statistics + of operations on table, separated by operation type and + execution status. If there wasn't any requests for table, + returns {}. If not specified, returns table with statistics + about all observed spaces. +

    +     +
    • +
+ +

Returns:

+
    + + table + Returns copy of metrics registry (or registry section). +
+ + + + +
+
+ + observe (space_name, latency, op, success) +
+
+

Increase requests count and update latency info. +
+

+ + +

Parameters:

+
    +
  • space_name + + (string) + +
    + +

    + Name of space. +

    +     +
    • +
  • latency + + (number) + +
    + +

    + Time of call execution. +

    +     +
    • +
  • op + + (string) + +
    + +

    + Label of registry collectors. + Use require('crud.stats').op to pick one. +

    +     +
    • +
  • success + + (string) + +
    + +

    + 'ok' if no errors on execution, 'error' otherwise. +

    +     +
    • +
+ +

Returns:

+
    + + boolean + Returns true. +
+ + + + +
+
+ + observe_fetch (space_name, tuples_fetched, tuples_lookup) +
+
+

Increase statistics of storage select/pairs calls +
+

+ + +

Parameters:

+
    +
  • space_name + + (string) + +
    + +

    + Name of space. +

    +     +
    • +
  • tuples_fetched + + (number) + +
    + +

    + Count of tuples fetched during storage call. +

    +     +
    • +
  • tuples_lookup + + (number) + +
    + +

    + Count of tuples looked up on storages while collecting response. +

    +     +
    • +
+ +

Returns:

+
    + + boolean + Returns true. +
+ + + + +
+
+ + observe_map_reduces (count, space_name) +
+
+

Increase statistics of planned map reduces during select/pairs +
+

+ + +

Parameters:

+
    +
  • count + + (number) + +
    + +

    + Count of map reduces planned. +

    +     +
    • +
  • space_name + + (string) + +
    + +

    + Name of space. +

    +     +
    • +
+ +

Returns:

+
    + + boolean + Returns true. +
+ + + + +
+
+ + +
+
+
+ + diff --git a/doc/apidoc/modules/crud.stats.metrics_registry.html b/doc/apidoc/modules/crud.stats.metrics_registry.html new file mode 100644 index 000000000..f329e0cc1 --- /dev/null +++ b/doc/apidoc/modules/crud.stats.metrics_registry.html @@ -0,0 +1,451 @@ + + + + + CRUD API reference + + + + +
+ +
+ +
+
+
+ + +
+ + + + + + + + + +
+ +

Module crud.stats.metrics_registry

+

Internal module used to store statistics in metrics registry.

+

+
+

+ + +

Functions

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
is_supported ()Check if application supports metrics rock for registry
init (opts)Initialize collectors in global metrics registry
destroy ()Unregister collectors in global metrics registry.
get ([space_name])Get copy of global metrics registry.
observe (space_name, latency, op, success)Increase requests count and update latency info.
observe_fetch (space_name, tuples_fetched, tuples_lookup)Increase statistics of storage select/pairs calls.
observe_map_reduces (count, space_name)Increase statistics of planned map reduces during select/pairs.
+ +
+
+ + +

Functions

+ +
+
+ + is_supported () +
+
+

Check if application supports metrics rock for registry +

metrics >= 0.10.0 is required. + metrics >= 0.9.0 is required to use summary quantiles with + age buckets. metrics >= 0.5.0, < 0.9.0 is unsupported + due to quantile overflow bug + (https://github.com/tarantool/metrics/issues/235). + metrics == 0.9.0 has bug that do not permits + to create summary collector without quantiles + (https://github.com/tarantool/metrics/issues/262). + In fact, user may use metrics >= 0.5.0, metrics != 0.9.0 + if he wants to use metrics without quantiles, and metrics >= 0.9.0 + if he wants to use metrics with quantiles. But this is confusing, + so we use a single restriction solving both cases. +

+ + + +

Returns:

+
    + + boolean + Returns true if metrics >= 0.10.0 found, false otherwise. +
+ + + + +
+
+ + init (opts) +
+
+

Initialize collectors in global metrics registry +

Registries are not meant to used explicitly + by users, init is not guaranteed to be idempotent. + Destroy collectors only through this registry methods. +

+ + +

Parameters:

+
    +
  • opts + + (tab) + + +
    + +

    +
    +

    +     +
      +
    • quantiles + + (bool) + +
      + +

      + If true, computes latency as 0.99 quantile with aging. +

      +       +
      • +
    +
+ +

Returns:

+
    + + boolean + Returns true. +
+ + + + +
+
+ + destroy () +
+
+

Unregister collectors in global metrics registry. +
+

+ Registries are not meant to used explicitly + by users, destroy is not guaranteed to be idempotent. + Destroy collectors only through this registry methods. +

+ + + +

Returns:

+
    + + boolean + Returns true. +
+ + + + +
+
+ + get ([space_name]) +
+
+

Get copy of global metrics registry. +
+

+ Registries are not meant to used explicitly + by users, get is not guaranteed to work without init. +

+ + +

Parameters:

+
    +
  • space_name + + (string, + + optional) + + +
    + +

    + If specified, returns table with statistics + of operations on table, separated by operation type and + execution status. If there wasn't any requests for table, + returns {}. If not specified, returns table with statistics + about all existing spaces, count of calls to spaces + that wasn't found and count of schema reloads. +

    +     +
    • +
+ +

Returns:

+
    + + table + Returns copy of metrics registry. +
+ + + + +
+
+ + observe (space_name, latency, op, success) +
+
+

Increase requests count and update latency info. +
+

+ + +

Parameters:

+
    +
  • space_name + + (string) + +
    + +

    + Name of space. +

    +     +
    • +
  • latency + + (number) + +
    + +

    + Time of call execution. +

    +     +
    • +
  • op + + (string) + +
    + +

    + Label of registry collectors. + Use require('crud.stats').op to pick one. +

    +     +
    • +
  • success + + (string) + +
    + +

    + 'ok' if no errors on execution, 'error' otherwise. +

    +     +
    • +
+ +

Returns:

+
    + + boolean + Returns true. +
+ + + + +
+
+ + observe_fetch (space_name, tuples_fetched, tuples_lookup) +
+
+

Increase statistics of storage select/pairs calls. +
+

+ + +

Parameters:

+
    +
  • space_name + + (string) + +
    + +

    + Name of space. +

    +     +
    • +
  • tuples_fetched + + (number) + +
    + +

    + Count of tuples fetched during storage call. +

    +     +
    • +
  • tuples_lookup + + (number) + +
    + +

    + Count of tuples looked up on storages while collecting response. +

    +     +
    • +
+ +

Returns:

+
    + + boolean + Returns true. +
+ + + + +
+
+ + observe_map_reduces (count, space_name) +
+
+

Increase statistics of planned map reduces during select/pairs. +
+

+ + +

Parameters:

+
    +
  • count + + (number) + +
    + +

    + Count of map reduces planned. +

    +     +
    • +
  • space_name + + (string) + +
    + +

    + Name of space. +

    +     +
    • +
+ +

Returns:

+
    + + boolean + Returns true. +
+ + + + +
+
+ + +
+
+
+ + diff --git a/doc/apidoc/modules/crud.stats.registry_utils.html b/doc/apidoc/modules/crud.stats.registry_utils.html new file mode 100644 index 000000000..f651953ff --- /dev/null +++ b/doc/apidoc/modules/crud.stats.registry_utils.html @@ -0,0 +1,192 @@ + + + + + CRUD API reference + + + + +
+ +
+ +
+
+
+ + +
+ + + + + + + + + +
+ +

Module crud.stats.registry_utils

+

Internal module used by statistics registries.

+

+
+

+ + +

Functions

+ + + + + + + + + +
build_collectors (op)Build collectors for local registry.
init_collectors_if_required (spaces, space_name, op)Initialize all statistic collectors for a space operation.
+ +
+
+ + +

Functions

+ +
+
+ + build_collectors (op) +
+
+

Build collectors for local registry. +
+

+ + +

Parameters:

+
    +
  • op + + (string) + +
    + +

    + Label of registry collectors. + Use require('crud.stats').op to pick one. +

    +     +
    • +
+ +

Returns:

+
    + + table + Returns collectors for success and error requests. + Collectors store 'count', 'latency' and 'time' values. Also + returns additional collectors for select operation. + +
+ + + + +
+
+ + init_collectors_if_required (spaces, space_name, op) +
+
+

Initialize all statistic collectors for a space operation. +
+

+ + +

Parameters:

+
    +
  • spaces + + (tab) + +
    + +

    + spaces section of registry. +

    +     +
    • +
  • space_name + + (string) + +
    + +

    + Name of space. +

    +     +
    • +
  • op + + (string) + +
    + +

    + Label of registry collectors. + Use require('crud.stats').op to pick one. +

    +     +
    • +
+ + + + + +
+
+ + +
+
+
+ + diff --git a/doc/apidoc/modules/crud.truncate.html b/doc/apidoc/modules/crud.truncate.html new file mode 100644 index 000000000..e36b86181 --- /dev/null +++ b/doc/apidoc/modules/crud.truncate.html @@ -0,0 +1,141 @@ + + + + + CRUD API reference + + + + +
+ +
+ +
+
+
+ + +
+ + + + + + + + + +
+ +

Module crud.truncate

+

Module

+

+
+

+ + +

Functions

+ + + + + +
truncate (space_name)Truncates specified space
+ +
+
+ + +

Functions

+ +
+
+ + truncate (space_name) +
+
+

Truncates specified space +
+

+ + +

Parameters:

+
    +
  • space_name + + (string) + +
    + +

    + A space name +

    +     +
    • +
+ +

Returns:

+
    + + boolean true +
+

Or

+
    +
  1. + nil + +
    +
    2. +
  2. + table + Error description
    3. +
+ + + + +
+
+ + +
+
+
+ + diff --git a/doc/apidoc/modules/crud.update.html b/doc/apidoc/modules/crud.update.html new file mode 100644 index 000000000..5de867bc0 --- /dev/null +++ b/doc/apidoc/modules/crud.update.html @@ -0,0 +1,161 @@ + + + + + CRUD API reference + + + + +
+ +
+ +
+
+
+ + +
+ + + + + + + + + +
+ +

Module crud.update

+

Module

+

+
+

+ + +

Functions

+ + + + + +
update (space_name, key, user_operations)Updates tuple in the specified space
+ +
+
+ + +

Functions

+ +
+
+ + update (space_name, key, user_operations) +
+
+

Updates tuple in the specified space +
+

+ + +

Parameters:

+
    +
  • space_name + + (string) + +
    + +

    + A space name +

    +     +
    • +
  • key +
    + +

    + Primary key value +

    +     +
    • +
  • user_operations + + (table) + +
    + +

    + Operations to be performed. + See space:update operations in Tarantool doc +

    +     +
    • +
+ +

Returns:

+
    + + object +
+

Or

+
    +
  1. + nil + +
    +
    2. +
  2. + table + Error description
    3. +
+ + + + +
+
+ + +
+
+
+ + diff --git a/doc/apidoc/modules/crud.upsert.html b/doc/apidoc/modules/crud.upsert.html new file mode 100644 index 000000000..755bc48cd --- /dev/null +++ b/doc/apidoc/modules/crud.upsert.html @@ -0,0 +1,80 @@ + + + + + CRUD API reference + + + + +
+ +
+ +
+
+
+ + +
+ + + + + + + + + +
+ +

Module crud.upsert

+

Module

+

+
+

+ + + +
+
+ + + + +
+
+
+ + diff --git a/test/integration/stats_test.lua b/test/integration/stats_test.lua index 5af04db91..cf937ca83 100644 --- a/test/integration/stats_test.lua +++ b/test/integration/stats_test.lua @@ -292,6 +292,80 @@ local simple_operation_cases = { }, } +local prepare_select_data = function(g) + helpers.insert_objects(g, space_name, { + -- Storage is s-2. + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, + -- Storage is s-2. + { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, + -- Storage is s-1. + { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, + -- Storage is s-2. + { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + } + }) +end + +local select_cases = { + select_by_primary_index = { + func = 'crud.select', + conditions = {{ '==', 'id_index', 3 }}, + map_reduces = 0, + tuples_fetched = 1, + tuples_lookup = 1, + }, + select_by_secondary_index = { + func = 'crud.select', + conditions = {{ '==', 'age_index', 46 }}, + map_reduces = 1, + tuples_fetched = 1, + tuples_lookup = 1, + }, + select_full_scan = { + func = 'crud.select', + conditions = {{ '>', 'id_index', 0 }, { '==', 'city', 'Kyoto' }}, + map_reduces = 1, + tuples_fetched = 0, + tuples_lookup = 4, + }, + pairs_by_primary_index = { + eval = eval.pairs, + conditions = {{ '==', 'id_index', 3 }}, + map_reduces = 0, + tuples_fetched = 1, + -- Since batch_size == 1, extra lookup is generated with + -- after_tuple scroll for second batch. + tuples_lookup = 2, + }, + pairs_by_secondary_index = { + eval = eval.pairs, + conditions = {{ '==', 'age_index', 46 }}, + map_reduces = 1, + tuples_fetched = 1, + -- Since batch_size == 1, extra lookup is generated with + -- after_tuple scroll for second batch. + tuples_lookup = 2, + }, + pairs_full_scan = { + eval = eval.pairs, + conditions = {{ '>', 'id_index', 0 }, { '==', 'city', 'Kyoto' }}, + map_reduces = 1, + tuples_fetched = 0, + tuples_lookup = 4, + }, +} + -- Generate non-null stats for all cases. local function generate_stats(g) for _, case in pairs(simple_operation_cases) do @@ -316,6 +390,19 @@ local function generate_stats(g) t.assert_not_equals(err, nil) end end + + -- Generate non-null select details. + prepare_select_data(g) + for _, case in pairs(select_cases) do + local _, err + if case.eval ~= nil then + _, err = g.router:eval(case.eval, { space_name, case.conditions }) + else + _, err = g.router:call(case.func, { space_name, case.conditions }) + end + + t.assert_equals(err, nil) + end end @@ -433,6 +520,53 @@ g.test_non_existing_space = function(g) end +for name, case in pairs(select_cases) do + local test_name = ('test_%s_details'):format(name) + + g.before_test(test_name, prepare_select_data) + + g[test_name] = function(g) + local op = 'select' + local space_name = space_name + + -- Collect stats before call. + local stats_before = g:get_stats(space_name) + t.assert_type(stats_before, 'table') + + -- Call operation. + local _, err + if case.eval ~= nil then + _, err = g.router:eval(case.eval, { space_name, case.conditions }) + else + _, err = g.router:call(case.func, { space_name, case.conditions }) + end + + t.assert_equals(err, nil) + + -- Collect stats after call. + local stats_after = g:get_stats(space_name) + t.assert_type(stats_after, 'table') + + local op_before = set_defaults_if_empty(stats_before, op) + local details_before = op_before.details + local op_after = set_defaults_if_empty(stats_after, op) + local details_after = op_after.details + + local tuples_fetched_diff = details_after.tuples_fetched - details_before.tuples_fetched + t.assert_equals(tuples_fetched_diff, case.tuples_fetched, + 'Expected count of tuples fetched') + + local tuples_lookup_diff = details_after.tuples_lookup - details_before.tuples_lookup + t.assert_equals(tuples_lookup_diff, case.tuples_lookup, + 'Expected count of tuples looked up on storage') + + local map_reduces_diff = details_after.map_reduces - details_before.map_reduces + t.assert_equals(map_reduces_diff, case.map_reduces, + 'Expected count of map reduces planned') + end +end + + g.before_test( 'test_role_reload_do_not_reset_observations', generate_stats) diff --git a/test/unit/select_executor_test.lua b/test/unit/select_executor_test.lua index ff09b9ca8..da0148751 100644 --- a/test/unit/select_executor_test.lua +++ b/test/unit/select_executor_test.lua @@ -105,7 +105,7 @@ g.test_one_condition_no_index = function() tarantool_iter = plan.tarantool_iter, scan_condition_num = plan.scan_condition_num, }) - t.assert_equals(get_ids(results), {2, 3}) + t.assert_equals(get_ids(results.tuples), {2, 3}) -- after tuple 2 local after_tuple = space:frommap(customers[2]):totable() @@ -115,7 +115,7 @@ g.test_one_condition_no_index = function() after_tuple = after_tuple, tarantool_iter = plan.tarantool_iter, }) - t.assert_equals(get_ids(results), {3}) + t.assert_equals(get_ids(results.tuples), {3}) -- after tuple 3 local after_tuple = space:frommap(customers[3]):totable() @@ -125,7 +125,7 @@ g.test_one_condition_no_index = function() after_tuple = after_tuple, tarantool_iter = plan.tarantool_iter, }) - t.assert_equals(#results, 0) + t.assert_equals(#results.tuples, 0) end g.test_one_condition_with_index = function() @@ -164,7 +164,7 @@ g.test_one_condition_with_index = function() scan_value = plan.scan_value, tarantool_iter = plan.tarantool_iter, }) - t.assert_equals(get_ids(results), {3, 2, 4}) -- in age order + t.assert_equals(get_ids(results.tuples), {3, 2, 4}) -- in age order -- after tuple 3 local after_tuple = space:frommap(customers[3]):totable() @@ -174,7 +174,7 @@ g.test_one_condition_with_index = function() after_tuple = after_tuple, tarantool_iter = plan.tarantool_iter, }) - t.assert_equals(get_ids(results), {2, 4}) -- in age order + t.assert_equals(get_ids(results.tuples), {2, 4}) -- in age order end g.test_multiple_conditions = function() @@ -220,7 +220,7 @@ g.test_multiple_conditions = function() scan_value = plan.scan_value, tarantool_iter = plan.tarantool_iter, }) - t.assert_equals(get_ids(results), {5, 2}) -- in age order + t.assert_equals(get_ids(results.tuples), {5, 2}) -- in age order -- after tuple 5 local after_tuple = space:frommap(customers[5]):totable() @@ -230,7 +230,7 @@ g.test_multiple_conditions = function() after_tuple = after_tuple, tarantool_iter = plan.tarantool_iter, }) - t.assert_equals(get_ids(results), {2}) + t.assert_equals(get_ids(results.tuples), {2}) end g.test_composite_index = function() @@ -271,7 +271,7 @@ g.test_composite_index = function() scan_value = plan.scan_value, tarantool_iter = plan.tarantool_iter, }) - t.assert_equals(get_ids(results), {2, 1, 4}) -- in full_name order + t.assert_equals(get_ids(results.tuples), {2, 1, 4}) -- in full_name order -- after tuple 2 local after_tuple = space:frommap(customers[2]):totable() @@ -281,7 +281,7 @@ g.test_composite_index = function() after_tuple = after_tuple, tarantool_iter = plan.tarantool_iter, }) - t.assert_equals(get_ids(results), {1, 4}) + t.assert_equals(get_ids(results.tuples), {1, 4}) end g.test_get_by_id = function() @@ -319,7 +319,7 @@ g.test_get_by_id = function() scan_value = plan.scan_value, tarantool_iter = plan.tarantool_iter, }) - t.assert_equals(get_ids(results), {2}) + t.assert_equals(get_ids(results.tuples), {2}) end g.test_early_exit = function() @@ -360,7 +360,7 @@ g.test_early_exit = function() scan_value = plan.scan_value, tarantool_iter = plan.tarantool_iter, }) - t.assert_equals(get_ids(results), {4, 2}) + t.assert_equals(get_ids(results.tuples), {4, 2}) end g.test_select_all = function() @@ -397,7 +397,7 @@ g.test_select_all = function() scan_value = plan.scan_value, tarantool_iter = plan.tarantool_iter, }) - t.assert_equals(get_ids(results), {1, 2, 3, 4}) + t.assert_equals(get_ids(results.tuples), {1, 2, 3, 4}) end g.test_limit = function() @@ -435,7 +435,7 @@ g.test_limit = function() tarantool_iter = plan.tarantool_iter, limit = 0, }) - t.assert_equals(#results, 0) + t.assert_equals(#results.tuples, 0) -- limit 2 local results = select_executor.execute(space, index, filter_func, { @@ -443,5 +443,5 @@ g.test_limit = function() tarantool_iter = plan.tarantool_iter, limit = 2, }) - t.assert_equals(get_ids(results), {1, 2}) + t.assert_equals(get_ids(results.tuples), {1, 2}) end diff --git a/test/unit/stats_test.lua b/test/unit/stats_test.lua index 628a9266f..8fced621d 100644 --- a/test/unit/stats_test.lua +++ b/test/unit/stats_test.lua @@ -171,6 +171,19 @@ for name, case in pairs(observe_cases) do }, 'Other status collectors initialized after observations' ) + + -- SELECT collectors have additional details section. + if op == stats_module.op.SELECT then + t.assert_equals( + op_stats.details, + { + tuples_fetched = 0, + tuples_lookup = 0, + map_reduces = 0, + }, + 'Detail collectors initialized after select observations' + ) + end end end end @@ -553,3 +566,52 @@ g.test_reset_for_disabled_stats_does_not_init_module = function(g) local stats_after = g:get_stats() t.assert_equals(stats_after, {}, "Stats is still empty") end + +g.test_fetch_stats_update = function(g) + local storage_cursor_stats = { tuples_fetched = 5, tuples_lookup = 25 } + + g.router:eval([[ stats_module.update_fetch_stats(...) ]], + { storage_cursor_stats, space_name }) + + local op = stats_module.op.SELECT + local stats = g:get_stats(space_name) + + t.assert_not_equals(stats[op], nil, + 'Fetch stats update inits SELECT collectors') + + local details = stats[op].details + + t.assert_equals(details.tuples_fetched, 5, + 'tuples_fetched is inremented by expected value') + t.assert_equals(details.tuples_lookup, 25, + 'tuples_lookup is inremented by expected value') +end + +g.test_disable_stats_do_not_break_fetch_stats_update_call = function(g) + local storage_cursor_stats = { tuples_fetched = 5, tuples_lookup = 25 } + + g:disable_stats() + + local _, err = g.router:eval([[ stats_module.update_fetch_stats(...) ]], + { storage_cursor_stats, space_name }) + t.assert_equals(err, nil) +end + +g.test_map_reduce_increment = function(g) + local op = stats_module.op.SELECT + + local _, err = g.router:eval([[ stats_module.update_map_reduces(...) ]], { space_name }) + t.assert_equals(err, nil) + + local stats = g:get_stats() + + t.assert_equals(stats.spaces[space_name][op].details.map_reduces, 1, + "Counter of map reduces incremented") +end + +g.test_disable_stats_do_not_break_map_reduce_update_call = function(g) + g:disable_stats() + + local _, err = g.router:eval([[ stats_module.update_map_reduces(...) ]], { space_name }) + t.assert_equals(err, nil) +end