crud: support async bootstrap on storage

All Tarantool 3.x instances start in ro mode. Simple
`if not box.info.ro` check isn't enough to properly bootstrap a user
since it may pass before master instance became rw. box.watch is
used to track the moment when instance became rw [1]. Watchers are
supported since Tarantool 2.10+.

This patch doesn't change existing behavior for Cartridge roles and
`crud.init_storage()` calls.

1. https://www.tarantool.io/ru/doc/latest/reference/reference_lua/box_events/

Part of #412
Part of #415

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Added
 * Storage-side API to wait for bootstrap:
   `crud.wait_until_storage_ready()` and `crud.init_storage{wait_until_ready = true}` (#412).
+* Asynchronous storage bootstrap for Tarantool 2.10+ (#412).
 
 ### Fixed
 * Compatibility with vshard configuration if UUIDs are omitted (#407).

README.md

@@ -93,6 +93,12 @@ enough access to modify some space, then you need to give access to the user.
 You can call `crud.init_storage{wait_until_ready = true}` or
 `crud.wait_until_storage_ready()` on storages to wait till API is bootstrapped.
 
+You can call `crud.init_storage{async = true}` to bootstrap procedures grants
+asynchronously. It is useful in case your application master instances may
+start in ro mode (for example, if you use Tarantool 3.x). Use additional
+`wait_until_ready = true` flag or `crud.wait_until_storage_ready()`
+handle to wait for async bootstrap finish.
+
 All VShard routers should call `crud.init_router()` after `vshard.router.cfg()`
 (or enable the `crud-router` role for Cartridge) to make `crud` functions
 callable via `net.box`. If a user is allowed to execute `crud` functions on

crud/storage.lua

@@ -88,8 +88,20 @@ local function get_operation_user()
     return utils.get_this_replica_user() or 'guest'
 end
 
+local function init_storage_api()
+    local user = get_operation_user()
+
+    for _, module in ipairs(modules_with_storage_api) do
+        init_storage_call(user, module.storage_api)
+    end
+end
+
 function storage.init(opts)
-    checks({wait_until_ready = '?boolean', timeout = '?number'})
+    checks({
+        async = '?boolean',
+        wait_until_ready = '?boolean',
+        timeout = '?number',
+    })
 
     opts = opts or {}
 
@@ -99,10 +111,11 @@ function storage.init(opts)
 
     rawset(_G, utils.STORAGE_NAMESPACE, {})
 
-    local user = get_operation_user()
-
-    for _, module in ipairs(modules_with_storage_api) do
-        init_storage_call(user, module.storage_api)
+    if opts.async then
+        -- It's ok if init_storage_api() will be triggered several times.
+        box.watch('box.status', init_storage_api)
+    else
+        init_storage_api()
     end
 
     if opts.wait_until_ready then

test/helper.lua

@@ -708,6 +708,7 @@ function helpers.start_default_cluster(g, srv_name)
         router_init = entrypoint_vshard(srv_name, 'router_init', false),
         all_init = entrypoint_vshard(srv_name, 'all_init', false),
         crud_init = true,
+        async_storage_start = helpers.async_storage_start(),
     }
 
     helpers.start_cluster(g, cartridge_cfg, vshard_cfg)
@@ -944,4 +945,18 @@ function helpers.assert_str_contains_pattern_with_replicaset_id(str, pattern)
     t.assert(found, ("string %q does not contain pattern %q"):format(str, pattern))
 end
 
+
+local function instance_start_in_ro_mode()
+    local tarantool_version = luatest_utils.get_tarantool_version()
+    return luatest_utils.version_ge(tarantool_version, luatest_utils.version(3, 0, 0))
+end
+
+local function async_storage_start_supported()
+    return box.watch ~= nil
+end
+
+function helpers.async_storage_start()
+    return instance_start_in_ro_mode() and async_storage_start_supported()
+end
+
 return helpers

test/vshard_helpers/vtest.lua

@@ -404,11 +404,13 @@ local function cluster_new(g, cfg)
     local router_init = cfg.router_init
     local all_init = cfg.all_init
     local crud_init = cfg.crud_init
+    local async_storage_start = cfg.async_storage_start
 
     cfg.storage_init = nil
     cfg.router_init = nil
     cfg.all_init = nil
     cfg.crud_init = nil
+    cfg.async_storage_start = nil
 
     for replicaset_id, replicaset in pairs(cfg.sharding) do
         -- Luatest depends on box.cfg being ready and listening. Need to
@@ -548,9 +550,12 @@ local function cluster_new(g, cfg)
 
     if crud_init then
         for _, replica in pairs(all_servers) do
-            replica:exec(function()
-                require('crud').init_storage{wait_until_ready = false}
-            end)
+            replica:exec(function(async)
+                require('crud').init_storage{
+                    async = async,
+                    wait_until_ready = false,
+                }
+            end, {async_storage_start})
         end
 
         for _, replica in pairs(all_servers) do