[Fast/Warm restart] Implement helper class for waiting restart done

common/Makefile.am

@@ -70,7 +70,9 @@ libswsscommon_la_SOURCES = \
     warm_restart.cpp          \
     luatable.cpp              \
     countertable.cpp          \
-    redisutility.cpp
+    redisutility.cpp          \
+    restart_waiter.cpp        \
+    redis_table_waiter.cpp
 
 libswsscommon_la_CXXFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(LIBNL_CFLAGS) $(CODE_COVERAGE_CXXFLAGS)
 libswsscommon_la_CPPFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(LIBNL_CPPFLAGS) $(CODE_COVERAGE_CPPFLAGS)

common/redis_table_waiter.cpp

@@ -0,0 +1,145 @@
+#include "redis_table_waiter.h"
+#include "select.h"
+#include "subscriberstatetable.h"
+
+using namespace swss;
+
+bool RedisTableWaiter::waitUntil(
+    DBConnector &db,
+    const std::string &tableName,
+    unsigned int maxWaitSec,
+    CheckFunc checkFunc)
+{
+    if (maxWaitSec == 0)
+    {
+        SWSS_LOG_ERROR("Error: invalid maxWaitSec value 0, must be larger than 0");
+        return false;
+    }
+
+    SubscriberStateTable table(&db, tableName);
+    Select s;
+    s.addSelectable(&table);
+
+    int maxWaitMs = static_cast<int>(maxWaitSec) * 1000;
+    int selectTimeout = maxWaitMs;
+    auto start = std::chrono::steady_clock::now();
+    while(1)
+    {
+        Selectable *sel = NULL;
+        int ret = s.select(&sel, selectTimeout, true);
+        if (ret == Select::OBJECT)
+        {
+            KeyOpFieldsValuesTuple kco;
+            table.pop(kco);
+            if (checkFunc(kco))
+            {
+                return true;
+            }
+        }
+        else if (ret == Select::ERROR)
+        {
+            SWSS_LOG_NOTICE("Error: wait redis table got error - %s!", strerror(errno));
+        }
+        else if (ret == Select::TIMEOUT)
+        {
+            SWSS_LOG_INFO("Timeout: wait redis table got select timeout");
+        }
+        else if (ret == Select::SIGNALINT)
+        {
+            return false;
+        }
+
+        auto end = std::chrono::steady_clock::now();
+        int delay = static_cast<int>(
+                    std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count());
+
+        if (delay >= maxWaitMs)
+        {
+            return false;
+        }
+
+        selectTimeout = maxWaitMs - delay;
+    }
+
+    return false;
+}
+
+bool RedisTableWaiter::waitUntilFieldSet(
+    DBConnector &db,
+    const std::string &tableName,
+    const std::string &key,
+    const std::string &fieldName,
+    unsigned int maxWaitSec,
+    ConditionFunc cond)
+{
+    auto sep = SonicDBConfig::getSeparator(&db);
+    auto value = db.hget(tableName + sep + key, fieldName);
+    if (value && cond(*value.get()))
+    {
+        return true;
+    }
+
+    auto checkFunc = [&](const KeyOpFieldsValuesTuple &kco) -> bool {
+        if (SET_COMMAND == kfvOp(kco))
+        {
+            if (key == kfvKey(kco))
+            {
+                auto& values = kfvFieldsValues(kco);
+                for (auto& fvt: values)
+                {
+                    if (fieldName == fvField(fvt))
+                    {
+                        return cond(fvValue(fvt));
+                    }
+                }
+            }
+        }
+
+        return false;
+    };
+    return waitUntil(db, tableName, maxWaitSec, checkFunc);
+}
+
+bool RedisTableWaiter::waitUntilKeySet(
+    DBConnector &db,
+    const std::string &tableName,
+    const std::string &key,
+    unsigned int maxWaitSec)
+{
+    auto sep = SonicDBConfig::getSeparator(&db);
+    if (db.exists(tableName + sep + key))
+    {
+        return true;
+    }
+
+    auto checkFunc = [&](const KeyOpFieldsValuesTuple &kco) -> bool {
+        if (SET_COMMAND == kfvOp(kco))
+        {
+            return key == kfvKey(kco);
+        }
+        return false;
+    };
+    return waitUntil(db, tableName, maxWaitSec, checkFunc);
+}
+
+bool RedisTableWaiter::waitUntilKeyDel(
+    DBConnector &db,
+    const std::string &tableName,
+    const std::string &key,
+    unsigned int maxWaitSec)
+{
+    auto sep = SonicDBConfig::getSeparator(&db);
+    if (!db.exists(tableName + sep + key))
+    {
+        return true;
+    }
+
+    auto checkFunc = [&](const KeyOpFieldsValuesTuple &kco) -> bool {
+        if (DEL_COMMAND == kfvOp(kco))
+        {
+            return key == kfvKey(kco);
+        }
+        return false;
+    };
+    return waitUntil(db, tableName, maxWaitSec, checkFunc);
+}

common/redis_table_waiter.h

@@ -0,0 +1,43 @@
+#pragma once
+
+#include <functional>
+#include <string>
+
+#include "dbconnector.h"
+
+namespace swss
+{
+
+class RedisTableWaiter
+{
+public:
+    typedef std::function<bool(const std::string &)> ConditionFunc;
+    typedef std::function<bool(const KeyOpFieldsValuesTuple &)> CheckFunc;
+
+    static bool waitUntilFieldSet(DBConnector &db,
+                          const std::string &tableName,
+                          const std::string &key,
+                          const std::string &fieldName,
+                          unsigned int maxWaitSec,
+                          ConditionFunc cond);
+
+
+    static bool waitUntilKeySet(DBConnector &db,
+                          const std::string &tableName,
+                          const std::string &key,
+                          unsigned int maxWaitSec);
+
+    static bool waitUntilKeyDel(DBConnector &db,
+                          const std::string &tableName,
+                          const std::string &key,
+                          unsigned int maxWaitSec);
+
+    static bool waitUntil(
+        DBConnector &db,
+        const std::string &tableName,
+        unsigned int maxWaitSec,
+        CheckFunc checkFunc);
+
+};
+
+}

common/restart_waiter.cpp

@@ -0,0 +1,92 @@
+#include "restart_waiter.h"
+#include "redis_table_waiter.h"
+#include "redispipeline.h"
+#include "schema.h"
+#include <boost/algorithm/string.hpp>
+#include <string>
+
+using namespace swss;
+
+static const std::string STATE_DB_NAME = "STATE_DB";
+static const std::string STATE_DB_SEPARATOR = "|";
+static const std::string RESTART_KEY = "system";
+static const std::string RESTART_ENABLE_FIELD = "enable";
+static const std::string FAST_REBOOT_TABLE_NAME = "FAST_REBOOT";
+
+// waitAdvancedBootDone
+bool RestartWaiter::waitAdvancedBootDone(
+    unsigned int maxWaitSec,
+    unsigned int dbTimeout,
+    bool isTcpConn)
+{
+    DBConnector stateDb(STATE_DB_NAME, dbTimeout, isTcpConn);
+    return isWarmOrFastBootInProgress(stateDb) ? doWait(stateDb, maxWaitSec) : true;
+}
+
+bool RestartWaiter::waitWarmBootDone(
+    unsigned int maxWaitSec,
+    unsigned int dbTimeout,
+    bool isTcpConn)
+{
+    DBConnector stateDb(STATE_DB_NAME, dbTimeout, isTcpConn);
+    if (isFastBootInProgress(stateDb))
+    {
+        // It is fast boot, just return
+        return true;
+    }
+
+    return isWarmOrFastBootInProgress(stateDb) ? doWait(stateDb, maxWaitSec) : true;
+}
+
+bool RestartWaiter::waitFastBootDone(
+    unsigned int maxWaitSec,
+    unsigned int dbTimeout,
+    bool isTcpConn)
+{
+    DBConnector stateDb(STATE_DB_NAME, dbTimeout, isTcpConn);
+    if (!isFastBootInProgress(stateDb))
+    {
+        // Fast boot is not in progress
+        return true;
+    }
+
+    return isWarmOrFastBootInProgress(stateDb) ? doWait(stateDb, maxWaitSec) : true;
+}
+
+bool RestartWaiter::doWait(DBConnector &stateDb,
+                           unsigned int maxWaitSec)
+{
+    auto condFunc = [](const std::string &value) -> bool {
+        std::string copy = value;
+        boost::to_lower(copy);
+        return copy == "false";
+    };
+    return RedisTableWaiter::waitUntilFieldSet(stateDb,
+                                               STATE_WARM_RESTART_ENABLE_TABLE_NAME,
+                                               RESTART_KEY,
+                                               RESTART_ENABLE_FIELD,
+                                               maxWaitSec,
+                                               condFunc);
+}
+
+bool RestartWaiter::isWarmOrFastBootInProgress(DBConnector &stateDb)
+{
+    auto ret = stateDb.hget(STATE_WARM_RESTART_ENABLE_TABLE_NAME + STATE_DB_SEPARATOR + RESTART_KEY, RESTART_ENABLE_FIELD);
+    if (ret) {
+        std::string value = *ret.get();
+        boost::to_lower(value);
+        return value == "true";
+    }
+    return false;
+}
+
+bool RestartWaiter::isFastBootInProgress(DBConnector &stateDb)
+{
+    auto ret = stateDb.get(FAST_REBOOT_TABLE_NAME + STATE_DB_SEPARATOR + RESTART_KEY);
+    return ret.get() != nullptr;
+}
+
+bool RestartWaiter::isWarmBootInProgress(swss::DBConnector &stateDb)
+{
+    return isWarmOrFastBootInProgress(stateDb) && !isFastBootInProgress(stateDb);
+}

common/restart_waiter.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include "dbconnector.h"
+
+namespace swss
+{
+
+// Helper class to wait for warm/fast reboot done
+class RestartWaiter
+{
+public:
+    static bool waitAdvancedBootDone(unsigned int maxWaitSec = 180,
+                                     unsigned int dbTimeout = 0,
+                                     bool isTcpConn = false);
+
+    static bool waitWarmBootDone(unsigned int maxWaitSec = 180,
+                                    unsigned int dbTimeout = 0,
+                                    bool isTcpConn = false);
+
+    static bool waitFastBootDone(unsigned int maxWaitSec = 180,
+                                    unsigned int dbTimeout = 0,
+                                    bool isTcpConn = false);
+
+    static bool isWarmOrFastBootInProgress(swss::DBConnector &stateDb);
+    static bool isFastBootInProgress(swss::DBConnector &stateDb);
+    static bool isWarmBootInProgress(swss::DBConnector &stateDb);
+
+private:
+    static bool doWait(swss::DBConnector &stateDb,
+                       unsigned int maxWaitSec);
+};
+
+}

pyext/swsscommon.i

@@ -38,6 +38,8 @@
 #include "events.h"
 #include "configdb.h"
 #include "status_code_util.h"
+#include "redis_table_waiter.h"
+#include "restart_waiter.h"
 %}
 
 %include <std_string.i>
@@ -219,3 +221,5 @@ T castSelectableObj(swss::Selectable *temp)
 %include "events.h"
 
 %include "status_code_util.h"
+#include "redis_table_waiter.h"
+%include "restart_waiter.h"

tests/Makefile.am

@@ -41,6 +41,8 @@ tests_SOURCES = redis_ut.cpp                \
                 events_common_ut.cpp        \
                 events_service_ut.cpp       \
                 events_ut.cpp               \
+                restart_waiter_ut.cpp       \
+                redis_table_waiter_ut.cpp   \
                 main.cpp
 
 tests_CFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(CFLAGS_GTEST) $(LIBNL_CFLAGS)