Add warm boot support for virtual switch (sonic-net#374)

* Add warm boot support for virtual switch

* Update default boot type to cold

meta/sai_meta.cpp

@@ -1745,6 +1745,21 @@ sai_status_t meta_generic_validation_remove(
 
     if (count != 0)
     {
+        if (object_type == SAI_OBJECT_TYPE_SWITCH)
+        {
+            /*
+             * We allow to remove switch object even if there are ROUTE_ENTRY
+             * created and refrencing this switch, since remove could be used
+             * in WARM boot scenario.
+             */
+
+            SWSS_LOG_WARN("removing switch object 0x%lx reference count is %d, removing all objects from meta DB", oid, count);
+
+            meta_init_db();
+
+            return SAI_STATUS_SUCCESS;
+        }
+
         SWSS_LOG_ERROR("object 0x%lx reference count is %d, can't remove", oid, count);
 
         return SAI_STATUS_INVALID_PARAMETER;
@@ -2719,6 +2734,16 @@ void meta_generic_validation_post_remove(
 {
     SWSS_LOG_ENTER();
 
+    if (meta_key.objecttype == SAI_OBJECT_TYPE_SWITCH)
+    {
+        /*
+         * If switch object was removed then meta db was cleared and there are
+         * no other attributes, no need for reference counting.
+         */
+
+        return;
+    }
+
     // get all attributes that was set
 
     for (auto&it: get_object_attributes(meta_key))
@@ -5456,7 +5481,7 @@ sai_status_t meta_sai_validate_oid(
 
     if (oid == SAI_NULL_OBJECT_ID)
     {
-        SWSS_LOG_ERROR("oid is set to null object id");
+        SWSS_LOG_ERROR("oid is set to null object id on %s", otname);
 
         return SAI_STATUS_INVALID_PARAMETER;
     }

saiplayer/saiplayer.cpp

@@ -726,6 +726,7 @@ void handle_get_response(
     if (status != SAI_STATUS_SUCCESS)
     {
         // TODO check if status is correct for get
+        SWSS_LOG_WARN("status is not success: %s", sai_serialize_status(status).c_str());
         return;
     }
 

syncd/syncd_applyview.cpp

@@ -6218,9 +6218,13 @@ void populateExistingObjects(
             continue;
         }
 
+        // TODO this object in current view may have some attributes, we need to copy them
+        // to temp view, so they match on compare, or in case of those objects matched
+        // just ignore operations ? what about attributes with oids?
+
         temporaryView.createDummyExistingObject(rid, vid);
 
-        SWSS_LOG_DEBUG("populate existing %s RID %s VID %s",
+        SWSS_LOG_INFO("populate existing %s RID %s VID %s",
                 sai_serialize_object_type(sai_object_type_query(rid)).c_str(),
                 sai_serialize_object_id(rid).c_str(),
                 sai_serialize_object_id(vid).c_str());

syncd/syncd_hard_reinit.cpp

@@ -47,12 +47,10 @@ static sai_object_id_t g_switch_vid = SAI_NULL_OBJECT_ID;
 
 static std::shared_ptr<SaiSwitch> g_sw;
 
-#ifdef SAITHRIFT
 /*
  * SAI switch global needed for RPC server
  */
 extern sai_object_id_t gSwitchId;
-#endif
 
 void processAttributesForOids(
         _In_ sai_object_type_t objectType,
@@ -425,10 +423,8 @@ void processSwitches()
 
         sai_status_t status = sai_metadata_sai_switch_api->create_switch(&switch_rid, attr_count, attr_list);
 
-#ifdef SAITHRIFT
         gSwitchId = switch_rid;
         SWSS_LOG_NOTICE("Initialize gSwitchId with ID = 0x%lx", gSwitchId);
-#endif
 
         if (status != SAI_STATUS_SUCCESS)
         {
@@ -1298,5 +1294,12 @@ void performWarmRestart()
 
     g_sw = sw;
 
+    /*
+     * Populate gSwitchId since it's needed if we want to make multiple warm
+     * starts in a row.
+     */
+
+    gSwitchId = g_switch_rid;
+
     startDiagShell();
 }

vslib/inc/sai_vs.h

@@ -27,6 +27,12 @@ extern "C" {
 #define SAI_VALUE_VS_SWITCH_TYPE_BCM56850     "SAI_VS_SWITCH_TYPE_BCM56850"
 #define SAI_VALUE_VS_SWITCH_TYPE_MLNX2700     "SAI_VS_SWITCH_TYPE_MLNX2700"
 
+#define SAI_VS_COLD_BOOT 0
+#define SAI_VS_WARM_BOOT 1
+#define SAI_VS_FAST_BOOT 2
+
+#define SAI_VS_MAX_PORTS 1024
+
 /**
  * @def SAI_VS_UNITTEST_CHANNEL
  *
@@ -71,6 +77,12 @@ extern bool                             g_vs_hostif_use_tap_device;
 extern sai_vs_switch_type_t             g_vs_switch_type;
 extern std::recursive_mutex             g_recursive_mutex;
 
+extern int g_vs_boot_type;
+
+extern const char *g_boot_type;
+extern const char *g_warm_boot_read_file;
+extern const char *g_warm_boot_write_file;
+
 extern const sai_acl_api_t              vs_acl_api;
 extern const sai_bfd_api_t              vs_bfd_api;
 extern const sai_bridge_api_t           vs_bridge_api;

vslib/inc/sai_vs_switch_BCM56850.h

@@ -6,7 +6,8 @@
 #include "meta/saiattributelist.h"
 
 void init_switch_BCM56850(
-        _In_ sai_object_id_t switch_id);
+        _In_ sai_object_id_t switch_id,
+        _In_ std::shared_ptr<SwitchState> warmBootState);
 
 void uninit_switch_BCM56850(
         _In_ sai_object_id_t switch_id);

vslib/inc/sai_vs_switch_MLNX2700.h

@@ -6,7 +6,8 @@
 #include "meta/saiattributelist.h"
 
 void init_switch_MLNX2700(
-        _In_ sai_object_id_t switch_id);
+        _In_ sai_object_id_t switch_id,
+        _In_ std::shared_ptr<SwitchState> warmBootState);
 
 void uninit_switch_MLNX2700(
         _In_ sai_object_id_t switch_id);

vslib/src/sai_vs_generic_create.cpp

@@ -207,6 +207,53 @@ sai_object_id_t vs_create_real_object_id(
     return object_id;
 }
 
+void vs_update_real_object_ids(
+        _In_ const std::shared_ptr<SwitchState> warmBootState)
+{
+    SWSS_LOG_ENTER();
+
+    /*
+     * Since we loaded state from warm boot, we need to update real object id's
+     * in case a new object will be created. We need this so new objects will
+     * not have the same ID as existing ones.
+     */
+
+    for (auto oh: warmBootState->objectHash)
+    {
+        sai_object_type_t ot = oh.first;
+
+        if (ot == SAI_OBJECT_TYPE_NULL)
+            continue;
+
+        auto oi = sai_metadata_get_object_type_info(ot);
+
+        if (oi == NULL)
+        {
+            SWSS_LOG_THROW("failed to find object type info for object type: %d", ot);
+        }
+
+        if (oi->isnonobjectid)
+            continue;
+
+        for (auto o: oh.second)
+        {
+            sai_object_id_t oid;
+
+            sai_deserialize_object_id(o.first, oid);
+
+            // lower 32 bits on VS is real id on specific object type
+            uint64_t real_id = oid & ((uint64_t)-1) >> (64 - OT_POSITION);
+
+            if (real_ids[ot] <= real_id)
+            {
+                real_ids[ot] = real_id + 1; // +1 since this will be next object number
+            }
+
+            SWSS_LOG_INFO("update %s:%s real id to from %lu to %lu", oi->objecttypename, o.first.c_str(), real_id, real_ids[ot]);
+        }
+    }
+}
+
 void vs_free_real_object_id(
         _In_ sai_object_id_t object_id)
 {
@@ -218,6 +265,108 @@ void vs_free_real_object_id(
     }
 }
 
+std::shared_ptr<SwitchState> vs_read_switch_database_for_warm_restart(
+        _In_ sai_object_id_t switch_id)
+{
+    SWSS_LOG_ENTER();
+
+    if (g_warm_boot_read_file == NULL)
+    {
+        SWSS_LOG_ERROR("warm boot read file is NULL");
+        return nullptr;
+    }
+
+    std::ifstream dumpFile;
+
+    dumpFile.open(g_warm_boot_read_file);
+
+    if (!dumpFile.is_open())
+    {
+        SWSS_LOG_ERROR("failed to open: %s, switching to cold boot", g_warm_boot_read_file);
+
+        g_vs_boot_type = SAI_VS_COLD_BOOT;
+
+        return nullptr;
+    }
+
+    std::shared_ptr<SwitchState> ss = std::make_shared<SwitchState>(switch_id);
+
+    size_t count = 1; // count is 1 since switch_id was inserted to objectHash in SwitchState constructor
+
+    std::string line;
+    while (std::getline(dumpFile, line))
+    {
+        // line format: OBJECT_TYPE OBJECT_ID ATTR_ID ATTR_VALUE
+        std::istringstream iss(line);
+
+        std::string str_object_type;
+        std::string str_object_id;
+        std::string str_attr_id;
+        std::string str_attr_value;
+
+        iss >> str_object_type >> str_object_id >> str_attr_id >> str_attr_value;
+
+        sai_object_meta_key_t meta_key;
+
+        sai_deserialize_object_meta_key(str_object_type + ":" + str_object_id, meta_key);
+
+        auto &objectHash = ss->objectHash.at(meta_key.objecttype);
+
+        if (objectHash.find(str_object_id) == objectHash.end())
+        {
+            count++;
+
+            objectHash[str_object_id] = {};
+        }
+
+        if (str_attr_id == "NULL")
+        {
+            // skip empty attributes
+            continue;
+        }
+
+        if (meta_key.objecttype == SAI_OBJECT_TYPE_SWITCH)
+        {
+            if (meta_key.objectkey.key.object_id != switch_id)
+            {
+                SWSS_LOG_THROW("created switch id is %s but warm boot serialized is %s",
+                        sai_serialize_object_id(switch_id).c_str(),
+                        str_object_id.c_str());
+            }
+        }
+
+        auto meta = sai_metadata_get_attr_metadata_by_attr_id_name(str_attr_id.c_str());
+
+        if (meta == NULL)
+        {
+            SWSS_LOG_THROW("failed to find metadata for %s", str_attr_id.c_str());
+        }
+
+        // populate attributes
+
+        sai_attribute_t attr;
+
+        attr.id = meta->attrid;
+
+        sai_deserialize_attr_value(str_attr_value.c_str(), *meta, attr, false);
+
+        auto a = std::make_shared<SaiAttrWrap>(meta_key.objecttype, &attr);
+
+        objectHash[str_object_id][a->getAttrMetadata()->attridname] = a;
+
+        // free possible list attributes
+        sai_deserialize_free_attribute_value(meta->attrvaluetype, attr);
+    }
+
+    // NOTE notification pointers should be restored by attr_list when creating switch
+
+    dumpFile.close();
+
+    SWSS_LOG_NOTICE("loaded %zu objects from: %s", count, g_warm_boot_read_file);
+
+    return ss;
+}
+
 sai_status_t internal_vs_generic_create(
         _In_ sai_object_type_t object_type,
         _In_ const std::string &serialized_object_id,
@@ -229,20 +378,32 @@ sai_status_t internal_vs_generic_create(
 
     if (object_type == SAI_OBJECT_TYPE_SWITCH)
     {
+        std::shared_ptr<SwitchState> warmBootState = nullptr;
+
+        if (g_vs_boot_type == SAI_VS_WARM_BOOT)
+        {
+            warmBootState = vs_read_switch_database_for_warm_restart(switch_id);
+        }
+
         switch (g_vs_switch_type)
         {
             case SAI_VS_SWITCH_TYPE_BCM56850:
-                init_switch_BCM56850(switch_id);
+                init_switch_BCM56850(switch_id, warmBootState);
                 break;
 
             case SAI_VS_SWITCH_TYPE_MLNX2700:
-                init_switch_MLNX2700(switch_id);
+                init_switch_MLNX2700(switch_id, warmBootState);
                 break;
 
             default:
                 SWSS_LOG_WARN("unknown switch type: %d", g_vs_switch_type);
                 return SAI_STATUS_FAILURE;
         }
+
+        if (warmBootState != nullptr)
+        {
+            vs_update_real_object_ids(warmBootState);
+        }
     }
 
     auto &objectHash = g_switch_state_map.at(switch_id)->objectHash.at(object_type);