[active-standby] add knob to enable/disable oscillation (#250)

### Approach #### What is the motivation for this PR? To avoid test flakiness. ##### Work item tracking - Microsoft ADO **(number only)**: - 28187403 #### How did you do it? 1. Add DB interface to enable/disable the oscillation feature 2. Add DB interface to config the oscillation interval #### How did you verify/test it? Tested on lab device: 1. Knob was default on 2. Turned it off 3. Turned it on 4. Changed the interval #### Any platform specific information?
sonic-net · May 25, 2024 · f96d40c · f96d40c
1 parent dd61844
commit f96d40c
Show file tree

Hide file tree

Showing 7 changed files with 141 additions and 2 deletions.
diff --git a/src/DbInterface.cpp b/src/DbInterface.cpp
@@ -1076,6 +1076,34 @@ void DbInterface::processMuxLinkmgrConfigNotifiction(std::deque<swss::KeyOpField
                     }
                 }
 
+                MUXLOGINFO(boost::format("key: %s, Operation: %s, f: %s, v: %s") %
+                    key %
+                    operation %
+                    f %
+                    v
+                );
+            }
+        } else if (key == "TIMED_OSCILLATION") {
+            std::string operation = kfvOp(entry);
+            std::vector<swss::FieldValueTuple> fieldValues = kfvFieldsValues(entry);
+
+            for (auto &fieldValue: fieldValues) {
+                std::string f = fvField(fieldValue);
+                std::string v = fvValue(fieldValue);
+                if (f == "oscillation_enabled") {
+                        if (v == "true") {
+                            mMuxManagerPtr->setOscillationEnabled(true);
+                        } else if (v == "false"){
+                            mMuxManagerPtr->setOscillationEnabled(false);
+                        }
+                } else if (f == "interval_sec") {
+                    try {
+                        mMuxManagerPtr->setOscillationInterval_sec(boost::lexical_cast<uint32_t> (v));
+                    } catch (boost::bad_lexical_cast const &badLexicalCast) {
+                        MUXLOGWARNING(boost::format("bad lexical cast: %s") % badLexicalCast.what());
+                    }
+                }
+
                 MUXLOGINFO(boost::format("key: %s, Operation: %s, f: %s, v: %s") %
                     key %
                     operation %

diff --git a/src/MuxManager.h b/src/MuxManager.h
@@ -123,6 +123,28 @@ class MuxManager
     */
     inline void setTimeoutIpv6_msec(uint32_t timeout_msec) {mMuxConfig.setTimeoutIpv6_msec(timeout_msec);};
 
+    /**
+    *@method setOscillationEnabled
+    *
+    *@brief setter for enable/disable oscillation
+    *
+    *@param enable (in)  true to enable oscillation
+    *
+    *@return none
+    */
+    inline void setOscillationEnabled(bool enable) {mMuxConfig.setOscillationEnabled(enable);};
+
+    /**
+    *@method setOscillationInterval_sec
+    *
+    *@brief setter for oscillation interval in sec
+    *
+    *@param interval_sec (in)  interval in sec
+    *
+    *@return none
+    */
+    inline void setOscillationInterval_sec(uint32_t interval_sec) {mMuxConfig.setOscillationInterval_sec(interval_sec);};
+
     /**
     *@method setLinkProberStatUpdateIntervalCount
     *

diff --git a/src/common/MuxConfig.h b/src/common/MuxConfig.h
@@ -97,6 +97,17 @@ class MuxConfig
     */
     inline void setTimeoutIpv6_msec(uint32_t timeout_msec) {mTimeoutIpv6_msec = timeout_msec;};
 
+    /**
+    *@method setOscillationEnabled
+    *
+    *@brief setter for enabling/disabling mux state oscillation
+    *
+    *@param enable (in)  enable/disable mux state oscillation
+    *
+    *@return none
+    */
+    inline void setOscillationEnabled(bool enable) {mEnableTimedOscillationWhenNoHeartbeat = enable;};
+
     /**
     *@method setLinkProberStatUpdateIntervalCount
     *
@@ -274,6 +285,15 @@ class MuxConfig
     */
     inline uint32_t getSuspendTimeout_msec() const {return (mNegativeStateChangeRetryCount + 1) * mTimeoutIpv4_msec;};
 
+    /**
+    *@method getIfOscillationEnabled
+    *
+    *@brief getter for oscillation flag
+    *
+    *@return oscillation flag
+    */
+    inline bool getIfOscillationEnabled() const {return mEnableTimedOscillationWhenNoHeartbeat;};
+
     /**
     *@method getOscillationInterval_sec
     *
@@ -433,7 +453,6 @@ class MuxConfig
 
 private:
     uint8_t mNumberOfThreads = 5;
-    uint32_t mOscillationTimeout_sec = 300;
     uint32_t mTimeoutIpv4_msec = 100;
     uint32_t mTimeoutIpv6_msec = 1000;
     uint32_t mPositiveStateChangeRetryCount = 1;
@@ -443,6 +462,9 @@ class MuxConfig
     uint32_t mMuxStateChangeRetryCount = 1;
     uint32_t mLinkStateChangeRetryCount = 1;
 
+    bool mEnableTimedOscillationWhenNoHeartbeat = true;
+    uint32_t mOscillationTimeout_sec = 300;
+
     bool mEnableSwitchoverMeasurement = false;
     uint32_t mDecreasedTimeoutIpv4_msec = 10;
 

diff --git a/src/common/MuxPortConfig.h b/src/common/MuxPortConfig.h
@@ -192,6 +192,15 @@ class MuxPortConfig
     */
     inline uint32_t getLinkWaitTimeout_msec() const {return mMuxConfig.getSuspendTimeout_msec();};
 
+    /**
+    *@method getIfOscillationEnabled
+    *
+    *@brief getter for mux state oscillation enable flag
+    *
+    *@return oscillation enable flag
+    */
+    inline bool getIfOscillationEnabled() const {return mMuxConfig.getIfOscillationEnabled();};
+
     /**
     *@method getOscillationInterval_sec
     *

diff --git a/src/link_manager/LinkManagerStateMachineActiveStandby.cpp b/src/link_manager/LinkManagerStateMachineActiveStandby.cpp
@@ -1083,7 +1083,8 @@ void ActiveStandbyStateMachine::handleOscillationTimeout(boost::system::error_co
 {
     MUXLOGDEBUG(mMuxPortConfig.getPortName());
 
-    if (errorCode == boost::system::errc::success &&
+    if (mMuxPortConfig.getIfOscillationEnabled() &&
+        errorCode == boost::system::errc::success &&
         ps(mCompositeState) == link_prober::LinkProberState::Label::Wait &&
         ms(mCompositeState) == mux_state::MuxState::Label::Active &&
         ls(mCompositeState) == link_state::LinkState::Label::Up) {

diff --git a/test/MuxManagerTest.cpp b/test/MuxManagerTest.cpp
@@ -479,6 +479,20 @@ void MuxManagerTest::resetUpdateEthernetFrameFn(const std::string &portName)
     }
 }
 
+uint32_t MuxManagerTest::getOscillationInterval_sec(std::string port)
+{
+    std::shared_ptr<mux::MuxPort> muxPortPtr = mMuxManagerPtr->mPortMap[port];
+
+    return muxPortPtr->mMuxPortConfig.getOscillationInterval_sec();
+}
+
+bool MuxManagerTest::getOscillationEnabled(std::string port)
+{
+    std::shared_ptr<mux::MuxPort> muxPortPtr = mMuxManagerPtr->mPortMap[port];
+
+    return muxPortPtr->mMuxPortConfig.getIfOscillationEnabled();
+}
+
 TEST_F(MuxManagerTest, UpdatePortCableTypeActiveStandby)
 {
     std::string port = MuxManagerTest::PortName;
@@ -663,6 +677,42 @@ TEST_F(MuxManagerTest, SrcMacAddressUpdate)
     EXPECT_TRUE(mFakeLinkProber->mUpdateEthernetFrameCallCount == updateEthernetFrameCallCountBefore + 1);
 }
 
+TEST_P(OscillationIntervalTest, OscillationInterval)
+{
+    std::string port = "Ethernet0";
+    createPort(port);
+
+    std::deque<swss::KeyOpFieldsValuesTuple> entries = {
+        {"TIMED_OSCILLATION", "SET", {{"interval_sec", std::get<0>(GetParam())}}}
+    };
+    processMuxLinkmgrConfigNotifiction(entries);
+
+    EXPECT_TRUE(getOscillationInterval_sec(port) == std::get<1>(GetParam()));
+}
+
+INSTANTIATE_TEST_CASE_P(
+    OscillationInterval,
+    OscillationIntervalTest,
+    ::testing::Values(
+        std::make_tuple("1200", 1200),
+        std::make_tuple("1", 300),
+        std::make_tuple("invalid", 300)
+    )
+);
+
+TEST_F(MuxManagerTest, OscillationDisabled)
+{
+    std::string port = "Ethernet0";
+    createPort(port);
+
+    std::deque<swss::KeyOpFieldsValuesTuple> entries = {
+        {"TIMED_OSCILLATION", "SET", {{"oscillation_enabled", "false"}}}
+    };
+    processMuxLinkmgrConfigNotifiction(entries);
+
+    EXPECT_TRUE(getOscillationEnabled(port) == false);
+}
+
 TEST_F(MuxManagerTest, ServerMacAddress)
 {
     std::string port = "Ethernet0";

diff --git a/test/MuxManagerTest.h b/test/MuxManagerTest.h
@@ -56,9 +56,11 @@ class MuxManagerTest: public testing::Test
     uint32_t getTimeoutIpv4_msec(std::string port);
     uint32_t getTimeoutIpv6_msec(std::string port);
     uint32_t getLinkWaitTimeout_msec(std::string port);
+    uint32_t getOscillationInterval_sec(std::string port);
     bool getIfUseWellKnownMac(std::string port);
     bool setUseWellKnownMacActiveActive(bool use);
     bool getIfUseToRMac(std::string port);
+    bool getOscillationEnabled(std::string port);
     boost::asio::ip::address getBladeIpv4Address(std::string port);
     std::array<uint8_t, ETHER_ADDR_LEN> getBladeMacAddress(std::string port);
     std::array<uint8_t, ETHER_ADDR_LEN> getLastUpdatedMacAddress(std::string port);
@@ -123,6 +125,11 @@ class MuxConfigUpdateTest: public MuxManagerTest,
 {
 };
 
+class OscillationIntervalTest: public MuxManagerTest,
+                               public testing::WithParamInterface<std::tuple<std::string, uint32_t>>
+{
+};
+
 } /* namespace test */
 
 #endif /* MUXMANAGERTEST_H_ */