diff --git a/common/Makefile.am b/common/Makefile.am index b47cfd217..7feb84ea5 100644 --- a/common/Makefile.am +++ b/common/Makefile.am @@ -70,7 +70,9 @@ libswsscommon_la_SOURCES = \ warm_restart.cpp \ luatable.cpp \ countertable.cpp \ - redisutility.cpp + redisutility.cpp \ + restart_waiter.cpp \ + redis_table_waiter.cpp libswsscommon_la_CXXFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(LIBNL_CFLAGS) $(CODE_COVERAGE_CXXFLAGS) libswsscommon_la_CPPFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(LIBNL_CPPFLAGS) $(CODE_COVERAGE_CPPFLAGS) diff --git a/common/redis_table_waiter.cpp b/common/redis_table_waiter.cpp new file mode 100644 index 000000000..3b7acfe92 --- /dev/null +++ b/common/redis_table_waiter.cpp @@ -0,0 +1,145 @@ +#include "redis_table_waiter.h" +#include "select.h" +#include "subscriberstatetable.h" + +using namespace swss; + +bool RedisTableWaiter::waitUntil( + DBConnector &db, + const std::string &tableName, + unsigned int maxWaitSec, + CheckFunc &checkFunc) +{ + if (maxWaitSec == 0) + { + SWSS_LOG_ERROR("Error: invalid maxWaitSec value 0, must be larger than 0"); + return false; + } + + SubscriberStateTable table(&db, tableName); + Select s; + s.addSelectable(&table); + + int maxWaitMs = static_cast(maxWaitSec) * 1000; + int selectTimeout = maxWaitMs; + auto start = std::chrono::steady_clock::now(); + while(1) + { + Selectable *sel = NULL; + int ret = s.select(&sel, selectTimeout, true); + if (ret == Select::OBJECT) + { + KeyOpFieldsValuesTuple kco; + table.pop(kco); + if (checkFunc(kco)) + { + return true; + } + } + else if (ret == Select::ERROR) + { + SWSS_LOG_NOTICE("Error: wait redis table got error - %s!", strerror(errno)); + } + else if (ret == Select::TIMEOUT) + { + SWSS_LOG_INFO("Timeout: wait redis table got select timeout"); + } + else if (ret == Select::SIGNALINT) + { + return false; + } + + auto end = std::chrono::steady_clock::now(); + int delay = static_cast( + std::chrono::duration_cast(end - start).count()); + + if (delay >= maxWaitMs) + { + return false; + } + + selectTimeout = maxWaitMs - delay; + } + + return false; +} + +bool RedisTableWaiter::waitUntilFieldSet( + DBConnector &db, + const std::string &tableName, + const std::string &key, + const std::string &fieldName, + unsigned int maxWaitSec, + ConditionFunc &cond) +{ + auto sep = SonicDBConfig::getSeparator(&db); + auto value = db.hget(tableName + sep + key, fieldName); + if (value && cond(*value.get())) + { + return true; + } + + CheckFunc checkFunc = [&](const KeyOpFieldsValuesTuple &kco) -> bool { + if (SET_COMMAND == kfvOp(kco)) + { + if (key == kfvKey(kco)) + { + auto& values = kfvFieldsValues(kco); + for (auto& fvt: values) + { + if (fieldName == fvField(fvt)) + { + return cond(fvValue(fvt)); + } + } + } + } + + return false; + }; + return waitUntil(db, tableName, maxWaitSec, checkFunc); +} + +bool RedisTableWaiter::waitUntilKeySet( + DBConnector &db, + const std::string &tableName, + const std::string &key, + unsigned int maxWaitSec) +{ + auto sep = SonicDBConfig::getSeparator(&db); + if (db.exists(tableName + sep + key)) + { + return true; + } + + CheckFunc checkFunc = [&](const KeyOpFieldsValuesTuple &kco) -> bool { + if (SET_COMMAND == kfvOp(kco)) + { + return key == kfvKey(kco); + } + return false; + }; + return waitUntil(db, tableName, maxWaitSec, checkFunc); +} + +bool RedisTableWaiter::waitUntilKeyDel( + DBConnector &db, + const std::string &tableName, + const std::string &key, + unsigned int maxWaitSec) +{ + auto sep = SonicDBConfig::getSeparator(&db); + if (!db.exists(tableName + sep + key)) + { + return true; + } + + CheckFunc checkFunc = [&](const KeyOpFieldsValuesTuple &kco) -> bool { + if (DEL_COMMAND == kfvOp(kco)) + { + return key == kfvKey(kco); + } + return false; + }; + return waitUntil(db, tableName, maxWaitSec, checkFunc); +} diff --git a/common/redis_table_waiter.h b/common/redis_table_waiter.h new file mode 100644 index 000000000..5cc773ae7 --- /dev/null +++ b/common/redis_table_waiter.h @@ -0,0 +1,43 @@ +#pragma once + +#include +#include + +#include "dbconnector.h" + +namespace swss +{ + +class RedisTableWaiter +{ +public: + typedef std::function ConditionFunc; + typedef std::function CheckFunc; + + static bool waitUntilFieldSet(DBConnector &db, + const std::string &tableName, + const std::string &key, + const std::string &fieldName, + unsigned int maxWaitSec, + ConditionFunc &cond); + + + static bool waitUntilKeySet(DBConnector &db, + const std::string &tableName, + const std::string &key, + unsigned int maxWaitSec); + + static bool waitUntilKeyDel(DBConnector &db, + const std::string &tableName, + const std::string &key, + unsigned int maxWaitSec); + + static bool waitUntil( + DBConnector &db, + const std::string &tableName, + unsigned int maxWaitSec, + CheckFunc &checkFunc); + +}; + +} \ No newline at end of file diff --git a/common/restart_waiter.cpp b/common/restart_waiter.cpp new file mode 100644 index 000000000..aca1c305b --- /dev/null +++ b/common/restart_waiter.cpp @@ -0,0 +1,92 @@ +#include "restart_waiter.h" +#include "redis_table_waiter.h" +#include "redispipeline.h" +#include "schema.h" +#include +#include + +using namespace swss; + +static const std::string STATE_DB_NAME = "STATE_DB"; +static const std::string STATE_DB_SEPARATOR = "|"; +static const std::string RESTART_KEY = "system"; +static const std::string RESTART_ENABLE_FIELD = "enable"; +static const std::string FAST_REBOOT_TABLE_NAME = "FAST_REBOOT"; + +// waitAdvancedBootDone +bool RestartWaiter::waitAdvancedBootDone( + unsigned int maxWaitSec, + unsigned int dbTimeout, + bool isTcpConn) +{ + DBConnector stateDb(STATE_DB_NAME, dbTimeout, isTcpConn); + return isAdvancedBootInProgress(stateDb) ? doWait(stateDb, maxWaitSec) : true; +} + +bool RestartWaiter::waitWarmBootDone( + unsigned int maxWaitSec, + unsigned int dbTimeout, + bool isTcpConn) +{ + DBConnector stateDb(STATE_DB_NAME, dbTimeout, isTcpConn); + if (isFastBootInProgress(stateDb)) + { + // It is fast boot, just return + return true; + } + + return isAdvancedBootInProgress(stateDb) ? doWait(stateDb, maxWaitSec) : true; +} + +bool RestartWaiter::waitFastBootDone( + unsigned int maxWaitSec, + unsigned int dbTimeout, + bool isTcpConn) +{ + DBConnector stateDb(STATE_DB_NAME, dbTimeout, isTcpConn); + if (!isFastBootInProgress(stateDb)) + { + // Fast boot is not in progress + return true; + } + + return isAdvancedBootInProgress(stateDb) ? doWait(stateDb, maxWaitSec) : true; +} + +bool RestartWaiter::doWait(DBConnector &stateDb, + unsigned int maxWaitSec) +{ + RedisTableWaiter::ConditionFunc condFunc = [](const std::string &value) -> bool { + std::string copy = value; + boost::to_lower(copy); + return copy == "false"; + }; + return RedisTableWaiter::waitUntilFieldSet(stateDb, + STATE_WARM_RESTART_ENABLE_TABLE_NAME, + RESTART_KEY, + RESTART_ENABLE_FIELD, + maxWaitSec, + condFunc); +} + +bool RestartWaiter::isAdvancedBootInProgress(DBConnector &stateDb) +{ + auto ret = stateDb.hget(STATE_WARM_RESTART_ENABLE_TABLE_NAME + STATE_DB_SEPARATOR + RESTART_KEY, RESTART_ENABLE_FIELD); + if (ret) { + std::string value = *ret.get(); + boost::to_lower(value); + return value == "true"; + } + return false; +} + +bool RestartWaiter::isFastBootInProgress(DBConnector &stateDb) +{ + auto ret = stateDb.get(FAST_REBOOT_TABLE_NAME + STATE_DB_SEPARATOR + RESTART_KEY); + return ret.get() != nullptr; +} + +bool RestartWaiter::isWarmBootInProgress(swss::DBConnector &stateDb) +{ + return isAdvancedBootInProgress(stateDb) && !isFastBootInProgress(stateDb); +} diff --git a/common/restart_waiter.h b/common/restart_waiter.h new file mode 100644 index 000000000..ea9eec7b6 --- /dev/null +++ b/common/restart_waiter.h @@ -0,0 +1,33 @@ +#pragma once + +#include "dbconnector.h" + +namespace swss +{ + +// Helper class to wait for warm/fast reboot done +class RestartWaiter +{ +public: + static bool waitAdvancedBootDone(unsigned int maxWaitSec = 180, + unsigned int dbTimeout = 0, + bool isTcpConn = false); + + static bool waitWarmBootDone(unsigned int maxWaitSec = 180, + unsigned int dbTimeout = 0, + bool isTcpConn = false); + + static bool waitFastBootDone(unsigned int maxWaitSec = 180, + unsigned int dbTimeout = 0, + bool isTcpConn = false); + + static bool isAdvancedBootInProgress(swss::DBConnector &stateDb); + static bool isFastBootInProgress(swss::DBConnector &stateDb); + static bool isWarmBootInProgress(swss::DBConnector &stateDb); + +private: + static bool doWait(swss::DBConnector &stateDb, + unsigned int maxWaitSec); +}; + +} diff --git a/pyext/swsscommon.i b/pyext/swsscommon.i index 41f2280cb..f7f0a8c3c 100644 --- a/pyext/swsscommon.i +++ b/pyext/swsscommon.i @@ -38,6 +38,8 @@ #include "events.h" #include "configdb.h" #include "status_code_util.h" +#include "redis_table_waiter.h" +#include "restart_waiter.h" %} %include @@ -219,3 +221,5 @@ T castSelectableObj(swss::Selectable *temp) %include "events.h" %include "status_code_util.h" +#include "redis_table_waiter.h" +%include "restart_waiter.h" diff --git a/tests/Makefile.am b/tests/Makefile.am index d53f88995..d6ddc291e 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -41,6 +41,8 @@ tests_SOURCES = redis_ut.cpp \ events_common_ut.cpp \ events_service_ut.cpp \ events_ut.cpp \ + restart_waiter_ut.cpp \ + redis_table_waiter_ut.cpp \ main.cpp tests_CFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(CFLAGS_GTEST) $(LIBNL_CFLAGS) diff --git a/tests/redis_table_waiter_ut.cpp b/tests/redis_table_waiter_ut.cpp new file mode 100644 index 000000000..6ed8116b3 --- /dev/null +++ b/tests/redis_table_waiter_ut.cpp @@ -0,0 +1,105 @@ +#include +#include +#include +#include + +#include "common/dbconnector.h" +#include "common/redis_table_waiter.h" +#include "common/table.h" + +using namespace swss; +using namespace std; + +const std::string dbName = "STATE_DB"; +const std::string tableName = "TestTable"; +const std::string key = "testKey"; +const std::string field = "testField"; +const std::string setValue = "1234"; + +static void set_field(int delay) +{ + if (delay > 0) + { + sleep(delay); + } + + DBConnector db(dbName, 0); + Table table(&db, tableName); + table.hset(key, field, setValue); +} + +static void del_key(int delay) +{ + if (delay > 0) + { + sleep(delay); + } + + DBConnector db(dbName, 0); + Table table(&db, tableName); + table.del(key); +} + +TEST(RedisTableWaiter, waitUntilFieldSet) +{ + del_key(0); + DBConnector db(dbName, 0); + RedisTableWaiter::ConditionFunc condFunc = [&](const std::string &value) -> bool { + return value == setValue; + }; + thread t(set_field, 1); + auto ret = RedisTableWaiter::waitUntilFieldSet(db, + tableName, + key, + field, + 3, + condFunc); + EXPECT_TRUE(ret); + t.join(); + // field already set + ret = RedisTableWaiter::waitUntilFieldSet(db, + tableName, + key, + field, + 3, + condFunc); + EXPECT_TRUE(ret); +} + +TEST(RedisTableWaiter, waitUntilKeySet) +{ + del_key(0); + DBConnector db(dbName, 0); + thread t(set_field, 1); + auto ret = RedisTableWaiter::waitUntilKeySet(db, + tableName, + key, + 3); + EXPECT_TRUE(ret); + t.join(); + // key already exist + ret = RedisTableWaiter::waitUntilKeySet(db, + tableName, + key, + 3); + EXPECT_TRUE(ret); +} + +TEST(RedisTableWaiter, waitUntilKeyDel) +{ + set_field(0); + DBConnector db(dbName, 0); + thread t(del_key, 1); + auto ret = RedisTableWaiter::waitUntilKeyDel(db, + tableName, + key, + 3); + EXPECT_TRUE(ret); + t.join(); + // key already removed + ret = RedisTableWaiter::waitUntilKeyDel(db, + tableName, + key, + 3); + EXPECT_TRUE(ret); +} diff --git a/tests/restart_waiter_ut.cpp b/tests/restart_waiter_ut.cpp new file mode 100644 index 000000000..47e619e48 --- /dev/null +++ b/tests/restart_waiter_ut.cpp @@ -0,0 +1,113 @@ +#include +#include +#include +#include +#include + +#include "common/dbconnector.h" +#include "common/restart_waiter.h" +#include "common/schema.h" +#include "common/table.h" + +using namespace swss; +using namespace std; + +static const string FAST_REBOOT_KEY = "FAST_REBOOT|system"; + +static void set_reboot_status(string status, int delay = 0) +{ + if (delay > 0) + { + sleep(delay); + } + + DBConnector db("STATE_DB", 0); + Table table(&db, STATE_WARM_RESTART_ENABLE_TABLE_NAME); + table.hset("system", "enable", status); +} + +class FastBootHelper +{ +public: + FastBootHelper(): db("STATE_DB", 0) + { + db.set(FAST_REBOOT_KEY, "1"); + } + + ~FastBootHelper() + { + db.del({FAST_REBOOT_KEY}); + } +private: + DBConnector db; +}; + +TEST(RestartWaiter, success) +{ + set_reboot_status("true"); + thread t(set_reboot_status, "false", 3); + EXPECT_TRUE(RestartWaiter::waitAdvancedBootDone()); + t.join(); +} + +TEST(RestartWaiter, successWarmReboot) +{ + set_reboot_status("true"); + thread t(set_reboot_status, "false", 3); + EXPECT_TRUE(RestartWaiter::waitWarmBootDone()); + t.join(); +} + +TEST(RestartWaiter, successFastReboot) +{ + FastBootHelper helper; + set_reboot_status("true"); + thread t(set_reboot_status, "false", 3); + EXPECT_TRUE(RestartWaiter::waitFastBootDone()); + t.join(); +} + +TEST(RestartWaiter, timeout) +{ + set_reboot_status("true"); + EXPECT_FALSE(RestartWaiter::waitAdvancedBootDone(1)); + EXPECT_FALSE(RestartWaiter::waitWarmBootDone(1)); + + FastBootHelper helper; + EXPECT_FALSE(RestartWaiter::waitFastBootDone(1)); + + set_reboot_status("false"); +} + +TEST(RestartWaiter, successNoDelay) +{ + set_reboot_status("false"); + EXPECT_TRUE(RestartWaiter::waitAdvancedBootDone()); + EXPECT_TRUE(RestartWaiter::waitWarmBootDone()); + + FastBootHelper helper; + EXPECT_TRUE(RestartWaiter::waitFastBootDone()); +} + +TEST(RestartWaiter, successNoKey) +{ + DBConnector db("STATE_DB", 0); + string key = string(STATE_WARM_RESTART_ENABLE_TABLE_NAME) + string("|system"); + db.del({key}); + EXPECT_TRUE(RestartWaiter::waitAdvancedBootDone()); + EXPECT_TRUE(RestartWaiter::waitWarmBootDone()); + + FastBootHelper helper; + EXPECT_TRUE(RestartWaiter::waitFastBootDone()); +} + +TEST(RestartWaiter, waitWarmButFastInProgress) +{ + FastBootHelper helper; + EXPECT_TRUE(RestartWaiter::waitWarmBootDone()); +} + +TEST(RestartWaiter, waitFastButFastNotInProgress) +{ + EXPECT_TRUE(RestartWaiter::waitFastBootDone()); +}