Improve decorator to support profile config.

common/configdb.h

@@ -245,7 +245,30 @@ class ConfigDBConnector_Native : public SonicV2Connector_Native
         def __init__(self, config_db_connector):
             self.connector = config_db_connector
             self.default_value_provider = DefaultValueProvider()
-        # helper methods for append default values to result.
+        # helper methods for append profile and default values to result.
+        def _append_profile(self, result):
+            client = self.connector.get_redis_client(self.connector.db_name)
+            profile_config = ProfileProvider.instance().getConfigs(client)
+            for table_name in profile_config:
+                profile_table = profile_config[table_name]
+                if table_name not in result:
+                    result[table_name] = profile_table
+                else:
+                    config_table = result[table_name]
+                    for profile_key in profile_table:
+                        if profile_key not in config_table:
+                            config_table[profile_key] = profile_table[profile_key]
+        def _append_profile_table(self, table, result):
+            client = self.connector.get_redis_client(self.connector.db_name)
+            profile_keys = ProfileProvider.instance().getKeys(table, client)
+            for profile_key in profile_keys:
+                if profile_key not in result:
+                    serialized_key = self.connector.serialize_key(profile_key)
+                    result[profile_key] = ProfileProvider.instance().getConfigs(table, serialized_key, client)
+        def _get_profile(self, table, key):
+            serialized_key = self.connector.serialize_key(key)
+            client = self.connector.get_redis_client(self.connector.db_name)
+            return ProfileProvider.instance().getConfigs(table, serialized_key, client)
         def _append_default_value(self, table, key, data):
             if data is None or len(data) == 0:
                 # empty entry means the entry been deleted
@@ -255,32 +278,95 @@ class ConfigDBConnector_Native : public SonicV2Connector_Native
             for field in defaultValues:
                 if field not in data:
                     data[field] = defaultValues[field]
+        def _try_delete_or_revert_profile(self, table, key, data):
+            serialized_key = self.connector.serialize_key(key)
+            client = self.connector.get_redis_client(self.connector.db_name)
+            if data is None or len(data) == 0:
+                # set a entry to empty will delete this entry
+                ProfileProvider.instance().tryDeleteItem(table, serialized_key, client)
+            else:
+                ProfileProvider.instance().tryRevertItem(table, serialized_key, client)
+        def _try_delete_profile_table(self, table):
+            client = self.connector.get_redis_client(self.connector.db_name)
+            keys = ProfileProvider.instance().getKeys(table, client)
+            for key in keys:
+                serialized_key = self.connector.serialize_key(key)
+                ProfileProvider.instance().tryDeleteItem(table, serialized_key, client)
+        def _try_delete_profile(self, config):
+            # the implementation of this method highly related with original mod_config behavior:
+            # 1. any table does not exist in config will not be changed.
+            # 2. any key remove from table will be deleted.
+            # 3. any key exist in table will be updated: if set data of that key to {}, the key also will be deleted.
+            client = self.connector.get_redis_client(self.connector.db_name)
+            profile_config = ProfileProvider.instance().getConfigs(client)
+            # delete/revert key in modified config tables, according to (1) not handle not existed tables
+            for tablename in config:
+                table = config[tablename]
+                if tablename not in profile_config:
+                    continue
+
+                profile_table = profile_config[tablename]
+                # delete removed key according to (2)
+                for key in profile_table:
+                    if key not in table:
+                        serialized_key = self.connector.serialize_key(key)
+                        ProfileProvider.instance().tryDeleteItem(tablename, serialized_key, client)
+                # try delete/revert still existed key according to (3)
+                for key in table:
+                    self._try_delete_or_revert_profile(tablename, key, table[key])
         # override read APIs
         def new_get_entry(self, table, key):
             result = self.connector.get_entry(table, key)
+            if result is None or len(result) == 0:
+                # when there are any user config, profile will be overwrite.
+                result = self._get_profile(table, key)
             self._append_default_value(table, key, result)
             return result
         def new_get_table(self, table):
             result = self.connector.get_table(table)
+            self._append_profile_table(table, result)
             for key in result:
                 self._append_default_value(table, key, result[key])
             return result
         def new_get_config(self):
             result = self.connector.get_config()
+            self._append_profile(result)
             for table in result:
                 for key in result[table]:
                     # Can not pass result[table][key] as parameter here, because python will create a copy. re-assign entry to result to bypass this issue.
                     entry = result[table][key]
                     self._append_default_value(table, key, entry)
                     result[table][key] = entry
             return result
+        # override write and delete APIs
+        def new_set_entry(self, table, key, data):
+            # set a entry to empty will delete this entry
+            self._try_delete_or_revert_profile(table, key, data)
+            return self.connector.set_entry(table, key, data)
+        def new_mod_entry(self, table, key, data):
+            self._try_delete_or_revert_profile(table, key, data)
+            return self.connector.mod_entry(table, key, data)
+        def new_delete_table(self, table):
+            self._try_delete_profile_table(table)
+            return self.connector.delete_table(table)
+        def new_mod_config(self, config):
+            self._try_delete_profile(config)
+            return self.connector.mod_config(config)
         def __getattr__(self, name):
             if name == "get_entry":
                 return self.new_get_entry
             elif name == "get_table":
                 return self.new_get_table
             elif name == "get_config":
                 return self.new_get_config
+            elif name == "set_entry":
+                return self.new_set_entry
+            elif name == "mod_entry":
+                return self.new_mod_entry
+            elif name == "delete_table":
+                return self.new_delete_table
+            elif name == "mod_config":
+                return self.new_mod_config
 
             originalMethod = self.connector.__getattribute__(name)
             return originalMethod

common/decoratorsubscriberstatetable.cpp

@@ -1,13 +1,21 @@
 #include <boost/algorithm/string.hpp>
 #include "common/decoratorsubscriberstatetable.h"
 #include "common/defaultvalueprovider.h"
+#include "common/profileprovider.h"
 
 using namespace std;
 using namespace swss;
 
 DecoratorSubscriberStateTable::DecoratorSubscriberStateTable(DBConnector *db, const string &tableName, int popBatchSize, int pri)
 :SubscriberStateTable(db, tableName, popBatchSize, pri)
 {
+    // Subscribe PROFILE_DELETE table for profile delete event
+    m_profile_keyspace = "__keyspace@";
+    m_profile_keyprefix = ProfileProvider::instance().getDeletedKeyName(tableName, "", db);
+    m_profile_keyspace += to_string(db->getDbId()) + "__:" + m_profile_keyprefix + "*";
+
+    m_subscribe->psubscribe(m_profile_keyspace);
+
     m_defaultValueProvider = std::make_shared<DefaultValueProvider>();
 }
 
@@ -32,3 +40,35 @@ void DecoratorSubscriberStateTable::appendDefaultValue(std::string &key, std::st
     auto table = getTableName();
     m_defaultValueProvider->appendDefaultValues(table, key, fvs);
 }
+
+void DecoratorSubscriberStateTable::onPopUnknownPattern(RedisMessage& message, deque<KeyOpFieldsValuesTuple> &vkco)
+{
+    if (message.pattern != m_profile_keyspace)
+    {
+        SWSS_LOG_ERROR("invalid pattern %s returned for pmessage of %s", message.pattern.c_str(), m_profile_keyspace.c_str());
+        SubscriberStateTable::onPopUnknownPattern(message, vkco);
+        return;
+    }
+
+    string op = message.data;
+    if ("del" == op)
+    {
+        // 'DEL' from PROFILE_DETETE table will revert profile config, ignore this event because there will always be a user config 'SET' event after this event.
+        return;
+    }
+
+    string msg = message.channel;
+    size_t pos = msg.find(m_profile_keyprefix);
+    if (pos == msg.npos)
+    {
+        SWSS_LOG_ERROR("invalid key returned for pmessage of %s", m_profile_keyspace.c_str());
+        return;
+    }
+
+    // 'SET' to PROFILE_DETETE table is delete profile config, convert this event to a config 'DEL' event
+    string key = msg.substr(pos + m_profile_keyprefix.length());
+    KeyOpFieldsValuesTuple kco;
+    kfvKey(kco) = key;
+    kfvOp(kco) = DEL_COMMAND;
+    vkco.push_back(kco);
+}

common/decoratorsubscriberstatetable.h

@@ -23,8 +23,15 @@ class DecoratorSubscriberStateTable : public SubscriberStateTable
     void pops(std::deque<KeyOpFieldsValuesTuple> &vkco, const std::string &prefix = EMPTY_PREFIX) override;
 
 private:
+    std::string m_profile_keyspace;
+
+    std::string m_profile_keyprefix;
+
     std::shared_ptr<DefaultValueProvider> m_defaultValueProvider;
 
+    /* Handle SubscriberStateTable unknown pattern */
+    void onPopUnknownPattern(RedisMessage& message, std::deque<KeyOpFieldsValuesTuple> &vkco) override;
+
     void appendDefaultValue(std::string &key, std::string &op, std::vector<FieldValueTuple> &fvs);
 };
 

common/decoratortable.cpp

@@ -1,6 +1,7 @@
 #include <boost/algorithm/string.hpp>
 
 #include "common/decoratortable.h"
+#include "common/profileprovider.h"
 
 using namespace std;
 using namespace swss;
@@ -21,10 +22,22 @@ DecoratorTable::DecoratorTable(RedisPipeline *pipeline, const string &tableName,
 bool DecoratorTable::get(const string &key, vector<pair<string, string>> &ovalues)
 {
     bool result = Table::get(key, ovalues);
+    auto connector = const_cast<DBConnector*>(m_pipe->getDBConnector());
     auto table = getTableName();
 
-    // Append default values
-    m_defaultValueProvider->appendDefaultValues(table, key, ovalues);
+    // Append profile config
+    bool append = ProfileProvider::instance().appendConfigs(table, key, ovalues, connector);
+    if (!result && append)
+    {
+        // No user config on this key, but found profile config on this key.
+        result = true;
+    }
+
+    // Append default values when key exist
+    if (result)
+    {
+        m_defaultValueProvider->appendDefaultValues(table, key, ovalues);
+    }
 
     return result;
 }
@@ -39,8 +52,17 @@ bool DecoratorTable::hget(const string &key, const string &field,  string &value
         return true;
     }
 
+    auto connector = const_cast<DBConnector*>(m_pipe->getDBConnector());
     auto table = getTableName();
 
+    // Try append profile config
+    auto profile = ProfileProvider::instance().getConfig(table, key, field, connector);
+    if (profile != nullptr)
+    {
+        value = *profile;
+        return true;
+    }
+
     // Try append default values
     auto default_value = m_defaultValueProvider->getDefaultValue(table, key, field);
     if (default_value != nullptr)
@@ -50,4 +72,79 @@ bool DecoratorTable::hget(const string &key, const string &field,  string &value
     }
 
     return false;
+}
+
+/* Get all the keys in the table */
+void DecoratorTable::getKeys(vector<string> &keys)
+{
+    Table::getKeys(keys);
+
+    // Append profile keys
+    auto connector = const_cast<DBConnector*>(m_pipe->getDBConnector());
+    auto table = getTableName();
+    auto profile_keys = ProfileProvider::instance().getKeys(table, connector);
+    for (auto &profile_key : profile_keys)
+    {
+        if(find(keys.begin(), keys.end(), profile_key) == keys.end())
+        {
+            keys.emplace_back(profile_key);
+        }
+    }
+}
+
+/* Set an entry in the DB directly and configure ttl for it (op not in use) */
+void DecoratorTable::set(const std::string &key,
+                 const std::vector<FieldValueTuple> &values, 
+                 const std::string &op,
+                 const std::string &prefix,
+                 const int64_t &ttl)
+{
+    auto connector = const_cast<DBConnector*>(m_pipe->getDBConnector());
+    auto table = getTableName();
+    if (values.size())
+    {
+        ProfileProvider::instance().tryRevertItem(table, key, connector);
+    }
+    else
+    {
+        // Set a entry to empty will delete entry.
+        ProfileProvider::instance().tryDeleteItem(table, key, connector);
+    }
+
+    Table::set(key,
+                 values, 
+                 op,
+                 prefix,
+                 ttl);
+}
+
+/* Delete an entry in the table */
+void DecoratorTable::del(const std::string &key,
+                 const std::string &op,
+                 const std::string &prefix)
+{
+    auto connector = const_cast<DBConnector*>(m_pipe->getDBConnector());
+    auto table = getTableName();
+    ProfileProvider::instance().tryDeleteItem(table, key, connector);
+
+    Table::del(key,
+                 op,
+                 prefix);
+}
+
+void DecoratorTable::hset(const std::string &key,
+                 const std::string &field,
+                 const std::string &value,
+                 const std::string &op,
+                 const std::string &prefix)
+{
+    auto connector = const_cast<DBConnector*>(m_pipe->getDBConnector());
+    auto table = getTableName();
+    ProfileProvider::instance().tryRevertItem(table, key, connector);
+
+    Table::hset(key,
+                 field,
+                 value,
+                 op,
+                 prefix);
 }

common/decoratortable.h

@@ -26,6 +26,31 @@ class DecoratorTable : public Table
     /* Get an entry field-value from the table */
     bool hget(const std::string &key, const std::string &field,  std::string &value) override;
 
+    /* Get all the keys from the table */
+    void getKeys(std::vector<std::string> &keys) override;
+
+    /* Set an entry in the DB directly and configure ttl for it (op not in use) */
+    void set(const std::string &key,
+                     const std::vector<FieldValueTuple> &values, 
+                     const std::string &op,
+                     const std::string &prefix,
+                     const int64_t &ttl) override;
+
+    /* Delete an entry in the table */
+    void del(const std::string &key,
+                     const std::string &op = "",
+                     const std::string &prefix = EMPTY_PREFIX) override;
+
+    /* Set an entry field in the table */
+    void hset(const std::string &key,
+                     const std::string &field,
+                     const std::string &value,
+                     const std::string &op = "",
+                     const std::string &prefix = EMPTY_PREFIX) override;
+
+    /* Unhide override 'set' methods in base class */
+    using Table::set;
+
 private:
     std::shared_ptr<DefaultValueProvider> m_defaultValueProvider;
 };

common/profileprovider.h

@@ -3,9 +3,9 @@
 #include <map>
 #include <string>
 #include <vector>
-#include "common/table.h"
-#include "common/dbconnector.h"
-#include "common/converter.h"
+#include "table.h"
+#include "dbconnector.h"
+#include "converter.h"
 
 namespace swss {
 

common/subscriberstatetable.cpp

@@ -118,7 +118,8 @@ void SubscriberStateTable::pops(deque<KeyOpFieldsValuesTuple> &vkco, const strin
         auto ctx = event->getContext()->element[1];
         if (message.pattern != m_keyspace)
         {
-            SWSS_LOG_ERROR("invalid pattern %s returned for pmessage of %s", message.pattern.c_str(), m_keyspace.c_str());
+            // Inherited class may subscribe more pattern and handle their pattern by override  onPopUnknownPattern method.
+            onPopUnknownPattern(message, vkco);
             continue;
         }
 
@@ -177,4 +178,9 @@ shared_ptr<RedisReply> SubscriberStateTable::popEventBuffer()
     return reply;
 }
 
+void SubscriberStateTable::onPopUnknownPattern(RedisMessage& message, deque<KeyOpFieldsValuesTuple> &vkco)
+{
+    SWSS_LOG_ERROR("invalid pattern %s returned for pmessage of %s", message.pattern.c_str(), m_keyspace.c_str());
+}
+
 }