Add ProfileProvider class to support read profile config from PROFILE_DB.

.azure-pipelines/build-sairedis-template.yml

@@ -20,6 +20,10 @@ parameters:
 - name: sonic_slave
   type: string
 
+- name: debian_version
+  type: string
+  default: buster
+
 - name: swss_common_artifact_name
   type: string
 
@@ -82,12 +86,31 @@ jobs:
       sudo sed -ri 's/^# unixsocket/unixsocket/' /etc/redis/redis.conf
       sudo sed -ri 's/^unixsocketperm .../unixsocketperm 777/' /etc/redis/redis.conf
       sudo sed -ri 's/redis-server.sock/redis.sock/' /etc/redis/redis.conf
+      sudo sed -ri 's/^databases .*/databases 17/' /etc/redis/redis.conf
       sudo service redis-server start
 
       sudo apt-get install -y rsyslog
       sudo service rsyslog start
 
     displayName: "Install dependencies"
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      source: specific
+      project: build
+      pipeline: Azure.sonic-buildimage.common_libs
+      runVersion: 'latestFromBranch'
+      runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      path: $(Build.ArtifactStagingDirectory)/download
+      artifact: common-lib
+      patterns: |
+        target/debs/${{ parameters.debian_version }}/libyang-*.deb
+        target/debs/${{ parameters.debian_version }}/libyang_*.deb
+    displayName: "Download libyang from common lib"
+  - script: |
+      set -ex
+      sudo dpkg -i $(find ./download -name *.deb)
+    workingDirectory: $(Build.ArtifactStagingDirectory)
+    displayName: "Install libyang from common lib"
   - task: DownloadPipelineArtifact@2
     inputs:
       artifact: ${{ parameters.swss_common_artifact_name }}
@@ -136,6 +159,7 @@ jobs:
         contents: 'syslog-all.tgz'
         targetFolder: $(Build.ArtifactStagingDirectory)
   - publish: $(Build.ArtifactStagingDirectory)/
-    artifact: ${{ parameters.syslog_artifact_name }}
+    # publish artifact with retry count, otherwise "artifact already exist" error will block job re-run.
+    artifact: ${{ parameters.syslog_artifact_name }}_$(System.JobAttempt)
     displayName: "Publish syslog artifacts"
     condition: always()

.azure-pipelines/build-swss-template.yml

@@ -103,14 +103,16 @@ jobs:
         target/debs/${{ parameters.debian_version }}/libnl-genl*.deb
         target/debs/${{ parameters.debian_version }}/libnl-route*.deb
         target/debs/${{ parameters.debian_version }}/libnl-nf*.deb
+        target/debs/${{ parameters.debian_version }}/libyang-*.deb
+        target/debs/${{ parameters.debian_version }}/libyang_*.deb
     displayName: "Download common libs"
 
   - script: |
       set -ex
       sudo dpkg -i $(find ./download -name *.deb)
       rm -rf download || true
     workingDirectory: $(Build.ArtifactStagingDirectory)
-    displayName: "Install libnl3, sonic swss common, and sairedis"
+    displayName: "Install libnl3, libyang, sonic swss common, and sairedis"
   - script: |
       set -ex
       rm ../*.deb || true

.azure-pipelines/build-template.yml

@@ -67,6 +67,47 @@ jobs:
         libnl-nf-3-dev \
         swig
     displayName: "Install dependencies"
+  - task: DownloadPipelineArtifact@2
+    # amd64 artifact name does not has arch suffix
+    condition: eq('${{ parameters.arch }}', 'amd64')
+    inputs:
+      source: specific
+      project: build
+      pipeline: Azure.sonic-buildimage.common_libs
+      runVersion: 'latestFromBranch'
+      runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      path: $(Build.ArtifactStagingDirectory)/download
+      artifact: common-lib
+      patterns: |
+        target/debs/${{ parameters.debian_version }}/libyang-*.deb
+        target/debs/${{ parameters.debian_version }}/libyang_*.deb
+    displayName: "Download libyang from amd64 common lib"
+  - task: DownloadPipelineArtifact@2
+    condition: ne('${{ parameters.arch }}', 'amd64')
+    inputs:
+      source: specific
+      project: build
+      pipeline: Azure.sonic-buildimage.common_libs
+      runVersion: 'latestFromBranch'
+      runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      path: $(Build.ArtifactStagingDirectory)/download
+      artifact: common-lib.${{ parameters.arch }}
+      patterns: |
+        target/debs/${{ parameters.debian_version }}/libyang-*.deb
+        target/debs/${{ parameters.debian_version }}/libyang_*.deb
+    displayName: "Download libyang from common lib"
+  - script: |
+      set -ex
+      sudo dpkg -i $(find ./download -name *.deb)
+    workingDirectory: $(Build.ArtifactStagingDirectory)
+    condition: ne('${{ parameters.debian_version }}', 'bullseye')
+    displayName: "Install libyang from common lib"
+    # Azure.sonic-buildimage.common_libs does not build libs for bullseye, so install libyang-dev by apt-get
+  - script: |
+      sudo apt-get install -qq -y \
+        libyang-dev
+    condition: eq('${{ parameters.debian_version }}', 'bullseye')
+    displayName: "Install libyang for bullseye"
   - script: |
       set -ex
       rm ../*.deb || true
@@ -93,7 +134,9 @@ jobs:
         sudo sed -ri 's/^# unixsocket/unixsocket/' /etc/redis/redis.conf
         sudo sed -ri 's/^unixsocketperm .../unixsocketperm 777/' /etc/redis/redis.conf
         sudo sed -ri 's/redis-server.sock/redis.sock/' /etc/redis/redis.conf
+        sudo sed -ri 's/^databases .*/databases 17/' /etc/redis/redis.conf
         sudo service redis-server restart
+        sudo mkdir /usr/local/yang-models
 
         sudo dpkg -i libswsscommon_*.deb
         sudo dpkg -i python-swsscommon_*.deb
@@ -103,6 +146,8 @@ jobs:
         pytest --cov=. --cov-report=xml
         mv coverage.xml tests/coverage.xml
         gcovr -r ./ -e ".*/swsscommon_wrap.cpp" --exclude-unreachable-branches --exclude-throw-branches -x --xml-pretty  -o coverage.xml
+
+        rm -rf $(Build.ArtifactStagingDirectory)/download
       displayName: "Run swss common unit tests"
   - publish: $(System.DefaultWorkingDirectory)/
     artifact: ${{ parameters.artifact_name }}

.azure-pipelines/test-docker-sonic-vs-template.yml

@@ -41,7 +41,7 @@ jobs:
       ls -l
       sudo sonic-swss-common/.azure-pipelines/build_and_install_module.sh
 
-      sudo apt-get install -y libhiredis0.14
+      sudo apt-get install -y libhiredis0.14 libyang0.16
       sudo dpkg -i --force-confask,confnew $(Build.ArtifactStagingDirectory)/download/libswsscommon_1.0.0_amd64.deb || apt-get install -f
       sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/python3-swsscommon_1.0.0_amd64.deb
 

BUILD

@@ -15,7 +15,7 @@ cc_library(
     includes = [
         "common",
     ],
-    linkopts = ["-lpthread -lhiredis -lnl-genl-3 -lnl-nf-3 -lnl-route-3 -lnl-3 -lzmq -lboost_serialization -luuid"],
+    linkopts = ["-lpthread -lhiredis -lnl-genl-3 -lnl-nf-3 -lnl-route-3 -lnl-3 -lzmq -lboost_serialization -luuid -lyang"],
     visibility = ["//visibility:public"],
 )
 

azure-pipelines.yml

@@ -68,7 +68,7 @@ stages:
         sudo apt-get install -y python3-pip
         sudo pip3 install pytest
         sudo apt-get install -y python
-        sudo apt-get install cmake libgtest-dev libgmock-dev
+        sudo apt-get install cmake libgtest-dev libgmock-dev libyang-dev
         cd /usr/src/gtest && sudo cmake . && sudo make
         ARCH=$(dpkg --print-architecture)
         set -x
@@ -95,6 +95,7 @@ stages:
       artifact_name: sonic-swss-common
       run_unit_test: true
       archive_gcov: true
+      debian_version: ${{ parameters.debian_version }}
 
 - stage: BuildArm
   dependsOn: Build
@@ -107,6 +108,7 @@ stages:
       pool: sonicbld-armhf
       sonic_slave: sonic-slave-${{ parameters.debian_version }}-armhf
       artifact_name: sonic-swss-common.armhf
+      debian_version: ${{ parameters.debian_version }}
 
   - template: .azure-pipelines/build-template.yml
     parameters:
@@ -115,6 +117,7 @@ stages:
       pool: sonicbld-arm64
       sonic_slave: sonic-slave-${{ parameters.debian_version }}-arm64
       artifact_name: sonic-swss-common.arm64
+      debian_version: ${{ parameters.debian_version }}
 
 - stage: BuildBullseye
   dependsOn: Build
@@ -139,6 +142,7 @@ stages:
       swss_common_artifact_name: sonic-swss-common
       artifact_name: sonic-sairedis
       syslog_artifact_name: sonic-sairedis.syslog
+      debian_version: ${{ parameters.debian_version }}
 
 - stage: BuildSwss
   dependsOn: BuildSairedis

common/Makefile.am

@@ -62,6 +62,7 @@ libswsscommon_la_SOURCES = \
     linkcache.cpp             \
     portmap.cpp               \
     pubsub.cpp                \
+    profileprovider.cpp       \
     tokenize.cpp              \
     exec.cpp                  \
     saiaclschema.cpp          \

common/configdb.cpp

@@ -244,24 +244,7 @@ void ConfigDBConnector_Native::mod_config(const map<string, map<string, map<stri
 map<string, map<string, map<string, string>>> ConfigDBConnector_Native::get_config()
 {
     auto& client = get_redis_client(m_db_name);
-    auto const& keys = client.keys("*");
-    map<string, map<string, map<string, string>>> data;
-    for (string key: keys)
-    {
-        size_t pos = key.find(m_table_name_separator);
-        if (pos == string::npos) {
-            continue;
-        }
-        string table_name = key.substr(0, pos);
-        string row = key.substr(pos + 1);
-        auto const& entry = client.hgetall<map<string, string>>(key);
-
-        if (!entry.empty())
-        {
-            data[table_name][row] = entry;
-        }
-    }
-    return data;
+    return client.getall();
 }
 
 std::string ConfigDBConnector_Native::getKeySeparator() const

common/database_config.json

@@ -91,6 +91,11 @@
             "id" : 14,
             "separator": ":",
             "instance" : "redis"
+        },
+        "PROFILE_DB" : {
+            "id" : 15,
+            "separator": ":",
+            "instance" : "redis"
         }
     },
     "VERSION" : "1.0"

common/dbconnector.cpp

@@ -905,3 +905,26 @@ void DBConnector::del(const std::vector<std::string>& keys)
 
     pipe.flush();
 }
+
+map<string, map<string, map<string, string>>> DBConnector::getall()
+{
+    const string separator = SonicDBConfig::getSeparator(this);
+    auto const& keys = this->keys("*");
+    map<string, map<string, map<string, string>>> data;
+    for (string key: keys)
+    {
+        size_t pos = key.find(separator);
+        if (pos == string::npos) {
+            continue;
+        }
+        string table_name = key.substr(0, pos);
+        string row = key.substr(pos + 1);
+        auto const& entry = this->hgetall<map<string, string>>(key);
+
+        if (!entry.empty())
+        {
+            data[table_name][row] = entry;
+        }
+    }
+    return data;
+}

common/dbconnector.h

@@ -248,6 +248,7 @@ class DBConnector : public RedisContext
 
     bool flushdb();
 
+    std::map<std::string, std::map<std::string, std::map<std::string, std::string>>> getall();
 private:
     void setNamespace(const std::string &netns);