[POC] [sonic-cfggen] Update sonic-cfggen to support generate profile config.

dockers/docker-database/database_config.json.j2

@@ -93,6 +93,11 @@
             "id" : 14,
             "separator": ":",
             "instance" : "redis"
+        },
+        "PROFILE_DB" : {
+            "id" : 15,
+            "separator": ":",
+            "instance" : "redis"
         }
     },
     "VERSION" : "1.0"

platform/vs/docker-sonic-vs/database_config.json

@@ -82,6 +82,11 @@
             "id" : 14,
             "separator": ":",
             "instance" : "redis"
+        },
+        "PROFILE_DB" : {
+            "id" : 15,
+            "separator": ":",
+            "instance" : "redis"
         }
     },
     "VERSION" : "1.0"

src/sonic-config-engine/sonic-cfggen

@@ -36,6 +36,7 @@ from sonic_py_common.multi_asic import get_asic_id_from_name, get_asic_device_id
 from sonic_py_common import device_info
 from swsscommon.swsscommon import SonicV2Connector, ConfigDBConnector, SonicDBConfig, ConfigDBPipeConnector
 
+profile_tables = ['BUFFER_POOL', 'BUFFER_PROFILE']
 
 PY3x = sys.version_info >= (3, 0)
 
@@ -254,6 +255,22 @@ def _get_jinja2_env(paths):
 
     return env
 
+def filter_profile_tables(data, generate_profile):
+    if generate_profile:
+        # remove all none profile tables
+        tables = list(data.keys())
+        for table in tables:
+            if table not in profile_tables:
+                del data[table]
+    # TODO: Enable following code after profile config related code migrated.
+    '''
+    else:
+        # remove profile tables
+        for table in profile_tables:
+            if table in data.keys():
+                del data[table]
+    '''
+
 def main():
     parser=argparse.ArgumentParser(description="Render configuration file from minigraph data and jinja2 template.")
     group = parser.add_mutually_exclusive_group()
@@ -270,6 +287,7 @@ def main():
     parser.add_argument("-d", "--from-db", help="read config from configdb", action='store_true')
     parser.add_argument("-H", "--platform-info", help="read platform and hardware info", action='store_true')
     parser.add_argument("-s", "--redis-unix-sock-file", help="unix sock file for redis connection")
+    parser.add_argument("-P", "--profile-config", help="generate profile config", action='store_true')
     group = parser.add_mutually_exclusive_group()
     group.add_argument("-t", "--template", help="render the data with the template file", action="append", default=[],
                        type=lambda opt_value: tuple(opt_value.split(',')) if ',' in opt_value else (opt_value, sys.stdout))
@@ -367,9 +385,12 @@ def main():
             SonicDBConfig.load_sonic_global_db_config(namespace=args.namespace)
             configdb = ConfigDBPipeConnector(use_unix_socket_path=use_unix_sock, namespace=args.namespace, **db_kwargs)
 
-        configdb.connect()
-        deep_update(data, FormatConverter.db_to_output(configdb.get_config()))
+        if args.profile_config:
+            configdb.db_connect("PROFILE_DB")
+        else:
+            configdb.connect()
 
+        deep_update(data, FormatConverter.db_to_output(configdb.get_config()))
 
     # the minigraph file must be provided to get the mac address for backend asics
     # or switch_type chassis_packet
@@ -416,7 +437,7 @@ def main():
         for template_file, dest_file in args.template:
             template = env.get_template(os.path.basename(template_file))
             template_data = template.render(data)
-            if dest_file == "config-db":
+            if dest_file == "config-db" or dest_file == "profile-db":
                 deep_update(data, FormatConverter.to_deserialized(json.loads(template_data)))
             else:
                 with smart_open(dest_file, 'w') as df:
@@ -426,6 +447,8 @@ def main():
         template = jinja2.Template('{{' + args.var + '}}')
         print(template.render(data))
 
+    filter_profile_tables(data, args.profile_config)
+
     if args.var_json is not None and args.var_json in data:
         if args.key is not None:
             print(json.dumps(FormatConverter.to_serialized(data[args.var_json], args.key), indent=4, cls=minigraph_encoder))
@@ -439,7 +462,11 @@ def main():
             SonicDBConfig.load_sonic_global_db_config(namespace=args.namespace)
             configdb = ConfigDBPipeConnector(use_unix_socket_path=True, namespace=args.namespace, **db_kwargs)
 
-        configdb.connect(False)
+        if args.profile_config:
+            configdb.db_connect("PROFILE_DB")
+        else:
+            configdb.connect(False)
+
         configdb.mod_config(FormatConverter.output_to_db(data))
 
     if args.print_data:

src/sonic-config-engine/tests/sample_output_profile/py2/buffers-dell6100.json

@@ -0,0 +1,37 @@
+{
+    "BUFFER_POOL": {
+        "ingress_lossless_pool": {
+            "size": "10875072",
+            "type": "ingress",
+            "mode": "dynamic",
+            "xoff": "4194112"
+        },
+        "egress_lossy_pool": {
+            "size": "9243812",
+            "type": "egress",
+            "mode": "dynamic"
+        },
+        "egress_lossless_pool": {
+            "size": "15982720",
+            "type": "egress",
+            "mode": "static"
+        }
+    },
+    "BUFFER_PROFILE": {
+        "ingress_lossy_profile": {
+            "pool": "ingress_lossless_pool",
+            "size": "0",
+            "dynamic_th": "3"
+        },
+        "egress_lossless_profile": {
+            "pool": "egress_lossless_pool",
+            "size": "1518",
+            "static_th": "15982720"
+        },
+        "egress_lossy_profile": {
+            "pool": "egress_lossy_pool",
+            "size": "1518",
+            "dynamic_th": "3"
+        }
+    }
+}

src/sonic-config-engine/tests/sample_output_profile/py3/buffers-dell6100.json

@@ -0,0 +1,37 @@
+{
+    "BUFFER_POOL": {
+        "ingress_lossless_pool": {
+            "size": "10875072",
+            "type": "ingress",
+            "mode": "dynamic",
+            "xoff": "4194112"
+        },
+        "egress_lossy_pool": {
+            "size": "9243812",
+            "type": "egress",
+            "mode": "dynamic"
+        },
+        "egress_lossless_pool": {
+            "size": "15982720",
+            "type": "egress",
+            "mode": "static"
+        }
+    },
+    "BUFFER_PROFILE": {
+        "ingress_lossy_profile": {
+            "pool": "ingress_lossless_pool",
+            "size": "0",
+            "dynamic_th": "3"
+        },
+        "egress_lossless_profile": {
+            "pool": "egress_lossless_pool",
+            "size": "1518",
+            "static_th": "15982720"
+        },
+        "egress_lossy_profile": {
+            "pool": "egress_lossy_pool",
+            "size": "1518",
+            "dynamic_th": "3"
+        }
+    }
+}

src/sonic-config-engine/tests/test_j2files.py

@@ -474,8 +474,44 @@ def _test_buffers_render_template(self, vendor, platform, sku, minigraph, buffer
 
         assert match, diff
 
+    def _test_buffers_profile_render_template(self, vendor, platform, sku, minigraph, buffer_template, expected):
+        file_exist, dir_exist = self.create_machine_conf(platform, vendor)
+        dir_path = os.path.join(self.test_dir, '..', '..', '..', 'device', vendor, platform, sku)
+        buffers_file = os.path.join(dir_path, buffer_template)
+        port_config_ini_file = os.path.join(dir_path, 'port_config.ini')
+
+        # copy buffers_config.j2 to the SKU directory to have all templates in one directory
+        buffers_config_file = os.path.join(self.test_dir, '..', '..', '..', 'files', 'build_templates', 'buffers_config.j2')
+        shutil.copy2(buffers_config_file, dir_path)
+
+        minigraph = os.path.join(self.test_dir, minigraph)
+        argument = '-P -m ' + minigraph + ' -p ' + port_config_ini_file + ' -t ' + buffers_file + ',profile-db --print-data > ' + self.output_file
+        self.run_script(argument)
+
+        # cleanup
+        buffers_config_file_new = os.path.join(dir_path, 'buffers_config.j2')
+        os.remove(buffers_config_file_new)
+        self.remove_machine_conf(file_exist, dir_exist)
+
+        out_file_dir = os.path.join(self.test_dir, 'sample_output_profile', utils.PYvX_DIR)
+        expected_files = [expected, self.modify_cable_len(expected, out_file_dir)]
+        match = False
+        diff = ''
+        for out_file in expected_files:
+            sample_output_file = os.path.join(out_file_dir, out_file)
+            if utils.cmp(sample_output_file, self.output_file):
+                match = True
+                break
+            else:
+                diff = diff + str(self.run_diff(sample_output_file, self.output_file))
+
+        os.remove(os.path.join(out_file_dir, expected_files[1]))
+
+        assert match, diff
+
     def test_buffers_dell6100_render_template(self):
         self._test_buffers_render_template('dell', 'x86_64-dell_s6100_c2538-r0', 'Force10-S6100', 'sample-dell-6100-t0-minigraph.xml', 'buffers.json.j2', 'buffers-dell6100.json')
+        self._test_buffers_profile_render_template('dell', 'x86_64-dell_s6100_c2538-r0', 'Force10-S6100', 'sample-dell-6100-t0-minigraph.xml', 'buffers.json.j2', 'buffers-dell6100.json')
 
     def test_buffers_mellanox2700_render_template(self):
         self._test_buffers_render_template('mellanox', 'x86_64-mlnx_msn2700-r0', 'Mellanox-SN2700-D48C8', 'sample-mellanox-2700-t0-minigraph.xml', 'buffers.json.j2', 'buffers-mellanox2700.json')