sonic-yang-mgmt: Generic Config Updater - performance dependencies

src/libyang/patch/libyang-leaf-must.patch

@@ -0,0 +1,29 @@
+diff --git a/swig/cpp/src/Tree_Schema.cpp b/swig/cpp/src/Tree_Schema.cpp
+index 3587320..8da206a 100644
+--- a/swig/cpp/src/Tree_Schema.cpp
++++ b/swig/cpp/src/Tree_Schema.cpp
+@@ -344,6 +344,7 @@ S_Schema_Node Schema_Node_Choice::dflt() {
+ Schema_Node_Leaf::~Schema_Node_Leaf() {};
+ S_Set Schema_Node_Leaf::backlinks() LY_NEW_CASTED(lys_node_leaf, node, backlinks, Set);
+ S_When Schema_Node_Leaf::when() LY_NEW_CASTED(lys_node_leaf, node, when, When);
++std::vector<S_Restr> Schema_Node_Leaf::must() LY_NEW_LIST_CASTED(lys_node_leaf, node, must, must_size, Restr);
+ S_Type Schema_Node_Leaf::type() {return std::make_shared<Type>(&((struct lys_node_leaf *)node)->type, deleter);}
+ S_Schema_Node_List Schema_Node_Leaf::is_key() {
+     uint8_t pos;
+diff --git a/swig/cpp/src/Tree_Schema.hpp b/swig/cpp/src/Tree_Schema.hpp
+index d506891..f8ecc50 100644
+--- a/swig/cpp/src/Tree_Schema.hpp
++++ b/swig/cpp/src/Tree_Schema.hpp
+@@ -683,8 +683,12 @@ public:
+     ~Schema_Node_Leaf();
+     /** get backlinks variable from [lys_node_leaf](@ref lys_node_leaf)*/
+     S_Set backlinks();
++    /** get must_size variable from [lys_node_leaf](@ref lys_node_leaf)*/
++    uint8_t must_size() {return ((struct lys_node_leaf *)node)->must_size;};
+     /** get when variable from [lys_node_leaf](@ref lys_node_leaf)*/
+     S_When when();
++    /** get must variable from [lys_node_leaf](@ref lys_node_leaf)*/
++    std::vector<S_Restr> must();
+     /** get type variable from [lys_node_leaf](@ref lys_node_leaf)*/
+     S_Type type();
+     /** get units variable from [lys_node_leaf](@ref lys_node_leaf)*/

src/libyang/patch/series

@@ -3,3 +3,4 @@ libyang_mgmt_framework.patch
 swig.patch
 large_file_support_arm32.patch
 debian-packaging-files.patch
+libyang-leaf-must.patch

src/sonic-yang-mgmt/setup.py

@@ -30,6 +30,7 @@
     install_requires = [
         'xmltodict==0.12.0',
         'ijson==3.2.3',
+        'jsonpointer>=1.9',
         'jsondiff>=1.2.0',
         'tabulate==0.9.0'
     ],
@@ -46,7 +47,7 @@
     include_package_data=True,
     keywords='sonic-yang-mgmt',
     name='sonic-yang-mgmt',
-    py_modules=['sonic_yang', 'sonic_yang_ext'],
+    py_modules=['sonic_yang', 'sonic_yang_ext', 'sonic_yang_path'],
     packages=find_packages(),
     version='1.0',
     zip_safe=False,

src/sonic-yang-mgmt/sonic_yang.py

@@ -4,20 +4,20 @@
 from json import dump
 from glob import glob
 from sonic_yang_ext import SonicYangExtMixin, SonicYangException
+from sonic_yang_path import SonicYangPathMixin
 
 """
 Yang schema and data tree python APIs based on libyang python
 Here, sonic_yang_ext_mixin extends funtionality of sonic_yang,
 i.e. it is mixin not parent class.
 """
-class SonicYang(SonicYangExtMixin):
+class SonicYang(SonicYangExtMixin, SonicYangPathMixin):
 
     def __init__(self, yang_dir, debug=False, print_log_enabled=True, sonic_yang_options=0):
         self.yang_dir = yang_dir
         self.ctx = None
         self.module = None
         self.root = None
-
         # logging vars
         self.SYSLOG_IDENTIFIER = "sonic_yang"
         self.DEBUG = debug
@@ -44,6 +44,12 @@ def __init__(self, yang_dir, debug=False, print_log_enabled=True, sonic_yang_opt
         # below dict will store preProcessed yang objects, which may be needed by
         # all yang modules, such as grouping.
         self.preProcessedYang = dict()
+        # Lazy caching for backlinks lookups
+        self.backlinkCache = dict()
+        # Lazy caching for must counts
+        self.mustCache = dict()
+        # Lazy caching for configdb to xpath
+        self.configPathCache = dict()
         # element path for CONFIG DB. An example for this list could be:
         # ['PORT', 'Ethernet0', 'speed']
         self.elementPath = []
@@ -492,13 +498,89 @@ def _find_data_nodes(self, data_xpath):
                 list.append(data_set.path())
             return list
 
+    """
+    find_schema_must_count():  find the number of must clauses for the schema path
+    input:    schema_xpath     of the schema node
+              match_ancestors  whether or not to treat the specified path as
+                               an ancestor rather than a full path.  If set to
+                               true, will add recursively.
+    returns:  - count of must statements encountered
+              - Exception if schema node not found
+    """
+    def find_schema_must_count(self, schema_xpath, match_ancestors: bool=False):
+        # See if we have this cached
+        key = ( schema_xpath, match_ancestors )
+        result = self.mustCache.get(key)
+        if result is not None:
+            return result
+
+        try:
+            schema_node = self._find_schema_node(schema_xpath)
+        except Exception as e:
+            self.sysLog(msg="Cound not find the schema node from xpath: " + str(schema_xpath), debug=syslog.LOG_ERR, doPrint=True)
+            self.fail(e)
+            return 0
+
+        # If not doing recursion, just return the result.  This will internally
+        # cache the child so no need to update the cache ourselves
+        if not match_ancestors:
+            return self.__find_schema_must_count_only(schema_node)
+
+        count = 0
+        # Recurse first
+        for elem in schema_node.tree_dfs():
+            count += self.__find_schema_must_count_only(elem)
+
+        # Pull self
+        count += self.__find_schema_must_count_only(schema_node)
+
+        # Save in cache
+        self.mustCache[key] = count
+
+        return count
+
+    def __find_schema_must_count_only(self, schema_node):
+        # Check non-recursive cache
+        key = ( schema_node.path(), False )
+        result = self.mustCache.get(key)
+        if result is not None:
+            return result
+
+        count = 0
+        if schema_node.nodetype() == ly.LYS_CONTAINER:
+            schema_leaf = ly.Schema_Node_Container(schema_node)
+            if schema_leaf.must() is not None:
+                count += 1
+        elif schema_node.nodetype() == ly.LYS_LEAF:
+            schema_leaf = ly.Schema_Node_Leaf(schema_node)
+            count += schema_leaf.must_size()
+        elif schema_node.nodetype() == ly.LYS_LEAFLIST:
+            schema_leaf = ly.Schema_Node_Leaflist(schema_node)
+            count += schema_leaf.must_size()
+        elif schema_node.nodetype() == ly.LYS_LIST:
+            schema_leaf = ly.Schema_Node_List(schema_node)
+            count += schema_leaf.must_size()
+
+        # Cache result
+        self.mustCache[key] = count
+        return count
+
     """
     find_schema_dependencies():  find the schema dependencies from schema xpath
-    input:    schema_xpath of the schema node
+    input:    schema_xpath     of the schema node
+              match_ancestors  whether or not to treat the specified path as
+                               an ancestor rather than a full path. If set to
+                               true, will add recursively.
     returns:  - list of xpath of the dependencies
               - Exception if schema node not found
     """
-    def _find_schema_dependencies(self, schema_xpath):
+    def find_schema_dependencies(self, schema_xpath, match_ancestors: bool=False):
+        # See if we have this cached
+        key = ( schema_xpath, match_ancestors )
+        result = self.backlinkCache.get(key)
+        if result is not None:
+            return result
+
         ref_list = []
         try:
             schema_node = self._find_schema_node(schema_xpath)
@@ -507,12 +589,46 @@ def _find_schema_dependencies(self, schema_xpath):
             self.fail(e)
             return ref_list
 
-        schema_node = ly.Schema_Node_Leaf(schema_node)
-        backlinks = schema_node.backlinks()
-        if backlinks.number() > 0:
-            for link in backlinks.schema():
-                self.sysLog(msg="backlink schema: {}".format(link.path()), doPrint=True)
-                ref_list.append(link.path())
+        # If not doing recursion, just return the result.  This will internally
+        # cache the child so no need to update the cache ourselves
+        if not match_ancestors:
+            return self.__find_schema_dependencies_only(schema_node)
+
+        # Recurse first
+        for elem in schema_node.tree_dfs():
+            ref_list.extend(self.__find_schema_dependencies_only(elem))
+
+        # Pull self
+        ref_list.extend(self.__find_schema_dependencies_only(schema_node))
+
+        # Save in cache
+        self.backlinkCache[key] = ref_list
+
+        return ref_list
+
+    def __find_schema_dependencies_only(self, schema_node):
+        # Check non-recursive cache
+        key = ( schema_node.path(), False )
+        result = self.backlinkCache.get(key)
+        if result is not None:
+            return result
+
+        # New lookup
+        ref_list = []
+        schema_leaf = None
+        if schema_node.nodetype() == ly.LYS_LEAF:
+            schema_leaf = ly.Schema_Node_Leaf(schema_node)
+        elif schema_node.nodetype() == ly.LYS_LEAFLIST:
+            schema_leaf = ly.Schema_Node_Leaflist(schema_node)
+
+        if schema_leaf is not None:
+            backlinks = schema_leaf.backlinks()
+            if backlinks is not None and backlinks.number() > 0:
+                for link in backlinks.schema():
+                    ref_list.append(link.path())
+
+        # Cache result
+        self.backlinkCache[key] = ref_list
         return ref_list
 
     """
@@ -533,11 +649,10 @@ def find_data_dependencies(self, data_xpath):
         try:
             value = str(self._find_data_node_value(data_xpath))
 
-            schema_node = ly.Schema_Node_Leaf(data_node.schema())
-            backlinks = schema_node.backlinks()
-            if backlinks is not None and backlinks.number() > 0:
-                for link in backlinks.schema():
-                     node_set = node.find_path(link.path())
+            backlinks = self.find_schema_dependencies(data_node.schema().path(), False)
+            if backlinks is not None and len(backlinks) > 0:
+                for link in backlinks:
+                     node_set = node.find_path(link)
                      for data_set in node_set.data():
                           data_set.schema()
                           casted = data_set.subtype()

src/sonic-yang-mgmt/sonic_yang_ext.py

@@ -7,6 +7,8 @@
 from json import dump, dumps, loads
 from xmltodict import parse
 from glob import glob
+import copy
+from sonic_yang_path import SonicYangPathMixin
 
 Type_1_list_maps_model = [
     'DSCP_TO_TC_MAP_LIST',
@@ -42,7 +44,7 @@ class SonicYangException(Exception):
     pass
 
 # class sonic_yang methods, use mixin to extend sonic_yang
-class SonicYangExtMixin:
+class SonicYangExtMixin(SonicYangPathMixin):
 
     """
     load all YANG models, create JSON of yang models. (Public function)
@@ -221,15 +223,21 @@ def _getModuleTLCcontainer(self, table):
     """
     Crop config as per yang models,
     This Function crops from config only those TABLEs, for which yang models is
-    provided. The Tables without YANG models are stored in
-    self.tablesWithOutYangModels.
+    provided. If there are tables to modify it will perform a deepcopy of the
+    original structure in case anyone is holding a reference.
+    The Tables without YANG models are stored in self.tablesWithOutYangModels.
     """
     def _cropConfigDB(self, croppedFile=None):
-
+        isCopy = False
         tables = list(self.jIn.keys())
         for table in tables:
             if table not in self.confDbYangMap:
-                # store in tablesWithOutYang
+                # Make sure we duplicate if we're modifying so if a caller
+                # has a reference we don't clobber it.
+                if not isCopy:
+                    isCopy = True
+                    self.jIn = copy.deepcopy(self.jIn)
+                # store in tablesWithOutYang and purge
                 self.tablesWithOutYang[table] = self.jIn[table]
                 del self.jIn[table]
 
@@ -1153,7 +1161,7 @@ def _findXpathList(self, xpath, list, keys):
 
     """
     load_data: load Config DB, crop, xlate and create data tree from it. (Public)
-    input:    data
+    input:    configdbJson - will NOT be modified
               debug Flag
     returns:  True - success   False - failed
     """
@@ -1168,7 +1176,8 @@ def loadData(self, configdbJson, debug=False):
           # reset xlate and tablesWithOutYang
           self.xlateJson = dict()
           self.tablesWithOutYang = dict()
-          # self.jIn will be cropped
+          # self.jIn will be cropped if needed, however it will duplicate the object
+          # so the original is not modified
           self._cropConfigDB()
           # xlated result will be in self.xlateJson
           self._xlateConfigDB(xlateFile=xlateFile)