RFC7895 yang module library implementation (#15)

Added new yang ietf-yang-library.yang, which defines data models for
RFC7895 yang library discovery feature.

New app module yanglib_app.go to handle ietf-yang-library.yang APIs.
REST and gNMI requests for '/ietf-yang-library:modules-state' and its
child paths will be serviced by this app module.

Parses all NB yangs using goyang parser and builds the module info data
tree as per ietf-yang-library.yang. Yangs are parsed upon first request
and cached thereafter.

Uses translib.GetModel() API to identify the yang modules that are
implemented by apps. Other modules are marked as 'import' in yang
library response data. Transformer annotation files (*_annot.yang) are
ignored -- they do not define any data model.

Uses hardcoded module-set-id value for now. Should be changed to use
"yang bundle version" number when yang versioning feature is in.

Yang schema URL is included in the response only if main program set the
root URL through translib.SetSchemaRootURL() API. Schema URL is prepared
by appending the yang file name to the root URL.

Author: sachinholla

Makefile

@@ -81,9 +81,9 @@ $(GOYANG_BIN): $(GO_DEPS)
 	cd vendor/github.com/openconfig/goyang && \
 		$(GO) build -o $@ *.go
 
-clean: models-clean translib-clean cvl-clean
+clean: models-clean translib-clean cvl-clean go-deps-clean
 	git check-ignore debian/* | xargs -r $(RM) -r
 	$(RM) -r $(BUILD_DIR)
 
-cleanall: clean go-deps-clean
+cleanall: clean
 	git clean -fdX tools

go.sum

@@ -35,6 +35,7 @@ github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:x
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
 github.com/golang/protobuf v1.4.0-rc.4 h1:+EOh4OY6tjM6ZueeUKinl1f0U2820HzQOuf1iqMnsks=
 github.com/golang/protobuf v1.4.0-rc.4/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0 h1:aRz0NBceriICVtjhCgKkDvl+RudKu1CT6h0ZvUTrNfE=
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -121,6 +122,7 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ
 google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
 google.golang.org/protobuf v1.20.1 h1:ESRXHgpUBG5D2I5mmsQIyYxB/tQIZfSZ8wLyFDf/N/U=
 google.golang.org/protobuf v1.20.1/go.mod h1:KqelGeouBkcbcuB3HCk4/YH2tmNLk6YSWA5LIWeI/lY=
+google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw=
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=

models/yang/ietf-yang-library.yang

@@ -0,0 +1,245 @@
+module ietf-yang-library {
+  namespace "urn:ietf:params:xml:ns:yang:ietf-yang-library";
+  prefix "yanglib";
+
+  import ietf-yang-types {
+    prefix yang;
+  }
+  import ietf-inet-types {
+    prefix inet;
+  }
+
+  organization
+    "IETF NETCONF (Network Configuration) Working Group";
+
+  contact
+    "WG Web:   <https://datatracker.ietf.org/wg/netconf/>
+     WG List:  <mailto:netconf@ietf.org>
+
+     WG Chair: Mehmet Ersue
+               <mailto:mehmet.ersue@nsn.com>
+
+     WG Chair: Mahesh Jethanandani
+               <mailto:mjethanandani@gmail.com>
+
+     Editor:   Andy Bierman
+               <mailto:andy@yumaworks.com>
+
+     Editor:   Martin Bjorklund
+               <mailto:mbj@tail-f.com>
+
+     Editor:   Kent Watsen
+               <mailto:kwatsen@juniper.net>";
+
+  description
+    "This module contains monitoring information about the YANG
+     modules and submodules that are used within a YANG-based
+     server.
+
+     Copyright (c) 2016 IETF Trust and the persons identified as
+     authors of the code.  All rights reserved.
+
+     Redistribution and use in source and binary forms, with or
+     without modification, is permitted pursuant to, and subject
+     to the license terms contained in, the Simplified BSD License
+     set forth in Section 4.c of the IETF Trust's Legal Provisions
+     Relating to IETF Documents
+     (http://trustee.ietf.org/license-info).
+
+     This version of this YANG module is part of RFC 7895; see
+     the RFC itself for full legal notices.";
+
+  revision 2016-06-21 {
+    description
+      "Initial revision.";
+    reference
+      "RFC 7895: YANG Module Library.";
+  }
+
+  /*
+   * Typedefs
+   */
+
+  typedef revision-identifier {
+    type string {
+      pattern '\d{4}-\d{2}-\d{2}';
+    }
+    description
+      "Represents a specific date in YYYY-MM-DD format.";
+  }
+
+  /*
+   * Groupings
+   */
+
+  grouping module-list {
+    description
+      "The module data structure is represented as a grouping
+       so it can be reused in configuration or another monitoring
+       data structure.";
+
+    grouping common-leafs {
+      description
+        "Common parameters for YANG modules and submodules.";
+
+      leaf name {
+        type yang:yang-identifier;
+        description
+          "The YANG module or submodule name.";
+      }
+      leaf revision {
+        type union {
+          type revision-identifier;
+          type string { length 0; }
+        }
+        description
+          "The YANG module or submodule revision date.
+           A zero-length string is used if no revision statement
+           is present in the YANG module or submodule.";
+      }
+    }
+
+    grouping schema-leaf {
+      description
+        "Common schema leaf parameter for modules and submodules.";
+
+      leaf schema {
+        type inet:uri;
+        description
+          "Contains a URL that represents the YANG schema
+           resource for this module or submodule.
+
+           This leaf will only be present if there is a URL
+           available for retrieval of the schema for this entry.";
+      }
+    }
+
+    list module {
+      key "name revision";
+      description
+        "Each entry represents one revision of one module
+         currently supported by the server.";
+
+      uses common-leafs;
+      uses schema-leaf;
+
+      leaf namespace {
+        type inet:uri;
+        mandatory true;
+        description
+          "The XML namespace identifier for this module.";
+      }
+      leaf-list feature {
+        type yang:yang-identifier;
+        description
+          "List of YANG feature names from this module that are
+           supported by the server, regardless of whether they are
+           defined in the module or any included submodule.";
+      }
+      list deviation {
+        key "name revision";
+        description
+          "List of YANG deviation module names and revisions
+           used by this server to modify the conformance of
+           the module associated with this entry.  Note that
+           the same module can be used for deviations for
+           multiple modules, so the same entry MAY appear
+           within multiple 'module' entries.
+
+           The deviation module MUST be present in the 'module'
+           list, with the same name and revision values.
+           The 'conformance-type' value will be 'implement' for
+           the deviation module.";
+        uses common-leafs;
+      }
+      leaf conformance-type {
+        type enumeration {
+          enum implement {
+            description
+              "Indicates that the server implements one or more
+               protocol-accessible objects defined in the YANG module
+               identified in this entry.  This includes deviation
+               statements defined in the module.
+
+               For YANG version 1.1 modules, there is at most one
+               module entry with conformance type 'implement' for a
+               particular module name, since YANG 1.1 requires that,
+               at most, one revision of a module is implemented.
+
+               For YANG version 1 modules, there SHOULD NOT be more
+               than one module entry for a particular module name.";
+          }
+          enum import {
+            description
+              "Indicates that the server imports reusable definitions
+               from the specified revision of the module but does
+               not implement any protocol-accessible objects from
+               this revision.
+
+               Multiple module entries for the same module name MAY
+               exist.  This can occur if multiple modules import the
+               same module but specify different revision dates in
+               the import statements.";
+          }
+        }
+        mandatory true;
+        description
+          "Indicates the type of conformance the server is claiming
+           for the YANG module identified by this entry.";
+      }
+      list submodule {
+        key "name revision";
+        description
+          "Each entry represents one submodule within the
+           parent module.";
+        uses common-leafs;
+        uses schema-leaf;
+      }
+    }
+  }
+
+  /*
+   * Operational state data nodes
+   */
+
+  container modules-state {
+    config false;
+    description
+      "Contains YANG module monitoring information.";
+
+    leaf module-set-id {
+      type string;
+      mandatory true;
+      description
+        "Contains a server-specific identifier representing
+         the current set of modules and submodules.  The
+         server MUST change the value of this leaf if the
+         information represented by the 'module' list instances
+         has changed.";
+    }
+
+    uses module-list;
+  }
+
+  /*
+   * Notifications
+   */
+
+  notification yang-library-change {
+    description
+      "Generated when the set of modules and submodules supported
+       by the server has changed.";
+    leaf module-set-id {
+      type leafref {
+        path "/yanglib:modules-state/yanglib:module-set-id";
+      }
+      mandatory true;
+      description
+        "Contains the module-set-id value representing the
+         set of modules and submodules supported at the server at
+         the time the notification is generated.";
+    }
+  }
+
+}
+

translib/path_utils.go

@@ -42,6 +42,12 @@ type PathInfo struct {
 	Vars     map[string]string
 }
 
+// HasVar checks if the PathInfo contains given variable.
+func (p *PathInfo) HasVar(name string) bool {
+	_, exists := p.Vars[name]
+	return exists
+}
+
 // Var returns the string value for a path variable. Returns
 // empty string if no such variable exists.
 func (p *PathInfo) Var(name string) string {
@@ -91,6 +97,14 @@ func NewPathInfo(path string) *PathInfo {
 
 		name := readUntil(r, '=')
 		value := readUntil(r, ']')
+
+		// Handle duplicate parameter names by suffixing "#N" to it.
+		// N is the number of occurance of that parameter name from left.
+		namePrefix := name
+		for k := 2; info.HasVar(name); k++ {
+			name = fmt.Sprintf("%s#%d", namePrefix, k)
+		}
+
 		if len(name) != 0 {
 			fmt.Fprintf(&template, "{}")
 			info.Vars[name] = value
@@ -104,12 +118,17 @@ func NewPathInfo(path string) *PathInfo {
 
 func readUntil(r *strings.Reader, delim byte) string {
 	var buff strings.Builder
+	var escaped bool
+
 	for {
 		c, err := r.ReadByte()
-		if err == nil && c != delim {
-			buff.WriteByte(c)
-		} else {
+		if err != nil || (c == delim && !escaped) {
 			break
+		} else if c == '\\' && !escaped {
+			escaped = true
+		} else {
+			escaped = false
+			buff.WriteByte(c)
 		}
 	}
 

translib/path_utils_test.go

@@ -162,3 +162,68 @@ func TestGetObjectFieldName(t *testing.T) {
 		}
 	}
 }
+
+func TestNewPathInfo_empty(t *testing.T) {
+	testPathInfo(t, "", "", mkmap())
+}
+
+func TestNewPathInfo_novar(t *testing.T) {
+	testPathInfo(t, "/test/simple", "/test/simple", mkmap())
+}
+
+func TestNewPathInfo_var1(t *testing.T) {
+	testPathInfo(t, "/test/xx[one=1]", "/test/xx{}", mkmap("one", "1"))
+}
+
+func TestNewPathInfo_vars(t *testing.T) {
+	testPathInfo(t, "/test/xx[one=1][two=2]/new[three=3]", "/test/xx{}{}/new{}",
+		mkmap("one", "1", "two", "2", "three", "3"))
+}
+
+func TestNewPathInfo_dup1(t *testing.T) {
+	testPathInfo(t, "/test/xx[one=1][two=2]/new[one=0001]", "/test/xx{}{}/new{}",
+		mkmap("one", "1", "two", "2", "one#2", "0001"))
+}
+
+func TestNewPathInfo_dups(t *testing.T) {
+	testPathInfo(t, "/test/one[xx=1]/two[yy=2]/three[xx=3]/four[zz=4]/five[yy=5]/six[xx=6]",
+		"/test/one{}/two{}/three{}/four{}/five{}/six{}",
+		mkmap("xx", "1", "yy", "2", "xx#2", "3", "zz", "4", "yy#2", "5", "xx#3", "6"))
+}
+
+func TestNewPathInfo_escaped_name(t *testing.T) {
+	testPathInfo(t, "/test/xx[one\\==1][two[\\]=2]", "/test/xx{}{}",
+		mkmap("one=", "1", "two[]", "2"))
+}
+
+func TestNewPathInfo_escaped_valu(t *testing.T) {
+	testPathInfo(t, "/test/xx[one=[1\\]][two=\\0\\02 [\\.\\D]", "/test/xx{}{}",
+		mkmap("one", "[1]", "two", "002 [.D"))
+}
+
+func testPathInfo(t *testing.T, path, expTemplate string, expVars map[string]string) {
+	info := NewPathInfo(path)
+	if info == nil {
+		t.Errorf("NewPathInfo() returned null!")
+	} else if info.Path != path {
+		t.Errorf("Expected info.Path = %s", path)
+		t.Errorf("Actual info.Path   = %s", info.Path)
+	} else if info.Template != expTemplate {
+		t.Errorf("Expected info.Template = %s", expTemplate)
+		t.Errorf("Actual info.Template   = %s", info.Template)
+	} else if reflect.DeepEqual(info.Vars, expVars) == false {
+		t.Errorf("Expected info.Vars = %v", expVars)
+		t.Errorf("Actual info.Vars   = %v", info.Vars)
+	}
+	if t.Failed() {
+		t.Fatalf("NewPathInfo() failed to parse \"%s\"", path)
+	}
+}
+
+func mkmap(args ...string) map[string]string {
+	m := make(map[string]string)
+	for i := 0; (i + 1) < len(args); i += 2 {
+		m[args[i]] = args[i+1]
+	}
+	return m
+}