Transformer Infra sonic yang singleton container support

translib/transformer/test/README.md

@@ -1,7 +1,9 @@
-# Transformer Unit Testing
+# Transformer Infra Unit Testing
 
-Following are the instructions on how to build and execute transformer unit test.
-The transformer folder (sonic-mgmt-common/translib/transformer) contains all the necessary source code files that includes transformer unit test framework files, callbacks to serve the annotations, unit-test file to exercise the transformer-translations.The test folder (sonic-mgmt-common/translib/transformer/test) contains the test-yangs and annotations. All these files are needed to build and execute transformer test binary.
+Following are the instructions on how to build and execute transformer infra unit test.
+The transformer folder (sonic-mgmt-common/translib/transformer) contains all the necessary source code files that includes transformer unit test framework files, callbacks to serve the annotations, transformer infra unit-test file to exercise the transformer-translations.The test folder (sonic-mgmt-common/translib/transformer/test) contains the test-yangs and annotations. All these files are needed to build and execute transformer infra test binary.
+
+Note: The transformer infra unit test files will use the build tag "xfmrtest". Other applications writing unit test cases will use the build tag "testapp". This is required to separate the app unit tests from transformer infra unit test files and not have the app tests duplicated in both the binaries if missing the correct build tag. The app unit test binary and transformer infra unit test binary are both added to the azure pipeline and will be run separately.
 
 * Generate transformer.test by building sonic-mgmt-common with the MAKE flag INCLUDE_TEST_MODELS=y to have the test yangs built
 * Copy the openconfig-test-xfmr.yang, openconfig-test-xfmr-annot.yang, sonic-test-xfmr.yang, sonic-test-xfmr-annot.yang to  mgmt-framework docker /usr/models/yang directory

translib/transformer/test/openconfig-test-xfmr-annot.yang

@@ -140,5 +140,12 @@ module openconfig-test-xfmr-annot {
         sonic-ext:virtual-table "true";
       }
     }
+
+    deviation /oc-test-xfmr:test-xfmr/oc-test-xfmr:global-sensor {
+      deviate add {
+	sonic-ext:table-name "TEST_SENSOR_GLOBAL";
+        sonic-ext:key-name "global_sensor";
+      }
+    }
 }
 

translib/transformer/test/openconfig-test-xfmr.yang

@@ -406,6 +406,25 @@ module openconfig-test-xfmr {
 
   }
 
+  grouping test-global-sensor-config {
+    container global-sensor {
+          description
+            "Configuration data for global sensor.";
+
+          leaf mode {
+            type string;
+            description
+              "global sensor mode";
+          }
+
+          leaf description {
+            type string;
+            description
+              "global sensor description";
+          }
+    }
+  }
+
   ///////////////////
 
 
@@ -418,6 +437,7 @@ module openconfig-test-xfmr {
     uses test-sensor-group-top;
     uses test-set-top;
     uses interfaces-top;
+    uses test-global-sensor-config;
   }
   // augment statements
 

translib/transformer/test/sonic-test-xfmr.yang

@@ -164,5 +164,16 @@ module sonic-test-xfmr {
                         }
                 }
 
+                container TEST_SENSOR_GLOBAL {
+                        container global_sensor {
+                                 leaf mode {
+                                        type string;
+                                 }
+                                 leaf description {
+                                        type string;
+                                }
+                        }
+                }
+
 	}
 }

translib/transformer/testxfmryang_test.go

@@ -355,3 +355,140 @@ func Test_leaflist_node(t *testing.T) {
 	loadDB(db.ConfigDB, pre_req_map)
 	time.Sleep(1 * time.Second)
 }
+
+func Test_node_exercising_singleton_container_and_keyname_mapping(t *testing.T) {
+        var pre_req_map, expected_map, cleanuptbl map[string]interface{}
+        var url, url_body_json string
+
+        t.Log("\n\n+++++++++++++ Performing Set on Yang Node Exercising Mapping to Sonic-Yang Singleton Container and Key-name  ++++++++++++")
+        url = "/openconfig-test-xfmr:test-xfmr/global-sensor"
+        url_body_json = "{ \"openconfig-test-xfmr:mode\": \"testmode\", \"openconfig-test-xfmr:description\": \"testdescription\"}"
+        expected_map = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "testmode", "description":"testdescription"}}}
+        cleanuptbl = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": ""}}
+        loadDB(db.ConfigDB, pre_req_map)
+        time.Sleep(1 * time.Second)
+        t.Run("Test set on yang node exercising mapping to sonic singleton conatiner and key-name.", processSetRequest(url, url_body_json, "POST", false, nil))
+        time.Sleep(1 * time.Second)
+        t.Run("Verify set on yang node exercising mapping to sonic-yang singleton conatiner and key-name.", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", expected_map, false))
+        unloadDB(db.ConfigDB, cleanuptbl)
+        time.Sleep(1 * time.Second)
+        t.Log("\n\n+++++++++++++ Done Performing Set on Yang Node Exercising Mapping to Sonic-Yang Singleton Container and Key-name  ++++++++++++")
+
+        t.Log("\n\n+++++++++++++ Performing Delete on Yang Node Exercising Mapping to Sonic-Yang Singleton Container and Key-name  ++++++++++++")
+        pre_req_map = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{
+                "mode":        "testmode",
+                "description": "testdescription"}}}
+        loadDB(db.ConfigDB, pre_req_map)
+        time.Sleep(1 * time.Second)
+        url = "/openconfig-test-xfmr:test-xfmr/global-sensor/description"
+        t.Run("Test delete on node exercising mapping to sonic-yang singleton conatiner and key-name.", processDeleteRequest(url, false))
+        time.Sleep(1 * time.Second)
+        expected_map = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{
+                "mode":   "testmode"}}}
+        t.Run("Verify delete on node exercising mapping to sonic-yang singleton conatiner and key-name.", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", expected_map, false))
+        unloadDB(db.ConfigDB, cleanuptbl)
+        time.Sleep(1 * time.Second)
+        t.Log("\n\n+++++++++++++ Done Performing Delete on Yang Node Exercising Mapping to Sonic-Yang Singleton Container and Key-name  ++++++++++++")
+
+	t.Log("\n\n+++++++++++++ Performing Get on Yang Node Exercising  Mapping to Sonic-Yang Singleton Container and Key-name ++++++++++++")
+        pre_req_map = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{
+                "mode":        "testmode",
+                "description": "testdescription"}}}
+        loadDB(db.ConfigDB, pre_req_map)
+        expected_get_json := "{\"openconfig-test-xfmr:global-sensor\": {\"description\": \"testdescription\",\"mode\": \"testmode\"}}"
+        url = "/openconfig-test-xfmr:test-xfmr/global-sensor"
+        t.Run("Test get on node exercising mapping to sonic-yang singleton conatiner and key-name.", processGetRequest(url, expected_get_json, false))
+        time.Sleep(1 * time.Second)
+        unloadDB(db.ConfigDB, cleanuptbl)
+        t.Log("\n\n+++++++++++++ Done Performing Get on Yang Node Exercising  mapping to sonic-yang singleton conatiner and key-name ++++++++++++")
+}
+
+func Test_singleton_sonic_yang_node_operations(t *testing.T) {
+
+        cleanuptbl := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": ""}}
+        url := "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL"
+
+        t.Log("++++++++++++++  Test_create_on_sonic_singleton_container_yang_node +++++++++++++")
+
+        // Setup - Prerequisite
+        unloadDB(db.ConfigDB, cleanuptbl)
+
+        // Payload
+        post_payload :=  "{ \"sonic-test-xfmr:global_sensor\": { \"mode\": \"testmode\", \"description\": \"testdescp\" }}"
+        post_sensor_global_expected := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "testmode", "description": "testdescp"}}}
+
+        t.Run("Create on singleton sonic table yang node", processSetRequest(url, post_payload, "POST", false))
+        time.Sleep(1 * time.Second)
+        t.Run("Verify Create on singleton sonic table yang node", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", post_sensor_global_expected, false))
+
+        // Teardown
+        unloadDB(db.ConfigDB, cleanuptbl)
+
+        t.Log("++++++++++++++  Test_patch_on_sonic_singleton_container_node +++++++++++++")
+
+        prereq := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "testmode", "description": "testdescp"}}}
+        url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL/global_sensor"
+
+        // Setup - Prerequisite
+        loadDB(db.ConfigDB, prereq)
+
+        // Payload
+        patch_payload :=  "{ \"sonic-test-xfmr:global_sensor\": { \"mode\": \"testmode\", \"description\": \"test description\" }}"
+        patch_sensor_global_expected := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "testmode", "description": "test description"}}}
+
+        t.Run("Patch on singleton sonic container yang node", processSetRequest(url, patch_payload, "PATCH", false))
+        time.Sleep(1 * time.Second)
+        t.Run("Verify patch on singleton sonic container yang node", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", patch_sensor_global_expected, false))
+
+        // Teardown
+        unloadDB(db.ConfigDB, cleanuptbl)
+
+        t.Log("++++++++++++++  Test_replace_on_sonic_singleton_container +++++++++++++")
+
+        url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL/global_sensor/mode"
+
+        // Setup - Prerequisite
+        loadDB(db.ConfigDB, prereq)
+
+        // Payload
+        put_payload :=  "{ \"sonic-test-xfmr:mode\": \"test_mode_1\"}"
+        put_sensor_global_expected := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "test_mode_1", "description": "testdescp"}}}
+
+        t.Run("Put on singleton sonic yang node", processSetRequest(url, put_payload, "PUT", false))
+        time.Sleep(1 * time.Second)
+        t.Run("Verify put on singleton sonic yang node", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", put_sensor_global_expected, false))
+
+        // Teardown
+        unloadDB(db.ConfigDB, cleanuptbl)
+
+
+        t.Log("++++++++++++++  Test_delete_on_singleton_sonic_container  +++++++++++++")
+
+        url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL/global_sensor"
+
+        // Setup - Prerequisite
+        loadDB(db.ConfigDB, prereq)
+
+        delete_expected := make(map[string]interface{})
+
+        t.Run("Delete on singleton sonic container", processDeleteRequest(url, false))
+        time.Sleep(1 * time.Second)
+        t.Run("Verify delete on sonic singleton container", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", delete_expected, false))
+
+        // Teardown
+        unloadDB(db.ConfigDB, cleanuptbl)
+
+        t.Log("++++++++++++++  Test_get_on_sonic_singleton_container  +++++++++++++")
+
+        prereq = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "mode_test", "description": "test description for single container"}}}
+        url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL"
+
+        // Setup - Prerequisite
+        loadDB(db.ConfigDB, prereq)
+
+        get_expected := "{\"sonic-test-xfmr:TEST_SENSOR_GLOBAL\":{ \"global_sensor\": { \"mode\": \"mode_test\", \"description\": \"test description for single container\" }}}"
+        t.Run("Get on Sonic singleton container", processGetRequest(url, get_expected, false))
+
+        // Teardown
+        unloadDB(db.ConfigDB, cleanuptbl)
+}

translib/transformer/transformer.go

@@ -36,16 +36,25 @@ var YangPath = "/usr/models/yang/" // OpenConfig-*.yang and sonic yang models pa
 var ModelsListFile = "models_list"
 var TblInfoJsonFile = "sonic_table_info.json"
 
-func getOcModelsList() []string {
+func getModelsList() ([]string, map[string]bool) {
 	var fileList []string
+	excludeSonicList := make(map[string]bool)
 	file, err := os.Open(YangPath + ModelsListFile)
 	if err != nil {
-		return fileList
+		return fileList, excludeSonicList
 	}
 	defer file.Close()
 	scanner := bufio.NewScanner(file)
 	for scanner.Scan() {
 		fileEntry := scanner.Text()
+		if strings.HasPrefix(fileEntry, "-sonic") {
+			_, err := os.Stat(YangPath + fileEntry[1:])
+			if err != nil {
+				continue
+			}
+			excludeSonicList[fileEntry[1:]] = true
+			continue
+		}
 		if !strings.HasPrefix(fileEntry, "#") {
 			_, err := os.Stat(YangPath + fileEntry)
 			if err != nil {
@@ -54,10 +63,10 @@ func getOcModelsList() []string {
 			fileList = append(fileList, fileEntry)
 		}
 	}
-	return fileList
+	return fileList, excludeSonicList
 }
 
-func getDefaultModelsList() []string {
+func getDefaultModelsList(excludeList map[string]bool) []string {
 	var files []string
 	fileInfo, err := ioutil.ReadDir(YangPath)
 	if err != nil {
@@ -66,7 +75,9 @@ func getDefaultModelsList() []string {
 
 	for _, file := range fileInfo {
 		if strings.HasPrefix(file.Name(), "sonic-") && !strings.HasSuffix(file.Name(), "-dev.yang") && filepath.Ext(file.Name()) == ".yang" {
-			files = append(files, file.Name())
+			if _, ok := excludeList[file.Name()]; !ok {
+				files = append(files, file.Name())
+			}
 		}
 	}
 	return files
@@ -75,9 +86,9 @@ func getDefaultModelsList() []string {
 func init() {
 	initYangModelsPath()
 	initRegex()
-	ocList := getOcModelsList()
-	yangFiles := getDefaultModelsList()
-	yangFiles = append(yangFiles, ocList...)
+	modelsList, excludeSncList := getModelsList()
+	yangFiles := getDefaultModelsList(excludeSncList)
+	yangFiles = append(yangFiles, modelsList...)
 	xfmrLogInfo("Yang model List: %v", yangFiles)
 	err := loadYangModules(yangFiles...)
 	if err != nil {

translib/transformer/xconst.go

@@ -23,7 +23,7 @@ const (
 	XFMR_EMPTY_STRING      = ""
 	XFMR_NONE_STRING       = "NONE"
 	SONIC_TABLE_INDEX      = 2
-	SONIC_LIST_INDEX       = 3
+	SONIC_TBL_CHILD_INDEX  = 3
 	SONIC_FIELD_INDEX      = 4
 	SONIC_TOPCONTR_INDEX   = 1
 	SONIC_MDL_PFX          = "sonic"