From 353fe57e6ccb4f70e5f20f5f5368bce73174ff85 Mon Sep 17 00:00:00 2001 From: ranjinidn Date: Thu, 25 May 2023 12:18:05 -0700 Subject: [PATCH 1/6] Transformer infra enhacements to support singleton container -Changes in DbSpec creation, inherit db-name from sonic table, use cvl db-name annotation for sonic yang, utility function changes -Support exclusion of sonic yangs xfmr support from models list --- translib/transformer/transformer.go | 25 +++++--- translib/transformer/xconst.go | 2 +- translib/transformer/xlate_from_db.go | 8 +-- translib/transformer/xlate_to_db.go | 23 +++++-- translib/transformer/xlate_utils.go | 87 +++++++++++++++++---------- translib/transformer/xspec.go | 34 +++++++---- 6 files changed, 120 insertions(+), 59 deletions(-) diff --git a/translib/transformer/transformer.go b/translib/transformer/transformer.go index 1088a9815..edbbb3956 100644 --- a/translib/transformer/transformer.go +++ b/translib/transformer/transformer.go @@ -36,16 +36,25 @@ var YangPath = "/usr/models/yang/" // OpenConfig-*.yang and sonic yang models pa var ModelsListFile = "models_list" var TblInfoJsonFile = "sonic_table_info.json" -func getOcModelsList() []string { +func getOcModelsList() ([]string, map[string]bool) { var fileList []string + excludeSonicList := make(map[string]bool) file, err := os.Open(YangPath + ModelsListFile) if err != nil { - return fileList + return fileList, excludeSonicList } defer file.Close() scanner := bufio.NewScanner(file) for scanner.Scan() { fileEntry := scanner.Text() + if strings.HasPrefix(fileEntry, "-sonic") { + _, err := os.Stat(YangPath + fileEntry[1:]) + if err != nil { + continue + } + excludeSonicList[fileEntry[1:]] = true + continue + } if !strings.HasPrefix(fileEntry, "#") { _, err := os.Stat(YangPath + fileEntry) if err != nil { @@ -54,10 +63,10 @@ func getOcModelsList() []string { fileList = append(fileList, fileEntry) } } - return fileList + return fileList, excludeSonicList } -func getDefaultModelsList() []string { +func getDefaultModelsList(excludeList map[string]bool) []string { var files []string fileInfo, err := ioutil.ReadDir(YangPath) if err != nil { @@ -66,7 +75,9 @@ func getDefaultModelsList() []string { for _, file := range fileInfo { if strings.HasPrefix(file.Name(), "sonic-") && !strings.HasSuffix(file.Name(), "-dev.yang") && filepath.Ext(file.Name()) == ".yang" { - files = append(files, file.Name()) + if _, ok := excludeList[file.Name()]; !ok { + files = append(files, file.Name()) + } } } return files @@ -75,8 +86,8 @@ func getDefaultModelsList() []string { func init() { initYangModelsPath() initRegex() - ocList := getOcModelsList() - yangFiles := getDefaultModelsList() + ocList, excludeSncList := getOcModelsList() + yangFiles := getDefaultModelsList(excludeSncList) yangFiles = append(yangFiles, ocList...) xfmrLogInfo("Yang model List: %v", yangFiles) err := loadYangModules(yangFiles...) diff --git a/translib/transformer/xconst.go b/translib/transformer/xconst.go index 5589c93a6..1980c3319 100644 --- a/translib/transformer/xconst.go +++ b/translib/transformer/xconst.go @@ -23,7 +23,7 @@ const ( XFMR_EMPTY_STRING = "" XFMR_NONE_STRING = "NONE" SONIC_TABLE_INDEX = 2 - SONIC_LIST_INDEX = 3 + SONIC_TBL_CHILD_INDEX = 3 SONIC_FIELD_INDEX = 4 SONIC_TOPCONTR_INDEX = 1 SONIC_MDL_PFX = "sonic" diff --git a/translib/transformer/xlate_from_db.go b/translib/transformer/xlate_from_db.go index cd2238372..bb4d17fba 100644 --- a/translib/transformer/xlate_from_db.go +++ b/translib/transformer/xlate_from_db.go @@ -98,10 +98,10 @@ func getLeafrefRefdYangType(yngTerminalNdDtType yang.TypeKind, fldXpath string) if len(pathList) > SONIC_FIELD_INDEX { xpath = pathList[SONIC_TABLE_INDEX] + "/" + pathList[SONIC_FIELD_INDEX] if xpath == fldXpath { - if sonicListInfo, ok := xDbSpecMap[pathList[SONIC_TABLE_INDEX]+"/"+pathList[SONIC_LIST_INDEX]]; ok { - if sonicListInfo.dbEntry != nil { - entry = sonicListInfo.dbEntry.Dir[pathList[SONIC_FIELD_INDEX]] - yngTerminalNdDtType = sonicListInfo.dbEntry.Dir[pathList[SONIC_FIELD_INDEX]].Type.Kind + if sonicTblChldInfo, ok := xDbSpecMap[pathList[SONIC_TABLE_INDEX]+"/"+pathList[SONIC_TBL_CHILD_INDEX]]; ok { + if sonicTblChldInfo.dbEntry != nil { + entry = sonicTblChldInfo.dbEntry.Dir[pathList[SONIC_FIELD_INDEX]] + yngTerminalNdDtType = sonicTblChldInfo.dbEntry.Dir[pathList[SONIC_FIELD_INDEX]].Type.Kind } } } else { diff --git a/translib/transformer/xlate_to_db.go b/translib/transformer/xlate_to_db.go index a3eb62ef0..c37bc9ac2 100644 --- a/translib/transformer/xlate_to_db.go +++ b/translib/transformer/xlate_to_db.go @@ -1007,7 +1007,6 @@ func yangReqToDbMapCreate(xlateParams xlateToParams) error { func verifyParentTableSonic(d *db.DB, dbs [db.MaxDB]*db.DB, oper Operation, uri string, dbData RedisDbMap) (bool, error) { var err error - pathList := splitUri(uri) xpath, dbKey, table := sonicXpathKeyExtract(uri) xfmrLogDebug("uri: %v xpath: %v table: %v, key: %v", uri, xpath, table, dbKey) @@ -1015,6 +1014,14 @@ func verifyParentTableSonic(d *db.DB, dbs [db.MaxDB]*db.DB, oper Operation, uri if (len(table) > 0) && (len(dbKey) > 0) { tableExists := false var derr error + + pathList := splitUri(uri) + hasSingletonContainer := SonicUriHasSingletonContainer(uri) + if hasSingletonContainer && oper != DELETE { + // No resource check required for singleton container for CRU cases + return true, err + } + if oper == GET { var cdb db.DBNum = db.ConfigDB dbInfo, ok := xDbSpecMap[table] @@ -1028,11 +1035,19 @@ func verifyParentTableSonic(d *db.DB, dbs [db.MaxDB]*db.DB, oper Operation, uri } else { // Valid table mapping exists. Read the table entry from DB tableExists, derr = dbTableExists(d, table, dbKey, oper) + if hasSingletonContainer && oper == DELETE { + // Special case when we delete at container that does'nt exist. Return true to skip translation. + if !tableExists { + return true, derr + } else { + return true, nil + } + } if derr != nil { return false, derr } } - if len(pathList) == SONIC_LIST_INDEX && (oper == UPDATE || oper == CREATE || oper == DELETE || oper == GET) && !tableExists { + if len(pathList) == SONIC_TBL_CHILD_INDEX && (oper == UPDATE || oper == CREATE || oper == DELETE || oper == GET) && !tableExists { // Uri is at /sonic-module:sonic-module/container-table/list // PATCH opertion permitted only if table exists in DB. // POST case since the URI is the parent, the parent needs to exist @@ -1041,8 +1056,8 @@ func verifyParentTableSonic(d *db.DB, dbs [db.MaxDB]*db.DB, oper Operation, uri log.Warningf("Parent table %v with key %v does not exist for oper %v in DB", table, dbKey, oper) err = tlerr.NotFound("Resource not found") return false, err - } else if len(pathList) > SONIC_LIST_INDEX && !tableExists { - // Uri is at /sonic-module/container-table/list or /sonic-module/container-table/list/leaf + } else if len(pathList) > SONIC_TBL_CHILD_INDEX && !tableExists { + // Uri is at /sonic-module/container-table/list/leaf // Parent table should exist for all CRUD cases log.Warningf("Parent table %v with key %v does not exist in DB", table, dbKey) err = tlerr.NotFound("Resource not found") diff --git a/translib/transformer/xlate_utils.go b/translib/transformer/xlate_utils.go index 12caebe7b..5251b332e 100644 --- a/translib/transformer/xlate_utils.go +++ b/translib/transformer/xlate_utils.go @@ -868,18 +868,15 @@ func sonicXpathKeyExtract(path string) (string, string, string) { var err error lpath := path xpath, _, err = XfmrRemoveXPATHPredicates(path) - if err != nil { + if err != nil || len(xpath) == 0 { return xpath, keyStr, tableName } - if xpath != "" { - fldPth := strings.Split(xpath, "/") - if len(fldPth) > SONIC_FIELD_INDEX { - fldNm = fldPth[SONIC_FIELD_INDEX] - xfmrLogDebug("Field Name : %v", fldNm) - } - } pathsubStr := strings.Split(xpath, "/") + if len(pathsubStr) > SONIC_FIELD_INDEX { + fldNm = pathsubStr[SONIC_FIELD_INDEX] + xfmrLogDebug("Field Name : %v", fldNm) + } if len(pathsubStr) > SONIC_TABLE_INDEX { if strings.Contains(pathsubStr[2], "[") { tableName = strings.Split(pathsubStr[SONIC_TABLE_INDEX], "[")[0] @@ -906,19 +903,23 @@ func sonicXpathKeyExtract(path string) (string, string, string) { lpath = "/" + strings.Join(pathLst[:SONIC_FIELD_INDEX-1], "/") xfmrLogDebug("path after removing the field portion %v", lpath) } - if len(pathsubStr) > SONIC_LIST_INDEX { - listNm := pathsubStr[SONIC_LIST_INDEX] - xfmrLogDebug("List Name : %v", listNm) - pathInfo := NewPathInfo(lpath) - listXpath := tableName + "/" + listNm - if (pathInfo != nil) && (len(pathInfo.Vars) > 0) { - if specListInfo, ok := xDbSpecMap[listXpath]; ok { - for idx, keyNm := range specListInfo.keyList { - if idx > 0 { - keyStr += dbOpts.KeySeparator - } - if pathInfo.HasVar(keyNm) { - keyStr += pathInfo.Var(keyNm) + if len(pathsubStr) > SONIC_TBL_CHILD_INDEX { + tblChldNm := pathsubStr[SONIC_TBL_CHILD_INDEX] + xfmrLogDebug("Table Child Name : %v", tblChldNm) + tblChldXpath := tableName + "/" + tblChldNm + if specTblChldInfo, ok := xDbSpecMap[tblChldXpath]; ok { + if specTblChldInfo.yangType == YANG_CONTAINER { + keyStr = tblChldNm + } else if specTblChldInfo.yangType == YANG_LIST { + pathInfo := NewPathInfo(lpath) + if (pathInfo != nil) && (len(pathInfo.Vars) > 0) { + for idx, keyNm := range specTblChldInfo.keyList { + if idx > 0 { + keyStr += dbOpts.KeySeparator + } + if pathInfo.HasVar(keyNm) { + keyStr += pathInfo.Var(keyNm) + } } } } @@ -1432,17 +1433,17 @@ func getXfmrSpecInfoFromUri(uri string) (interface{}, error) { tokens := strings.Split(xpath, "/") fieldName := "" tableName := "" - listName := "" + tblChldName := "" dbSpecXpath := "" if len(tokens) > SONIC_FIELD_INDEX { fieldName = tokens[SONIC_FIELD_INDEX] tableName = tokens[SONIC_TABLE_INDEX] dbSpecXpath = tableName + "/" + fieldName specInfo, xpathInSpecMapOk = xDbSpecMap[dbSpecXpath] - } else if len(tokens) > SONIC_LIST_INDEX { + } else if len(tokens) > SONIC_TBL_CHILD_INDEX { tableName = tokens[SONIC_TABLE_INDEX] - listName = tokens[SONIC_LIST_INDEX] - dbSpecXpath = tableName + "/" + listName + tblChldName = tokens[SONIC_TBL_CHILD_INDEX] + dbSpecXpath = tableName + "/" + tblChldName specInfo, xpathInSpecMapOk = xDbSpecMap[dbSpecXpath] } else if len(tokens) > SONIC_TABLE_INDEX { tableName = tokens[SONIC_TABLE_INDEX] @@ -1547,21 +1548,21 @@ func getYangEntryForXPath(xpath string) *yang.Entry { field := dbPathList[1] yNd, tok := xDbSpecMap[table] if tok && yNd.yangType == YANG_CONTAINER && yNd.dbEntry != nil { - for _, lstNode := range yNd.dbEntry.Dir { - if lstNode == nil { + for _, chldNode := range yNd.dbEntry.Dir { + if chldNode == nil { continue } - if chEntry, cok := lstNode.Dir[field]; cok { + if chEntry, cok := chldNode.Dir[field]; cok { entry = chEntry break } } if entry == nil { - for _, lstNode := range yNd.dbEntry.Dir { - if lstNode == nil { + for _, tblChldNode := range yNd.dbEntry.Dir { + if tblChldNode == nil { continue } - for _, chNode := range lstNode.Dir { + for _, chNode := range tblChldNode.Dir { if chNode == nil { continue } @@ -1673,3 +1674,27 @@ func (oper Operation) String() string { } return ret } + +func SonicUriHasSingletonContainer(uri string) bool { + hasSingletonContainer := false + if !strings.HasPrefix(uri, "/sonic") { + return hasSingletonContainer + } + + xpath, _, err := XfmrRemoveXPATHPredicates(uri) + if err != nil || len(xpath) == 0 { + return hasSingletonContainer + } + + pathList := strings.Split(xpath, "/") + + if len(pathList) > SONIC_TBL_CHILD_INDEX { + tblChldXpath := pathList[SONIC_TABLE_INDEX] + "/" + pathList[SONIC_TBL_CHILD_INDEX] + if specTblChldInfo, ok := xDbSpecMap[tblChldXpath]; ok { + if specTblChldInfo.yangType == YANG_CONTAINER { + hasSingletonContainer = true + } + } + } + return hasSingletonContainer +} diff --git a/translib/transformer/xspec.go b/translib/transformer/xspec.go index 091985e80..ceb873a38 100644 --- a/translib/transformer/xspec.go +++ b/translib/transformer/xspec.go @@ -540,25 +540,36 @@ func dbMapFill(tableName string, curPath string, moduleNm string, xDbSpecMap map } entryType := getYangTypeIntId(entry) - var xDbSpecPath string + tblDbIndex := db.ConfigDB + var tblSpecInfo *dbInfo + tblOk := false + if entry.Name != moduleNm { - if entryType == YANG_CONTAINER { - tableName = entry.Name + dbXpath := "" + tableContainer := false + if tableName == "" && entryType == YANG_CONTAINER { + tableContainer = true } - dbXpath := tableName - if entryType != YANG_CONTAINER { + if tableContainer { + tableName = entry.Name + dbXpath = tableName + } else { + // This includes all nodes which are not module or table containers dbXpath = tableName + "/" + entry.Name + if tblSpecInfo, tblOk = xDbSpecMap[tableName]; tblOk { + tblDbIndex = xDbSpecMap[tableName].dbIndex + } } xDbSpecPath = dbXpath xDbSpecMap[dbXpath] = new(dbInfo) - xDbSpecMap[dbXpath].dbIndex = db.MaxDB + xDbSpecMap[dbXpath].dbIndex = tblDbIndex xDbSpecMap[dbXpath].yangType = entryType xDbSpecMap[dbXpath].dbEntry = entry xDbSpecMap[dbXpath].module = moduleNm xDbSpecMap[dbXpath].cascadeDel = XFMR_INVALID - if entryType == YANG_CONTAINER { - xDbSpecMap[dbXpath].dbIndex = db.ConfigDB + if tableContainer { + xDbSpecMap[dbXpath].dbIndex = tblDbIndex if entry.Exts != nil && len(entry.Exts) > 0 { for _, ext := range entry.Exts { dataTagArr := strings.Split(ext.Keyword, ":") @@ -578,7 +589,7 @@ func dbMapFill(tableName string, curPath string, moduleNm string, xDbSpecMap map } } } - } else if tblSpecInfo, ok := xDbSpecMap[tableName]; ok && (entryType == YANG_LIST && len(entry.Key) != 0) { + } else if tblOk && (entryType == YANG_LIST && len(entry.Key) != 0) { tblSpecInfo.listName = append(tblSpecInfo.listName, entry.Name) xDbSpecMap[dbXpath].keyList = append(xDbSpecMap[dbXpath].keyList, strings.Split(entry.Key, " ")...) } else if entryType == YANG_LEAF || entryType == YANG_LEAF_LIST { @@ -629,6 +640,7 @@ func dbMapFill(tableName string, curPath string, moduleNm string, xDbSpecMap map xDbSpecMap[moduleXpath].yangType = entryType xDbSpecMap[moduleXpath].module = moduleNm xDbSpecMap[moduleXpath].cascadeDel = XFMR_INVALID + xDbSpecMap[moduleXpath].dbIndex = db.MaxDB for { done := true sncTblInfo := new(sonicTblSeqnInfo) @@ -961,8 +973,6 @@ func annotDbSpecMapFill(xDbSpecMap map[string]*dbInfo, dbXpath string, entry *ya dbXpathData.keyName = new(string) } *dbXpathData.keyName = ext.NName() - case "db-name": - dbXpathData.dbIndex = db.GetdbNameToIndex(ext.NName()) case "value-transformer": fieldName := pname[len(pname)-1] fieldXpath := tableName + "/" + fieldName @@ -989,7 +999,7 @@ func annotDbSpecMapFill(xDbSpecMap map[string]*dbInfo, dbXpath string, entry *ya dbXpathData.cascadeDel = XFMR_DISABLE } case "key-transformer": - listName := pname[SONIC_LIST_INDEX] + listName := pname[SONIC_TBL_CHILD_INDEX] listXpath := tableName + "/" + listName if listXpathData, ok := xDbSpecMap[listXpath]; ok { listXpathData.xfmrKey = ext.NName() From 8aee90204098a62497ae45b5e40e81138a63493d Mon Sep 17 00:00:00 2001 From: amrutasali Date: Thu, 25 May 2023 14:00:36 -0700 Subject: [PATCH 2/6] Sonic yang singleton container support in GET and SET/CRU code flow --- translib/transformer/xlate_from_db.go | 95 +++++++++++++-------------- translib/transformer/xlate_to_db.go | 16 ++++- 2 files changed, 57 insertions(+), 54 deletions(-) diff --git a/translib/transformer/xlate_from_db.go b/translib/transformer/xlate_from_db.go index bb4d17fba..c9d05f39d 100644 --- a/translib/transformer/xlate_from_db.go +++ b/translib/transformer/xlate_from_db.go @@ -352,26 +352,28 @@ func sonicDbToYangDataFill(inParamsForGet xlateFromDbParams) { fldName = fldName + "@" } curUri := inParamsForGet.uri + "/" + yangChldName - linParamsForGet := formXlateFromDbParams(nil, inParamsForGet.dbs, dbIdx, inParamsForGet.ygRoot, curUri, inParamsForGet.requestUri, curUri, inParamsForGet.oper, table, key, dbDataMap, inParamsForGet.txCache, resultMap, inParamsForGet.validate) + linParamsForGet := formXlateFromDbParams(nil, inParamsForGet.dbs, dbIdx, inParamsForGet.ygRoot, curUri, inParamsForGet.requestUri, chldXpath, inParamsForGet.oper, table, key, dbDataMap, inParamsForGet.txCache, resultMap, inParamsForGet.validate) dbEntry := yangNode.dbEntry.Dir[yangChldName] sonicDbToYangTerminalNodeFill(fldName, linParamsForGet, dbEntry) resultMap = linParamsForGet.resultMap inParamsForGet.resultMap = resultMap } else if chldYangType == YANG_CONTAINER { curMap := make(map[string]interface{}) - curUri := xpath + "/" + yangChldName + curUri := uri + "/" + yangChldName // container can have a static key, so extract key for current container _, curKey, curTable := sonicXpathKeyExtract(curUri) if _, specmapOk := xDbSpecMap[curTable]; !specmapOk || xDbSpecMap[curTable].dbEntry == nil { xfmrLogDebug("Yang entry not found for %v", curTable) continue } - // use table-name as xpath from now on d := inParamsForGet.dbs[xDbSpecMap[curTable].dbIndex] - linParamsForGet := formXlateFromDbParams(d, inParamsForGet.dbs, xDbSpecMap[curTable].dbIndex, inParamsForGet.ygRoot, curUri, inParamsForGet.requestUri, curTable, inParamsForGet.oper, curTable, curKey, dbDataMap, inParamsForGet.txCache, curMap, inParamsForGet.validate) + linParamsForGet := formXlateFromDbParams(d, inParamsForGet.dbs, xDbSpecMap[curTable].dbIndex, inParamsForGet.ygRoot, curUri, inParamsForGet.requestUri, chldXpath, inParamsForGet.oper, curTable, curKey, dbDataMap, inParamsForGet.txCache, curMap, inParamsForGet.validate) sonicDbToYangDataFill(linParamsForGet) curMap = linParamsForGet.resultMap dbDataMap = linParamsForGet.dbDataMap + if _, ok := (*dbDataMap)[xDbSpecMap[curTable].dbIndex][curTable][curKey]; ok { + delete((*dbDataMap)[xDbSpecMap[curTable].dbIndex][curTable], curKey) + } if len(curMap) > 0 { resultMap[yangChldName] = curMap } else { @@ -380,34 +382,25 @@ func sonicDbToYangDataFill(inParamsForGet xlateFromDbParams) { inParamsForGet.dbDataMap = linParamsForGet.dbDataMap inParamsForGet.resultMap = resultMap } else if chldYangType == YANG_LIST { - pathList := strings.Split(uri, "/") - // Skip the list entries if the URI has specific list query - if len(pathList) > SONIC_TABLE_INDEX+1 && !strings.Contains(uri, yangChldName) { - xfmrLogDebug("Skipping yangChldName: %v, pathList:%v, len:%v", yangChldName, pathList, len(pathList)) - } else { - var mapSlice []typeMapOfInterface - curUri := xpath + "/" + yangChldName - inParamsForGet.uri = curUri - inParamsForGet.xpath = curUri - mapSlice = sonicDbToYangListFill(inParamsForGet) - dbDataMap = inParamsForGet.dbDataMap - if len(key) > 0 && len(mapSlice) == 1 { // Single instance query. Don't return array of maps - for k, val := range mapSlice[0] { - resultMap[k] = val - } - - } else if len(mapSlice) > 0 { - resultMap[yangChldName] = mapSlice - } else { - xfmrLogDebug("Empty list for xpath(%v)", curUri) + var mapSlice []typeMapOfInterface + curUri := uri + "/" + yangChldName + inParamsForGet.uri = curUri + inParamsForGet.xpath = chldXpath + mapSlice = sonicDbToYangListFill(inParamsForGet) + dbDataMap = inParamsForGet.dbDataMap + if len(key) > 0 && len(mapSlice) == 1 { // Single instance query. Don't return array of maps + for k, val := range mapSlice[0] { + resultMap[k] = val } - inParamsForGet.resultMap = resultMap + + } else if len(mapSlice) > 0 { + resultMap[yangChldName] = mapSlice + } else { + xfmrLogDebug("Empty list for xpath(%v)", curUri) } + inParamsForGet.resultMap = resultMap } else if chldYangType == YANG_CHOICE || chldYangType == YANG_CASE { - curUri := table + "/" + yangChldName - inParamsForGet.uri = curUri - inParamsForGet.xpath = curUri - inParamsForGet.curDb = xDbSpecMap[table].dbIndex + inParamsForGet.xpath = chldXpath sonicDbToYangDataFill(inParamsForGet) dbDataMap = inParamsForGet.dbDataMap resultMap = inParamsForGet.resultMap @@ -434,25 +427,29 @@ func directDbToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, e fieldName := "" if len(xpath) > 0 { - var dbNode *dbInfo - - if len(table) > 0 { - tokens := strings.Split(xpath, "/") - if tokens[SONIC_TABLE_INDEX] == table { - fieldName = tokens[len(tokens)-1] - dbSpecField := table + "/" + fieldName - _, ok := xDbSpecMap[dbSpecField] - if ok && (xDbSpecMap[dbSpecField].yangType == YANG_LEAF || xDbSpecMap[dbSpecField].yangType == YANG_LEAF_LIST) { - dbNode = xDbSpecMap[dbSpecField] - xpath = dbSpecField - inParamsForGet.xpath = xpath - } else { - dbNode = xDbSpecMap[table] - } - } - } else { - dbNode = xDbSpecMap[xpath] + tokens := strings.Split(xpath, "/") + if len(tokens) > SONIC_FIELD_INDEX { + // Request is at levelf/leaflist level + xpath = table + "/" + tokens[SONIC_FIELD_INDEX] + fieldName = tokens[SONIC_FIELD_INDEX] + xfmrLogDebug("Request is at terminal node(leaf/leaf-list) - %v.", uri) + } else if len(tokens) > SONIC_TBL_CHILD_INDEX { + // Request is at list level + xpath = table + "/" + tokens[SONIC_TBL_CHILD_INDEX] + xfmrLogDebug("Request is at immediate child node of table level container - %v.", uri) + } else if len(tokens) > SONIC_TABLE_INDEX { + // Request is at table level + xpath = table + xfmrLogDebug("Request is at table level container - %v.", uri) + } else { + // Request is at top level container prefixed by module name + xfmrLogDebug("Request is at top level container - %v.", uri) } + dbNode, ok := xDbSpecMap[xpath] + if !ok { + xfmrLogInfo("xDbSpecMap doesn't contain entry for xpath - %v", xpath) + } + inParamsForGet.xpath = xpath if dbNode != nil { cdb := db.ConfigDB @@ -471,10 +468,6 @@ func directDbToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, e sonicDbToYangTerminalNodeFill(fieldName, linParamsForGet, dbEntry) resultMap = linParamsForGet.resultMap } else if yangType == YANG_CONTAINER { - if len(table) > 0 { - xpath = table - inParamsForGet.xpath = xpath - } sonicDbToYangDataFill(inParamsForGet) resultMap = inParamsForGet.resultMap } else if yangType == YANG_LIST { diff --git a/translib/transformer/xlate_to_db.go b/translib/transformer/xlate_to_db.go index c37bc9ac2..4c11fcb55 100644 --- a/translib/transformer/xlate_to_db.go +++ b/translib/transformer/xlate_to_db.go @@ -314,7 +314,7 @@ func dbMapDataFill(uri string, tableName string, keyName string, d map[string]in } } -func dbMapListDataFill(uri string, tableName string, dbEntry *yang.Entry, jsonData interface{}, result map[string]map[string]db.Value) { +func dbMapTableChildListDataFill(uri string, tableName string, dbEntry *yang.Entry, jsonData interface{}, result map[string]map[string]db.Value) { data := reflect.ValueOf(jsonData) tblKeyName := strings.Split(dbEntry.Key, " ") for idx := 0; idx < data.Len(); idx++ { @@ -337,6 +337,13 @@ func dbMapListDataFill(uri string, tableName string, dbEntry *yang.Entry, jsonDa } } +func dbMapTableChildContainerDataFill(uri string, tableName string, dbEntry *yang.Entry, jsonData interface{}, result map[string]map[string]db.Value) { + data := reflect.ValueOf(jsonData).Interface().(map[string]interface{}) + keyName := dbEntry.Name + xfmrLogDebug("Container name %v will become table key.", keyName) + dbMapDataFill(uri, tableName, keyName, data, result) +} + func directDbMapData(uri string, tableName string, jsonData interface{}, result map[string]map[string]db.Value) bool { _, ok := xDbSpecMap[tableName] if ok && xDbSpecMap[tableName].dbEntry != nil { @@ -359,8 +366,11 @@ func directDbMapData(uri string, tableName string, jsonData interface{}, result eType := curDbSpecData.yangType switch eType { case YANG_LIST: - xfmrLogDebug("Fill data for list uri(%v)", uri) - dbMapListDataFill(uri, tableName, curDbSpecData.dbEntry, v, result) + xfmrLogDebug("Fill data for list %v child of table level node %v", k, tableName) + dbMapTableChildListDataFill(uri, tableName, curDbSpecData.dbEntry, v, result) + case YANG_CONTAINER: + xfmrLogDebug("Fill data for container %v child of table level node %v", k, tableName) + dbMapTableChildContainerDataFill(uri, tableName, curDbSpecData.dbEntry, v, result) default: xfmrLogDebug("Invalid node type for uri(%v)", uri) } From 64a47368909facd0f0fe88906803ce8f8ea584cc Mon Sep 17 00:00:00 2001 From: amrutasali Date: Thu, 25 May 2023 14:21:22 -0700 Subject: [PATCH 3/6] Added a container in openconfig-test-xfmr.yang that maps to a sonic yang table containing a singleton container and UT cases exercising this mapping using table-name and key-name annotations --- .../test/openconfig-test-xfmr-annot.yang | 7 +++ .../test/openconfig-test-xfmr.yang | 20 ++++++++ translib/transformer/testxfmryang_test.go | 48 +++++++++++++++++++ 3 files changed, 75 insertions(+) diff --git a/translib/transformer/test/openconfig-test-xfmr-annot.yang b/translib/transformer/test/openconfig-test-xfmr-annot.yang index 1b09c2368..3190ba6db 100644 --- a/translib/transformer/test/openconfig-test-xfmr-annot.yang +++ b/translib/transformer/test/openconfig-test-xfmr-annot.yang @@ -140,5 +140,12 @@ module openconfig-test-xfmr-annot { sonic-ext:virtual-table "true"; } } + + deviation /oc-test-xfmr:test-xfmr/oc-test-xfmr:global-sensor { + deviate add { + sonic-ext:table-name "TEST_SENSOR_GLOBAL"; + sonic-ext:key-name "global_sensor"; + } + } } diff --git a/translib/transformer/test/openconfig-test-xfmr.yang b/translib/transformer/test/openconfig-test-xfmr.yang index d8ba46a54..f04fff4e9 100644 --- a/translib/transformer/test/openconfig-test-xfmr.yang +++ b/translib/transformer/test/openconfig-test-xfmr.yang @@ -406,6 +406,25 @@ module openconfig-test-xfmr { } + grouping test-global-sensor-config { + container global-sensor { + description + "Configuration data for global sensor."; + + leaf mode { + type string; + description + "global sensor mode"; + } + + leaf description { + type string; + description + "global sensor description"; + } + } + } + /////////////////// @@ -418,6 +437,7 @@ module openconfig-test-xfmr { uses test-sensor-group-top; uses test-set-top; uses interfaces-top; + uses test-global-sensor-config; } // augment statements diff --git a/translib/transformer/testxfmryang_test.go b/translib/transformer/testxfmryang_test.go index d1102abd5..a04a62dbf 100644 --- a/translib/transformer/testxfmryang_test.go +++ b/translib/transformer/testxfmryang_test.go @@ -355,3 +355,51 @@ func Test_leaflist_node(t *testing.T) { loadDB(db.ConfigDB, pre_req_map) time.Sleep(1 * time.Second) } + +func Test_node_exercising_singleton_container_and_keyname_mapping(t *testing.T) { + var pre_req_map, expected_map, cleanuptbl map[string]interface{} + var url, url_body_json string + + t.Log("\n\n+++++++++++++ Performing Set on Yang Node Exercising Mapping to Sonic-Yang Singleton Container and Key-name ++++++++++++") + url = "/openconfig-test-xfmr:test-xfmr/global-sensor" + url_body_json = "{ \"openconfig-test-xfmr:mode\": \"testmode\", \"openconfig-test-xfmr:description\": \"testdescription\"}" + expected_map = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "testmode", "description":"testdescription"}}} + cleanuptbl = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": ""}} + loadDB(db.ConfigDB, pre_req_map) + time.Sleep(1 * time.Second) + t.Run("Test set on yang node exercising mapping to sonic singleton conatiner and key-name.", processSetRequest(url, url_body_json, "POST", false, nil)) + time.Sleep(1 * time.Second) + t.Run("Verify set on yang node exercising mapping to sonic-yang singleton conatiner and key-name.", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", expected_map, false)) + unloadDB(db.ConfigDB, cleanuptbl) + time.Sleep(1 * time.Second) + t.Log("\n\n+++++++++++++ Done Performing Set on Yang Node Exercising Mapping to Sonic-Yang Singleton Container and Key-name ++++++++++++") + + t.Log("\n\n+++++++++++++ Performing Delete on Yang Node Exercising Mapping to Sonic-Yang Singleton Container and Key-name ++++++++++++") + pre_req_map = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{ + "mode": "testmode", + "description": "testdescription"}}} + loadDB(db.ConfigDB, pre_req_map) + time.Sleep(1 * time.Second) + url = "/openconfig-test-xfmr:test-xfmr/global-sensor/description" + t.Run("Test delete on node exercising mapping to sonic-yang singleton conatiner and key-name.", processDeleteRequest(url, false)) + time.Sleep(1 * time.Second) + expected_map = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{ + "mode": "testmode"}}} + t.Run("Verify delete on node exercising mapping to sonic-yang singleton conatiner and key-name.", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", expected_map, false)) + unloadDB(db.ConfigDB, cleanuptbl) + time.Sleep(1 * time.Second) + t.Log("\n\n+++++++++++++ Done Performing Delete on Yang Node Exercising Mapping to Sonic-Yang Singleton Container and Key-name ++++++++++++") + + t.Log("\n\n+++++++++++++ Performing Get on Yang Node Exercising Mapping to Sonic-Yang Singleton Container and Key-name ++++++++++++") + pre_req_map = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{ + "mode": "testmode", + "description": "testdescription"}}} + loadDB(db.ConfigDB, pre_req_map) + expected_get_json := "{\"openconfig-test-xfmr:global-sensor\": {\"description\": \"testdescription\",\"mode\": \"testmode\"}}" + url = "/openconfig-test-xfmr:test-xfmr/global-sensor" + t.Run("Test get on node exercising mapping to sonic-yang singleton conatiner and key-name.", processGetRequest(url, expected_get_json, false)) + time.Sleep(1 * time.Second) + unloadDB(db.ConfigDB, cleanuptbl) + t.Log("\n\n+++++++++++++ Done Performing Get on Yang Node Exercising mapping to sonic-yang singleton conatiner and key-name ++++++++++++") +} + From b2ff5dba660f710605234c6976b78c2a2c50336b Mon Sep 17 00:00:00 2001 From: ranjinidn Date: Thu, 25 May 2023 14:45:43 -0700 Subject: [PATCH 4/6] Add sonic singleton container in sonic yang and UT cases for CRUD and GET --- .../transformer/test/sonic-test-xfmr.yang | 11 +++ translib/transformer/testxfmryang_test.go | 89 +++++++++++++++++++ 2 files changed, 100 insertions(+) diff --git a/translib/transformer/test/sonic-test-xfmr.yang b/translib/transformer/test/sonic-test-xfmr.yang index df833bae6..afe30ad0e 100644 --- a/translib/transformer/test/sonic-test-xfmr.yang +++ b/translib/transformer/test/sonic-test-xfmr.yang @@ -164,5 +164,16 @@ module sonic-test-xfmr { } } + container TEST_SENSOR_GLOBAL { + container global_sensor { + leaf mode { + type string; + } + leaf description { + type string; + } + } + } + } } diff --git a/translib/transformer/testxfmryang_test.go b/translib/transformer/testxfmryang_test.go index a04a62dbf..bf8dc4b2d 100644 --- a/translib/transformer/testxfmryang_test.go +++ b/translib/transformer/testxfmryang_test.go @@ -403,3 +403,92 @@ func Test_node_exercising_singleton_container_and_keyname_mapping(t *testing.T) t.Log("\n\n+++++++++++++ Done Performing Get on Yang Node Exercising mapping to sonic-yang singleton conatiner and key-name ++++++++++++") } +func Test_singleton_sonic_yang_node_operations(t *testing.T) { + + cleanuptbl := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": ""}} + url := "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL" + + t.Log("++++++++++++++ Test_create_on_sonic_singleton_container_yang_node +++++++++++++") + + // Setup - Prerequisite + unloadDB(db.ConfigDB, cleanuptbl) + + // Payload + post_payload := "{ \"sonic-test-xfmr:global_sensor\": { \"mode\": \"testmode\", \"description\": \"testdescp\" }}" + post_sensor_global_expected := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "testmode", "description": "testdescp"}}} + + t.Run("Create on singleton sonic table yang node", processSetRequest(url, post_payload, "POST", false)) + time.Sleep(1 * time.Second) + t.Run("Verify Create on singleton sonic table yang node", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", post_sensor_global_expected, false)) + + // Teardown + unloadDB(db.ConfigDB, cleanuptbl) + + t.Log("++++++++++++++ Test_patch_on_sonic_singleton_container_node +++++++++++++") + + prereq := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "testmode", "description": "testdescp"}}} + url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL/global_sensor" + + // Setup - Prerequisite + loadDB(db.ConfigDB, prereq) + + // Payload + patch_payload := "{ \"sonic-test-xfmr:global_sensor\": { \"mode\": \"testmode\", \"description\": \"test description\" }}" + patch_sensor_global_expected := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "testmode", "description": "test description"}}} + + t.Run("Patch on singleton sonic container yang node", processSetRequest(url, patch_payload, "PATCH", false)) + time.Sleep(1 * time.Second) + t.Run("Verify patch on singleton sonic container yang node", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", patch_sensor_global_expected, false)) + + // Teardown + unloadDB(db.ConfigDB, cleanuptbl) + + t.Log("++++++++++++++ Test_replace_on_sonic_singleton_container +++++++++++++") + + url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL/global_sensor/mode" + + // Setup - Prerequisite + loadDB(db.ConfigDB, prereq) + + // Payload + put_payload := "{ \"sonic-test-xfmr:mode\": \"test_mode_1\"}" + put_sensor_global_expected := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "test_mode_1", "description": "testdescp"}}} + + t.Run("Put on singleton sonic yang node", processSetRequest(url, put_payload, "PUT", false)) + time.Sleep(1 * time.Second) + t.Run("Verify put on singleton sonic yang node", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", put_sensor_global_expected, false)) + + // Teardown + unloadDB(db.ConfigDB, cleanuptbl) + + + t.Log("++++++++++++++ Test_delete_on_singleton_sonic_container +++++++++++++") + + url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL/global_sensor" + + // Setup - Prerequisite + loadDB(db.ConfigDB, prereq) + + delete_expected := make(map[string]interface{}) + + t.Run("Delete on singleton sonic container", processDeleteRequest(url, false)) + time.Sleep(1 * time.Second) + t.Run("Verify delete on sonic singleton container", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", delete_expected, false)) + + // Teardown + unloadDB(db.ConfigDB, cleanuptbl) + + t.Log("++++++++++++++ Test_get_on_sonic_singleton_container +++++++++++++") + + prereq = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "mode_test", "description": "test description for single container"}}} + url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL" + + // Setup - Prerequisite + loadDB(db.ConfigDB, prereq) + + get_expected := "{\"sonic-test-xfmr:TEST_SENSOR_GLOBAL\":{ \"global_sensor\": { \"mode\": \"mode_test\", \"description\": \"test description for single container\" }}}" + t.Run("Get on Sonic singleton container", processGetRequest(url, get_expected, false)) + + // Teardown + unloadDB(db.ConfigDB, cleanuptbl) +} From b200825be61daed57889000f603927388b4dab73 Mon Sep 17 00:00:00 2001 From: ranjinidn Date: Fri, 2 Jun 2023 09:49:56 -0700 Subject: [PATCH 5/6] Update unit test README file with the build tag information --- translib/transformer/test/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/translib/transformer/test/README.md b/translib/transformer/test/README.md index affde2a8f..45809dc89 100644 --- a/translib/transformer/test/README.md +++ b/translib/transformer/test/README.md @@ -1,7 +1,9 @@ -# Transformer Unit Testing +# Transformer Infra Unit Testing -Following are the instructions on how to build and execute transformer unit test. -The transformer folder (sonic-mgmt-common/translib/transformer) contains all the necessary source code files that includes transformer unit test framework files, callbacks to serve the annotations, unit-test file to exercise the transformer-translations.The test folder (sonic-mgmt-common/translib/transformer/test) contains the test-yangs and annotations. All these files are needed to build and execute transformer test binary. +Following are the instructions on how to build and execute transformer infra unit test. +The transformer folder (sonic-mgmt-common/translib/transformer) contains all the necessary source code files that includes transformer unit test framework files, callbacks to serve the annotations, transformer infra unit-test file to exercise the transformer-translations.The test folder (sonic-mgmt-common/translib/transformer/test) contains the test-yangs and annotations. All these files are needed to build and execute transformer infra test binary. + +Note: The transformer infra unit test files will use the build tag "xfmrtest". Other applications writing unit test cases will use the build tag "testapp". This is required to separate the app unit tests from transformer infra unit test files and not have the app tests duplicated in both the binaries if missing the correct build tag. The app unit test binary and transformer infra unit test binary are both added to the azure pipeline and will be run separately. * Generate transformer.test by building sonic-mgmt-common with the MAKE flag INCLUDE_TEST_MODELS=y to have the test yangs built * Copy the openconfig-test-xfmr.yang, openconfig-test-xfmr-annot.yang, sonic-test-xfmr.yang, sonic-test-xfmr-annot.yang to mgmt-framework docker /usr/models/yang directory From 6a3b8d270d1c48023d8ba20925f9d553dad0ce16 Mon Sep 17 00:00:00 2001 From: ranjinidn Date: Fri, 9 Jun 2023 05:55:44 -0700 Subject: [PATCH 6/6] address review comment --- translib/transformer/transformer.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/translib/transformer/transformer.go b/translib/transformer/transformer.go index edbbb3956..5c7c1e4c0 100644 --- a/translib/transformer/transformer.go +++ b/translib/transformer/transformer.go @@ -36,7 +36,7 @@ var YangPath = "/usr/models/yang/" // OpenConfig-*.yang and sonic yang models pa var ModelsListFile = "models_list" var TblInfoJsonFile = "sonic_table_info.json" -func getOcModelsList() ([]string, map[string]bool) { +func getModelsList() ([]string, map[string]bool) { var fileList []string excludeSonicList := make(map[string]bool) file, err := os.Open(YangPath + ModelsListFile) @@ -86,9 +86,9 @@ func getDefaultModelsList(excludeList map[string]bool) []string { func init() { initYangModelsPath() initRegex() - ocList, excludeSncList := getOcModelsList() + modelsList, excludeSncList := getModelsList() yangFiles := getDefaultModelsList(excludeSncList) - yangFiles = append(yangFiles, ocList...) + yangFiles = append(yangFiles, modelsList...) xfmrLogInfo("Yang model List: %v", yangFiles) err := loadYangModules(yangFiles...) if err != nil {