sonic-mgmt-common: port to libyang3

azure-pipelines.yml

@@ -64,7 +64,7 @@ stages:
         sudo service redis-server start
 
         # LIBYANG
-        sudo dpkg -i ../target/debs/bookworm/libyang*1.0.73*.deb
+        sudo dpkg -i ../target/debs/bookworm/libyang*3.*.deb
       displayName: "Install dependency"
 
     - script: |

cvl/cvl.go

@@ -800,7 +800,9 @@ func getRedisTblToYangList(tableName, key string) (yangList string) {
 
 // Convert Redis key to Yang keys, if multiple key components are there,
 // they are separated based on Yang schema
-func getRedisToYangKeys(tableName string, redisKey string) []keyValuePairStruct {
+func getRedisToYangKeys(tableName string, redisKey string) ([]keyValuePairStruct, CVLErrorInfo) {
+	var cvlErrObj CVLErrorInfo
+
 	keyNames := modelInfo.tableInfo[tableName].keys
 	//First split all the keys components
 	keyVals := strings.Split(redisKey, modelInfo.tableInfo[tableName].redisKeyDelim) //split by DB separator
@@ -809,7 +811,16 @@ func getRedisToYangKeys(tableName string, redisKey string) []keyValuePairStruct
 		modelInfo.tableInfo[tableName].redisKeyDelim) //split by DB separator
 
 	if len(keyNames) != len(keyVals) {
-		return nil //number key names and values does not match
+		cvlErrObj.ErrCode = CVL_SEMANTIC_KEY_INVALID
+		cvlErrObj.TableName = tableName
+		cvlErrObj.Keys = keyNames
+		if len(keyNames) < len(keyVals) {
+			cvlErrObj.Msg = "Too many keys specified."
+		} else {
+			cvlErrObj.Msg = "Missing Key \"" + keyNames[len(keyVals)] + "\"."
+			cvlErrObj.Field = keyNames[len(keyVals)]
+		}
+		return nil, cvlErrObj //number key names and values does not match
 	}
 
 	mkeys := []keyValuePairStruct{}
@@ -829,7 +840,7 @@ func getRedisToYangKeys(tableName string, redisKey string) []keyValuePairStruct
 	TRACE_LOG(TRACE_SYNTAX, "getRedisToYangKeys() returns %v "+
 		"from Redis Table '%s', Key '%s'", mkeys, tableName, redisKey)
 
-	return mkeys
+	return mkeys, cvlErrObj
 }
 
 // Checks field map values and removes "NULL" entry, create array for leaf-list
@@ -885,6 +896,7 @@ func (c *CVL) generateYangParserData(jsonNode *jsonquery.Node, root **yparser.YP
 		if *root, errObj = c.yp.MergeSubtree(*root, topNode); errObj.ErrCode != yparser.YP_SUCCESS {
 			CVL_LOG(WARNING, "Unable to merge translated YANG data(libyang) "+
 				"while translating from request data to YANG format")
+			defer c.yp.FreeNode(topNode)
 			cvlErrObj.ErrCode = CVL_SYNTAX_ERROR
 			return cvlErrObj
 		}
@@ -945,6 +957,7 @@ func (c *CVL) translateToYang(jsonMap *map[string]interface{}, buildSyntaxDOM bo
 			//Visit each top level list in a loop for creating table data
 			cvlError = c.generateYangParserData(jsonNode, &root)
 			if cvlError.ErrCode != CVL_SUCCESS {
+				defer c.yp.FreeNode(root)
 				return nil, cvlError
 			}
 		}
@@ -961,9 +974,10 @@ func (c *CVL) validate(data *yparser.YParserNode) CVLRetCode {
 	if IsTraceAllowed(TRACE_LIBYANG) {
 		TRACE_LOG(TRACE_LIBYANG, "\nValidate1 data=%v\n", c.yp.NodeDump(data))
 	}
-	errObj := c.yp.ValidateSyntax(data, depData)
-	if yparser.YP_SUCCESS != errObj.ErrCode {
-		return CVL_FAILURE
+
+	errObj, _ := c.validateSyntax(data, depData)
+	if CVL_SUCCESS != errObj.ErrCode {
+		return errObj.ErrCode
 	}
 
 	cvlErrObj := c.validateCfgSemantics(c.yv.root)
@@ -1041,6 +1055,7 @@ func (c *CVL) addCfgDataItem(configData *map[string]interface{},
 	var cfgData map[string]interface{} = *configData
 
 	tblName, key := splitRedisKey(cfgDataItem.Key)
+	CVL_LOG(WARNING, "addCfgDataItem(): table: %s, key: %s", tblName, key)
 	if tblName == "" || key == "" {
 		//Bad redis key
 		return "", ""

cvl/cvl_api.go

@@ -200,7 +200,6 @@ func ValidationSessOpen(dbAccess cmn.DBAccess) (*CVL, CVLRetCode) {
 }
 
 func ValidationSessClose(c *CVL) CVLRetCode {
-	c.yp.DestroyCache()
 	c = nil
 
 	return CVL_SUCCESS
@@ -233,12 +232,11 @@ func (c *CVL) ValidateIncrementalConfig(jsonData string) CVLRetCode {
 	defer c.yp.FreeNode(root)
 	if root == nil {
 		return CVL_SYNTAX_ERROR
-
 	}
 
-	errObj := c.yp.ValidateSyntax(root, nil)
-	if yparser.YP_SUCCESS != errObj.ErrCode {
-		return CVL_FAILURE
+	errObj, _ := c.validateSyntax(root, nil)
+	if CVL_SUCCESS != errObj.ErrCode {
+		return errObj.ErrCode
 	}
 
 	//Add and fetch entries if already exists in Redis
@@ -252,7 +250,7 @@ func (c *CVL) ValidateIncrementalConfig(jsonData string) CVLRetCode {
 
 	//Merge existing data for update syntax or checking duplicate entries
 	if existingData != nil {
-		if _, errObj = c.yp.MergeSubtree(root, existingData); errObj.ErrCode != yparser.YP_SUCCESS {
+		if _, err := c.yp.MergeSubtree(root, existingData); err.ErrCode != yparser.YP_SUCCESS {
 			return CVL_ERROR
 		}
 	}
@@ -313,7 +311,6 @@ func (c *CVL) ValidateEditConfig(cfgData []cmn.CVLEditConfigData) (cvlErr CVLErr
 		c.clearTmpDbCache()
 		c.yv.root = &xmlquery.Node{Type: xmlquery.DocumentNode}
 		c.depDataCache = make(DepDataCacheType)
-		c.yp.DestroyCache()
 	}()
 
 	var cvlErrObj CVLErrorInfo
@@ -517,8 +514,6 @@ func (c *CVL) ValidateEditConfig(cfgData []cmn.CVLEditConfigData) (cvlErr CVLErr
 				CVL_LOG(WARNING, "\nValidateEditConfig(): OP_CREATE - Err = %v ", err1)
 			}
 
-			c.yp.SetOperation("CREATE")
-
 		case cmn.OP_UPDATE:
 			n, err1 := c.dbAccess.Exists(cfgData[i].Key).Result()
 			if err1 != nil || n == 0 { //key must exists
@@ -537,8 +532,6 @@ func (c *CVL) ValidateEditConfig(cfgData []cmn.CVLEditConfigData) (cvlErr CVLErr
 				continue
 			}
 
-			c.yp.SetOperation("UPDATE")
-
 		case cmn.OP_DELETE:
 			n, err1 := c.dbAccess.Exists(cfgData[i].Key).Result()
 			if err1 != nil || n == 0 { //key must exists
@@ -550,8 +543,6 @@ func (c *CVL) ValidateEditConfig(cfgData []cmn.CVLEditConfigData) (cvlErr CVLErr
 				cvlErrObj.Keys = splitKeyComponents(tbl, key)
 				return cvlErrObj, CVL_SEMANTIC_KEY_NOT_EXIST
 			}
-
-			c.yp.SetOperation("DELETE")
 		}
 
 		yangListName := getRedisTblToYangList(tbl, key)

cvl/cvl_error_test.go

@@ -67,18 +67,24 @@ func expandMessagePatterns(ex *CVLErrorInfo) {
 		ex.Msg = strings.ReplaceAll(ex.Msg, "{{field}}", ex.Field)
 		ex.Msg = strings.ReplaceAll(ex.Msg, "{{value}}", ex.Value)
 		ex.Msg = strings.TrimSuffix(ex.Msg, " \"\"") // if value is empty
+	case invalidValueNoValueErrMessage:
+		ex.Msg = strings.ReplaceAll(ex.Msg, "{{field}}", ex.Field)
 	case unknownFieldErrMessage:
 		ex.Msg = strings.ReplaceAll(ex.Msg, "{{field}}", ex.Field)
+	case missingKeyErrMessage:
+		ex.Msg = strings.ReplaceAll(ex.Msg, "{{field}}", ex.Field)
 	}
 }
 
 const (
-	invalidValueErrMessage   = "Field \"{{field}}\" has invalid value \"{{value}}\""
-	unknownFieldErrMessage   = "Unknown field \"{{field}}\""
-	genericValueErrMessage   = "Data validation failed"
-	mustExpressionErrMessage = "Must expression validation failed"
-	whenExpressionErrMessage = "When expression validation failed"
-	instanceInUseErrMessage  = "Validation failed for Delete operation, given instance is in use"
+	invalidValueErrMessage        = "Field \"{{field}}\" has invalid value \"{{value}}\""
+	invalidValueNoValueErrMessage = "Field \"{{field}}\" has invalid value"
+	unknownFieldErrMessage        = "Unknown field \"{{field}}\""
+	genericValueErrMessage        = "Data validation failed"
+	mustExpressionErrMessage      = "Must expression validation failed"
+	whenExpressionErrMessage      = "When expression validation failed"
+	instanceInUseErrMessage       = "Validation failed for Delete operation, given instance is in use"
+	missingKeyErrMessage          = "Missing Key \"{{field}}\"."
 )
 
 func verifyValidateEditConfig(t *testing.T, data []CVLEditConfigData, exp CVLErrorInfo) {

cvl/cvl_semantics.go

@@ -281,7 +281,10 @@ func (c *CVL) generateYangListData(jsonNode *jsonquery.Node,
 			listConatinerNode = c.addYangNode(origTableName, topNode, origTableName, "")
 			topNodesAdded = true
 		}
-		keyValuePair := getRedisToYangKeys(tableName, redisKey)
+		keyValuePair, cvlErrObj := getRedisToYangKeys(tableName, redisKey)
+		if cvlErrObj.ErrCode != CVL_SUCCESS {
+			return nil, cvlErrObj
+		}
 		keyCompCount := len(keyValuePair)
 		totalKeyComb := 1
 		var keyIndices []int
@@ -1527,8 +1530,8 @@ func (c *CVL) checkDepDataCompatible(tblName, key, reftblName, refTblKey, leafRe
 	}
 
 	// Fetch key value pair for both current table and ref table
-	tblKeysKVP := getRedisToYangKeys(tblName, key)
-	refTblKeysKVP := getRedisToYangKeys(reftblName, refTblKey)
+	tblKeysKVP, _ := getRedisToYangKeys(tblName, key)
+	refTblKeysKVP, _ := getRedisToYangKeys(reftblName, refTblKey)
 
 	// Determine the leafref
 	leafRef := getLeafRefInfo(reftblName, leafRefField, tblName)

cvl/cvl_syntax.go

@@ -99,13 +99,12 @@ func (c *CVL) checkMaxElemConstraint(op cmn.CVLOperation, tableName string, key
 }
 
 // Add child node to a parent node
-func (c *CVL) addChildNode(tableName string, parent *yparser.YParserNode, name string) *yparser.YParserNode {
+func (c *CVL) addListNode(tableName string, parent *yparser.YParserNode, name string, keys []*yparser.YParserLeafValue) (*yparser.YParserNode, yparser.YParserError) {
 
-	//return C.lyd_new(parent, modelInfo.tableInfo[tableName].module, C.CString(name))
-	return c.yp.AddChildNode(modelInfo.tableInfo[tableName].module, parent, name)
+	return c.yp.AddListNode(modelInfo.tableInfo[tableName].module, parent, name, keys)
 }
 
-func (c *CVL) addChildLeaf(config bool, tableName string, parent *yparser.YParserNode, name string, value string, multileaf *[]*yparser.YParserLeafValue) {
+func (c *CVL) appendLeafValue(name string, value string, multileaf *[]*yparser.YParserLeafValue) {
 
 	/* If there is no value then assign default space string. */
 	if len(value) == 0 {
@@ -127,21 +126,23 @@ func (c *CVL) generateTableFieldsData(config bool, tableName string, jsonNode *j
 			jsonFieldNode.FirstChild != nil &&
 			jsonFieldNode.FirstChild.Type == jsonquery.TextNode {
 
-			if len(modelInfo.tableInfo[tableName].mapLeaf) == 2 { //mapping should have two leaf always
-				batchInnerListLeaf := make([]*yparser.YParserLeafValue, 0)
+			if len(modelInfo.tableInfo[tableName].mapLeaf) == 2 { //mapping should have two leaf always, first is key
+				batchInnerListLeafKeys := make([]*yparser.YParserLeafValue, 0)
+				batchInnerListLeafNonKeys := make([]*yparser.YParserLeafValue, 0)
+
 				//Values should be stored inside another list as map table
-				listNode := c.addChildNode(tableName, parent, tableName) //Add the list to the top node
-				c.addChildLeaf(config, tableName,
-					listNode, modelInfo.tableInfo[tableName].mapLeaf[0],
-					jsonFieldNode.Data, &batchInnerListLeaf)
 
-				c.addChildLeaf(config, tableName,
-					listNode, modelInfo.tableInfo[tableName].mapLeaf[1],
-					jsonFieldNode.FirstChild.Data, &batchInnerListLeaf)
+				c.appendLeafValue(modelInfo.tableInfo[tableName].mapLeaf[0], jsonFieldNode.Data, &batchInnerListLeafKeys)
+				c.appendLeafValue(modelInfo.tableInfo[tableName].mapLeaf[1], jsonFieldNode.FirstChild.Data, &batchInnerListLeafNonKeys)
+
+				listNode, err := c.addListNode(tableName, parent, tableName, batchInnerListLeafKeys)
+				if err.ErrCode != yparser.YP_SUCCESS {
+					cvlErrObj = createCVLErrObj(err, jsonNode)
+					return cvlErrObj
+				}
 
-				if errObj := c.yp.AddMultiLeafNodes(modelInfo.tableInfo[tableName].module, listNode, batchInnerListLeaf); errObj.ErrCode != yparser.YP_SUCCESS {
+				if errObj := c.yp.AddMultiLeafNodes(modelInfo.tableInfo[tableName].module, listNode, batchInnerListLeafNonKeys); errObj.ErrCode != yparser.YP_SUCCESS {
 					cvlErrObj = createCVLErrObj(errObj, jsonNode)
-					CVL_LOG(WARNING, "Failed to create innner list leaf nodes, data = %v", batchInnerListLeaf)
 					return cvlErrObj
 				}
 			} else {
@@ -150,12 +151,10 @@ func (c *CVL) generateTableFieldsData(config bool, tableName string, jsonNode *j
 
 				if len(hashRefMatch) == 3 {
 
-					c.addChildLeaf(config, tableName,
-						parent, jsonFieldNode.Data,
+					c.appendLeafValue(jsonFieldNode.Data,
 						hashRefMatch[2], multileaf) //take hashref key value
 				} else {
-					c.addChildLeaf(config, tableName,
-						parent, jsonFieldNode.Data,
+					c.appendLeafValue(jsonFieldNode.Data,
 						jsonFieldNode.FirstChild.Data, multileaf)
 				}
 			}
@@ -165,8 +164,7 @@ func (c *CVL) generateTableFieldsData(config bool, tableName string, jsonNode *j
 			jsonFieldNode.FirstChild.Type == jsonquery.ElementNode {
 			//Array data e.g. VLAN members
 			for arrayNode := jsonFieldNode.FirstChild; arrayNode != nil; arrayNode = arrayNode.NextSibling {
-				c.addChildLeaf(config, tableName,
-					parent, jsonFieldNode.Data,
+				c.appendLeafValue(jsonFieldNode.Data,
 					arrayNode.FirstChild.Data, multileaf)
 			}
 		}
@@ -189,17 +187,18 @@ func (c *CVL) generateTableData(config bool, jsonNode *jsonquery.Node) (*yparser
 	//E.g. ACL_RULE is mapped as
 	// container ACL_RULE { list ACL_RULE_LIST {} }
 	var topNode *yparser.YParserNode
-	var listConatinerNode *yparser.YParserNode
+	var listContainerNode *yparser.YParserNode
+	var err yparser.YParserError
 
 	//Traverse each key instance
 	for jsonNode = jsonNode.FirstChild; jsonNode != nil; jsonNode = jsonNode.NextSibling {
-
 		//For each field check if is key
 		//If it is key, create list as child of top container
 		// Get all key name/value pairs
 		if yangListName := getRedisTblToYangList(origTableName, jsonNode.Data); yangListName != "" {
 			tableName = yangListName
 		}
+
 		if _, exists := modelInfo.tableInfo[tableName]; !exists {
 			CVL_LOG(WARNING, "Schema details not found for %s", tableName)
 			cvlErrObj.ErrCode = CVL_SYNTAX_ERROR
@@ -209,16 +208,27 @@ func (c *CVL) generateTableData(config bool, jsonNode *jsonquery.Node) (*yparser
 		}
 		if !topNodesAdded {
 			// Add top most conatiner e.g. 'container sonic-acl {...}'
-			topNode = c.yp.AddChildNode(modelInfo.tableInfo[tableName].module,
+			topNode, err = c.yp.AddContainerNode(modelInfo.tableInfo[tableName].module,
 				nil, modelInfo.tableInfo[tableName].modelName)
+			if err.ErrCode != yparser.YP_SUCCESS {
+				cvlErrObj = createCVLErrObj(err, jsonNode)
+				return nil, cvlErrObj
+			}
 
 			//Add the container node for each list
 			//e.g. 'container ACL_TABLE { list ACL_TABLE_LIST ...}
-			listConatinerNode = c.yp.AddChildNode(modelInfo.tableInfo[tableName].module,
+			listContainerNode, err = c.yp.AddContainerNode(modelInfo.tableInfo[tableName].module,
 				topNode, origTableName)
+			if err.ErrCode != yparser.YP_SUCCESS {
+				cvlErrObj = createCVLErrObj(err, jsonNode)
+				return nil, cvlErrObj
+			}
 			topNodesAdded = true
 		}
-		keyValuePair := getRedisToYangKeys(tableName, jsonNode.Data)
+		keyValuePair, cvlErrObj := getRedisToYangKeys(tableName, jsonNode.Data)
+		if cvlErrObj.ErrCode != CVL_SUCCESS {
+			return nil, cvlErrObj
+		}
 		keyCompCount := len(keyValuePair)
 		totalKeyComb := 1
 		var keyIndices []int
@@ -236,15 +246,17 @@ func (c *CVL) generateTableData(config bool, jsonNode *jsonquery.Node) (*yparser
 			//Ideally they are same except when one Redis table is split
 			//into multiple YANG lists
 
-			//Add table i.e. create list element
-			listNode := c.addChildNode(tableName, listConatinerNode, tableName+"_LIST") //Add the list to the top node
-
 			//For each key combination
 			//Add keys as leaf to the list
+			keyList := make([]*yparser.YParserLeafValue, 0)
 			for idx = 0; idx < keyCompCount; idx++ {
-				c.addChildLeaf(config, tableName,
-					listNode, keyValuePair[idx].key,
-					keyValuePair[idx].values[keyIndices[idx]], &c.batchLeaf)
+				c.appendLeafValue(keyValuePair[idx].key,
+					keyValuePair[idx].values[keyIndices[idx]], &keyList)
+			}
+			listNode, err := c.addListNode(tableName, listContainerNode, tableName+"_LIST", keyList) //Add the list to the top node
+			if err.ErrCode != yparser.YP_SUCCESS {
+				cvlErrObj = createCVLErrObj(err, jsonNode)
+				return nil, cvlErrObj
 			}
 
 			//Get all fields under the key field and add them as children of the list
@@ -273,7 +285,6 @@ func (c *CVL) generateTableData(config bool, jsonNode *jsonquery.Node) (*yparser
 			//process batch leaf creation
 			if errObj := c.yp.AddMultiLeafNodes(modelInfo.tableInfo[tableName].module, listNode, c.batchLeaf); errObj.ErrCode != yparser.YP_SUCCESS {
 				cvlErrObj = createCVLErrObj(errObj, jsonNode)
-				CVL_LOG(WARNING, "Failed to create leaf nodes, data = %v", c.batchLeaf)
 				return nil, cvlErrObj
 			}