Add Go generation flag ignore_unsupported (#712)

* Add Go generation flag `ignore_unsupported`

This allows generating from YANG files with unsupported statements such
as `notification` without erroring out.

* Add tests and logging

* Fix integration test error message

Author: wenovus

generator/generator.go

@@ -78,6 +78,7 @@ var (
 	trimEnumOpenConfigPrefix             = flag.Bool("trim_enum_openconfig_prefix", false, `If set to true when compressPaths=true, the organizational prefix "openconfig-" is trimmed from the module part of the name of enumerated names in the generated code`)
 	includeDescriptions                  = flag.Bool("include_descriptions", false, "If set to true when generateSchema=true, the YANG descriptions will be included in the generated code artefact.")
 	enumOrgPrefixesToTrim                []string
+	ignoreUnsupportedStatements          = flag.Bool("ignore_unsupported", false, "If set to true, unsupported YANG statements are ignored.")
 
 	// Flags used for GoStruct generation only.
 	generateFakeRoot        = flag.Bool("generate_fakeroot", false, "If set to true, a fake element at the root of the data tree is generated. By default the fake root entity is named Device, its name can be controlled with the fakeroot_name flag.")
@@ -329,7 +330,8 @@ func main() {
 			"",
 			ygen.IROptions{
 				ParseOptions: ygen.ParseOpts{
-					ExcludeModules: modsExcluded,
+					IgnoreUnsupportedStatements: *ignoreUnsupportedStatements,
+					ExcludeModules:              modsExcluded,
 					YANGParseOptions: yang.Options{
 						IgnoreSubmoduleCircularDependencies: *ignoreCircDeps,
 					},
@@ -439,6 +441,7 @@ func main() {
 		FakeRootName:                         *fakeRootName,
 		PathStructSuffix:                     *pathStructSuffix,
 		ExcludeModules:                       modsExcluded,
+		IgnoreUnsupportedStatements:          *ignoreUnsupportedStatements,
 		YANGParseOptions: yang.Options{
 			IgnoreSubmoduleCircularDependencies: *ignoreCircDeps,
 		},

gogen/codegen_test.go

@@ -139,6 +139,49 @@ func TestSimpleStructs(t *testing.T) {
 			},
 		},
 		wantStructsCodeFile: filepath.Join(TestRoot, "testdata/structs/openconfig-simple-no-compress.trimmed-enum.formatted-txt"),
+	}, {
+		name:    "simple openconfig test with unsupported statements, don't tolerate",
+		inFiles: []string{filepath.Join(datapath, "openconfig-simple-with-unsupported.yang")},
+		inConfig: CodeGenerator{
+			IROptions: ygen.IROptions{
+				TransformationOptions: ygen.TransformationOpts{
+					CompressBehaviour:                    genutil.PreferIntendedConfig,
+					ShortenEnumLeafNames:                 true,
+					EnumOrgPrefixesToTrim:                []string{"openconfig"},
+					UseDefiningModuleForTypedefEnumNames: true,
+					EnumerationsUseUnderscores:           true,
+				},
+			},
+			GoOptions: GoOpts{
+				GenerateSimpleUnions:    true,
+				GenerateLeafGetters:     true,
+				GeneratePopulateDefault: true,
+			},
+		},
+		wantErrSubstring: "unsupported statement type (Notification)",
+	}, {
+		name:    "simple openconfig test with unsupported statements, tolerate",
+		inFiles: []string{filepath.Join(datapath, "openconfig-simple-with-unsupported.yang")},
+		inConfig: CodeGenerator{
+			IROptions: ygen.IROptions{
+				ParseOptions: ygen.ParseOpts{
+					IgnoreUnsupportedStatements: true,
+				},
+				TransformationOptions: ygen.TransformationOpts{
+					CompressBehaviour:                    genutil.PreferIntendedConfig,
+					ShortenEnumLeafNames:                 true,
+					EnumOrgPrefixesToTrim:                []string{"openconfig"},
+					UseDefiningModuleForTypedefEnumNames: true,
+					EnumerationsUseUnderscores:           true,
+				},
+			},
+			GoOptions: GoOpts{
+				GenerateSimpleUnions:    true,
+				GenerateLeafGetters:     true,
+				GeneratePopulateDefault: true,
+			},
+		},
+		wantStructsCodeFile: filepath.Join(TestRoot, "testdata/structs/openconfig-simple-with-unsupported.formatted-txt"),
 	}, {
 		name:    "OpenConfig leaf-list defaults test, with compression",
 		inFiles: []string{filepath.Join(datapath, "openconfig-leaflist-default.yang")},

gogen/testdata/structs/openconfig-simple-with-unsupported.formatted-txt

@@ -0,0 +1,273 @@
+/*
+Package ocstructs is a generated package which contains definitions
+of structs which represent a YANG schema. The generated schema can be
+compressed by a series of transformations (compression was true
+in this case).
+
+This package was generated by codegen-tests
+using the following YANG input files:
+	- ../testdata/modules/openconfig-simple-with-unsupported.yang
+Imported modules were sourced from:
+*/
+package ocstructs
+
+import (
+	"encoding/json"
+	"fmt"
+	"reflect"
+
+	"github.com/openconfig/ygot/ygot"
+)
+
+// Binary is a type that is used for fields that have a YANG type of
+// binary. It is used such that binary fields can be distinguished from
+// leaf-lists of uint8s (which are mapped to []uint8, equivalent to
+// []byte in reflection).
+type Binary []byte
+
+// YANGEmpty is a type that is used for fields that have a YANG type of
+// empty. It is used such that empty fields can be distinguished from boolean fields
+// in the generated code.
+type YANGEmpty bool
+
+// UnionInt8 is an int8 type assignable to unions of which it is a subtype.
+type UnionInt8 int8
+
+// UnionInt16 is an int16 type assignable to unions of which it is a subtype.
+type UnionInt16 int16
+
+// UnionInt32 is an int32 type assignable to unions of which it is a subtype.
+type UnionInt32 int32
+
+// UnionInt64 is an int64 type assignable to unions of which it is a subtype.
+type UnionInt64 int64
+
+// UnionUint8 is a uint8 type assignable to unions of which it is a subtype.
+type UnionUint8 uint8
+
+// UnionUint16 is a uint16 type assignable to unions of which it is a subtype.
+type UnionUint16 uint16
+
+// UnionUint32 is a uint32 type assignable to unions of which it is a subtype.
+type UnionUint32 uint32
+
+// UnionUint64 is a uint64 type assignable to unions of which it is a subtype.
+type UnionUint64 uint64
+
+// UnionFloat64 is a float64 type assignable to unions of which it is a subtype.
+type UnionFloat64 float64
+
+// UnionString is a string type assignable to unions of which it is a subtype.
+type UnionString string
+
+// UnionBool is a bool type assignable to unions of which it is a subtype.
+type UnionBool bool
+
+// UnionUnsupported is an interface{} wrapper type for unsupported types. It is
+// assignable to unions of which it is a subtype.
+type UnionUnsupported struct {
+	Value interface{}
+}
+
+// Parent represents the /openconfig-simple/parent YANG schema element.
+type Parent struct {
+	Child	*Parent_Child	`path:"child" module:"openconfig-simple"`
+}
+
+// IsYANGGoStruct ensures that Parent implements the yang.GoStruct
+// interface. This allows functions that need to handle this struct to
+// identify it as being generated by ygen.
+func (*Parent) IsYANGGoStruct() {}
+
+// PopulateDefaults recursively populates unset leaf fields in the Parent
+// with default values as specified in the YANG schema, instantiating any nil
+// container fields.
+func (t *Parent) PopulateDefaults() {
+	if (t == nil) {
+		return
+	}
+	ygot.BuildEmptyTree(t)
+	t.Child.PopulateDefaults()
+}
+
+// ΛBelongingModule returns the name of the module that defines the namespace
+// of Parent.
+func (*Parent) ΛBelongingModule() string {
+	return "openconfig-simple"
+}
+
+// Parent_Child represents the /openconfig-simple/parent/child YANG schema element.
+type Parent_Child struct {
+	Four	Binary	`path:"config/four" module:"openconfig-simple/openconfig-simple"`
+	One	*string	`path:"config/one" module:"openconfig-simple/openconfig-simple"`
+	Three	E_Child_Three	`path:"config/three" module:"openconfig-simple/openconfig-simple"`
+	Two	*string	`path:"state/two" module:"openconfig-simple/openconfig-simple"`
+}
+
+// IsYANGGoStruct ensures that Parent_Child implements the yang.GoStruct
+// interface. This allows functions that need to handle this struct to
+// identify it as being generated by ygen.
+func (*Parent_Child) IsYANGGoStruct() {}
+
+// GetFour retrieves the value of the leaf Four from the Parent_Child
+// struct. If the field is unset but has a default value in the YANG schema,
+// then the default value will be returned.
+// Caution should be exercised whilst using this method since when without a
+// default value, it will return the Go zero value if the field is explicitly
+// unset. If the caller explicitly does not care if Four is set, it can
+// safely use t.GetFour() to retrieve the value. In the case that the
+// caller has different actions based on whether the leaf is set or unset, it
+// should use 'if t.Four == nil' before retrieving the leaf's value.
+func (t *Parent_Child) GetFour() Binary {
+	if t == nil || t.Four ==  nil {
+		return nil
+	}
+	return t.Four
+}
+
+// GetOne retrieves the value of the leaf One from the Parent_Child
+// struct. If the field is unset but has a default value in the YANG schema,
+// then the default value will be returned.
+// Caution should be exercised whilst using this method since when without a
+// default value, it will return the Go zero value if the field is explicitly
+// unset. If the caller explicitly does not care if One is set, it can
+// safely use t.GetOne() to retrieve the value. In the case that the
+// caller has different actions based on whether the leaf is set or unset, it
+// should use 'if t.One == nil' before retrieving the leaf's value.
+func (t *Parent_Child) GetOne() string {
+	if t == nil || t.One == nil {
+		return ""
+	}
+	return *t.One
+}
+
+// GetThree retrieves the value of the leaf Three from the Parent_Child
+// struct. If the field is unset but has a default value in the YANG schema,
+// then the default value will be returned.
+// Caution should be exercised whilst using this method since when without a
+// default value, it will return the Go zero value if the field is explicitly
+// unset. If the caller explicitly does not care if Three is set, it can
+// safely use t.GetThree() to retrieve the value. In the case that the
+// caller has different actions based on whether the leaf is set or unset, it
+// should use 'if t.Three == nil' before retrieving the leaf's value.
+func (t *Parent_Child) GetThree() E_Child_Three {
+	if t == nil || t.Three ==  0 {
+		return 0
+	}
+	return t.Three
+}
+
+// GetTwo retrieves the value of the leaf Two from the Parent_Child
+// struct. If the field is unset but has a default value in the YANG schema,
+// then the default value will be returned.
+// Caution should be exercised whilst using this method since when without a
+// default value, it will return the Go zero value if the field is explicitly
+// unset. If the caller explicitly does not care if Two is set, it can
+// safely use t.GetTwo() to retrieve the value. In the case that the
+// caller has different actions based on whether the leaf is set or unset, it
+// should use 'if t.Two == nil' before retrieving the leaf's value.
+func (t *Parent_Child) GetTwo() string {
+	if t == nil || t.Two == nil {
+		return ""
+	}
+	return *t.Two
+}
+
+// PopulateDefaults recursively populates unset leaf fields in the Parent_Child
+// with default values as specified in the YANG schema, instantiating any nil
+// container fields.
+func (t *Parent_Child) PopulateDefaults() {
+	if (t == nil) {
+		return
+	}
+	ygot.BuildEmptyTree(t)
+}
+
+// ΛBelongingModule returns the name of the module that defines the namespace
+// of Parent_Child.
+func (*Parent_Child) ΛBelongingModule() string {
+	return "openconfig-simple"
+}
+
+// RemoteContainer represents the /openconfig-simple/remote-container YANG schema element.
+type RemoteContainer struct {
+	ALeaf	*string	`path:"config/a-leaf" module:"openconfig-simple/openconfig-simple"`
+}
+
+// IsYANGGoStruct ensures that RemoteContainer implements the yang.GoStruct
+// interface. This allows functions that need to handle this struct to
+// identify it as being generated by ygen.
+func (*RemoteContainer) IsYANGGoStruct() {}
+
+// GetALeaf retrieves the value of the leaf ALeaf from the RemoteContainer
+// struct. If the field is unset but has a default value in the YANG schema,
+// then the default value will be returned.
+// Caution should be exercised whilst using this method since when without a
+// default value, it will return the Go zero value if the field is explicitly
+// unset. If the caller explicitly does not care if ALeaf is set, it can
+// safely use t.GetALeaf() to retrieve the value. In the case that the
+// caller has different actions based on whether the leaf is set or unset, it
+// should use 'if t.ALeaf == nil' before retrieving the leaf's value.
+func (t *RemoteContainer) GetALeaf() string {
+	if t == nil || t.ALeaf == nil {
+		return ""
+	}
+	return *t.ALeaf
+}
+
+// PopulateDefaults recursively populates unset leaf fields in the RemoteContainer
+// with default values as specified in the YANG schema, instantiating any nil
+// container fields.
+func (t *RemoteContainer) PopulateDefaults() {
+	if (t == nil) {
+		return
+	}
+	ygot.BuildEmptyTree(t)
+}
+
+// ΛBelongingModule returns the name of the module that defines the namespace
+// of RemoteContainer.
+func (*RemoteContainer) ΛBelongingModule() string {
+	return "openconfig-simple"
+}
+
+// E_Child_Three is a derived int64 type which is used to represent
+// the enumerated node Child_Three. An additional value named
+// Child_Three_UNSET is added to the enumeration which is used as
+// the nil value, indicating that the enumeration was not explicitly set by
+// the program importing the generated structures.
+type E_Child_Three int64
+
+// IsYANGGoEnum ensures that Child_Three implements the yang.GoEnum
+// interface. This ensures that Child_Three can be identified as a
+// mapped type for a YANG enumeration.
+func (E_Child_Three) IsYANGGoEnum() {}
+
+// ΛMap returns the value lookup map associated with  Child_Three.
+func (E_Child_Three) ΛMap() map[string]map[int64]ygot.EnumDefinition { return ΛEnum; }
+
+// String returns a logging-friendly string for E_Child_Three.
+func (e E_Child_Three) String() string {
+	return ygot.EnumLogString(e, int64(e), "E_Child_Three")
+}
+
+const (
+	// Child_Three_UNSET corresponds to the value UNSET of Child_Three
+	Child_Three_UNSET E_Child_Three = 0
+	// Child_Three_ONE corresponds to the value ONE of Child_Three
+	Child_Three_ONE E_Child_Three = 1
+	// Child_Three_TWO corresponds to the value TWO of Child_Three
+	Child_Three_TWO E_Child_Three = 2
+)
+
+// ΛEnum is a map, keyed by the name of the type defined for each enum in the
+// generated Go code, which provides a mapping between the constant int64 value
+// of each value of the enumeration, and the string that is used to represent it
+// in the YANG schema. The map is named ΛEnum in order to avoid clash with any
+// valid YANG identifier.
+var ΛEnum = map[string]map[int64]ygot.EnumDefinition{
+	"E_Child_Three": {
+		1: {Name: "ONE"},
+		2: {Name: "TWO"},
+	},
+}

testdata/modules/openconfig-simple-with-unsupported.yang

@@ -0,0 +1,46 @@
+module openconfig-simple {
+  prefix "ocs";
+  namespace "urn:ocs";
+  description
+    "A simple test module with the OpenConfig structure.";
+
+  import openconfig-remote { prefix "ocr"; }
+
+  grouping parent-config {
+    leaf one { type string; }
+    leaf three {
+      type enumeration {
+        enum ONE;
+        enum TWO;
+      }
+    }
+    leaf four {
+      type binary;
+    }
+  }
+
+  container parent {
+    description
+      "I am a parent container
+       that has 4 children.";
+    container child {
+      container config {
+        uses parent-config;
+      }
+      container state {
+        config false;
+        uses parent-config;
+        leaf two { type string; }
+      }
+    }
+  }
+
+  notification update {
+    description "update notification";
+    anyxml data {
+      description "updated stuff";
+    }
+  }
+
+  uses ocr:a-grouping;
+}