fix: support import system package in go extensions

Author: sunxilin

core/src/ten_runtime/binding/go/tools/build/main.go

@@ -528,7 +528,11 @@ func Generate(location string, envs []string, verbose bool) error {
 	return nil
 }
 
-func ModAddLocalModule(module *ExtensionModule, target string) error {
+func ModAddLocalModule(
+	module *TenPackageModule,
+	target string,
+	moduleType string,
+) error {
 	// go mod edit -replace <mod>=<full path>
 	//
 	// Note that the module must be replaced with the full path, as the module
@@ -540,8 +544,9 @@ func ModAddLocalModule(module *ExtensionModule, target string) error {
 			"edit",
 			"-replace",
 			fmt.Sprintf(
-				"%s=./ten_packages/extension/%s",
+				"%s=./ten_packages/%s/%s",
 				module.module,
+				moduleType,
 				path.Base(module.location),
 			),
 		},
@@ -613,11 +618,12 @@ func (b *BuildOption) Valid() error {
 // -------------- builder ----------------
 
 type AppBuilder struct {
-	pkgName           string
-	acquiredGoVersion string
-	options           *BuildOption
-	cachedEnv         map[string]string
-	extensions        []*ExtensionModule
+	pkgName                 string
+	acquiredGoVersion       string
+	options                 *BuildOption
+	cachedEnv               map[string]string
+	extensionPackageModules []*TenPackageModule
+	systemPackageModules    []*TenPackageModule
 }
 
 func NewAppBuilder(
@@ -626,18 +632,19 @@ func NewAppBuilder(
 	options *BuildOption,
 ) *AppBuilder {
 	return &AppBuilder{
-		pkgName:           pkgName,
-		acquiredGoVersion: goVersion,
-		options:           options,
-		cachedEnv:         make(map[string]string),
-		extensions:        make([]*ExtensionModule, 0),
+		pkgName:                 pkgName,
+		acquiredGoVersion:       goVersion,
+		options:                 options,
+		cachedEnv:               make(map[string]string),
+		extensionPackageModules: make([]*TenPackageModule, 0),
+		systemPackageModules:    make([]*TenPackageModule, 0),
 	}
 }
 
 // runTidyAndGenerate executes 'go mod tidy' and 'go generate' on GO app and all
 // GO extensions.
 func (ab *AppBuilder) runTidyAndGenerate(envs []string) error {
-	for _, ext := range ab.extensions {
+	for _, ext := range ab.extensionPackageModules {
 		if err := ModTidy(ext.location, envs, ab.options.Verbose); err != nil {
 			return err
 		}
@@ -690,9 +697,16 @@ func (ab *AppBuilder) Build() error {
 		return fmt.Errorf("precheck failed. Root cause: \n\t%w", err)
 	}
 
-	if err := ab.autoDetectExtensions(); err != nil {
+	if err := ab.autoDetectExtensionPackageModules(); err != nil {
 		return fmt.Errorf(
-			"auto detect extensions failed. Root cause: \n\t%w",
+			"auto detect extension packages failed. Root cause: \n\t%w",
+			err,
+		)
+	}
+
+	if err := ab.autoDetectSystemPackageModules(); err != nil {
+		return fmt.Errorf(
+			"auto detect system packages failed. Root cause: \n\t%w",
 			err,
 		)
 	}
@@ -713,7 +727,7 @@ func (ab *AppBuilder) Build() error {
 		return err
 	}
 
-	if err := ab.requireExtensionModules(); err != nil {
+	if err := ab.requireExtensionAndSystemPackageModules(); err != nil {
 		return err
 	}
 
@@ -967,21 +981,69 @@ func (ab *AppBuilder) buildExecEnvs() []string {
 
 // -------------- extension --------------
 
-type ExtensionModule struct {
+type TenPackageModule struct {
 	// The module name in go.mod
 	module string
 
 	location string
 }
 
-func (em *ExtensionModule) String() string {
-	return fmt.Sprintf("%s @ %s", em.module, em.location)
+func (tpm *TenPackageModule) String() string {
+	return fmt.Sprintf("%s @ %s", tpm.module, tpm.location)
 }
 
-func LoadExtensionModule(
+func LoadSystemPackageModule(
 	location string,
 	verbose bool,
-) (*ExtensionModule, error) {
+) (*TenPackageModule, error) {
+	// Check if the folder contains a manifest.json to determine if it is a
+	// system package.
+	if !IsFilePresent(path.Join(location, "manifest.json")) {
+		if verbose {
+			log.Printf(
+				"%s is not a system package, no manifest.json.\n",
+				location,
+			)
+		}
+
+		return nil, nil
+	}
+
+	// Check if the folder contains a go.mod to determine if it is a GO system
+	// package.
+	modFile := path.Join(location, "go.mod")
+	if !IsFilePresent(modFile) {
+		if verbose {
+			log.Printf("%s is not a GO system package, no go.mod", location)
+		}
+
+		return nil, nil
+	}
+
+	bytes, err := os.ReadFile(modFile)
+	if err != nil {
+		return nil, fmt.Errorf(
+			"%s system package is invalid. \n\t%w",
+			location,
+			err,
+		)
+	}
+
+	module := ModulePath(bytes)
+	if len(module) == 0 {
+		return nil, fmt.Errorf("no mod is detected in %s/go.mod", location)
+	}
+
+	return &TenPackageModule{
+		module:   module,
+		location: location,
+	}, nil
+}
+
+func LoadExtensionPackageModule(
+	location string,
+	verbose bool,
+) (*TenPackageModule, error) {
 	// Check if the folder contains a manifest.json to determine if it is an
 	// extension.
 	if !IsFilePresent(path.Join(location, "manifest.json")) {
@@ -1024,13 +1086,58 @@ func LoadExtensionModule(
 		return nil, fmt.Errorf("no mod is detected in %s/go.mod", location)
 	}
 
-	return &ExtensionModule{
+	return &TenPackageModule{
 		module:   module,
 		location: location,
 	}, nil
 }
 
-func (ab *AppBuilder) autoDetectExtensions() error {
+func (ab *AppBuilder) autoDetectSystemPackageModules() error {
+	sysBaseDir := path.Join(ab.options.AppDir, "ten_packages/system")
+	if !IsDirPresent(sysBaseDir) {
+		if ab.options.Verbose {
+			log.Println(
+				"The base directory [ten_packages/system] is absent, no system packages.",
+			)
+		}
+
+		return nil
+	}
+
+	entries, err := os.ReadDir(sysBaseDir)
+	if err != nil {
+		return err
+	}
+
+	// Check if the system package is a GO module.
+	for _, entry := range entries {
+		if entry.IsDir() {
+			sysModFile := path.Join(sysBaseDir, entry.Name(), "go.mod")
+			if !IsFilePresent(sysModFile) {
+				continue
+			}
+
+			sysModDir := path.Join(sysBaseDir, entry.Name())
+			sysMod, err := LoadSystemPackageModule(
+				sysModDir,
+				ab.options.Verbose,
+			)
+			if err != nil {
+				return err
+			}
+
+			if sysMod == nil {
+				continue
+			}
+
+			ab.systemPackageModules = append(ab.systemPackageModules, sysMod)
+		}
+	}
+
+	return nil
+}
+
+func (ab *AppBuilder) autoDetectExtensionPackageModules() error {
 	extBaseDir := path.Join(ab.options.AppDir, "ten_packages/extension")
 	if !IsDirPresent(extBaseDir) {
 		if ab.options.Verbose {
@@ -1136,7 +1243,7 @@ func (ab *AppBuilder) autoDetectExtensions() error {
 		}
 
 		extDir := path.Join(extBaseDir, entry.Name())
-		ext, err := LoadExtensionModule(extDir, ab.options.Verbose)
+		ext, err := LoadExtensionPackageModule(extDir, ab.options.Verbose)
 		if err != nil {
 			return err
 		}
@@ -1156,12 +1263,12 @@ func (ab *AppBuilder) autoDetectExtensions() error {
 		}
 
 		uniqueModules[ext.module] = ext.location
-		ab.extensions = append(ab.extensions, ext)
+		ab.extensionPackageModules = append(ab.extensionPackageModules, ext)
 	}
 
-	if ab.options.Verbose && len(ab.extensions) > 0 {
+	if ab.options.Verbose && len(ab.extensionPackageModules) > 0 {
 		log.Println("Go Extensions are detected:")
-		for _, ext := range ab.extensions {
+		for _, ext := range ab.extensionPackageModules {
 			log.Printf("\t%s\n", ext)
 		}
 	}
@@ -1241,7 +1348,7 @@ func (ab *AppBuilder) generateAutoImportFile() error {
 		}
 	}
 
-	if len(ab.extensions) == 0 {
+	if len(ab.extensionPackageModules) == 0 {
 		log.Println(
 			"No extension is detected, no need to generate import file.",
 		)
@@ -1259,7 +1366,7 @@ func (ab *AppBuilder) generateAutoImportFile() error {
 	_, _ = f.WriteString("// Code generated by app builder. DO NOT EDIT.\n\n")
 	_, _ = f.WriteString(fmt.Sprintf("package %s\n\n", ab.pkgName))
 
-	for _, ext := range ab.extensions {
+	for _, ext := range ab.extensionPackageModules {
 		_, _ = f.WriteString(fmt.Sprintf("import _ \"%s\"\n", ext.module))
 	}
 
@@ -1270,10 +1377,11 @@ func (ab *AppBuilder) generateAutoImportFile() error {
 	return nil
 }
 
-func (ab *AppBuilder) requireExtensionModules() error {
-	if len(ab.extensions) == 0 {
+func (ab *AppBuilder) requireExtensionAndSystemPackageModules() error {
+	if len(ab.extensionPackageModules) == 0 &&
+		len(ab.systemPackageModules) == 0 {
 		log.Println(
-			"No extension is detected, no need to require extension modules.",
+			"No extension or system package is detected, no need to require extension and system modules.",
 		)
 
 		return nil
@@ -1299,9 +1407,9 @@ func (ab *AppBuilder) requireExtensionModules() error {
 		}
 	}
 
-	// Add GO extensions as the module of the app.
-	for _, ext := range ab.extensions {
-		if err := ModAddLocalModule(ext, ab.options.AppDir); err != nil {
+	// Add GO extension packages as the module of the app.
+	for _, ext := range ab.extensionPackageModules {
+		if err := ModAddLocalModule(ext, ab.options.AppDir, "extension"); err != nil {
 			return fmt.Errorf(
 				"Failed to add %s as go module of the app. \n\t%w",
 				ext.location,
@@ -1310,6 +1418,17 @@ func (ab *AppBuilder) requireExtensionModules() error {
 		}
 	}
 
+	// Add GO system packages as the module of the app.
+	for _, sys := range ab.systemPackageModules {
+		if err := ModAddLocalModule(sys, ab.options.AppDir, "system"); err != nil {
+			return fmt.Errorf(
+				"Failed to add %s as go module of the app. \n\t%w",
+				sys.location,
+				err,
+			)
+		}
+	}
+
 	if ab.options.Verbose {
 		log.Println("Add extension modules to app successfully.")
 	}