Skip to content

fix: support import system package in go extensions #1058

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 10, 2025
Merged

fix: support import system package in go extensions #1058

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
189 changes: 154 additions & 35 deletions core/src/ten_runtime/binding/go/tools/build/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -528,7 +528,11 @@ func Generate(location string, envs []string, verbose bool) error {
return nil
}

func ModAddLocalModule(module *ExtensionModule, target string) error {
func ModAddLocalModule(
module *TenPackageModule,
target string,
moduleType string,
) error {
// go mod edit -replace <mod>=<full path>
//
// Note that the module must be replaced with the full path, as the module
Expand All @@ -540,8 +544,9 @@ func ModAddLocalModule(module *ExtensionModule, target string) error {
"edit",
"-replace",
fmt.Sprintf(
"%s=./ten_packages/extension/%s",
"%s=./ten_packages/%s/%s",
module.module,
moduleType,
path.Base(module.location),
),
},
Expand Down Expand Up @@ -613,11 +618,12 @@ func (b *BuildOption) Valid() error {
// -------------- builder ----------------

type AppBuilder struct {
pkgName string
acquiredGoVersion string
options *BuildOption
cachedEnv map[string]string
extensions []*ExtensionModule
pkgName string
acquiredGoVersion string
options *BuildOption
cachedEnv map[string]string
extensionPackageModules []*TenPackageModule
systemPackageModules []*TenPackageModule
}

func NewAppBuilder(
Expand All @@ -626,18 +632,19 @@ func NewAppBuilder(
options *BuildOption,
) *AppBuilder {
return &AppBuilder{
pkgName: pkgName,
acquiredGoVersion: goVersion,
options: options,
cachedEnv: make(map[string]string),
extensions: make([]*ExtensionModule, 0),
pkgName: pkgName,
acquiredGoVersion: goVersion,
options: options,
cachedEnv: make(map[string]string),
extensionPackageModules: make([]*TenPackageModule, 0),
systemPackageModules: make([]*TenPackageModule, 0),
}
}

// runTidyAndGenerate executes 'go mod tidy' and 'go generate' on GO app and all
// GO extensions.
func (ab *AppBuilder) runTidyAndGenerate(envs []string) error {
for _, ext := range ab.extensions {
for _, ext := range ab.extensionPackageModules {
if err := ModTidy(ext.location, envs, ab.options.Verbose); err != nil {
return err
}
Expand Down Expand Up @@ -690,9 +697,16 @@ func (ab *AppBuilder) Build() error {
return fmt.Errorf("precheck failed. Root cause: \n\t%w", err)
}

if err := ab.autoDetectExtensions(); err != nil {
if err := ab.autoDetectExtensionPackageModules(); err != nil {
return fmt.Errorf(
"auto detect extensions failed. Root cause: \n\t%w",
"auto detect extension packages failed. Root cause: \n\t%w",
err,
)
}

if err := ab.autoDetectSystemPackageModules(); err != nil {
return fmt.Errorf(
"auto detect system packages failed. Root cause: \n\t%w",
err,
)
}
Expand All @@ -713,7 +727,7 @@ func (ab *AppBuilder) Build() error {
return err
}

if err := ab.requireExtensionModules(); err != nil {
if err := ab.requireExtensionAndSystemPackageModules(); err != nil {
return err
}

Expand Down Expand Up @@ -967,21 +981,69 @@ func (ab *AppBuilder) buildExecEnvs() []string {

// -------------- extension --------------

type ExtensionModule struct {
type TenPackageModule struct {
// The module name in go.mod
module string

location string
}

func (em *ExtensionModule) String() string {
return fmt.Sprintf("%s @ %s", em.module, em.location)
func (tpm *TenPackageModule) String() string {
return fmt.Sprintf("%s @ %s", tpm.module, tpm.location)
}

func LoadExtensionModule(
func LoadSystemPackageModule(
location string,
verbose bool,
) (*ExtensionModule, error) {
) (*TenPackageModule, error) {
// Check if the folder contains a manifest.json to determine if it is a
// system package.
if !IsFilePresent(path.Join(location, "manifest.json")) {
if verbose {
log.Printf(
"%s is not a system package, no manifest.json.\n",
location,
)
}

return nil, nil
}

// Check if the folder contains a go.mod to determine if it is a GO system
// package.
modFile := path.Join(location, "go.mod")
if !IsFilePresent(modFile) {
if verbose {
log.Printf("%s is not a GO system package, no go.mod", location)
}

return nil, nil
}

bytes, err := os.ReadFile(modFile)
if err != nil {
return nil, fmt.Errorf(
"%s system package is invalid. \n\t%w",
location,
err,
)
}

module := ModulePath(bytes)
if len(module) == 0 {
return nil, fmt.Errorf("no mod is detected in %s/go.mod", location)
}

return &TenPackageModule{
module: module,
location: location,
}, nil
}

func LoadExtensionPackageModule(
location string,
verbose bool,
) (*TenPackageModule, error) {
// Check if the folder contains a manifest.json to determine if it is an
// extension.
if !IsFilePresent(path.Join(location, "manifest.json")) {
Expand Down Expand Up @@ -1024,13 +1086,58 @@ func LoadExtensionModule(
return nil, fmt.Errorf("no mod is detected in %s/go.mod", location)
}

return &ExtensionModule{
return &TenPackageModule{
module: module,
location: location,
}, nil
}

func (ab *AppBuilder) autoDetectExtensions() error {
func (ab *AppBuilder) autoDetectSystemPackageModules() error {
sysBaseDir := path.Join(ab.options.AppDir, "ten_packages/system")
if !IsDirPresent(sysBaseDir) {
if ab.options.Verbose {
log.Println(
"The base directory [ten_packages/system] is absent, no system packages.",
)
}

return nil
}

entries, err := os.ReadDir(sysBaseDir)
if err != nil {
return err
}

// Check if the system package is a GO module.
for _, entry := range entries {
if entry.IsDir() {
sysModFile := path.Join(sysBaseDir, entry.Name(), "go.mod")
if !IsFilePresent(sysModFile) {
continue
}

sysModDir := path.Join(sysBaseDir, entry.Name())
sysMod, err := LoadSystemPackageModule(
sysModDir,
ab.options.Verbose,
)
if err != nil {
return err
}

if sysMod == nil {
continue
}

ab.systemPackageModules = append(ab.systemPackageModules, sysMod)
}
}

return nil
}

func (ab *AppBuilder) autoDetectExtensionPackageModules() error {
extBaseDir := path.Join(ab.options.AppDir, "ten_packages/extension")
if !IsDirPresent(extBaseDir) {
if ab.options.Verbose {
Expand Down Expand Up @@ -1136,7 +1243,7 @@ func (ab *AppBuilder) autoDetectExtensions() error {
}

extDir := path.Join(extBaseDir, entry.Name())
ext, err := LoadExtensionModule(extDir, ab.options.Verbose)
ext, err := LoadExtensionPackageModule(extDir, ab.options.Verbose)
if err != nil {
return err
}
Expand All @@ -1156,12 +1263,12 @@ func (ab *AppBuilder) autoDetectExtensions() error {
}

uniqueModules[ext.module] = ext.location
ab.extensions = append(ab.extensions, ext)
ab.extensionPackageModules = append(ab.extensionPackageModules, ext)
}

if ab.options.Verbose && len(ab.extensions) > 0 {
if ab.options.Verbose && len(ab.extensionPackageModules) > 0 {
log.Println("Go Extensions are detected:")
for _, ext := range ab.extensions {
for _, ext := range ab.extensionPackageModules {
log.Printf("\t%s\n", ext)
}
}
Expand Down Expand Up @@ -1241,7 +1348,7 @@ func (ab *AppBuilder) generateAutoImportFile() error {
}
}

if len(ab.extensions) == 0 {
if len(ab.extensionPackageModules) == 0 {
log.Println(
"No extension is detected, no need to generate import file.",
)
Expand All @@ -1259,7 +1366,7 @@ func (ab *AppBuilder) generateAutoImportFile() error {
_, _ = f.WriteString("// Code generated by app builder. DO NOT EDIT.\n\n")
_, _ = f.WriteString(fmt.Sprintf("package %s\n\n", ab.pkgName))

for _, ext := range ab.extensions {
for _, ext := range ab.extensionPackageModules {
_, _ = f.WriteString(fmt.Sprintf("import _ \"%s\"\n", ext.module))
}

Expand All @@ -1270,10 +1377,11 @@ func (ab *AppBuilder) generateAutoImportFile() error {
return nil
}

func (ab *AppBuilder) requireExtensionModules() error {
if len(ab.extensions) == 0 {
func (ab *AppBuilder) requireExtensionAndSystemPackageModules() error {
if len(ab.extensionPackageModules) == 0 &&
len(ab.systemPackageModules) == 0 {
log.Println(
"No extension is detected, no need to require extension modules.",
"No extension or system package is detected, no need to require extension and system modules.",
)

return nil
Expand All @@ -1299,9 +1407,9 @@ func (ab *AppBuilder) requireExtensionModules() error {
}
}

// Add GO extensions as the module of the app.
for _, ext := range ab.extensions {
if err := ModAddLocalModule(ext, ab.options.AppDir); err != nil {
// Add GO extension packages as the module of the app.
for _, ext := range ab.extensionPackageModules {
if err := ModAddLocalModule(ext, ab.options.AppDir, "extension"); err != nil {
return fmt.Errorf(
"Failed to add %s as go module of the app. \n\t%w",
ext.location,
Expand All @@ -1310,6 +1418,17 @@ func (ab *AppBuilder) requireExtensionModules() error {
}
}

// Add GO system packages as the module of the app.
for _, sys := range ab.systemPackageModules {
if err := ModAddLocalModule(sys, ab.options.AppDir, "system"); err != nil {
return fmt.Errorf(
"Failed to add %s as go module of the app. \n\t%w",
sys.location,
err,
)
}
}

if ab.options.Verbose {
log.Println("Add extension modules to app successfully.")
}
Expand Down
Loading