Add support for large packages in go_repos (#320)

toastwaffle · chrisnovakovic · web-flow · commit 492ed8323901 · 2025-10-13T19:06:04.000+01:00
Co-authored-by: Chris Novakovic &lt;chris@chrisn.me.uk&gt;
diff --git a/.plzconfig b/.plzconfig
@@ -1,5 +1,5 @@
 [Please]
-Version = >=17.11.0
+Version = >=17.21.0
 
 [build]
 hashcheckers = sha256
@@ -20,7 +20,7 @@ Target = //plugins:cc
 
 [Plugin "e2e"]
 Target = //plugins:e2e
-PleaseVersion = 17.10.1
+PleaseVersion = 17.21.0
 
 [PluginDefinition]
 Name = go
diff --git a/build_defs/go.build_defs b/build_defs/go.build_defs
@@ -240,7 +240,8 @@ def go_library(name:str, srcs:list, resources:list=[], asm_srcs:list=None, hdrs:
                _needs_transitive_deps=False, _all_srcs=False, cover:bool=True,
                filter_srcs:bool=True, _link_private:bool=False, _link_extra:bool=True, _abi:str=None,
                _generate_import_config:bool=True, _generate_pkg_info:bool=CONFIG.GO.PKG_INFO,
-               import_path:str='', labels:list=[], package:str=None, pgo_file:str=None, _module:str='', _subrepo:str=''):
+               import_path:str='', labels:list=[], package:str=None, pgo_file:str=None, _module:str='',
+               _subrepo:str='', _is_large_package:bool=False):
     """Generates a Go library which can be reused by other rules.
 
     Args:
@@ -464,6 +465,7 @@ def go_library(name:str, srcs:list, resources:list=[], asm_srcs:list=None, hdrs:
             package = package,
             test_only = test_only,
             visibility = visibility,
+            large_package = _is_large_package,
         )
         deps += [pkg_info]
         provides["pkg_info"] = pkg_info
@@ -484,7 +486,18 @@ def go_library(name:str, srcs:list, resources:list=[], asm_srcs:list=None, hdrs:
     if pgo_file:
         srcs['pgo'] = [pgo_file]
 
-    cmds, tools = _go_library_cmds(name, import_path=package_path, complete=complete, all_srcs=_all_srcs, cover=cover, filter_srcs=filter_srcs, abi=_abi, embedcfg=embedcfg, pgo_file=pgo_file)
+    cmds, tools = _go_library_cmds(
+        name,
+        import_path=package_path,
+        complete=complete,
+        all_srcs=_all_srcs,
+        cover=cover,
+        filter_srcs=filter_srcs,
+        abi=_abi,
+        embedcfg=embedcfg,
+        pgo_file=pgo_file,
+        large_package=_is_large_package,
+    )
 
     return build_rule(
         name = name,
@@ -502,6 +515,7 @@ def go_library(name:str, srcs:list, resources:list=[], asm_srcs:list=None, hdrs:
         tools = tools,
         needs_transitive_deps = _needs_transitive_deps,
         exported_deps = exported_deps,
+        src_list_files = _is_large_package,
     )
 
 def _generate_pkg_import_cfg_cmd(name, out, pkg_location, handle_basename_eq_dirname=False):
@@ -1176,7 +1190,7 @@ def _module_rule_name(module):
 
 def go_repo(module: str, version:str='', download:str=None, name:str=None, install:list=[], requirements:list=[],
             licences:list=None, patch:list=None, visibility:list=["PUBLIC"], deps:list=[], build_tags:list=CONFIG.GO.BUILD_TAGS,
-            third_party_path:str="third_party/go", strip:list=None, labels:list=[]):
+            third_party_path:str="third_party/go", strip:list=None, labels:list=[], large_packages:list=[]):
     """Adds a third party go module to the build graph as a subrepo. This is designed to be closer to how the `go.mod`
     file works, requiring only the module name and version to be specified. Unlike go_module, each package is compiled
     individually, and dependencies between packages are inferred by convention.
@@ -1214,6 +1228,9 @@ def go_repo(module: str, version:str='', download:str=None, name:str=None, insta
       third_party_path (str): Optional path of third_party directory.
       strip (list): A list of directories to strip from the repo
       labels (list): Labels for this rule.
+      large_packages (list): List of relative package names which should be marked as large in the
+                             generated go_library rules (i.e. packages which have large number of
+                             source files)
     """
     subrepo_name = _module_rule_name(module)
 
@@ -1251,6 +1268,7 @@ def go_repo(module: str, version:str='', download:str=None, name:str=None, insta
 
     build_tag_args = " ".join([f"--build_tag {build_tag}" for build_tag in build_tags])
     label_args = " ".join([f"--label '{label}'" for label in labels])
+    large_package_args = " ".join([f"--large_package '{pkg}'" for pkg in large_packages])
 
     pkgRoot = f"pkg/{CONFIG.OS}_{CONFIG.ARCH}/{module}"
 
@@ -1259,7 +1277,7 @@ def go_repo(module: str, version:str='', download:str=None, name:str=None, insta
         "find $SRCS_DOWNLOAD -name BUILD -delete",
         f"mkdir -p $(dirname {pkgRoot})",
         f"mv $SRCS_DOWNLOAD {pkgRoot}",
-        f"$TOOL generate {modFileArg} --module {module} --version '{version}' {build_tag_args} {label_args} --src_root={pkgRoot} --third_part_folder='{third_party_path}' --subrepo '{pkg_name}/{subrepo_name}' {install_args} {requirements}",
+        f"$TOOL generate {modFileArg} --module {module} --version '{version}' {build_tag_args} {label_args} {large_package_args} --src_root={pkgRoot} --third_part_folder='{third_party_path}' --subrepo '{pkg_name}/{subrepo_name}' {install_args} {requirements}",
         f"mv {pkgRoot} $OUT",
     ]
     cmd = " && ".join(cmds)
@@ -1559,7 +1577,7 @@ def _go_modinfo(name:str, test_only:bool=False, deps:list):
 
 
 def _go_pkg_info(name:str, srcs:list, visibility:list, importconfig:str=None, import_path:str="", package:str="",
-                 module:str="", subrepo:str="", test_only:bool=False, install:list=[]):
+                 module:str="", subrepo:str="", test_only:bool=False, install:list=[], large_package:bool=False):
     """Internal-only function for generating the pkg_info files"""
     install_flags = ""
     for i in install:
@@ -1594,6 +1612,7 @@ def _go_pkg_info(name:str, srcs:list, visibility:list, importconfig:str=None, im
         labels = ["go_pkg_info"],
         test_only = test_only,
         visibility=visibility,
+        src_list_files=large_package,
     )
 
 
@@ -1620,11 +1639,12 @@ def _set_go_env():
     return f"export CGO_ENABLED=1 && {cmd}" if CONFIG.GO.CGO_ENABLED else cmd
 
 
-def _go_library_cmds(name, import_path:str="", complete=True, all_srcs=False, cover=True, filter_srcs=True, abi=False, embedcfg=None, pgo_file=None):
+def _go_library_cmds(name, import_path:str="", complete=True, all_srcs=False, cover=True, filter_srcs=True, abi=False, embedcfg=None, pgo_file=None, large_package=False):
     """Returns the commands to run for building a Go library."""
     complete_flag = '-complete ' if complete else ''
     embed_flag = ' -embedcfg $SRCS_EMBED' if embedcfg else ''
     out_cmd = ' -o "$OUTS_O" -symabis $SRCS_ABI -asmhdr "$OUTS_H"' if abi else ' -o "$OUT"'
+    srcs_var = '$(cat _plz/named_srcs/go)' if large_package else '$SRCS_GO'
 
     if CONFIG.GO.GO_COMPILE_TOOL:
         tool = '"$TOOLS_COMPILE"'
@@ -1635,10 +1655,10 @@ def _go_library_cmds(name, import_path:str="", complete=True, all_srcs=False, co
 
     if filter_srcs:
         build_tags = '-t ' + ' -t '.join(CONFIG.GO.BUILD_TAGS) if CONFIG.GO.BUILD_TAGS else ''
-        filter_cmd = f'export SRCS_GO="$(\"${TOOLS_PLEASE_GO}\" filter {build_tags} $SRCS_GO)"; '
+        filter_cmd = f'filtered_srcs="$(\"${TOOLS_PLEASE_GO}\" filter {build_tags} {srcs_var})"; '
         tools['please_go'] = [CONFIG.GO.PLEASE_GO_TOOL]
     else:
-        filter_cmd = ''
+        filter_cmd = f'filtered_srcs={srcs_var}; '
 
     compile_cmd = f'{tool} -importcfg importconfig -trimpath "$TMP_DIR" {complete_flag}{embed_flag} -pack {out_cmd}'
     package_flag = f" -p {import_path}" if import_path else ""
@@ -1654,11 +1674,11 @@ def _go_library_cmds(name, import_path:str="", complete=True, all_srcs=False, co
     prefix = ('export SRCS_GO="$PKG_DIR/*.go"; ' + gen_import_cfg) if all_srcs else gen_import_cfg
 
     cmds = {
-        'dbg': f'{prefix}; {filter_cmd}{compile_cmd}{package_flag} -N -l $SRCS_GO',
-        'opt': f'{prefix}; {filter_cmd}{compile_cmd}{package_flag} $SRCS_GO',
+        'dbg': f'{prefix}; {filter_cmd}{compile_cmd}{package_flag} -N -l $filtered_srcs',
+        'opt': f'{prefix}; {filter_cmd}{compile_cmd}{package_flag} $filtered_srcs',
     }
     if cover:
-        cmds['cover'] = f'{prefix}; {filter_cmd} $TOOLS_PLEASE_GO cover -c covcfg.json -o covfiles.txt {package_flag} $SRCS_GO && {compile_cmd} -coveragecfg covcfg.json {package_flag} `cat covfiles.txt`'
+        cmds['cover'] = f'{prefix}; {filter_cmd} $TOOLS_PLEASE_GO cover -c covcfg.json -o covfiles.txt {package_flag} $filtered_srcs && {compile_cmd} -coveragecfg covcfg.json {package_flag} `cat covfiles.txt`'
         tools['please_go'] = [CONFIG.GO.PLEASE_GO_TOOL]
         if CONFIG.GO.GO_COVER_TOOL:
             tools['cover'] = CONFIG.GO.GO_COVER_TOOL
diff --git a/test/large_packages/BUILD b/test/large_packages/BUILD
@@ -0,0 +1,8 @@
+subinclude("//test/build_defs:e2e")
+
+please_repo_e2e_test(
+    name = "large_package_test",
+    # Package known to have ~1800 source files
+    plz_command = "plz --noupdate build ///third_party/go/github.com_elastic_go-elasticsearch_v9//typedapi/types",
+    repo = "test_repo",
+)
diff --git a/test/large_packages/test_repo/.plzconfig b/test/large_packages/test_repo/.plzconfig
@@ -0,0 +1,14 @@
+[Parse]
+BuildFileName = BUILD_FILE
+BuildFileName = BUILD
+
+[Plugin "go"]
+Target = //plugins:go
+PleaseGoTool = <pleasegotool>
+GoTool = <gotool>
+FeatureFlags = go_get
+Stdlib = //third_party/go:std
+Modfile = //third_party/go:modfile
+
+[FeatureFlags]
+ExcludeGoRules = true
diff --git a/test/large_packages/test_repo/plugins/BUILD_FILE b/test/large_packages/test_repo/plugins/BUILD_FILE
@@ -0,0 +1,4 @@
+plugin_repo(
+    name = "go",
+    revision = "master",
+)
diff --git a/test/large_packages/test_repo/third_party/go/BUILD_FILE b/test/large_packages/test_repo/third_party/go/BUILD_FILE
@@ -0,0 +1,17 @@
+subinclude("///go//build_defs:go")
+
+filegroup(
+    name = "modfile",
+    srcs = ["go.mod"],
+)
+
+go_stdlib(
+    name = "std",
+)
+
+go_repo(
+    licences = ["Apache-2.0"],
+    module = "github.com/elastic/go-elasticsearch/v9",
+    version = "v9.0.0",
+    large_packages = ["typedapi/types"],
+)
diff --git a/test/large_packages/test_repo/third_party/go/go.mod b/test/large_packages/test_repo/third_party/go/go.mod
@@ -0,0 +1,7 @@
+module gorepolargepackages
+
+go 1.21
+
+require (
+    github.com/elastic/go-elasticsearch/v9 v9.0.0
+)
diff --git a/tools/please_go/ChangeLog b/tools/please_go/ChangeLog
@@ -1,3 +1,7 @@
+Version 1.19.0
+-------------
+    * Enable `go_library` to compile packages containing large numbers of files (#320)
+
 Version 1.18.0
 -------------
     * Migrate to `go_repo` and upgrade golang.org/x packages (#316)
diff --git a/tools/please_go/VERSION b/tools/please_go/VERSION
@@ -1 +1 @@
-1.18.0
+1.19.0
diff --git a/tools/please_go/generate/generate.go b/tools/please_go/generate/generate.go
@@ -9,6 +9,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"slices"
 	"strings"
 
 	bazelbuild "github.com/bazelbuild/buildtools/build"
@@ -31,9 +32,10 @@ type Generate struct {
 	thirdPartyFolder   string
 	install            []string
 	labels             []string
+	largePackages      []string
 }
 
-func New(srcRoot, thirdPartyFolder, hostModFile, module, version, subrepo string, buildFileNames, moduleDeps, install []string, buildTags []string, labels []string) *Generate {
+func New(srcRoot, thirdPartyFolder, hostModFile, module, version, subrepo string, buildFileNames, moduleDeps, install []string, buildTags []string, labels []string, largePackages []string) *Generate {
 	moduleArg := module
 	if version != "" {
 		moduleArg += "@" + version
@@ -55,6 +57,7 @@ func New(srcRoot, thirdPartyFolder, hostModFile, module, version, subrepo string
 		moduleArg:          moduleArg,
 		subrepo:            subrepo,
 		labels:             labels,
+		largePackages:      largePackages,
 	}
 }
 
@@ -401,6 +404,13 @@ func NewStringExpr(s string) *bazelbuild.StringExpr {
 	return &bazelbuild.StringExpr{Value: s}
 }
 
+func NewBoolExpr(v bool) bazelbuild.Expr {
+	if v {
+		return &bazelbuild.LiteralExpr{Token: "True"}
+	}
+	return &bazelbuild.LiteralExpr{Token: "False"}
+}
+
 func NewStringList(ss []string) *bazelbuild.ListExpr {
 	l := new(bazelbuild.ListExpr)
 	for _, s := range ss {
@@ -447,21 +457,22 @@ func (g *Generate) ruleForPackage(pkg *build.Package, dir string) *Rule {
 	}
 
 	return &Rule{
-		name:          name,
-		kind:          packageKind(pkg),
-		srcs:          pkg.GoFiles,
-		module:        g.moduleArg,
-		subrepo:       g.subrepo,
-		cgoSrcs:       pkg.CgoFiles,
-		cSrcs:         pkg.CFiles,
-		compilerFlags: pkg.CgoCFLAGS,
-		linkerFlags:   orderLinkerFlags(pkg.CgoLDFLAGS),
-		pkgConfigs:    pkg.CgoPkgConfig,
-		asmFiles:      pkg.SFiles,
-		hdrs:          pkg.HFiles,
-		deps:          deps,
-		embedPatterns: pkg.EmbedPatterns,
-		isCMD:         pkg.IsCommand(),
+		name:           name,
+		kind:           packageKind(pkg),
+		srcs:           pkg.GoFiles,
+		module:         g.moduleArg,
+		subrepo:        g.subrepo,
+		cgoSrcs:        pkg.CgoFiles,
+		cSrcs:          pkg.CFiles,
+		compilerFlags:  pkg.CgoCFLAGS,
+		linkerFlags:    orderLinkerFlags(pkg.CgoLDFLAGS),
+		pkgConfigs:     pkg.CgoPkgConfig,
+		asmFiles:       pkg.SFiles,
+		hdrs:           pkg.HFiles,
+		deps:           deps,
+		embedPatterns:  pkg.EmbedPatterns,
+		isCMD:          pkg.IsCommand(),
+		isLargePackage: slices.Contains(g.largePackages, dir),
 	}
 }
 
diff --git a/tools/please_go/generate/rules.go b/tools/please_go/generate/rules.go
@@ -3,21 +3,22 @@ package generate
 import "github.com/bazelbuild/buildtools/build"
 
 type Rule struct {
-	name          string
-	kind          string
-	module        string
-	subrepo       string
-	srcs          []string
-	cgoSrcs       []string
-	cSrcs         []string
-	compilerFlags []string
-	linkerFlags   []string
-	pkgConfigs    []string
-	asmFiles      []string
-	hdrs          []string
-	deps          []string
-	embedPatterns []string
-	isCMD         bool
+	name           string
+	kind           string
+	module         string
+	subrepo        string
+	srcs           []string
+	cgoSrcs        []string
+	cSrcs          []string
+	compilerFlags  []string
+	linkerFlags    []string
+	pkgConfigs     []string
+	asmFiles       []string
+	hdrs           []string
+	deps           []string
+	embedPatterns  []string
+	isCMD          bool
+	isLargePackage bool
 }
 
 func populateRule(r *build.Rule, targetState *Rule) {
@@ -60,4 +61,7 @@ func populateRule(r *build.Rule, targetState *Rule) {
 		r.SetAttr("_module", NewStringExpr(targetState.module))
 		r.SetAttr("_subrepo", NewStringExpr(targetState.subrepo))
 	}
+	if targetState.isLargePackage {
+		r.SetAttr("_is_large_package", NewBoolExpr(true))
+	}
 }
diff --git a/tools/please_go/please_go.go b/tools/please_go/please_go.go
@@ -96,6 +96,7 @@ var opts = struct {
 		BuildTags        []string `long:"build_tag" description:"Any build tags to apply to the build"`
 		Subrepo          string   `long:"subrepo" description:"The subrepo root to output into"`
 		Labels           []string `long:"label" description:"Additional labels to attach to subrepo targets"`
+		LargePackages    []string `long:"large_package" description:"Relative names of packages which have lots of input files (meaning the go_library target should be marked as large)"`
 		Args             struct {
 			Requirements []string `positional-arg-name:"requirements" description:"Any module requirements not included in the go.mod"`
 		} `positional-args:"true"`
@@ -168,7 +169,7 @@ var subCommands = map[string]func() int{
 	},
 	"generate": func() int {
 		gen := opts.Generate
-		g := generate.New(gen.SrcRoot, gen.ThirdPartyFolder, gen.ModFile, gen.Module, gen.Version, gen.Subrepo, []string{"BUILD", "BUILD.plz"}, gen.Args.Requirements, gen.Install, gen.BuildTags, gen.Labels)
+		g := generate.New(gen.SrcRoot, gen.ThirdPartyFolder, gen.ModFile, gen.Module, gen.Version, gen.Subrepo, []string{"BUILD", "BUILD.plz"}, gen.Args.Requirements, gen.Install, gen.BuildTags, gen.Labels, gen.LargePackages)
 		if err := g.Generate(); err != nil {
 			log.Fatalf("failed to generate go rules: %v", err)
 		}
