From 9ad0c3b535e3f855f589699b12b2f4579b62d290 Mon Sep 17 00:00:00 2001 From: Chayan Das <01chayandas@gmail.com> Date: Sat, 7 Mar 2026 13:14:56 +0530 Subject: [PATCH 01/11] feat(dag): add --local-only to dag export and import - Export: only export blocks present locally; skip missing (partial CAR). --local-only with --offline. Support both binary and base58 link keys. - Import: support partial CARs; --local-only with -- pin-roots=false (error if both --pin-roots and --local-only set). - Fix cidFromBinString to accept base58 key format from link implementations. Signed-off-by: Chayan Das <01chayandas@gmail.com> --- core/commands/dag/dag.go | 6 ++++++ core/commands/dag/export.go | 36 +++++++++++++++++++++++++++++------- core/commands/dag/import.go | 4 ++++ go.mod | 2 ++ go.sum | 2 -- 5 files changed, 41 insertions(+), 9 deletions(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index a256213ecd0..f5548bddcce 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -23,6 +23,7 @@ const ( statsOptionName = "stats" fastProvideRootOptionName = "fast-provide-root" fastProvideWaitOptionName = "fast-provide-wait" + localOnlyOptionName = "local-only" ) // DagCmd provides a subset of commands for interacting with ipld dag objects @@ -192,6 +193,9 @@ Note: currently present in the blockstore does not represent a complete DAG, pinning of that individual root will fail. + Use --local-only with --pin-roots=false when importing a partial CAR (e.g. from + 'dag export --local-only'). + FAST PROVIDE OPTIMIZATION: Root CIDs from CAR headers are immediately provided to the DHT in addition @@ -213,6 +217,7 @@ Specification of CAR formats: https://ipld.io/specs/transport/car/ }, Options: []cmds.Option{ cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing.").WithDefault(true), + cmds.BoolOption(localOnlyOptionName, "Import a partial CAR without pinning roots (for CARs from dag export --local-only)."), cmds.BoolOption(silentOptionName, "No output."), cmds.BoolOption(statsOptionName, "Output stats."), cmds.BoolOption(fastProvideRootOptionName, "Immediately provide root CIDs to DHT in addition to regular queue, for faster discovery. Default: Import.FastProvideRoot"), @@ -285,6 +290,7 @@ CAR file follows the CARv1 format: https://ipld.io/specs/transport/car/carv1/ }, Options: []cmds.Option{ cmds.BoolOption(progressOptionName, "p", "Display progress on CLI. Defaults to true when STDERR is a TTY."), + cmds.BoolOption(localOnlyOptionName, "If set, only blocks present locally are exported; missing blocks are skipped (partial CAR). Use with --offline for a local-only DAG walk."), }, Run: dagExport, PostRun: cmds.PostRunMap{ diff --git a/core/commands/dag/export.go b/core/commands/dag/export.go index 48223f86083..9104ae74c57 100644 --- a/core/commands/dag/export.go +++ b/core/commands/dag/export.go @@ -1,6 +1,7 @@ package dagcmd import ( + "bytes" "context" "errors" "fmt" @@ -16,7 +17,10 @@ import ( "github.com/ipfs/kubo/core/commands/cmdutils" iface "github.com/ipfs/kubo/core/coreiface" gocar "github.com/ipld/go-car/v2" + "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/linking" cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipld/go-ipld-prime/traversal" selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" ) @@ -27,6 +31,7 @@ func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment return err } + localOnly, _ := req.Options[localOnlyOptionName].(bool) api, err := cmdenv.GetApi(env, req) if err != nil { return err @@ -51,7 +56,25 @@ func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment }() lsys := cidlink.DefaultLinkSystem() - lsys.SetReadStorage(&dagStore{dag: api.Dag(), ctx: req.Context}) + ds := &dagStore{dag: api.Dag(), ctx: req.Context} + if localOnly { + lsys.StorageReadOpener = func(lctx linking.LinkContext, lnk datamodel.Link) (io.Reader, error) { + cl, ok := lnk.(cidlink.Link) + if !ok { + return nil, fmt.Errorf("unsupported link type: %T", lnk) + } + data, err := ds.Get(lctx.Ctx, cl.Cid.String()) + if err != nil { + if ipld.IsNotFound(err) { + return nil, traversal.SkipMe{} + } + return nil, fmt.Errorf("local block read failed: %w", err) + } + return bytes.NewReader(data), nil + } + } else { + lsys.SetReadStorage(ds) + } // Uncomment the following to support CARv2 output. /* @@ -189,12 +212,11 @@ func (ds *dagStore) Has(ctx context.Context, key string) (bool, error) { } func cidFromBinString(key string) (cid.Cid, error) { - l, k, err := cid.CidFromBytes([]byte(key)) - if err != nil { - return cid.Undef, fmt.Errorf("dagStore: key was not a cid: %w", err) + if l, k, err := cid.CidFromBytes([]byte(key)); err == nil && l == len(key) { + return k, nil } - if l != len(key) { - return cid.Undef, fmt.Errorf("dagSore: key was not a cid: had %d bytes leftover", len(key)-l) + if c, decodeErr := cid.Decode(key); decodeErr == nil { + return c, nil } - return k, nil + return cid.Undef, fmt.Errorf("dagStore: key was not a cid (binary or base58)") } diff --git a/core/commands/dag/import.go b/core/commands/dag/import.go index 032b9e52a6c..fde3e9157f8 100644 --- a/core/commands/dag/import.go +++ b/core/commands/dag/import.go @@ -49,7 +49,11 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment } doPinRoots, _ := req.Options[pinRootsOptionName].(bool) + localOnly, _ := req.Options[localOnlyOptionName].(bool) + if doPinRoots && localOnly { + return fmt.Errorf("cannot pass both --%s and --%s", pinRootsOptionName, localOnlyOptionName) + } fastProvideRoot, fastProvideRootSet := req.Options[fastProvideRootOptionName].(bool) fastProvideWait, fastProvideWaitSet := req.Options[fastProvideWaitOptionName].(bool) diff --git a/go.mod b/go.mod index cfffd2b69b4..9c4a0877c68 100644 --- a/go.mod +++ b/go.mod @@ -280,3 +280,5 @@ exclude ( github.com/ipfs/go-ipfs-cmds v2.0.1+incompatible github.com/libp2p/go-libp2p v6.0.23+incompatible ) + +replace github.com/ipld/go-car/v2 => ../go-car/v2 diff --git a/go.sum b/go.sum index 819906d270e..4bdc7d0fadb 100644 --- a/go.sum +++ b/go.sum @@ -458,8 +458,6 @@ github.com/ipfs/go-test v0.2.3 h1:Z/jXNAReQFtCYyn7bsv/ZqUwS6E7iIcSpJ2CuzCvnrc= github.com/ipfs/go-test v0.2.3/go.mod h1:QW8vSKkwYvWFwIZQLGQXdkt9Ud76eQXRQ9Ao2H+cA1o= github.com/ipfs/go-unixfsnode v1.10.3 h1:c8sJjuGNkxXAQH75P+f5ngPda/9T+DrboVA0TcDGvGI= github.com/ipfs/go-unixfsnode v1.10.3/go.mod h1:2Jlc7DoEwr12W+7l8Hr6C7XF4NHST3gIkqSArLhGSxU= -github.com/ipld/go-car/v2 v2.16.0 h1:LWe0vmN/QcQmUU4tr34W5Nv5mNraW+G6jfN2s+ndBco= -github.com/ipld/go-car/v2 v2.16.0/go.mod h1:RqFGWN9ifcXVmCrTAVnfnxiWZk1+jIx67SYhenlmL34= github.com/ipld/go-codec-dagpb v1.7.0 h1:hpuvQjCSVSLnTnHXn+QAMR0mLmb1gA6wl10LExo2Ts0= github.com/ipld/go-codec-dagpb v1.7.0/go.mod h1:rD3Zg+zub9ZnxcLwfol/OTQRVjaLzXypgy4UqHQvilM= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= From b4027962057b38c4c32fcc35be0ccca06dae1432 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 9 Mar 2026 00:00:05 +0100 Subject: [PATCH 02/11] chore(deps): update go-car/v2 to latest master - remove local replace directive for go-car/v2 - upgrade to v2.16.1-0.20260306172652-7d2f4aceb070 --- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 4 +--- go.sum | 2 ++ test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index a0be7d15588..0e46deab170 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -99,7 +99,7 @@ require ( github.com/ipfs/go-peertaskqueue v0.8.3 // indirect github.com/ipfs/go-test v0.2.3 // indirect github.com/ipfs/go-unixfsnode v1.10.3 // indirect - github.com/ipld/go-car/v2 v2.16.0 // indirect + github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070 // indirect github.com/ipld/go-codec-dagpb v1.7.0 // indirect github.com/ipld/go-ipld-prime v0.22.0 // indirect github.com/ipshipyard/p2p-forge v0.7.0 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 969cf1187db..5af55edb1ca 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -416,8 +416,8 @@ github.com/ipfs/go-test v0.2.3 h1:Z/jXNAReQFtCYyn7bsv/ZqUwS6E7iIcSpJ2CuzCvnrc= github.com/ipfs/go-test v0.2.3/go.mod h1:QW8vSKkwYvWFwIZQLGQXdkt9Ud76eQXRQ9Ao2H+cA1o= github.com/ipfs/go-unixfsnode v1.10.3 h1:c8sJjuGNkxXAQH75P+f5ngPda/9T+DrboVA0TcDGvGI= github.com/ipfs/go-unixfsnode v1.10.3/go.mod h1:2Jlc7DoEwr12W+7l8Hr6C7XF4NHST3gIkqSArLhGSxU= -github.com/ipld/go-car/v2 v2.16.0 h1:LWe0vmN/QcQmUU4tr34W5Nv5mNraW+G6jfN2s+ndBco= -github.com/ipld/go-car/v2 v2.16.0/go.mod h1:RqFGWN9ifcXVmCrTAVnfnxiWZk1+jIx67SYhenlmL34= +github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070 h1:aUJhNVqQZV3PhxQX/BPAkurYnsxBmMLx2a7YPsgd5NU= +github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070/go.mod h1:eN3661vGFqfxzIfPLrCQIdKcY9iIOCgZON1CedVtvwk= github.com/ipld/go-codec-dagpb v1.7.0 h1:hpuvQjCSVSLnTnHXn+QAMR0mLmb1gA6wl10LExo2Ts0= github.com/ipld/go-codec-dagpb v1.7.0/go.mod h1:rD3Zg+zub9ZnxcLwfol/OTQRVjaLzXypgy4UqHQvilM= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= diff --git a/go.mod b/go.mod index 9c4a0877c68..92e528910f5 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/ipfs/go-metrics-prometheus v0.1.0 github.com/ipfs/go-test v0.2.3 github.com/ipfs/go-unixfsnode v1.10.3 - github.com/ipld/go-car/v2 v2.16.0 + github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070 github.com/ipld/go-codec-dagpb v1.7.0 github.com/ipld/go-ipld-prime v0.22.0 github.com/ipshipyard/p2p-forge v0.7.0 @@ -280,5 +280,3 @@ exclude ( github.com/ipfs/go-ipfs-cmds v2.0.1+incompatible github.com/libp2p/go-libp2p v6.0.23+incompatible ) - -replace github.com/ipld/go-car/v2 => ../go-car/v2 diff --git a/go.sum b/go.sum index 4bdc7d0fadb..bff2d977bf6 100644 --- a/go.sum +++ b/go.sum @@ -458,6 +458,8 @@ github.com/ipfs/go-test v0.2.3 h1:Z/jXNAReQFtCYyn7bsv/ZqUwS6E7iIcSpJ2CuzCvnrc= github.com/ipfs/go-test v0.2.3/go.mod h1:QW8vSKkwYvWFwIZQLGQXdkt9Ud76eQXRQ9Ao2H+cA1o= github.com/ipfs/go-unixfsnode v1.10.3 h1:c8sJjuGNkxXAQH75P+f5ngPda/9T+DrboVA0TcDGvGI= github.com/ipfs/go-unixfsnode v1.10.3/go.mod h1:2Jlc7DoEwr12W+7l8Hr6C7XF4NHST3gIkqSArLhGSxU= +github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070 h1:aUJhNVqQZV3PhxQX/BPAkurYnsxBmMLx2a7YPsgd5NU= +github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070/go.mod h1:eN3661vGFqfxzIfPLrCQIdKcY9iIOCgZON1CedVtvwk= github.com/ipld/go-codec-dagpb v1.7.0 h1:hpuvQjCSVSLnTnHXn+QAMR0mLmb1gA6wl10LExo2Ts0= github.com/ipld/go-codec-dagpb v1.7.0/go.mod h1:rD3Zg+zub9ZnxcLwfol/OTQRVjaLzXypgy4UqHQvilM= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index bb890a1c195..b562acf1222 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -149,7 +149,7 @@ require ( github.com/ipfs/go-metrics-interface v0.3.0 // indirect github.com/ipfs/go-unixfsnode v1.10.3 // indirect github.com/ipfs/kubo v0.31.0 // indirect - github.com/ipld/go-car/v2 v2.16.0 // indirect + github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070 // indirect github.com/ipld/go-codec-dagpb v1.7.0 // indirect github.com/ipld/go-ipld-prime v0.22.0 // indirect github.com/ipshipyard/p2p-forge v0.7.0 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index 2c131ae4576..7c396f08660 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -496,8 +496,8 @@ github.com/ipfs/iptb v1.4.1 h1:faXd3TKGPswbHyZecqqg6UfbES7RDjTKQb+6VFPKDUo= github.com/ipfs/iptb v1.4.1/go.mod h1:nTsBMtVYFEu0FjC5DgrErnABm3OG9ruXkFXGJoTV5OA= github.com/ipfs/iptb-plugins v0.5.1 h1:11PNTNEt2+SFxjUcO5qpyCTXqDj6T8Tx9pU/G4ytCIQ= github.com/ipfs/iptb-plugins v0.5.1/go.mod h1:mscJAjRnu4g16QK6oUBn9RGpcp8ueJmLfmPxIG/At78= -github.com/ipld/go-car/v2 v2.16.0 h1:LWe0vmN/QcQmUU4tr34W5Nv5mNraW+G6jfN2s+ndBco= -github.com/ipld/go-car/v2 v2.16.0/go.mod h1:RqFGWN9ifcXVmCrTAVnfnxiWZk1+jIx67SYhenlmL34= +github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070 h1:aUJhNVqQZV3PhxQX/BPAkurYnsxBmMLx2a7YPsgd5NU= +github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070/go.mod h1:eN3661vGFqfxzIfPLrCQIdKcY9iIOCgZON1CedVtvwk= github.com/ipld/go-codec-dagpb v1.7.0 h1:hpuvQjCSVSLnTnHXn+QAMR0mLmb1gA6wl10LExo2Ts0= github.com/ipld/go-codec-dagpb v1.7.0/go.mod h1:rD3Zg+zub9ZnxcLwfol/OTQRVjaLzXypgy4UqHQvilM= github.com/ipld/go-ipld-prime v0.22.0 h1:YJhDhjEOvOYaqshd3b4atIWUoRg/rKrgmwCyUHwlbuY= From 7b99db60393fa8373d9b6ff3edde7416ee3f09af Mon Sep 17 00:00:00 2001 From: Chayan Das <01chayandas@gmail.com> Date: Tue, 10 Mar 2026 20:06:20 +0530 Subject: [PATCH 03/11] fix(dag): avoid CID round-trip in export and fix ci failure Signed-off-by: Chayan Das <01chayandas@gmail.com> --- core/commands/dag/dag.go | 4 ++-- core/commands/dag/export.go | 15 ++++++++------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index f5548bddcce..eab70aa3332 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -193,7 +193,7 @@ Note: currently present in the blockstore does not represent a complete DAG, pinning of that individual root will fail. - Use --local-only with --pin-roots=false when importing a partial CAR (e.g. from + Use --local-only and --pin-roots=false for partial CARs (e.g. from 'dag export --local-only'). FAST PROVIDE OPTIMIZATION: @@ -217,7 +217,7 @@ Specification of CAR formats: https://ipld.io/specs/transport/car/ }, Options: []cmds.Option{ cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing.").WithDefault(true), - cmds.BoolOption(localOnlyOptionName, "Import a partial CAR without pinning roots (for CARs from dag export --local-only)."), + cmds.BoolOption(localOnlyOptionName, "Import partial CAR without pinning roots (e.g. from dag export --local-only)."), cmds.BoolOption(silentOptionName, "No output."), cmds.BoolOption(statsOptionName, "Output stats."), cmds.BoolOption(fastProvideRootOptionName, "Immediately provide root CIDs to DHT in addition to regular queue, for faster discovery. Default: Import.FastProvideRoot"), diff --git a/core/commands/dag/export.go b/core/commands/dag/export.go index 9104ae74c57..4967b47ab41 100644 --- a/core/commands/dag/export.go +++ b/core/commands/dag/export.go @@ -63,14 +63,14 @@ func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment if !ok { return nil, fmt.Errorf("unsupported link type: %T", lnk) } - data, err := ds.Get(lctx.Ctx, cl.Cid.String()) + block, err := ds.dag.Get(lctx.Ctx, cl.Cid) if err != nil { if ipld.IsNotFound(err) { return nil, traversal.SkipMe{} } return nil, fmt.Errorf("local block read failed: %w", err) } - return bytes.NewReader(data), nil + return bytes.NewReader(block.RawData()), nil } } else { lsys.SetReadStorage(ds) @@ -212,11 +212,12 @@ func (ds *dagStore) Has(ctx context.Context, key string) (bool, error) { } func cidFromBinString(key string) (cid.Cid, error) { - if l, k, err := cid.CidFromBytes([]byte(key)); err == nil && l == len(key) { - return k, nil + l, k, err := cid.CidFromBytes([]byte(key)) + if err != nil { + return cid.Undef, fmt.Errorf("dagStore: key was not a cid: %w", err) } - if c, decodeErr := cid.Decode(key); decodeErr == nil { - return c, nil + if l != len(key) { + return cid.Undef, fmt.Errorf("dagStore: key was not a cid: had %d bytes leftover", len(key)-l) } - return cid.Undef, fmt.Errorf("dagStore: key was not a cid (binary or base58)") + return k, nil } From 3b27eed8b0be97ed8e29781c0b56a165c9622bc3 Mon Sep 17 00:00:00 2001 From: Chayan Das <01chayandas@gmail.com> Date: Wed, 11 Mar 2026 12:08:14 +0530 Subject: [PATCH 04/11] dag: add validation and tests for --local-only flag Signed-off-by: Chayan Das <01chayandas@gmail.com> --- test/cli/dag_test.go | 155 +++++++++++++++++++++++++++++++++++++++ test/cli/harness/ipfs.go | 8 +- 2 files changed, 160 insertions(+), 3 deletions(-) diff --git a/test/cli/dag_test.go b/test/cli/dag_test.go index 38457318a0c..a834767ba20 100644 --- a/test/cli/dag_test.go +++ b/test/cli/dag_test.go @@ -1,9 +1,14 @@ package cli import ( + "context" "encoding/json" + "fmt" "io" "os" + "os/exec" + "path/filepath" + "strings" "testing" "time" @@ -305,3 +310,153 @@ func TestDagImportFastProvide(t *testing.T) { require.Contains(t, daemonLog, "fast-provide-root: skipped") }) } + +// dagRefs returns root plus recursive ref CIDs from "ipfs refs -r --unique root". +func dagRefs(node *harness.Node, root string) []string { + refsRes := node.IPFS("refs", "-r", "--unique", root) + refs := []string{root} + for _, line := range testutils.SplitLines(strings.TrimSpace(refsRes.Stdout.String())) { + if line != "" { + refs = append(refs, line) + } + } + return refs +} + +func parseImportedBlockCount(stdout string) int { + var n int + for _, line := range testutils.SplitLines(stdout) { + if _, err := fmt.Sscanf(line, "Imported %d blocks", &n); err == nil { + return n + } + } + return 0 +} + +func TestDagExportLocalOnly(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + root := node.IPFSAddDeterministic("300KiB", "dag-export-local-only", "--raw-leaves") + refs := dagRefs(node, root) + require.GreaterOrEqual(t, len(refs), 2, "need at least root and one child block") + + fullCarPath := filepath.Join(node.Dir, "full.car") + require.NoError(t, node.IPFSDagExport(root, fullCarPath)) + require.Equal(t, 0, node.RunIPFS("pin", "rm", root).ExitCode()) + require.Equal(t, 0, node.RunIPFS("block", "rm", refs[1]).ExitCode()) + + // Export --offline should fail; discard output (no file needed). + res := node.Runner.Run(harness.RunRequest{ + Path: node.IPFSBin, + Args: []string{"dag", "export", "--offline", root}, + CmdOpts: []harness.CmdOpt{harness.RunWithStdout(io.Discard)}, + }) + require.NotEqual(t, 0, res.ExitCode(), "export --offline without --local-only should fail when a block is missing") + require.Contains(t, res.Stderr.String(), "block was not found locally") + + partialCarPath := filepath.Join(node.Dir, "partial.car") + require.NoError(t, node.IPFSDagExport(root, partialCarPath, "--local-only", "--offline")) + + nodeFull := harness.NewT(t).NewNode().Init().StartDaemon() + defer nodeFull.StopDaemon() + fullCAR, err := os.Open(fullCarPath) + require.NoError(t, err) + defer fullCAR.Close() + fullRes := nodeFull.Runner.Run(harness.RunRequest{ + Path: nodeFull.IPFSBin, + Args: []string{"dag", "import", "--pin-roots=false", "--stats"}, + CmdOpts: []harness.CmdOpt{harness.RunWithStdin(fullCAR)}, + }) + require.Equal(t, 0, fullRes.ExitCode()) + fullCount := parseImportedBlockCount(fullRes.Stdout.String()) + require.Greater(t, fullCount, 0, "expected 'Imported N blocks' in output: %s", fullRes.Stdout.String()) + + nodePartial := harness.NewT(t).NewNode().Init().StartDaemon() + defer nodePartial.StopDaemon() + partialCAR, err := os.Open(partialCarPath) + require.NoError(t, err) + defer partialCAR.Close() + partialRes := nodePartial.Runner.Run(harness.RunRequest{ + Path: nodePartial.IPFSBin, + Args: []string{"dag", "import", "--pin-roots=false", "--stats"}, + CmdOpts: []harness.CmdOpt{harness.RunWithStdin(partialCAR)}, + }) + require.Equal(t, 0, partialRes.ExitCode()) + partialCount := parseImportedBlockCount(partialRes.Stdout.String()) + require.Greater(t, partialCount, 0, "expected 'Imported N blocks' in output: %s", partialRes.Stdout.String()) + + require.Less(t, partialCount, fullCount, "partial CAR should have fewer blocks than full DAG") +} + +func TestDagExportLocalOnlyRequiresOffline(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + root := node.IPFSAddDeterministic("300KiB", "dag-local-only-requires-offline", "--raw-leaves") + refs := dagRefs(node, root) + + require.GreaterOrEqual(t, len(refs), 2) + require.Equal(t, 0, node.RunIPFS("pin", "rm", root).ExitCode()) + require.Equal(t, 0, node.RunIPFS("block", "rm", refs[1]).ExitCode()) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + cmd := exec.CommandContext(ctx, node.IPFSBin, "dag", "export", "--local-only", root) + cmd.Env = append(os.Environ(), "IPFS_PATH="+node.Dir) + cmd.Stdout = io.Discard + + err := cmd.Run() + + require.Error(t, err) // command should fail +} + +func TestDagImportPartialCAR(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + root := node.IPFSAddDeterministic("300KiB", "dag-import-partial", "--raw-leaves") + refs := dagRefs(node, root) + require.GreaterOrEqual(t, len(refs), 2) + + require.Equal(t, 0, node.RunIPFS("pin", "rm", root).ExitCode()) + require.Equal(t, 0, node.RunIPFS("block", "rm", refs[1]).ExitCode()) + + partialCarPath := filepath.Join(node.Dir, "partial.car") + require.NoError(t, node.IPFSDagExport(root, partialCarPath, "--local-only", "--offline")) + + imp := harness.NewT(t).NewNode().Init().StartDaemon() + defer imp.StopDaemon() + partialCAR, err := os.Open(partialCarPath) + require.NoError(t, err) + defer partialCAR.Close() + require.NoError(t, imp.IPFSDagImport(partialCAR, root)) +} +func TestDagImportLocalOnlyPinRootsConflict(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + r, err := os.Open(fixtureFile) + require.NoError(t, err) + defer r.Close() + + res := node.Runner.Run(harness.RunRequest{ + Path: node.IPFSBin, + Args: []string{"dag", "import", "--local-only", "--pin-roots"}, + CmdOpts: []harness.CmdOpt{harness.RunWithStdin(r)}, + }) + + require.Equal(t, 1, res.ExitCode()) + require.Error(t, res.Err) + + errOutput := res.Stderr.String() + + require.Contains(t, errOutput, "cannot pass both") + require.Contains(t, errOutput, "pin-roots") + require.Contains(t, errOutput, "local-only") +} diff --git a/test/cli/harness/ipfs.go b/test/cli/harness/ipfs.go index d7470b4e764..637b316867b 100644 --- a/test/cli/harness/ipfs.go +++ b/test/cli/harness/ipfs.go @@ -162,17 +162,19 @@ func (n *Node) IPFSDagImport(content io.Reader, cid string, args ...string) erro } // IPFSDagExport exports a DAG rooted at cid to a CAR file at carPath. -func (n *Node) IPFSDagExport(cid string, carPath string) error { - log.Debugf("node %d dag export of %s to %q", n.ID, cid, carPath) +func (n *Node) IPFSDagExport(cid string, carPath string, args ...string) error { + log.Debugf("node %d dag export of %s to %q with args: %v", n.ID, cid, carPath, args) car, err := os.Create(carPath) if err != nil { return err } defer car.Close() + fullArgs := append([]string{"dag", "export"}, args...) + fullArgs = append(fullArgs, cid) res := n.Runner.MustRun(RunRequest{ Path: n.IPFSBin, - Args: []string{"dag", "export", cid}, + Args: fullArgs, CmdOpts: []CmdOpt{RunWithStdout(car)}, }) return res.Err From 51689a1629eee4c25ae4b652f972145fbdc5256d Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 25 May 2026 16:56:22 +0200 Subject: [PATCH 05/11] chore(deps): bump go-car/v2 to latest master --- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 59c87d10154..60f7edbd06f 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -101,7 +101,7 @@ require ( github.com/ipfs/go-peertaskqueue v0.8.3 // indirect github.com/ipfs/go-test v0.3.0 // indirect github.com/ipfs/go-unixfsnode v1.10.4 // indirect - github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070 // indirect + github.com/ipld/go-car/v2 v2.16.1-0.20260428045700-c4b9f366f20c // indirect github.com/ipld/go-codec-dagpb v1.7.0 // indirect github.com/ipld/go-ipld-prime v0.23.0 // indirect github.com/ipshipyard/p2p-forge v0.8.0 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 1dae0441445..7171803a3bd 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -422,8 +422,8 @@ github.com/ipfs/go-test v0.3.0 h1:0Y4Uve3tp9HI+2lIJjfOliOrOgv/YpXg/l1y3P4DEYE= github.com/ipfs/go-test v0.3.0/go.mod h1:JK+U8pRpATZb7lsYNSJlCj3WYB3cFfWIbI6nWRM/GFk= github.com/ipfs/go-unixfsnode v1.10.4 h1:cMmMyOrSjQkPVQbQvt8trErIn6jhayNf9pBA9oOwfxY= github.com/ipfs/go-unixfsnode v1.10.4/go.mod h1:Vu1e/s7ToALBBRo38sJ8DwUVWmSeQMTdxk5/rcHl7d0= -github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070 h1:aUJhNVqQZV3PhxQX/BPAkurYnsxBmMLx2a7YPsgd5NU= -github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070/go.mod h1:eN3661vGFqfxzIfPLrCQIdKcY9iIOCgZON1CedVtvwk= +github.com/ipld/go-car/v2 v2.16.1-0.20260428045700-c4b9f366f20c h1:ZFONxHSj6bzzB9eKIu+yS2AazTJe7j9FPesfy4sZSE0= +github.com/ipld/go-car/v2 v2.16.1-0.20260428045700-c4b9f366f20c/go.mod h1:/4HY8tFZ1q42Mw54ILLPQfjkUqMJxFKqY1yMDKHlYko= github.com/ipld/go-codec-dagpb v1.7.0 h1:hpuvQjCSVSLnTnHXn+QAMR0mLmb1gA6wl10LExo2Ts0= github.com/ipld/go-codec-dagpb v1.7.0/go.mod h1:rD3Zg+zub9ZnxcLwfol/OTQRVjaLzXypgy4UqHQvilM= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= diff --git a/go.mod b/go.mod index 9f0e1411edb..abdd79bded0 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/ipfs/go-metrics-prometheus v0.1.0 github.com/ipfs/go-test v0.3.0 github.com/ipfs/go-unixfsnode v1.10.4 - github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070 + github.com/ipld/go-car/v2 v2.16.1-0.20260428045700-c4b9f366f20c github.com/ipld/go-codec-dagpb v1.7.0 github.com/ipld/go-ipld-prime v0.23.0 github.com/ipshipyard/p2p-forge v0.8.0 diff --git a/go.sum b/go.sum index 4d389223e83..549a575784b 100644 --- a/go.sum +++ b/go.sum @@ -465,8 +465,8 @@ github.com/ipfs/go-test v0.3.0 h1:0Y4Uve3tp9HI+2lIJjfOliOrOgv/YpXg/l1y3P4DEYE= github.com/ipfs/go-test v0.3.0/go.mod h1:JK+U8pRpATZb7lsYNSJlCj3WYB3cFfWIbI6nWRM/GFk= github.com/ipfs/go-unixfsnode v1.10.4 h1:cMmMyOrSjQkPVQbQvt8trErIn6jhayNf9pBA9oOwfxY= github.com/ipfs/go-unixfsnode v1.10.4/go.mod h1:Vu1e/s7ToALBBRo38sJ8DwUVWmSeQMTdxk5/rcHl7d0= -github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070 h1:aUJhNVqQZV3PhxQX/BPAkurYnsxBmMLx2a7YPsgd5NU= -github.com/ipld/go-car/v2 v2.16.1-0.20260306172652-7d2f4aceb070/go.mod h1:eN3661vGFqfxzIfPLrCQIdKcY9iIOCgZON1CedVtvwk= +github.com/ipld/go-car/v2 v2.16.1-0.20260428045700-c4b9f366f20c h1:ZFONxHSj6bzzB9eKIu+yS2AazTJe7j9FPesfy4sZSE0= +github.com/ipld/go-car/v2 v2.16.1-0.20260428045700-c4b9f366f20c/go.mod h1:/4HY8tFZ1q42Mw54ILLPQfjkUqMJxFKqY1yMDKHlYko= github.com/ipld/go-codec-dagpb v1.7.0 h1:hpuvQjCSVSLnTnHXn+QAMR0mLmb1gA6wl10LExo2Ts0= github.com/ipld/go-codec-dagpb v1.7.0/go.mod h1:rD3Zg+zub9ZnxcLwfol/OTQRVjaLzXypgy4UqHQvilM= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= From 8871b6b4c2f3eeebc93be06dd29995730151aded Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 25 May 2026 17:17:45 +0200 Subject: [PATCH 06/11] feat(dag): --local-only auto-sets companion flags Pass --local-only without pairing it with --offline (export) or --pin-roots=false (import); the companion is now implicit. Explicit opposites (--offline=false, --pin-roots=true) are rejected so the intent stays unambiguous. * export: imply --offline so missing blocks are not fetched over the network, which would defeat --local-only * import: imply --pin-roots=false since a partial CAR has no full DAG to pin * tests: cover the new implications and the rejected explicit-opposite combinations; drop the brittle exec.CommandContext path in favor of the existing harness --- core/commands/dag/dag.go | 11 +++-- core/commands/dag/export.go | 9 ++++ core/commands/dag/import.go | 18 +++++-- test/cli/dag_test.go | 93 ++++++++++++++++++++++++++++--------- 4 files changed, 102 insertions(+), 29 deletions(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index e549ae2c481..e7067781e81 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -194,8 +194,9 @@ Note: currently present in the blockstore does not represent a complete DAG, pinning of that individual root will fail. - Use --local-only and --pin-roots=false for partial CARs (e.g. from - 'dag export --local-only'). + Use --local-only to import a partial CAR (e.g. from 'dag export + --local-only'). --local-only implies --pin-roots=false because a partial + CAR has no full DAG to pin. FAST PROVIDE OPTIMIZATION: @@ -217,8 +218,8 @@ Specification of CAR formats: https://ipld.io/specs/transport/car/ cmds.FileArg("path", true, true, "The path of a .car file.").EnableStdin(), }, Options: []cmds.Option{ - cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing.").WithDefault(true), - cmds.BoolOption(localOnlyOptionName, "Import partial CAR without pinning roots (e.g. from dag export --local-only)."), + cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing. Default: true."), + cmds.BoolOption(localOnlyOptionName, "Import a partial CAR (e.g. from 'dag export --local-only'). Implies --pin-roots=false."), cmds.BoolOption(silentOptionName, "No output."), cmds.BoolOption(statsOptionName, "Output stats."), cmds.BoolOption(fastProvideRootOptionName, "Immediately provide root CIDs to DHT in addition to regular queue, for faster discovery. Default: Import.FastProvideRoot"), @@ -292,7 +293,7 @@ CAR file follows the CARv1 format: https://ipld.io/specs/transport/car/carv1/ }, Options: []cmds.Option{ cmds.BoolOption(progressOptionName, "p", "Stream progress data. Defaults to true when stderr is a terminal."), - cmds.BoolOption(localOnlyOptionName, "If set, only blocks present locally are exported; missing blocks are skipped (partial CAR). Use with --offline for a local-only DAG walk."), + cmds.BoolOption(localOnlyOptionName, "Export only blocks present in the local blockstore; missing blocks are skipped (partial CAR). Implies --offline."), }, Run: dagExport, PostRun: cmds.PostRunMap{ diff --git a/core/commands/dag/export.go b/core/commands/dag/export.go index ebda01998e2..e8513ba476e 100644 --- a/core/commands/dag/export.go +++ b/core/commands/dag/export.go @@ -39,6 +39,15 @@ func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment } localOnly, _ := req.Options[localOnlyOptionName].(bool) + if localOnly { + offlineVal, offlineSet := req.Options["offline"].(bool) + if offlineSet && !offlineVal { + return fmt.Errorf("--%s implies --offline and cannot be combined with --offline=false; please drop one of them", localOnlyOptionName) + } + // --local-only implies --offline: a partial CAR is local-only by + // definition, so missing blocks must not be fetched over the network. + req.Options["offline"] = true + } api, err := cmdenv.GetApi(env, req) if err != nil { return err diff --git a/core/commands/dag/import.go b/core/commands/dag/import.go index 5cb446e1386..5ec9b0a29e5 100644 --- a/core/commands/dag/import.go +++ b/core/commands/dag/import.go @@ -48,11 +48,23 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment return err } - doPinRoots, _ := req.Options[pinRootsOptionName].(bool) + pinRootsVal, pinRootsSet := req.Options[pinRootsOptionName].(bool) localOnly, _ := req.Options[localOnlyOptionName].(bool) - if doPinRoots && localOnly { - return fmt.Errorf("cannot pass both --%s and --%s", pinRootsOptionName, localOnlyOptionName) + // --pin-roots defaults to true; the default is applied here (not via + // .WithDefault) so we can tell apart "user explicitly passed true" from + // "no value provided". + doPinRoots := true + if pinRootsSet { + doPinRoots = pinRootsVal + } + + if localOnly { + if pinRootsSet && pinRootsVal { + return fmt.Errorf("--%s implies --%s=false and cannot be combined with --%s=true; please drop one of them", localOnlyOptionName, pinRootsOptionName, pinRootsOptionName) + } + // --local-only implies --pin-roots=false: a partial CAR has no full DAG to pin. + doPinRoots = false } fastProvideRoot, fastProvideRootSet := req.Options[fastProvideRootOptionName].(bool) fastProvideDAG, fastProvideDAGSet := req.Options[fastProvideDAGOptionName].(bool) diff --git a/test/cli/dag_test.go b/test/cli/dag_test.go index 30e2335fa75..6dae4eff673 100644 --- a/test/cli/dag_test.go +++ b/test/cli/dag_test.go @@ -1,12 +1,10 @@ package cli import ( - "context" "encoding/json" "fmt" "io" "os" - "os/exec" "path/filepath" "strings" "testing" @@ -429,28 +427,46 @@ func TestDagExportLocalOnly(t *testing.T) { require.Less(t, partialCount, fullCount, "partial CAR should have fewer blocks than full DAG") } -func TestDagExportLocalOnlyRequiresOffline(t *testing.T) { +// TestDagExportLocalOnlyImpliesOffline verifies that --local-only on its own +// makes a partial-DAG export succeed: it implies --offline so the user does +// not have to pass both flags. +func TestDagExportLocalOnlyImpliesOffline(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() - root := node.IPFSAddDeterministic("300KiB", "dag-local-only-requires-offline", "--raw-leaves") + root := node.IPFSAddDeterministic("300KiB", "dag-export-local-only-implies", "--raw-leaves") refs := dagRefs(node, root) - require.GreaterOrEqual(t, len(refs), 2) require.Equal(t, 0, node.RunIPFS("pin", "rm", root).ExitCode()) require.Equal(t, 0, node.RunIPFS("block", "rm", refs[1]).ExitCode()) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() + // Export with only --local-only (no --offline). Should succeed because + // --local-only implies --offline. + partialCarPath := filepath.Join(node.Dir, "partial.car") + require.NoError(t, node.IPFSDagExport(root, partialCarPath, "--local-only")) - cmd := exec.CommandContext(ctx, node.IPFSBin, "dag", "export", "--local-only", root) - cmd.Env = append(os.Environ(), "IPFS_PATH="+node.Dir) - cmd.Stdout = io.Discard + // Sanity check: the partial CAR is non-empty and importable. + st, err := os.Stat(partialCarPath) + require.NoError(t, err) + require.Greater(t, st.Size(), int64(0)) +} - err := cmd.Run() +// TestDagExportLocalOnlyConflictsWithOnline verifies that explicitly asking +// for online mode together with --local-only is rejected, since the two +// settings contradict each other. +func TestDagExportLocalOnlyConflictsWithOnline(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() - require.Error(t, err) // command should fail + root := node.IPFSAddDeterministic("300KiB", "dag-export-local-only-online", "--raw-leaves") + + res := node.RunIPFS("dag", "export", "--local-only", "--offline=false", root) + require.NotEqual(t, 0, res.ExitCode(), "dag export --local-only --offline=false should be rejected") + stderr := res.Stderr.String() + require.Contains(t, stderr, "--local-only") + require.Contains(t, stderr, "--offline") } func TestDagImportPartialCAR(t *testing.T) { @@ -475,6 +491,45 @@ func TestDagImportPartialCAR(t *testing.T) { defer partialCAR.Close() require.NoError(t, imp.IPFSDagImport(partialCAR, root)) } + +// TestDagImportLocalOnlyImpliesNoPin verifies that --local-only on its own +// makes a partial-CAR import succeed: it implies --pin-roots=false so the +// user does not have to pass both flags. +func TestDagImportLocalOnlyImpliesNoPin(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + root := node.IPFSAddDeterministic("300KiB", "dag-import-local-only-implies", "--raw-leaves") + refs := dagRefs(node, root) + require.GreaterOrEqual(t, len(refs), 2) + require.Equal(t, 0, node.RunIPFS("pin", "rm", root).ExitCode()) + require.Equal(t, 0, node.RunIPFS("block", "rm", refs[1]).ExitCode()) + + partialCarPath := filepath.Join(node.Dir, "partial.car") + require.NoError(t, node.IPFSDagExport(root, partialCarPath, "--local-only", "--offline")) + + imp := harness.NewT(t).NewNode().Init().StartDaemon() + defer imp.StopDaemon() + partialCAR, err := os.Open(partialCarPath) + require.NoError(t, err) + defer partialCAR.Close() + + // Import with only --local-only (no --pin-roots=false). Should succeed + // because --local-only implies --pin-roots=false. + res := imp.Runner.Run(harness.RunRequest{ + Path: imp.IPFSBin, + Args: []string{"dag", "import", "--local-only"}, + CmdOpts: []harness.CmdOpt{harness.RunWithStdin(partialCAR)}, + }) + require.Equal(t, 0, res.ExitCode(), "dag import --local-only on a partial CAR should succeed; stderr: %s", res.Stderr.String()) + // No pinning happened, so no "Pinned root" line in stdout/stderr. + require.NotContains(t, res.Stdout.String(), "Pinned root") +} + +// TestDagImportLocalOnlyPinRootsConflict verifies that --local-only is +// rejected when combined with an explicit --pin-roots=true. The two are +// mutually exclusive: --local-only is for partial CARs (no full DAG to pin). func TestDagImportLocalOnlyPinRootsConflict(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() @@ -486,16 +541,12 @@ func TestDagImportLocalOnlyPinRootsConflict(t *testing.T) { res := node.Runner.Run(harness.RunRequest{ Path: node.IPFSBin, - Args: []string{"dag", "import", "--local-only", "--pin-roots"}, + Args: []string{"dag", "import", "--local-only", "--pin-roots=true"}, CmdOpts: []harness.CmdOpt{harness.RunWithStdin(r)}, }) - require.Equal(t, 1, res.ExitCode()) - require.Error(t, res.Err) - - errOutput := res.Stderr.String() - - require.Contains(t, errOutput, "cannot pass both") - require.Contains(t, errOutput, "pin-roots") - require.Contains(t, errOutput, "local-only") + require.NotEqual(t, 0, res.ExitCode()) + stderr := res.Stderr.String() + require.Contains(t, stderr, "--local-only") + require.Contains(t, stderr, "--pin-roots") } From bcc4457bcc3cda9f3e52cd0a5c9a1bb1c1ded17e Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 25 May 2026 17:56:53 +0200 Subject: [PATCH 07/11] refactor(dag): use boxo/walker for --local-only export The --local-only branch now uses walker.WalkDAG with WithLocality(bs.Has) and carstorage.NewWritable, matching the MFS+unique provider in core/node/provider.go. Semantics: any input-side read error during the walk (missing block, decode failure, post-locality race) is treated as "not available locally" and the block plus its subtree are skipped. Output-side errors (writable.Put) are still surfaced. --help is updated to call out the best-effort nature. The non-local-only path is unchanged. --- core/commands/dag/dag.go | 6 ++- core/commands/dag/export.go | 84 +++++++++++++++++++++++++++---------- 2 files changed, 68 insertions(+), 22 deletions(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index e7067781e81..ec042af2794 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -283,6 +283,10 @@ var DagExportCmd = &cmds.Command{ Note that at present only single root selections / .car files are supported. The output of blocks happens in strict DAG-traversal, first-seen, order. CAR file follows the CARv1 format: https://ipld.io/specs/transport/car/carv1/ + +Use --local-only for a best-effort export from the local blockstore: blocks +that are missing or unreadable locally (and their subtrees) are skipped, so +the resulting CAR is partial. --local-only implies --offline. `, HTTP: &cmds.HTTPHelpText{ ResponseContentType: "application/vnd.ipld.car", @@ -293,7 +297,7 @@ CAR file follows the CARv1 format: https://ipld.io/specs/transport/car/carv1/ }, Options: []cmds.Option{ cmds.BoolOption(progressOptionName, "p", "Stream progress data. Defaults to true when stderr is a terminal."), - cmds.BoolOption(localOnlyOptionName, "Export only blocks present in the local blockstore; missing blocks are skipped (partial CAR). Implies --offline."), + cmds.BoolOption(localOnlyOptionName, "Best-effort export of locally-available blocks; missing or unreadable blocks (and their subtrees) are skipped. Implies --offline."), }, Run: dagExport, PostRun: cmds.PostRunMap{ diff --git a/core/commands/dag/export.go b/core/commands/dag/export.go index e8513ba476e..4dfe1e18d75 100644 --- a/core/commands/dag/export.go +++ b/core/commands/dag/export.go @@ -1,7 +1,6 @@ package dagcmd import ( - "bytes" "context" "errors" "fmt" @@ -10,6 +9,8 @@ import ( "time" "github.com/cheggaaa/pb/v3" + blockstore "github.com/ipfs/boxo/blockstore" + "github.com/ipfs/boxo/dag/walker" cid "github.com/ipfs/go-cid" cmds "github.com/ipfs/go-ipfs-cmds" ipld "github.com/ipfs/go-ipld-format" @@ -17,10 +18,8 @@ import ( "github.com/ipfs/kubo/core/commands/cmdutils" iface "github.com/ipfs/kubo/core/coreiface" gocar "github.com/ipld/go-car/v2" - "github.com/ipld/go-ipld-prime/datamodel" - "github.com/ipld/go-ipld-prime/linking" + carstorage "github.com/ipld/go-car/v2/storage" cidlink "github.com/ipld/go-ipld-prime/linking/cid" - "github.com/ipld/go-ipld-prime/traversal" selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" ) @@ -60,6 +59,15 @@ func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment } c := b.Path().RootCid() + var bs blockstore.Blockstore + if localOnly { + node, err := cmdenv.GetNode(env) + if err != nil { + return err + } + bs = node.Blockstore + } + pipeR, pipeW := io.Pipe() errCh := make(chan error, 2) // we only report the 1st error @@ -71,27 +79,16 @@ func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment close(errCh) }() - lsys := cidlink.DefaultLinkSystem() - ds := &dagStore{dag: api.Dag(), ctx: req.Context} if localOnly { - lsys.StorageReadOpener = func(lctx linking.LinkContext, lnk datamodel.Link) (io.Reader, error) { - cl, ok := lnk.(cidlink.Link) - if !ok { - return nil, fmt.Errorf("unsupported link type: %T", lnk) - } - block, err := ds.dag.Get(lctx.Ctx, cl.Cid) - if err != nil { - if ipld.IsNotFound(err) { - return nil, traversal.SkipMe{} - } - return nil, fmt.Errorf("local block read failed: %w", err) - } - return bytes.NewReader(block.RawData()), nil + if err := exportPartialCAR(req.Context, bs, c, pipeW); err != nil { + errCh <- err } - } else { - lsys.SetReadStorage(ds) + return } + lsys := cidlink.DefaultLinkSystem() + lsys.SetReadStorage(&dagStore{dag: api.Dag(), ctx: req.Context}) + // Uncomment the following to support CARv2 output. /* car, err := gocar.NewSelectiveWriter(req.Context, &lsys, c, selectorparse.CommonSelector_ExploreAllRecursively, gocar.AllowDuplicatePuts(false)) @@ -137,6 +134,51 @@ func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment return err } +// exportPartialCAR is the best-effort engine behind `dag export --local-only`. +// It walks the DAG rooted at root using only the local blockstore and writes +// the visited blocks to w as a CARv1 stream. Any block that is missing or +// unreadable locally (and its entire subtree) is treated as "not available +// locally" and skipped. The resulting CAR is therefore partial by design. +// +// Errors writing the CAR itself (i.e. emit failures) are surfaced — those +// are output problems, not local-availability problems. +// +// This mirrors the MFS+unique provider in core/node/provider.go. +func exportPartialCAR(ctx context.Context, bs blockstore.Blockstore, root cid.Cid, w io.Writer) error { + writable, err := carstorage.NewWritable(w, []cid.Cid{root}, gocar.WriteAsCarV1(true)) + if err != nil { + return err + } + + // Capture the first emit (write-side) error so the walk stops cleanly. + var emitErr error + emit := func(k cid.Cid) bool { + blk, err := bs.Get(ctx, k) + if err != nil { + // Any read error after locality passed (e.g. GC race, + // corruption) is treated as "not available locally" — skip + // the block and keep streaming the rest of the partial CAR. + return true + } + if err := writable.Put(ctx, k.KeyString(), blk.RawData()); err != nil { + emitErr = err + return false + } + return true + } + + // walker.WithLocality + walker.LinksFetcherFromBlockstore also skip-and-log + // on locality/fetch errors, which matches the best-effort semantics here. + if err := walker.WalkDAG(ctx, root, + walker.LinksFetcherFromBlockstore(bs), + emit, + walker.WithLocality(func(ctx context.Context, k cid.Cid) (bool, error) { return bs.Has(ctx, k) }), + ); err != nil { + return err + } + return emitErr +} + func finishCLIExport(res cmds.Response, re cmds.ResponseEmitter) error { if !cmdenv.ShouldShowProgress(res.Request(), progressOptionName) { return cmds.Copy(re, res) From 16bb699eb4a2d321d9a2270f4680f4dba4fa701b Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 25 May 2026 17:57:01 +0200 Subject: [PATCH 08/11] test(dag): tighten --local-only tests, add subtree-skip case Pin chunker and max-file-links via a shared shallowDAGArgs so block counts are deterministic regardless of Import.* defaults or active profiles. Tighten existing assertions: * TestDagExportLocalOnly: assert exact fullCount=3 and partialCount=fullCount-1 instead of partialCount 0 (proves --offline was applied) Add TestDagExportLocalOnlySkipsSubtree: builds a 259-block DAG with depth>1 (256 chunks under 2 intermediates), removes an intermediate, and verifies the partial CAR is missing the intermediate plus all 174 of its descendants. Existing tests only exercised leaf removal. Extract countCARBlocks and makePartialDAG helpers used across tests. --- test/cli/dag_test.go | 192 ++++++++++++++++++++++++++++--------------- 1 file changed, 127 insertions(+), 65 deletions(-) diff --git a/test/cli/dag_test.go b/test/cli/dag_test.go index 6dae4eff673..5daadc18cfb 100644 --- a/test/cli/dag_test.go +++ b/test/cli/dag_test.go @@ -360,71 +360,94 @@ func dagRefs(node *harness.Node, root string) []string { return refs } -func parseImportedBlockCount(stdout string) int { +// countCARBlocks imports the CAR at carPath onto a fresh node and returns the +// number of blocks reported by `dag import --stats`. The fresh node guarantees +// the count reflects what is in the CAR, not what was already in the store. +func countCARBlocks(t *testing.T, carPath string) int { + t.Helper() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + car, err := os.Open(carPath) + require.NoError(t, err) + defer car.Close() + + res := node.Runner.Run(harness.RunRequest{ + Path: node.IPFSBin, + Args: []string{"dag", "import", "--pin-roots=false", "--stats"}, + CmdOpts: []harness.CmdOpt{harness.RunWithStdin(car)}, + }) + require.Equal(t, 0, res.ExitCode(), "dag import --stats failed: %s", res.Stderr.String()) + var n int - for _, line := range testutils.SplitLines(stdout) { + for _, line := range testutils.SplitLines(res.Stdout.String()) { if _, err := fmt.Sscanf(line, "Imported %d blocks", &n); err == nil { - return n + break } } - return 0 + require.Greater(t, n, 0, "expected 'Imported N blocks' in stdout: %q", res.Stdout.String()) + return n } +// shallowDAGArgs are the `ipfs add` args used by the partial-DAG helpers +// below. Chunker and max-file-links are pinned so the resulting DAG shape +// (root + 2 raw leaves) is independent of changes to Import.* defaults or +// applied profiles. +var shallowDAGArgs = []string{"--raw-leaves", "--chunker=size-262144", "--max-file-links=174"} + +// makePartialDAG adds a 300 KiB file with shallowDAGArgs (yielding root + 2 +// raw leaves) and then deletes the first leaf so the node holds a DAG with +// one missing block. Returns the root CID and the CID that was removed. +func makePartialDAG(t *testing.T, node *harness.Node, seed string, addArgs ...string) (root, removed string) { + t.Helper() + root = node.IPFSAddDeterministic("300KiB", seed, append(shallowDAGArgs, addArgs...)...) + refs := dagRefs(node, root) + require.Equal(t, 3, len(refs), "expected exactly root + 2 raw leaves with pinned chunker/max-links, got %v", refs) + require.Equal(t, 0, node.RunIPFS("pin", "rm", root).ExitCode()) + require.Equal(t, 0, node.RunIPFS("block", "rm", refs[1]).ExitCode()) + return root, refs[1] +} + +// TestDagExportLocalOnly verifies the core promise of --local-only: a DAG +// with a single missing leaf can still be exported as a partial CAR, and +// the partial CAR contains exactly the full DAG minus the removed block. func TestDagExportLocalOnly(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() - root := node.IPFSAddDeterministic("300KiB", "dag-export-local-only", "--raw-leaves") - refs := dagRefs(node, root) - require.GreaterOrEqual(t, len(refs), 2, "need at least root and one child block") - + // Snapshot the full DAG to a CAR before the block is removed, so we + // have a baseline block count to compare against. + root := node.IPFSAddDeterministic("300KiB", "dag-export-local-only", shallowDAGArgs...) fullCarPath := filepath.Join(node.Dir, "full.car") require.NoError(t, node.IPFSDagExport(root, fullCarPath)) + fullCount := countCARBlocks(t, fullCarPath) + require.Equal(t, 3, fullCount, "expected root + 2 raw leaves (full=%d)", fullCount) + + // Drop one leaf so the local DAG is partial. + refs := dagRefs(node, root) require.Equal(t, 0, node.RunIPFS("pin", "rm", root).ExitCode()) require.Equal(t, 0, node.RunIPFS("block", "rm", refs[1]).ExitCode()) - // Export --offline should fail; discard output (no file needed). + // Sanity: plain --offline (without --local-only) must fail loudly + // when a block is missing. This guards the existing behavior. res := node.Runner.Run(harness.RunRequest{ Path: node.IPFSBin, Args: []string{"dag", "export", "--offline", root}, CmdOpts: []harness.CmdOpt{harness.RunWithStdout(io.Discard)}, }) - require.NotEqual(t, 0, res.ExitCode(), "export --offline without --local-only should fail when a block is missing") + require.NotEqual(t, 0, res.ExitCode(), "dag export --offline must fail when a block is missing") require.Contains(t, res.Stderr.String(), "block was not found locally") + // --local-only must succeed and produce a CAR with exactly the + // full DAG minus the one removed leaf. partialCarPath := filepath.Join(node.Dir, "partial.car") require.NoError(t, node.IPFSDagExport(root, partialCarPath, "--local-only", "--offline")) + partialCount := countCARBlocks(t, partialCarPath) - nodeFull := harness.NewT(t).NewNode().Init().StartDaemon() - defer nodeFull.StopDaemon() - fullCAR, err := os.Open(fullCarPath) - require.NoError(t, err) - defer fullCAR.Close() - fullRes := nodeFull.Runner.Run(harness.RunRequest{ - Path: nodeFull.IPFSBin, - Args: []string{"dag", "import", "--pin-roots=false", "--stats"}, - CmdOpts: []harness.CmdOpt{harness.RunWithStdin(fullCAR)}, - }) - require.Equal(t, 0, fullRes.ExitCode()) - fullCount := parseImportedBlockCount(fullRes.Stdout.String()) - require.Greater(t, fullCount, 0, "expected 'Imported N blocks' in output: %s", fullRes.Stdout.String()) - - nodePartial := harness.NewT(t).NewNode().Init().StartDaemon() - defer nodePartial.StopDaemon() - partialCAR, err := os.Open(partialCarPath) - require.NoError(t, err) - defer partialCAR.Close() - partialRes := nodePartial.Runner.Run(harness.RunRequest{ - Path: nodePartial.IPFSBin, - Args: []string{"dag", "import", "--pin-roots=false", "--stats"}, - CmdOpts: []harness.CmdOpt{harness.RunWithStdin(partialCAR)}, - }) - require.Equal(t, 0, partialRes.ExitCode()) - partialCount := parseImportedBlockCount(partialRes.Stdout.String()) - require.Greater(t, partialCount, 0, "expected 'Imported N blocks' in output: %s", partialRes.Stdout.String()) - - require.Less(t, partialCount, fullCount, "partial CAR should have fewer blocks than full DAG") + require.Equal(t, fullCount-1, partialCount, + "partial CAR should be exactly the full DAG minus the one removed leaf (full=%d, partial=%d)", + fullCount, partialCount) } // TestDagExportLocalOnlyImpliesOffline verifies that --local-only on its own @@ -435,21 +458,64 @@ func TestDagExportLocalOnlyImpliesOffline(t *testing.T) { node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() - root := node.IPFSAddDeterministic("300KiB", "dag-export-local-only-implies", "--raw-leaves") + root, _ := makePartialDAG(t, node, "dag-export-local-only-implies") + + // Export with only --local-only (no --offline) and confirm the + // resulting CAR has the right number of blocks (full DAG minus one). + partialCarPath := filepath.Join(node.Dir, "partial.car") + require.NoError(t, node.IPFSDagExport(root, partialCarPath, "--local-only")) + + // 300KiB --raw-leaves yields root + 2 leaves, so removing one leaf + // leaves 2 blocks. Asserting the exact count proves --offline was + // actually applied (without it, the export would either fetch the + // missing block or fail differently). + require.Equal(t, 2, countCARBlocks(t, partialCarPath)) +} + +// TestDagExportLocalOnlySkipsSubtree verifies that when a non-leaf block is +// missing, --local-only skips the entire subtree under it, not just the +// missing block. Uses a small chunk size to force a depth>1 DAG so removing +// an intermediate prunes many descendant blocks. +func TestDagExportLocalOnlySkipsSubtree(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + // chunker=size-256 + 64 KiB → 256 leaves; max-file-links=174 forces + // at least one intermediate dag-pb layer between root and leaves + // (256 > 174). Both values are pinned so the DAG shape (and the + // counts below) survives any change to Import.* defaults or profiles. + root := node.IPFSAddDeterministic("64KiB", "dag-export-local-only-subtree", + "--raw-leaves", "--chunker=size-256", "--max-file-links=174") + fullCarPath := filepath.Join(node.Dir, "full.car") + require.NoError(t, node.IPFSDagExport(root, fullCarPath)) + fullCount := countCARBlocks(t, fullCarPath) + // 1 root + 2 intermediates (174 + 82 children) + 256 leaves = 259. + require.Equal(t, 259, fullCount, "expected root + 2 intermediates + 256 leaves, got %d", fullCount) + + // Find the first intermediate ref: a non-leaf whose codec is dag-pb. + // "ipfs refs -r --unique" lists CIDs depth-first; the root's first + // child in a balanced UnixFS DAG with >174 leaves is an intermediate. refs := dagRefs(node, root) - require.GreaterOrEqual(t, len(refs), 2) + intermediate := refs[1] + intermediateChildren := dagRefs(node, intermediate) + require.Greater(t, len(intermediateChildren), 10, + "expected refs[1] to be a non-leaf with many children, got %d", len(intermediateChildren)) + + // Remove the intermediate. Its subtree blocks remain locally, but + // without the intermediate the walker cannot reach them, so they + // must be skipped along with it. require.Equal(t, 0, node.RunIPFS("pin", "rm", root).ExitCode()) - require.Equal(t, 0, node.RunIPFS("block", "rm", refs[1]).ExitCode()) + require.Equal(t, 0, node.RunIPFS("block", "rm", intermediate).ExitCode()) - // Export with only --local-only (no --offline). Should succeed because - // --local-only implies --offline. partialCarPath := filepath.Join(node.Dir, "partial.car") require.NoError(t, node.IPFSDagExport(root, partialCarPath, "--local-only")) + partialCount := countCARBlocks(t, partialCarPath) - // Sanity check: the partial CAR is non-empty and importable. - st, err := os.Stat(partialCarPath) - require.NoError(t, err) - require.Greater(t, st.Size(), int64(0)) + expectedDropped := len(intermediateChildren) // includes the intermediate itself + require.Equal(t, fullCount-expectedDropped, partialCount, + "removing intermediate %s should drop it and its %d descendants (full=%d, partial=%d)", + intermediate, expectedDropped-1, fullCount, partialCount) } // TestDagExportLocalOnlyConflictsWithOnline verifies that explicitly asking @@ -469,17 +535,16 @@ func TestDagExportLocalOnlyConflictsWithOnline(t *testing.T) { require.Contains(t, stderr, "--offline") } +// TestDagImportPartialCAR is the round-trip happy path: a partial CAR from +// --local-only can be imported on a fresh node with default flags (the +// IPFSDagImport harness helper passes --pin-roots=false). The helper also +// confirms the root resolves offline on the receiver. func TestDagImportPartialCAR(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() - root := node.IPFSAddDeterministic("300KiB", "dag-import-partial", "--raw-leaves") - refs := dagRefs(node, root) - require.GreaterOrEqual(t, len(refs), 2) - - require.Equal(t, 0, node.RunIPFS("pin", "rm", root).ExitCode()) - require.Equal(t, 0, node.RunIPFS("block", "rm", refs[1]).ExitCode()) + root, _ := makePartialDAG(t, node, "dag-import-partial") partialCarPath := filepath.Join(node.Dir, "partial.car") require.NoError(t, node.IPFSDagExport(root, partialCarPath, "--local-only", "--offline")) @@ -500,12 +565,7 @@ func TestDagImportLocalOnlyImpliesNoPin(t *testing.T) { node := harness.NewT(t).NewNode().Init().StartDaemon() defer node.StopDaemon() - root := node.IPFSAddDeterministic("300KiB", "dag-import-local-only-implies", "--raw-leaves") - refs := dagRefs(node, root) - require.GreaterOrEqual(t, len(refs), 2) - require.Equal(t, 0, node.RunIPFS("pin", "rm", root).ExitCode()) - require.Equal(t, 0, node.RunIPFS("block", "rm", refs[1]).ExitCode()) - + root, _ := makePartialDAG(t, node, "dag-import-local-only-implies") partialCarPath := filepath.Join(node.Dir, "partial.car") require.NoError(t, node.IPFSDagExport(root, partialCarPath, "--local-only", "--offline")) @@ -515,16 +575,18 @@ func TestDagImportLocalOnlyImpliesNoPin(t *testing.T) { require.NoError(t, err) defer partialCAR.Close() - // Import with only --local-only (no --pin-roots=false). Should succeed - // because --local-only implies --pin-roots=false. + // Import with only --local-only (no --pin-roots=false). Should + // succeed because --local-only implies --pin-roots=false, and the + // receiver must not attempt to pin (pin would fail on a partial DAG). res := imp.Runner.Run(harness.RunRequest{ Path: imp.IPFSBin, Args: []string{"dag", "import", "--local-only"}, CmdOpts: []harness.CmdOpt{harness.RunWithStdin(partialCAR)}, }) - require.Equal(t, 0, res.ExitCode(), "dag import --local-only on a partial CAR should succeed; stderr: %s", res.Stderr.String()) - // No pinning happened, so no "Pinned root" line in stdout/stderr. - require.NotContains(t, res.Stdout.String(), "Pinned root") + require.Equal(t, 0, res.ExitCode(), + "dag import --local-only on a partial CAR should succeed; stderr: %s", res.Stderr.String()) + require.NotContains(t, res.Stdout.String(), "Pinned root", + "import must not pin when --local-only is set") } // TestDagImportLocalOnlyPinRootsConflict verifies that --local-only is From 6fcf6cac3c5e77305a01ff638863ba023e04b034 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 25 May 2026 18:02:14 +0200 Subject: [PATCH 09/11] docs: changelog entry for --local-only dag export/import --- docs/changelogs/v0.42.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/changelogs/v0.42.md b/docs/changelogs/v0.42.md index 09c35e473cf..c1bb64a7200 100644 --- a/docs/changelogs/v0.42.md +++ b/docs/changelogs/v0.42.md @@ -11,6 +11,7 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team. - [Overview](#overview) - [🔦 Highlights](#-highlights) - [🎯 Announce CIDs on demand with `ipfs provide once`](#-announce-cids-on-demand-with-ipfs-provide-once) + - [🧩 Export and import partial CARs with `--local-only`](#-export-and-import-partial-cars-with---local-only) - [⚙️ `Provide.DHT.Interval=0` no longer disables providing](#%EF%B8%8F-providedhtinterval0-no-longer-disables-providing) - [🐛 Fixed pin operations hanging under pinned reprovide strategies](#-fixed-pin-operations-hanging-under-pinned-reprovide-strategies) - [🐛 Smoother first-run upgrades from very old repos](#-smoother-first-run-upgrades-from-very-old-repos) @@ -46,6 +47,17 @@ In a terminal, the command shows a running count of queued CIDs. With `--enc=jso `ipfs routing provide` keeps working but is deprecated. See `ipfs provide once --help` for usage and migration notes. +#### 🧩 Export and import partial CARs with `--local-only` + +`ipfs dag export --local-only` writes a CAR with only the blocks you have locally; any missing blocks (and their subtrees) are skipped instead of failing the export. `ipfs dag import --local-only` reads such a partial CAR without trying to pin its roots. + +This is useful when: + +- you want to share part of a DAG (for example an MFS tree) that is only partly cached locally +- you fetched a partial CAR from a gateway that supports [IPIP-0402](https://specs.ipfs.tech/ipips/ipip-0402/) and want to add what you got to your local store + +`--local-only` sets the matching companion flag automatically: on export it implies `--offline`; on import it implies `--pin-roots=false`. See `ipfs dag export --help` and `ipfs dag import --help` for details. + #### ⚙️ `Provide.DHT.Interval=0` no longer disables providing `Provide.DHT.Interval=0` now disables only the periodic reprovide schedule. New CIDs still announce via fast-provide-root and `ipfs provide once`. To fully disable providing, set [`Provide.Enabled=false`](https://github.com/ipfs/kubo/blob/master/docs/config.md#provideenabled). From 1f39245d49a9d06a07212b79b6804cf43439af1e Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 25 May 2026 21:31:04 +0200 Subject: [PATCH 10/11] refactor(dag): wrap API explicitly for --local-only Replace the req.Options["offline"] = true mutation with an explicit api.WithOptions(options.Api.Offline(true)) wrap after GetApi, matching the pattern already used in core/commands/dag/import.go. Clarify in comments that the walker reads from the raw blockstore (not via the kubo CoreAPI or DAGService) and therefore cannot trigger a network fetch by construction. The --offline implication exists for api.Block().Stat path resolution, not for the DAG walk itself. --- core/commands/dag/export.go | 44 ++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/core/commands/dag/export.go b/core/commands/dag/export.go index 4dfe1e18d75..a79136e3c2b 100644 --- a/core/commands/dag/export.go +++ b/core/commands/dag/export.go @@ -17,6 +17,7 @@ import ( "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" iface "github.com/ipfs/kubo/core/coreiface" + "github.com/ipfs/kubo/core/coreiface/options" gocar "github.com/ipld/go-car/v2" carstorage "github.com/ipld/go-car/v2/storage" cidlink "github.com/ipld/go-ipld-prime/linking/cid" @@ -39,18 +40,26 @@ func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment localOnly, _ := req.Options[localOnlyOptionName].(bool) if localOnly { - offlineVal, offlineSet := req.Options["offline"].(bool) - if offlineSet && !offlineVal { + // --local-only and --offline=false contradict each other. + if offline, set := req.Options["offline"].(bool); set && !offline { return fmt.Errorf("--%s implies --offline and cannot be combined with --offline=false; please drop one of them", localOnlyOptionName) } - // --local-only implies --offline: a partial CAR is local-only by - // definition, so missing blocks must not be fetched over the network. - req.Options["offline"] = true } + api, err := cmdenv.GetApi(env, req) if err != nil { return err } + if localOnly { + // --local-only implies --offline so api.Block().Stat below cannot + // reach out for path resolution. The DAG walk itself uses the raw + // blockstore via walker (see exportPartialCAR) and is local by + // construction regardless of this setting. + api, err = api.WithOptions(options.Api.Offline(true)) + if err != nil { + return err + } + } // Resolve path and confirm the root block is available, fail fast if not b, err := api.Block().Stat(req.Context, p) @@ -135,13 +144,16 @@ func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment } // exportPartialCAR is the best-effort engine behind `dag export --local-only`. -// It walks the DAG rooted at root using only the local blockstore and writes -// the visited blocks to w as a CARv1 stream. Any block that is missing or -// unreadable locally (and its entire subtree) is treated as "not available -// locally" and skipped. The resulting CAR is therefore partial by design. +// It walks the DAG rooted at root and writes the visited blocks to w as a +// CARv1 stream. +// +// The walker reads from the raw blockstore directly (not via the kubo +// CoreAPI or DAGService), so it is structurally incapable of triggering a +// network fetch. Any block missing or unreadable locally, plus its entire +// subtree, is silently skipped: the resulting CAR is partial by design. // -// Errors writing the CAR itself (i.e. emit failures) are surfaced — those -// are output problems, not local-availability problems. +// Errors writing the CAR itself (emit failures) are surfaced: those are +// output problems, not local-availability problems. // // This mirrors the MFS+unique provider in core/node/provider.go. func exportPartialCAR(ctx context.Context, bs blockstore.Blockstore, root cid.Cid, w io.Writer) error { @@ -155,8 +167,8 @@ func exportPartialCAR(ctx context.Context, bs blockstore.Blockstore, root cid.Ci emit := func(k cid.Cid) bool { blk, err := bs.Get(ctx, k) if err != nil { - // Any read error after locality passed (e.g. GC race, - // corruption) is treated as "not available locally" — skip + // Any read error after locality passed (e.g. GC race or + // corruption) is treated as "not available locally": skip // the block and keep streaming the rest of the partial CAR. return true } @@ -167,8 +179,10 @@ func exportPartialCAR(ctx context.Context, bs blockstore.Blockstore, root cid.Ci return true } - // walker.WithLocality + walker.LinksFetcherFromBlockstore also skip-and-log - // on locality/fetch errors, which matches the best-effort semantics here. + // Both the locality check (bs.Has) and the link fetcher read straight + // from the blockstore, so the walk cannot reach the network. Errors + // inside walker (locality, fetch) are skip-and-log, matching the + // best-effort semantics here. if err := walker.WalkDAG(ctx, root, walker.LinksFetcherFromBlockstore(bs), emit, From d4fce1db727d090c68ad5cd5e1369063e1ce124b Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 25 May 2026 23:07:35 +0200 Subject: [PATCH 11/11] fix(provider): quiet context.Canceled on shutdown ResetCids returns ctx.Err() straight from its ctx-done select, so a shutdown-during-sync surfaces as err="context canceled" while the outer ctx.Err() check at the classifier sometimes races behind the propagation and logs at Error. Classify context.Canceled the same way as keystore.ErrClosed so the message lands at Debug. Applied to both the startup and periodic classifiers. DeadlineExceeded is intentionally not included: nothing in the current call chain imposes a deadline, and a future timeout would be a real failure worth logging at Error. Closes the flake in TestProviderKeystoreSyncShutdownQuiet (10/10 local soak now green; CI hit the race 3 reruns in a row). --- core/node/provider.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/core/node/provider.go b/core/node/provider.go index c07927d0906..6c87d8af6bc 100644 --- a/core/node/provider.go +++ b/core/node/provider.go @@ -880,11 +880,13 @@ func SweepingProviderOpt(cfg *config.Config) fx.Option { strategy := cfg.Provide.Strategy.WithDefault(config.DefaultProvideStrategy) providerLog.Infow("provider keystore sync started", "strategy", strategy) if err := syncKeystore(ctx); err != nil { - // ErrClosed means the keystore was closed by the shutdown - // hook while this goroutine was still in flight: the - // OnStart ctx is not cancelled yet, so we classify the - // failure as shutdown explicitly. - if ctx.Err() != nil || errors.Is(err, keystore.ErrClosed) { + // Shutdown can race ahead of ctx.Err() becoming + // visible here: ResetCids returns ctx.Err() + // straight from its own ctx-done select, and the + // keystore can also close mid-sync (ErrClosed) + // before the OnStart ctx is cancelled. Classify + // both as shutdown. + if ctx.Err() != nil || errors.Is(err, context.Canceled) || errors.Is(err, keystore.ErrClosed) { providerLog.Debugw("provider keystore sync interrupted by shutdown", "err", err, "strategy", strategy) } else { providerLog.Errorw("provider keystore sync failed", "err", err, "strategy", strategy) @@ -908,7 +910,11 @@ func SweepingProviderOpt(cfg *config.Config) fx.Option { return case <-ticker.C: if err := syncKeystore(gcCtx); err != nil { - if gcCtx.Err() != nil || errors.Is(err, keystore.ErrClosed) { + // See classifier note on the startup-sync + // branch above: context.Canceled can + // arrive ahead of gcCtx.Err() becoming + // visible to this goroutine. + if gcCtx.Err() != nil || errors.Is(err, context.Canceled) || errors.Is(err, keystore.ErrClosed) { providerLog.Debugw("provider keystore sync interrupted by shutdown", "err", err) } else { providerLog.Errorw("provider keystore sync failed", "err", err)