diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index e4103581971..ec042af2794 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -24,6 +24,7 @@ const ( fastProvideRootOptionName = "fast-provide-root" fastProvideDAGOptionName = "fast-provide-dag" fastProvideWaitOptionName = "fast-provide-wait" + localOnlyOptionName = "local-only" ) // DagCmd provides a subset of commands for interacting with ipld dag objects @@ -193,6 +194,10 @@ Note: currently present in the blockstore does not represent a complete DAG, pinning of that individual root will fail. + Use --local-only to import a partial CAR (e.g. from 'dag export + --local-only'). --local-only implies --pin-roots=false because a partial + CAR has no full DAG to pin. + FAST PROVIDE OPTIMIZATION: Root CIDs from CAR headers are immediately provided to the DHT in addition @@ -213,7 +218,8 @@ Specification of CAR formats: https://ipld.io/specs/transport/car/ cmds.FileArg("path", true, true, "The path of a .car file.").EnableStdin(), }, Options: []cmds.Option{ - cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing.").WithDefault(true), + cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing. Default: true."), + cmds.BoolOption(localOnlyOptionName, "Import a partial CAR (e.g. from 'dag export --local-only'). Implies --pin-roots=false."), cmds.BoolOption(silentOptionName, "No output."), cmds.BoolOption(statsOptionName, "Output stats."), cmds.BoolOption(fastProvideRootOptionName, "Immediately provide root CIDs to DHT in addition to regular queue, for faster discovery. Default: Import.FastProvideRoot"), @@ -277,6 +283,10 @@ var DagExportCmd = &cmds.Command{ Note that at present only single root selections / .car files are supported. The output of blocks happens in strict DAG-traversal, first-seen, order. CAR file follows the CARv1 format: https://ipld.io/specs/transport/car/carv1/ + +Use --local-only for a best-effort export from the local blockstore: blocks +that are missing or unreadable locally (and their subtrees) are skipped, so +the resulting CAR is partial. --local-only implies --offline. `, HTTP: &cmds.HTTPHelpText{ ResponseContentType: "application/vnd.ipld.car", @@ -287,6 +297,7 @@ CAR file follows the CARv1 format: https://ipld.io/specs/transport/car/carv1/ }, Options: []cmds.Option{ cmds.BoolOption(progressOptionName, "p", "Stream progress data. Defaults to true when stderr is a terminal."), + cmds.BoolOption(localOnlyOptionName, "Best-effort export of locally-available blocks; missing or unreadable blocks (and their subtrees) are skipped. Implies --offline."), }, Run: dagExport, PostRun: cmds.PostRunMap{ diff --git a/core/commands/dag/export.go b/core/commands/dag/export.go index 3efad02f46b..a79136e3c2b 100644 --- a/core/commands/dag/export.go +++ b/core/commands/dag/export.go @@ -9,13 +9,17 @@ import ( "time" "github.com/cheggaaa/pb/v3" + blockstore "github.com/ipfs/boxo/blockstore" + "github.com/ipfs/boxo/dag/walker" cid "github.com/ipfs/go-cid" cmds "github.com/ipfs/go-ipfs-cmds" ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" iface "github.com/ipfs/kubo/core/coreiface" + "github.com/ipfs/kubo/core/coreiface/options" gocar "github.com/ipld/go-car/v2" + carstorage "github.com/ipld/go-car/v2/storage" cidlink "github.com/ipld/go-ipld-prime/linking/cid" selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" ) @@ -34,10 +38,28 @@ func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment return err } + localOnly, _ := req.Options[localOnlyOptionName].(bool) + if localOnly { + // --local-only and --offline=false contradict each other. + if offline, set := req.Options["offline"].(bool); set && !offline { + return fmt.Errorf("--%s implies --offline and cannot be combined with --offline=false; please drop one of them", localOnlyOptionName) + } + } + api, err := cmdenv.GetApi(env, req) if err != nil { return err } + if localOnly { + // --local-only implies --offline so api.Block().Stat below cannot + // reach out for path resolution. The DAG walk itself uses the raw + // blockstore via walker (see exportPartialCAR) and is local by + // construction regardless of this setting. + api, err = api.WithOptions(options.Api.Offline(true)) + if err != nil { + return err + } + } // Resolve path and confirm the root block is available, fail fast if not b, err := api.Block().Stat(req.Context, p) @@ -46,6 +68,15 @@ func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment } c := b.Path().RootCid() + var bs blockstore.Blockstore + if localOnly { + node, err := cmdenv.GetNode(env) + if err != nil { + return err + } + bs = node.Blockstore + } + pipeR, pipeW := io.Pipe() errCh := make(chan error, 2) // we only report the 1st error @@ -57,6 +88,13 @@ func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment close(errCh) }() + if localOnly { + if err := exportPartialCAR(req.Context, bs, c, pipeW); err != nil { + errCh <- err + } + return + } + lsys := cidlink.DefaultLinkSystem() lsys.SetReadStorage(&dagStore{dag: api.Dag(), ctx: req.Context}) @@ -105,6 +143,56 @@ func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment return err } +// exportPartialCAR is the best-effort engine behind `dag export --local-only`. +// It walks the DAG rooted at root and writes the visited blocks to w as a +// CARv1 stream. +// +// The walker reads from the raw blockstore directly (not via the kubo +// CoreAPI or DAGService), so it is structurally incapable of triggering a +// network fetch. Any block missing or unreadable locally, plus its entire +// subtree, is silently skipped: the resulting CAR is partial by design. +// +// Errors writing the CAR itself (emit failures) are surfaced: those are +// output problems, not local-availability problems. +// +// This mirrors the MFS+unique provider in core/node/provider.go. +func exportPartialCAR(ctx context.Context, bs blockstore.Blockstore, root cid.Cid, w io.Writer) error { + writable, err := carstorage.NewWritable(w, []cid.Cid{root}, gocar.WriteAsCarV1(true)) + if err != nil { + return err + } + + // Capture the first emit (write-side) error so the walk stops cleanly. + var emitErr error + emit := func(k cid.Cid) bool { + blk, err := bs.Get(ctx, k) + if err != nil { + // Any read error after locality passed (e.g. GC race or + // corruption) is treated as "not available locally": skip + // the block and keep streaming the rest of the partial CAR. + return true + } + if err := writable.Put(ctx, k.KeyString(), blk.RawData()); err != nil { + emitErr = err + return false + } + return true + } + + // Both the locality check (bs.Has) and the link fetcher read straight + // from the blockstore, so the walk cannot reach the network. Errors + // inside walker (locality, fetch) are skip-and-log, matching the + // best-effort semantics here. + if err := walker.WalkDAG(ctx, root, + walker.LinksFetcherFromBlockstore(bs), + emit, + walker.WithLocality(func(ctx context.Context, k cid.Cid) (bool, error) { return bs.Has(ctx, k) }), + ); err != nil { + return err + } + return emitErr +} + func finishCLIExport(res cmds.Response, re cmds.ResponseEmitter) error { if !cmdenv.ShouldShowProgress(res.Request(), progressOptionName) { return cmds.Copy(re, res) @@ -185,7 +273,7 @@ func cidFromBinString(key string) (cid.Cid, error) { return cid.Undef, fmt.Errorf("dagStore: key was not a cid: %w", err) } if l != len(key) { - return cid.Undef, fmt.Errorf("dagSore: key was not a cid: had %d bytes leftover", len(key)-l) + return cid.Undef, fmt.Errorf("dagStore: key was not a cid: had %d bytes leftover", len(key)-l) } return k, nil } diff --git a/core/commands/dag/import.go b/core/commands/dag/import.go index 472533be3c1..5ec9b0a29e5 100644 --- a/core/commands/dag/import.go +++ b/core/commands/dag/import.go @@ -48,8 +48,24 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment return err } - doPinRoots, _ := req.Options[pinRootsOptionName].(bool) + pinRootsVal, pinRootsSet := req.Options[pinRootsOptionName].(bool) + localOnly, _ := req.Options[localOnlyOptionName].(bool) + + // --pin-roots defaults to true; the default is applied here (not via + // .WithDefault) so we can tell apart "user explicitly passed true" from + // "no value provided". + doPinRoots := true + if pinRootsSet { + doPinRoots = pinRootsVal + } + if localOnly { + if pinRootsSet && pinRootsVal { + return fmt.Errorf("--%s implies --%s=false and cannot be combined with --%s=true; please drop one of them", localOnlyOptionName, pinRootsOptionName, pinRootsOptionName) + } + // --local-only implies --pin-roots=false: a partial CAR has no full DAG to pin. + doPinRoots = false + } fastProvideRoot, fastProvideRootSet := req.Options[fastProvideRootOptionName].(bool) fastProvideDAG, fastProvideDAGSet := req.Options[fastProvideDAGOptionName].(bool) fastProvideWait, fastProvideWaitSet := req.Options[fastProvideWaitOptionName].(bool) diff --git a/core/node/provider.go b/core/node/provider.go index c07927d0906..6c87d8af6bc 100644 --- a/core/node/provider.go +++ b/core/node/provider.go @@ -880,11 +880,13 @@ func SweepingProviderOpt(cfg *config.Config) fx.Option { strategy := cfg.Provide.Strategy.WithDefault(config.DefaultProvideStrategy) providerLog.Infow("provider keystore sync started", "strategy", strategy) if err := syncKeystore(ctx); err != nil { - // ErrClosed means the keystore was closed by the shutdown - // hook while this goroutine was still in flight: the - // OnStart ctx is not cancelled yet, so we classify the - // failure as shutdown explicitly. - if ctx.Err() != nil || errors.Is(err, keystore.ErrClosed) { + // Shutdown can race ahead of ctx.Err() becoming + // visible here: ResetCids returns ctx.Err() + // straight from its own ctx-done select, and the + // keystore can also close mid-sync (ErrClosed) + // before the OnStart ctx is cancelled. Classify + // both as shutdown. + if ctx.Err() != nil || errors.Is(err, context.Canceled) || errors.Is(err, keystore.ErrClosed) { providerLog.Debugw("provider keystore sync interrupted by shutdown", "err", err, "strategy", strategy) } else { providerLog.Errorw("provider keystore sync failed", "err", err, "strategy", strategy) @@ -908,7 +910,11 @@ func SweepingProviderOpt(cfg *config.Config) fx.Option { return case <-ticker.C: if err := syncKeystore(gcCtx); err != nil { - if gcCtx.Err() != nil || errors.Is(err, keystore.ErrClosed) { + // See classifier note on the startup-sync + // branch above: context.Canceled can + // arrive ahead of gcCtx.Err() becoming + // visible to this goroutine. + if gcCtx.Err() != nil || errors.Is(err, context.Canceled) || errors.Is(err, keystore.ErrClosed) { providerLog.Debugw("provider keystore sync interrupted by shutdown", "err", err) } else { providerLog.Errorw("provider keystore sync failed", "err", err) diff --git a/docs/changelogs/v0.42.md b/docs/changelogs/v0.42.md index 09c35e473cf..c1bb64a7200 100644 --- a/docs/changelogs/v0.42.md +++ b/docs/changelogs/v0.42.md @@ -11,6 +11,7 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team. - [Overview](#overview) - [🔦 Highlights](#-highlights) - [🎯 Announce CIDs on demand with `ipfs provide once`](#-announce-cids-on-demand-with-ipfs-provide-once) + - [🧩 Export and import partial CARs with `--local-only`](#-export-and-import-partial-cars-with---local-only) - [⚙️ `Provide.DHT.Interval=0` no longer disables providing](#%EF%B8%8F-providedhtinterval0-no-longer-disables-providing) - [🐛 Fixed pin operations hanging under pinned reprovide strategies](#-fixed-pin-operations-hanging-under-pinned-reprovide-strategies) - [🐛 Smoother first-run upgrades from very old repos](#-smoother-first-run-upgrades-from-very-old-repos) @@ -46,6 +47,17 @@ In a terminal, the command shows a running count of queued CIDs. With `--enc=jso `ipfs routing provide` keeps working but is deprecated. See `ipfs provide once --help` for usage and migration notes. +#### 🧩 Export and import partial CARs with `--local-only` + +`ipfs dag export --local-only` writes a CAR with only the blocks you have locally; any missing blocks (and their subtrees) are skipped instead of failing the export. `ipfs dag import --local-only` reads such a partial CAR without trying to pin its roots. + +This is useful when: + +- you want to share part of a DAG (for example an MFS tree) that is only partly cached locally +- you fetched a partial CAR from a gateway that supports [IPIP-0402](https://specs.ipfs.tech/ipips/ipip-0402/) and want to add what you got to your local store + +`--local-only` sets the matching companion flag automatically: on export it implies `--offline`; on import it implies `--pin-roots=false`. See `ipfs dag export --help` and `ipfs dag import --help` for details. + #### ⚙️ `Provide.DHT.Interval=0` no longer disables providing `Provide.DHT.Interval=0` now disables only the periodic reprovide schedule. New CIDs still announce via fast-provide-root and `ipfs provide once`. To fully disable providing, set [`Provide.Enabled=false`](https://github.com/ipfs/kubo/blob/master/docs/config.md#provideenabled). diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 1582969aa12..60f7edbd06f 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -101,7 +101,7 @@ require ( github.com/ipfs/go-peertaskqueue v0.8.3 // indirect github.com/ipfs/go-test v0.3.0 // indirect github.com/ipfs/go-unixfsnode v1.10.4 // indirect - github.com/ipld/go-car/v2 v2.16.0 // indirect + github.com/ipld/go-car/v2 v2.16.1-0.20260428045700-c4b9f366f20c // indirect github.com/ipld/go-codec-dagpb v1.7.0 // indirect github.com/ipld/go-ipld-prime v0.23.0 // indirect github.com/ipshipyard/p2p-forge v0.8.0 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index b926368cfda..7171803a3bd 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -422,8 +422,8 @@ github.com/ipfs/go-test v0.3.0 h1:0Y4Uve3tp9HI+2lIJjfOliOrOgv/YpXg/l1y3P4DEYE= github.com/ipfs/go-test v0.3.0/go.mod h1:JK+U8pRpATZb7lsYNSJlCj3WYB3cFfWIbI6nWRM/GFk= github.com/ipfs/go-unixfsnode v1.10.4 h1:cMmMyOrSjQkPVQbQvt8trErIn6jhayNf9pBA9oOwfxY= github.com/ipfs/go-unixfsnode v1.10.4/go.mod h1:Vu1e/s7ToALBBRo38sJ8DwUVWmSeQMTdxk5/rcHl7d0= -github.com/ipld/go-car/v2 v2.16.0 h1:LWe0vmN/QcQmUU4tr34W5Nv5mNraW+G6jfN2s+ndBco= -github.com/ipld/go-car/v2 v2.16.0/go.mod h1:RqFGWN9ifcXVmCrTAVnfnxiWZk1+jIx67SYhenlmL34= +github.com/ipld/go-car/v2 v2.16.1-0.20260428045700-c4b9f366f20c h1:ZFONxHSj6bzzB9eKIu+yS2AazTJe7j9FPesfy4sZSE0= +github.com/ipld/go-car/v2 v2.16.1-0.20260428045700-c4b9f366f20c/go.mod h1:/4HY8tFZ1q42Mw54ILLPQfjkUqMJxFKqY1yMDKHlYko= github.com/ipld/go-codec-dagpb v1.7.0 h1:hpuvQjCSVSLnTnHXn+QAMR0mLmb1gA6wl10LExo2Ts0= github.com/ipld/go-codec-dagpb v1.7.0/go.mod h1:rD3Zg+zub9ZnxcLwfol/OTQRVjaLzXypgy4UqHQvilM= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= diff --git a/go.mod b/go.mod index 05ee8b0ad58..abdd79bded0 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/ipfs/go-metrics-prometheus v0.1.0 github.com/ipfs/go-test v0.3.0 github.com/ipfs/go-unixfsnode v1.10.4 - github.com/ipld/go-car/v2 v2.16.0 + github.com/ipld/go-car/v2 v2.16.1-0.20260428045700-c4b9f366f20c github.com/ipld/go-codec-dagpb v1.7.0 github.com/ipld/go-ipld-prime v0.23.0 github.com/ipshipyard/p2p-forge v0.8.0 diff --git a/go.sum b/go.sum index 0c9b8fd84bb..549a575784b 100644 --- a/go.sum +++ b/go.sum @@ -465,8 +465,8 @@ github.com/ipfs/go-test v0.3.0 h1:0Y4Uve3tp9HI+2lIJjfOliOrOgv/YpXg/l1y3P4DEYE= github.com/ipfs/go-test v0.3.0/go.mod h1:JK+U8pRpATZb7lsYNSJlCj3WYB3cFfWIbI6nWRM/GFk= github.com/ipfs/go-unixfsnode v1.10.4 h1:cMmMyOrSjQkPVQbQvt8trErIn6jhayNf9pBA9oOwfxY= github.com/ipfs/go-unixfsnode v1.10.4/go.mod h1:Vu1e/s7ToALBBRo38sJ8DwUVWmSeQMTdxk5/rcHl7d0= -github.com/ipld/go-car/v2 v2.16.0 h1:LWe0vmN/QcQmUU4tr34W5Nv5mNraW+G6jfN2s+ndBco= -github.com/ipld/go-car/v2 v2.16.0/go.mod h1:RqFGWN9ifcXVmCrTAVnfnxiWZk1+jIx67SYhenlmL34= +github.com/ipld/go-car/v2 v2.16.1-0.20260428045700-c4b9f366f20c h1:ZFONxHSj6bzzB9eKIu+yS2AazTJe7j9FPesfy4sZSE0= +github.com/ipld/go-car/v2 v2.16.1-0.20260428045700-c4b9f366f20c/go.mod h1:/4HY8tFZ1q42Mw54ILLPQfjkUqMJxFKqY1yMDKHlYko= github.com/ipld/go-codec-dagpb v1.7.0 h1:hpuvQjCSVSLnTnHXn+QAMR0mLmb1gA6wl10LExo2Ts0= github.com/ipld/go-codec-dagpb v1.7.0/go.mod h1:rD3Zg+zub9ZnxcLwfol/OTQRVjaLzXypgy4UqHQvilM= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= diff --git a/test/cli/dag_test.go b/test/cli/dag_test.go index 7e28435f364..5daadc18cfb 100644 --- a/test/cli/dag_test.go +++ b/test/cli/dag_test.go @@ -2,8 +2,11 @@ package cli import ( "encoding/json" + "fmt" "io" "os" + "path/filepath" + "strings" "testing" "time" @@ -344,3 +347,268 @@ func TestDagImportFastProvide(t *testing.T) { require.Contains(t, daemonLog, "fast-provide-root: skipped") }) } + +// dagRefs returns root plus recursive ref CIDs from "ipfs refs -r --unique root". +func dagRefs(node *harness.Node, root string) []string { + refsRes := node.IPFS("refs", "-r", "--unique", root) + refs := []string{root} + for _, line := range testutils.SplitLines(strings.TrimSpace(refsRes.Stdout.String())) { + if line != "" { + refs = append(refs, line) + } + } + return refs +} + +// countCARBlocks imports the CAR at carPath onto a fresh node and returns the +// number of blocks reported by `dag import --stats`. The fresh node guarantees +// the count reflects what is in the CAR, not what was already in the store. +func countCARBlocks(t *testing.T, carPath string) int { + t.Helper() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + car, err := os.Open(carPath) + require.NoError(t, err) + defer car.Close() + + res := node.Runner.Run(harness.RunRequest{ + Path: node.IPFSBin, + Args: []string{"dag", "import", "--pin-roots=false", "--stats"}, + CmdOpts: []harness.CmdOpt{harness.RunWithStdin(car)}, + }) + require.Equal(t, 0, res.ExitCode(), "dag import --stats failed: %s", res.Stderr.String()) + + var n int + for _, line := range testutils.SplitLines(res.Stdout.String()) { + if _, err := fmt.Sscanf(line, "Imported %d blocks", &n); err == nil { + break + } + } + require.Greater(t, n, 0, "expected 'Imported N blocks' in stdout: %q", res.Stdout.String()) + return n +} + +// shallowDAGArgs are the `ipfs add` args used by the partial-DAG helpers +// below. Chunker and max-file-links are pinned so the resulting DAG shape +// (root + 2 raw leaves) is independent of changes to Import.* defaults or +// applied profiles. +var shallowDAGArgs = []string{"--raw-leaves", "--chunker=size-262144", "--max-file-links=174"} + +// makePartialDAG adds a 300 KiB file with shallowDAGArgs (yielding root + 2 +// raw leaves) and then deletes the first leaf so the node holds a DAG with +// one missing block. Returns the root CID and the CID that was removed. +func makePartialDAG(t *testing.T, node *harness.Node, seed string, addArgs ...string) (root, removed string) { + t.Helper() + root = node.IPFSAddDeterministic("300KiB", seed, append(shallowDAGArgs, addArgs...)...) + refs := dagRefs(node, root) + require.Equal(t, 3, len(refs), "expected exactly root + 2 raw leaves with pinned chunker/max-links, got %v", refs) + require.Equal(t, 0, node.RunIPFS("pin", "rm", root).ExitCode()) + require.Equal(t, 0, node.RunIPFS("block", "rm", refs[1]).ExitCode()) + return root, refs[1] +} + +// TestDagExportLocalOnly verifies the core promise of --local-only: a DAG +// with a single missing leaf can still be exported as a partial CAR, and +// the partial CAR contains exactly the full DAG minus the removed block. +func TestDagExportLocalOnly(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + // Snapshot the full DAG to a CAR before the block is removed, so we + // have a baseline block count to compare against. + root := node.IPFSAddDeterministic("300KiB", "dag-export-local-only", shallowDAGArgs...) + fullCarPath := filepath.Join(node.Dir, "full.car") + require.NoError(t, node.IPFSDagExport(root, fullCarPath)) + fullCount := countCARBlocks(t, fullCarPath) + require.Equal(t, 3, fullCount, "expected root + 2 raw leaves (full=%d)", fullCount) + + // Drop one leaf so the local DAG is partial. + refs := dagRefs(node, root) + require.Equal(t, 0, node.RunIPFS("pin", "rm", root).ExitCode()) + require.Equal(t, 0, node.RunIPFS("block", "rm", refs[1]).ExitCode()) + + // Sanity: plain --offline (without --local-only) must fail loudly + // when a block is missing. This guards the existing behavior. + res := node.Runner.Run(harness.RunRequest{ + Path: node.IPFSBin, + Args: []string{"dag", "export", "--offline", root}, + CmdOpts: []harness.CmdOpt{harness.RunWithStdout(io.Discard)}, + }) + require.NotEqual(t, 0, res.ExitCode(), "dag export --offline must fail when a block is missing") + require.Contains(t, res.Stderr.String(), "block was not found locally") + + // --local-only must succeed and produce a CAR with exactly the + // full DAG minus the one removed leaf. + partialCarPath := filepath.Join(node.Dir, "partial.car") + require.NoError(t, node.IPFSDagExport(root, partialCarPath, "--local-only", "--offline")) + partialCount := countCARBlocks(t, partialCarPath) + + require.Equal(t, fullCount-1, partialCount, + "partial CAR should be exactly the full DAG minus the one removed leaf (full=%d, partial=%d)", + fullCount, partialCount) +} + +// TestDagExportLocalOnlyImpliesOffline verifies that --local-only on its own +// makes a partial-DAG export succeed: it implies --offline so the user does +// not have to pass both flags. +func TestDagExportLocalOnlyImpliesOffline(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + root, _ := makePartialDAG(t, node, "dag-export-local-only-implies") + + // Export with only --local-only (no --offline) and confirm the + // resulting CAR has the right number of blocks (full DAG minus one). + partialCarPath := filepath.Join(node.Dir, "partial.car") + require.NoError(t, node.IPFSDagExport(root, partialCarPath, "--local-only")) + + // 300KiB --raw-leaves yields root + 2 leaves, so removing one leaf + // leaves 2 blocks. Asserting the exact count proves --offline was + // actually applied (without it, the export would either fetch the + // missing block or fail differently). + require.Equal(t, 2, countCARBlocks(t, partialCarPath)) +} + +// TestDagExportLocalOnlySkipsSubtree verifies that when a non-leaf block is +// missing, --local-only skips the entire subtree under it, not just the +// missing block. Uses a small chunk size to force a depth>1 DAG so removing +// an intermediate prunes many descendant blocks. +func TestDagExportLocalOnlySkipsSubtree(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + // chunker=size-256 + 64 KiB → 256 leaves; max-file-links=174 forces + // at least one intermediate dag-pb layer between root and leaves + // (256 > 174). Both values are pinned so the DAG shape (and the + // counts below) survives any change to Import.* defaults or profiles. + root := node.IPFSAddDeterministic("64KiB", "dag-export-local-only-subtree", + "--raw-leaves", "--chunker=size-256", "--max-file-links=174") + fullCarPath := filepath.Join(node.Dir, "full.car") + require.NoError(t, node.IPFSDagExport(root, fullCarPath)) + fullCount := countCARBlocks(t, fullCarPath) + // 1 root + 2 intermediates (174 + 82 children) + 256 leaves = 259. + require.Equal(t, 259, fullCount, "expected root + 2 intermediates + 256 leaves, got %d", fullCount) + + // Find the first intermediate ref: a non-leaf whose codec is dag-pb. + // "ipfs refs -r --unique" lists CIDs depth-first; the root's first + // child in a balanced UnixFS DAG with >174 leaves is an intermediate. + refs := dagRefs(node, root) + intermediate := refs[1] + intermediateChildren := dagRefs(node, intermediate) + require.Greater(t, len(intermediateChildren), 10, + "expected refs[1] to be a non-leaf with many children, got %d", len(intermediateChildren)) + + // Remove the intermediate. Its subtree blocks remain locally, but + // without the intermediate the walker cannot reach them, so they + // must be skipped along with it. + require.Equal(t, 0, node.RunIPFS("pin", "rm", root).ExitCode()) + require.Equal(t, 0, node.RunIPFS("block", "rm", intermediate).ExitCode()) + + partialCarPath := filepath.Join(node.Dir, "partial.car") + require.NoError(t, node.IPFSDagExport(root, partialCarPath, "--local-only")) + partialCount := countCARBlocks(t, partialCarPath) + + expectedDropped := len(intermediateChildren) // includes the intermediate itself + require.Equal(t, fullCount-expectedDropped, partialCount, + "removing intermediate %s should drop it and its %d descendants (full=%d, partial=%d)", + intermediate, expectedDropped-1, fullCount, partialCount) +} + +// TestDagExportLocalOnlyConflictsWithOnline verifies that explicitly asking +// for online mode together with --local-only is rejected, since the two +// settings contradict each other. +func TestDagExportLocalOnlyConflictsWithOnline(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + root := node.IPFSAddDeterministic("300KiB", "dag-export-local-only-online", "--raw-leaves") + + res := node.RunIPFS("dag", "export", "--local-only", "--offline=false", root) + require.NotEqual(t, 0, res.ExitCode(), "dag export --local-only --offline=false should be rejected") + stderr := res.Stderr.String() + require.Contains(t, stderr, "--local-only") + require.Contains(t, stderr, "--offline") +} + +// TestDagImportPartialCAR is the round-trip happy path: a partial CAR from +// --local-only can be imported on a fresh node with default flags (the +// IPFSDagImport harness helper passes --pin-roots=false). The helper also +// confirms the root resolves offline on the receiver. +func TestDagImportPartialCAR(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + root, _ := makePartialDAG(t, node, "dag-import-partial") + + partialCarPath := filepath.Join(node.Dir, "partial.car") + require.NoError(t, node.IPFSDagExport(root, partialCarPath, "--local-only", "--offline")) + + imp := harness.NewT(t).NewNode().Init().StartDaemon() + defer imp.StopDaemon() + partialCAR, err := os.Open(partialCarPath) + require.NoError(t, err) + defer partialCAR.Close() + require.NoError(t, imp.IPFSDagImport(partialCAR, root)) +} + +// TestDagImportLocalOnlyImpliesNoPin verifies that --local-only on its own +// makes a partial-CAR import succeed: it implies --pin-roots=false so the +// user does not have to pass both flags. +func TestDagImportLocalOnlyImpliesNoPin(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + root, _ := makePartialDAG(t, node, "dag-import-local-only-implies") + partialCarPath := filepath.Join(node.Dir, "partial.car") + require.NoError(t, node.IPFSDagExport(root, partialCarPath, "--local-only", "--offline")) + + imp := harness.NewT(t).NewNode().Init().StartDaemon() + defer imp.StopDaemon() + partialCAR, err := os.Open(partialCarPath) + require.NoError(t, err) + defer partialCAR.Close() + + // Import with only --local-only (no --pin-roots=false). Should + // succeed because --local-only implies --pin-roots=false, and the + // receiver must not attempt to pin (pin would fail on a partial DAG). + res := imp.Runner.Run(harness.RunRequest{ + Path: imp.IPFSBin, + Args: []string{"dag", "import", "--local-only"}, + CmdOpts: []harness.CmdOpt{harness.RunWithStdin(partialCAR)}, + }) + require.Equal(t, 0, res.ExitCode(), + "dag import --local-only on a partial CAR should succeed; stderr: %s", res.Stderr.String()) + require.NotContains(t, res.Stdout.String(), "Pinned root", + "import must not pin when --local-only is set") +} + +// TestDagImportLocalOnlyPinRootsConflict verifies that --local-only is +// rejected when combined with an explicit --pin-roots=true. The two are +// mutually exclusive: --local-only is for partial CARs (no full DAG to pin). +func TestDagImportLocalOnlyPinRootsConflict(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + r, err := os.Open(fixtureFile) + require.NoError(t, err) + defer r.Close() + + res := node.Runner.Run(harness.RunRequest{ + Path: node.IPFSBin, + Args: []string{"dag", "import", "--local-only", "--pin-roots=true"}, + CmdOpts: []harness.CmdOpt{harness.RunWithStdin(r)}, + }) + + require.NotEqual(t, 0, res.ExitCode()) + stderr := res.Stderr.String() + require.Contains(t, stderr, "--local-only") + require.Contains(t, stderr, "--pin-roots") +} diff --git a/test/cli/harness/ipfs.go b/test/cli/harness/ipfs.go index d7470b4e764..637b316867b 100644 --- a/test/cli/harness/ipfs.go +++ b/test/cli/harness/ipfs.go @@ -162,17 +162,19 @@ func (n *Node) IPFSDagImport(content io.Reader, cid string, args ...string) erro } // IPFSDagExport exports a DAG rooted at cid to a CAR file at carPath. -func (n *Node) IPFSDagExport(cid string, carPath string) error { - log.Debugf("node %d dag export of %s to %q", n.ID, cid, carPath) +func (n *Node) IPFSDagExport(cid string, carPath string, args ...string) error { + log.Debugf("node %d dag export of %s to %q with args: %v", n.ID, cid, carPath, args) car, err := os.Create(carPath) if err != nil { return err } defer car.Close() + fullArgs := append([]string{"dag", "export"}, args...) + fullArgs = append(fullArgs, cid) res := n.Runner.MustRun(RunRequest{ Path: n.IPFSBin, - Args: []string{"dag", "export", cid}, + Args: fullArgs, CmdOpts: []CmdOpt{RunWithStdout(car)}, }) return res.Err diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 4a2598a3282..c6ce9b92f85 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -149,7 +149,7 @@ require ( github.com/ipfs/go-metrics-interface v0.3.0 // indirect github.com/ipfs/go-unixfsnode v1.10.4 // indirect github.com/ipfs/kubo v0.31.0 // indirect - github.com/ipld/go-car/v2 v2.16.0 // indirect + github.com/ipld/go-car/v2 v2.16.1-0.20260428045700-c4b9f366f20c // indirect github.com/ipld/go-codec-dagpb v1.7.0 // indirect github.com/ipld/go-ipld-prime v0.23.0 // indirect github.com/ipshipyard/p2p-forge v0.8.0 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index 3a8cb5db2f7..e9af42b4539 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -500,8 +500,8 @@ github.com/ipfs/iptb v1.4.1 h1:faXd3TKGPswbHyZecqqg6UfbES7RDjTKQb+6VFPKDUo= github.com/ipfs/iptb v1.4.1/go.mod h1:nTsBMtVYFEu0FjC5DgrErnABm3OG9ruXkFXGJoTV5OA= github.com/ipfs/iptb-plugins v0.5.1 h1:11PNTNEt2+SFxjUcO5qpyCTXqDj6T8Tx9pU/G4ytCIQ= github.com/ipfs/iptb-plugins v0.5.1/go.mod h1:mscJAjRnu4g16QK6oUBn9RGpcp8ueJmLfmPxIG/At78= -github.com/ipld/go-car/v2 v2.16.0 h1:LWe0vmN/QcQmUU4tr34W5Nv5mNraW+G6jfN2s+ndBco= -github.com/ipld/go-car/v2 v2.16.0/go.mod h1:RqFGWN9ifcXVmCrTAVnfnxiWZk1+jIx67SYhenlmL34= +github.com/ipld/go-car/v2 v2.16.1-0.20260428045700-c4b9f366f20c h1:ZFONxHSj6bzzB9eKIu+yS2AazTJe7j9FPesfy4sZSE0= +github.com/ipld/go-car/v2 v2.16.1-0.20260428045700-c4b9f366f20c/go.mod h1:/4HY8tFZ1q42Mw54ILLPQfjkUqMJxFKqY1yMDKHlYko= github.com/ipld/go-codec-dagpb v1.7.0 h1:hpuvQjCSVSLnTnHXn+QAMR0mLmb1gA6wl10LExo2Ts0= github.com/ipld/go-codec-dagpb v1.7.0/go.mod h1:rD3Zg+zub9ZnxcLwfol/OTQRVjaLzXypgy4UqHQvilM= github.com/ipld/go-ipld-prime v0.23.0 h1:csqdPZH60BsTC+AZrv7fpa27v+09I/oTqyHYYYE27eE=