diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a638e3..0757b45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,9 +42,16 @@ All notable changes to kage are recorded here. The format follows the URL that was originally discovered. Consumed `href` attributes are removed from every `` so they cannot re-root rewritten links when the saved page opens, while `target` behavior is preserved. - After a cross-host redirect, - relative references resolve to the other host and remain absolute when that - host is outside the crawl scope, so those resources are not localised. + A redirect that leaves the crawl scope no longer takes the page's links with + it. `urlx.SameSite` matches hostnames exactly, so a seed redirecting apex→www + (or www→apex) resolved every relative link onto a host the scope rejects: + the links stayed absolute, nothing was enqueued, and the crawl saved the seed + page and stopped. Assets are matched on the registrable domain and kept + downloading, so the run ended with a complete-looking single page and no + error. The resolution base now falls back to the document ``, and + then to the enqueued URL, whenever the redirect target is out of scope. + A redirect that genuinely leaves the site is unchanged: those references + resolve to the other host and stay absolute, so they are not localised. - `--resume` picks an interrupted crawl back up instead of doing nothing ([#36](https://github.com/tamnd/kage/issues/36)). `state.json` persisted only the visited set, and the frontier was rebuilt purely by re-rendering pages and following their links, which resume exists to avoid. So a resumed run found its seed already visited, `enqueuePage` turned it down, nothing was queued, and the run printed `pages 0` and exited successfully with most of the site still missing. diff --git a/clone/cloner.go b/clone/cloner.go index bc2ad7b..9cc4bf4 100644 --- a/clone/cloner.go +++ b/clone/cloner.go @@ -321,7 +321,7 @@ func (c *Cloner) processPage(ctx context.Context, j pageItem) { // links that pointed at /old still resolve. Cross-host redirects leave the // resolve base as the final location for relative refs; scope checks still // use that absolute URL. - resolveBase := pageResolveBase(j.u, res.FinalURL, root) + resolveBase := scopedResolveBase(c.seed, j.u, res.FinalURL, root, c.cfg.scope()) localFile := urlx.LocalPath(c.seedHost, j.u, urlx.Page, c.cfg.Reserved) fileDir := urlx.Dir(localFile) @@ -383,6 +383,9 @@ func (c *Cloner) waitForCrawlDelay(ctx context.Context) bool { // // The page is still written under the enqueued URL so offline links discovered // as /old keep working when the server redirected /old → /new. +// +// Callers should prefer scopedResolveBase, which keeps an off-scope redirect +// from taking the page's links with it. func pageResolveBase(enqueued *url.URL, finalURL string, root *html.Node) *url.URL { base := enqueued if finalURL != "" { @@ -400,6 +403,31 @@ func pageResolveBase(enqueued *url.URL, finalURL string, root *html.Node) *url.U return base } +// scopedResolveBase is pageResolveBase constrained to the crawl scope. +// +// urlx.SameSite matches hostnames exactly, so a seed that redirects apex→www +// (or www→apex) lands on a host the scope rejects. Resolving the page's links +// against that host would put every one of them out of scope: they stay +// absolute, nothing is enqueued, and the crawl saves the seed page and stops. +// The failure is quiet because assets are matched with SameRegistrableDomain +// and still download, so the run ends with a complete-looking single page. +// +// A redirect that genuinely leaves the site (example.com/go/x → partner.com) +// keeps the old behaviour: those links are out of scope either way, and they +// are re-fetched and re-redirected if reached on their own. +func scopedResolveBase(seed, enqueued *url.URL, finalURL string, root *html.Node, scope urlx.ScopeConfig) *url.URL { + base := pageResolveBase(enqueued, finalURL, root) + if urlx.InScope(seed, base, scope) { + return base + } + // Drop the redirect target but keep any document , which is the + // page's own statement about where its links point. + if b := pageResolveBase(enqueued, "", root); urlx.InScope(seed, b, scope) { + return b + } + return enqueued +} + // documentBaseHref returns the first in document order, or "". func documentBaseHref(root *html.Node) string { var found string diff --git a/clone/resolve_test.go b/clone/resolve_test.go index 42d59dc..b74d243 100644 --- a/clone/resolve_test.go +++ b/clone/resolve_test.go @@ -6,6 +6,8 @@ import ( "testing" "golang.org/x/net/html" + + "github.com/tamnd/kage/urlx" ) func TestPageResolveBaseUsesFinalURL(t *testing.T) { @@ -33,6 +35,56 @@ func TestPageResolveBasePrefersDocumentBase(t *testing.T) { } } +// An apex→www redirect (or the reverse) must not carry the page's links out of +// scope, or the crawl saves the seed page and stops. +func TestScopedResolveBaseKeepsRedirectInScope(t *testing.T) { + doc := func(s string) *html.Node { + root, err := html.Parse(strings.NewReader(s)) + if err != nil { + t.Fatal(err) + } + return root + } + plain := `a` + + cases := []struct { + name string + seed string + enqueued string + finalURL string + html string + want string + }{ + {"apex redirects to www", "https://ex.com/", "https://ex.com/", "https://www.ex.com/", plain, "https://ex.com/"}, + {"www redirects to apex", "https://www.ex.com/", "https://www.ex.com/", "https://ex.com/", plain, "https://www.ex.com/"}, + {"same-host redirect still wins", "https://ex.com/", "https://ex.com/old", "https://ex.com/new/", plain, "https://ex.com/new/"}, + {"off-scope document base is dropped", "https://ex.com/", "https://ex.com/p", "", + ``, "https://ex.com/p"}, + {"in-scope document base survives an off-scope redirect", "https://ex.com/", "https://ex.com/p", "https://www.ex.com/p", + ``, "https://ex.com/dir/"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + seed, err := url.Parse(tc.seed) + if err != nil { + t.Fatal(err) + } + enqueued, err := url.Parse(tc.enqueued) + if err != nil { + t.Fatal(err) + } + got := scopedResolveBase(seed, enqueued, tc.finalURL, doc(tc.html), urlx.ScopeConfig{}) + if got.String() != tc.want { + t.Fatalf("scopedResolveBase = %q, want %q", got, tc.want) + } + if !urlx.InScope(seed, got, urlx.ScopeConfig{}) { + t.Fatalf("resolve base %q is out of scope, links would never be enqueued", got) + } + }) + } +} + func TestDocumentBaseHref(t *testing.T) { root, err := html.Parse(strings.NewReader( ``))