Keep an out-of-scope redirect from stopping the crawl

CHANGELOG.md

@@ -42,9 +42,16 @@ All notable changes to kage are recorded here. The format follows
   the URL that was originally discovered. Consumed `href` attributes are
   removed from every `<base>` so they cannot re-root rewritten links when the
   saved page opens, while `target` behavior is preserved.
-  After a cross-host redirect,
-  relative references resolve to the other host and remain absolute when that
-  host is outside the crawl scope, so those resources are not localised.
+  A redirect that leaves the crawl scope no longer takes the page's links with
+  it. `urlx.SameSite` matches hostnames exactly, so a seed redirecting apex→www
+  (or www→apex) resolved every relative link onto a host the scope rejects:
+  the links stayed absolute, nothing was enqueued, and the crawl saved the seed
+  page and stopped. Assets are matched on the registrable domain and kept
+  downloading, so the run ended with a complete-looking single page and no
+  error. The resolution base now falls back to the document `<base href>`, and
+  then to the enqueued URL, whenever the redirect target is out of scope.
+  A redirect that genuinely leaves the site is unchanged: those references
+  resolve to the other host and stay absolute, so they are not localised.
 - `--resume` picks an interrupted crawl back up instead of doing nothing ([#36](https://github.com/tamnd/kage/issues/36)).
   `state.json` persisted only the visited set, and the frontier was rebuilt purely by re-rendering pages and following their links, which resume exists to avoid.
   So a resumed run found its seed already visited, `enqueuePage` turned it down, nothing was queued, and the run printed `pages 0` and exited successfully with most of the site still missing.

clone/cloner.go

@@ -321,7 +321,7 @@ func (c *Cloner) processPage(ctx context.Context, j pageItem) {
 	// links that pointed at /old still resolve. Cross-host redirects leave the
 	// resolve base as the final location for relative refs; scope checks still
 	// use that absolute URL.
-	resolveBase := pageResolveBase(j.u, res.FinalURL, root)
+	resolveBase := scopedResolveBase(c.seed, j.u, res.FinalURL, root, c.cfg.scope())
 
 	localFile := urlx.LocalPath(c.seedHost, j.u, urlx.Page, c.cfg.Reserved)
 	fileDir := urlx.Dir(localFile)
@@ -383,6 +383,9 @@ func (c *Cloner) waitForCrawlDelay(ctx context.Context) bool {
 //
 // The page is still written under the enqueued URL so offline links discovered
 // as /old keep working when the server redirected /old → /new.
+//
+// Callers should prefer scopedResolveBase, which keeps an off-scope redirect
+// from taking the page's links with it.
 func pageResolveBase(enqueued *url.URL, finalURL string, root *html.Node) *url.URL {
 	base := enqueued
 	if finalURL != "" {
@@ -400,6 +403,31 @@ func pageResolveBase(enqueued *url.URL, finalURL string, root *html.Node) *url.U
 	return base
 }
 
+// scopedResolveBase is pageResolveBase constrained to the crawl scope.
+//
+// urlx.SameSite matches hostnames exactly, so a seed that redirects apex→www
+// (or www→apex) lands on a host the scope rejects. Resolving the page's links
+// against that host would put every one of them out of scope: they stay
+// absolute, nothing is enqueued, and the crawl saves the seed page and stops.
+// The failure is quiet because assets are matched with SameRegistrableDomain
+// and still download, so the run ends with a complete-looking single page.
+//
+// A redirect that genuinely leaves the site (example.com/go/x → partner.com)
+// keeps the old behaviour: those links are out of scope either way, and they
+// are re-fetched and re-redirected if reached on their own.
+func scopedResolveBase(seed, enqueued *url.URL, finalURL string, root *html.Node, scope urlx.ScopeConfig) *url.URL {
+	base := pageResolveBase(enqueued, finalURL, root)
+	if urlx.InScope(seed, base, scope) {
+		return base
+	}
+	// Drop the redirect target but keep any document <base href>, which is the
+	// page's own statement about where its links point.
+	if b := pageResolveBase(enqueued, "", root); urlx.InScope(seed, b, scope) {
+		return b
+	}
+	return enqueued
+}
+
 // documentBaseHref returns the first <base href> in document order, or "".
 func documentBaseHref(root *html.Node) string {
 	var found string

clone/resolve_test.go

@@ -6,6 +6,8 @@ import (
 	"testing"
 
 	"golang.org/x/net/html"
+
+	"github.com/tamnd/kage/urlx"
 )
 
 func TestPageResolveBaseUsesFinalURL(t *testing.T) {
@@ -33,6 +35,56 @@ func TestPageResolveBasePrefersDocumentBase(t *testing.T) {
 	}
 }
 
+// An apex→www redirect (or the reverse) must not carry the page's links out of
+// scope, or the crawl saves the seed page and stops.
+func TestScopedResolveBaseKeepsRedirectInScope(t *testing.T) {
+	doc := func(s string) *html.Node {
+		root, err := html.Parse(strings.NewReader(s))
+		if err != nil {
+			t.Fatal(err)
+		}
+		return root
+	}
+	plain := `<html><head></head><body><a href="/about">a</a></body></html>`
+
+	cases := []struct {
+		name     string
+		seed     string
+		enqueued string
+		finalURL string
+		html     string
+		want     string
+	}{
+		{"apex redirects to www", "https://ex.com/", "https://ex.com/", "https://www.ex.com/", plain, "https://ex.com/"},
+		{"www redirects to apex", "https://www.ex.com/", "https://www.ex.com/", "https://ex.com/", plain, "https://www.ex.com/"},
+		{"same-host redirect still wins", "https://ex.com/", "https://ex.com/old", "https://ex.com/new/", plain, "https://ex.com/new/"},
+		{"off-scope document base is dropped", "https://ex.com/", "https://ex.com/p", "",
+			`<html><head><base href="https://cdn.other.test/app/"></head></html>`, "https://ex.com/p"},
+		{"in-scope document base survives an off-scope redirect", "https://ex.com/", "https://ex.com/p", "https://www.ex.com/p",
+			`<html><head><base href="https://ex.com/dir/"></head></html>`, "https://ex.com/dir/"},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			seed, err := url.Parse(tc.seed)
+			if err != nil {
+				t.Fatal(err)
+			}
+			enqueued, err := url.Parse(tc.enqueued)
+			if err != nil {
+				t.Fatal(err)
+			}
+			got := scopedResolveBase(seed, enqueued, tc.finalURL, doc(tc.html), urlx.ScopeConfig{})
+			if got.String() != tc.want {
+				t.Fatalf("scopedResolveBase = %q, want %q", got, tc.want)
+			}
+			if !urlx.InScope(seed, got, urlx.ScopeConfig{}) {
+				t.Fatalf("resolve base %q is out of scope, links would never be enqueued", got)
+			}
+		})
+	}
+}
+
 func TestDocumentBaseHref(t *testing.T) {
 	root, err := html.Parse(strings.NewReader(
 		`<html><head><base href="/subdir/"><base href="https://ignored.example/"></head></html>`))