diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9a638e3..0757b45 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -42,9 +42,16 @@ All notable changes to kage are recorded here. The format follows
the URL that was originally discovered. Consumed `href` attributes are
removed from every `` so they cannot re-root rewritten links when the
saved page opens, while `target` behavior is preserved.
- After a cross-host redirect,
- relative references resolve to the other host and remain absolute when that
- host is outside the crawl scope, so those resources are not localised.
+ A redirect that leaves the crawl scope no longer takes the page's links with
+ it. `urlx.SameSite` matches hostnames exactly, so a seed redirecting apex→www
+ (or www→apex) resolved every relative link onto a host the scope rejects:
+ the links stayed absolute, nothing was enqueued, and the crawl saved the seed
+ page and stopped. Assets are matched on the registrable domain and kept
+ downloading, so the run ended with a complete-looking single page and no
+ error. The resolution base now falls back to the document ``, and
+ then to the enqueued URL, whenever the redirect target is out of scope.
+ A redirect that genuinely leaves the site is unchanged: those references
+ resolve to the other host and stay absolute, so they are not localised.
- `--resume` picks an interrupted crawl back up instead of doing nothing ([#36](https://github.com/tamnd/kage/issues/36)).
`state.json` persisted only the visited set, and the frontier was rebuilt purely by re-rendering pages and following their links, which resume exists to avoid.
So a resumed run found its seed already visited, `enqueuePage` turned it down, nothing was queued, and the run printed `pages 0` and exited successfully with most of the site still missing.
diff --git a/clone/cloner.go b/clone/cloner.go
index bc2ad7b..9cc4bf4 100644
--- a/clone/cloner.go
+++ b/clone/cloner.go
@@ -321,7 +321,7 @@ func (c *Cloner) processPage(ctx context.Context, j pageItem) {
// links that pointed at /old still resolve. Cross-host redirects leave the
// resolve base as the final location for relative refs; scope checks still
// use that absolute URL.
- resolveBase := pageResolveBase(j.u, res.FinalURL, root)
+ resolveBase := scopedResolveBase(c.seed, j.u, res.FinalURL, root, c.cfg.scope())
localFile := urlx.LocalPath(c.seedHost, j.u, urlx.Page, c.cfg.Reserved)
fileDir := urlx.Dir(localFile)
@@ -383,6 +383,9 @@ func (c *Cloner) waitForCrawlDelay(ctx context.Context) bool {
//
// The page is still written under the enqueued URL so offline links discovered
// as /old keep working when the server redirected /old → /new.
+//
+// Callers should prefer scopedResolveBase, which keeps an off-scope redirect
+// from taking the page's links with it.
func pageResolveBase(enqueued *url.URL, finalURL string, root *html.Node) *url.URL {
base := enqueued
if finalURL != "" {
@@ -400,6 +403,31 @@ func pageResolveBase(enqueued *url.URL, finalURL string, root *html.Node) *url.U
return base
}
+// scopedResolveBase is pageResolveBase constrained to the crawl scope.
+//
+// urlx.SameSite matches hostnames exactly, so a seed that redirects apex→www
+// (or www→apex) lands on a host the scope rejects. Resolving the page's links
+// against that host would put every one of them out of scope: they stay
+// absolute, nothing is enqueued, and the crawl saves the seed page and stops.
+// The failure is quiet because assets are matched with SameRegistrableDomain
+// and still download, so the run ends with a complete-looking single page.
+//
+// A redirect that genuinely leaves the site (example.com/go/x → partner.com)
+// keeps the old behaviour: those links are out of scope either way, and they
+// are re-fetched and re-redirected if reached on their own.
+func scopedResolveBase(seed, enqueued *url.URL, finalURL string, root *html.Node, scope urlx.ScopeConfig) *url.URL {
+ base := pageResolveBase(enqueued, finalURL, root)
+ if urlx.InScope(seed, base, scope) {
+ return base
+ }
+ // Drop the redirect target but keep any document , which is the
+ // page's own statement about where its links point.
+ if b := pageResolveBase(enqueued, "", root); urlx.InScope(seed, b, scope) {
+ return b
+ }
+ return enqueued
+}
+
// documentBaseHref returns the first in document order, or "".
func documentBaseHref(root *html.Node) string {
var found string
diff --git a/clone/resolve_test.go b/clone/resolve_test.go
index 42d59dc..b74d243 100644
--- a/clone/resolve_test.go
+++ b/clone/resolve_test.go
@@ -6,6 +6,8 @@ import (
"testing"
"golang.org/x/net/html"
+
+ "github.com/tamnd/kage/urlx"
)
func TestPageResolveBaseUsesFinalURL(t *testing.T) {
@@ -33,6 +35,56 @@ func TestPageResolveBasePrefersDocumentBase(t *testing.T) {
}
}
+// An apex→www redirect (or the reverse) must not carry the page's links out of
+// scope, or the crawl saves the seed page and stops.
+func TestScopedResolveBaseKeepsRedirectInScope(t *testing.T) {
+ doc := func(s string) *html.Node {
+ root, err := html.Parse(strings.NewReader(s))
+ if err != nil {
+ t.Fatal(err)
+ }
+ return root
+ }
+ plain := `
a`
+
+ cases := []struct {
+ name string
+ seed string
+ enqueued string
+ finalURL string
+ html string
+ want string
+ }{
+ {"apex redirects to www", "https://ex.com/", "https://ex.com/", "https://www.ex.com/", plain, "https://ex.com/"},
+ {"www redirects to apex", "https://www.ex.com/", "https://www.ex.com/", "https://ex.com/", plain, "https://www.ex.com/"},
+ {"same-host redirect still wins", "https://ex.com/", "https://ex.com/old", "https://ex.com/new/", plain, "https://ex.com/new/"},
+ {"off-scope document base is dropped", "https://ex.com/", "https://ex.com/p", "",
+ ``, "https://ex.com/p"},
+ {"in-scope document base survives an off-scope redirect", "https://ex.com/", "https://ex.com/p", "https://www.ex.com/p",
+ ``, "https://ex.com/dir/"},
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ seed, err := url.Parse(tc.seed)
+ if err != nil {
+ t.Fatal(err)
+ }
+ enqueued, err := url.Parse(tc.enqueued)
+ if err != nil {
+ t.Fatal(err)
+ }
+ got := scopedResolveBase(seed, enqueued, tc.finalURL, doc(tc.html), urlx.ScopeConfig{})
+ if got.String() != tc.want {
+ t.Fatalf("scopedResolveBase = %q, want %q", got, tc.want)
+ }
+ if !urlx.InScope(seed, got, urlx.ScopeConfig{}) {
+ t.Fatalf("resolve base %q is out of scope, links would never be enqueued", got)
+ }
+ })
+ }
+}
+
func TestDocumentBaseHref(t *testing.T) {
root, err := html.Parse(strings.NewReader(
``))