Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,16 @@ All notable changes to kage are recorded here. The format follows

## [Unreleased]

### Fixed

- Saved pages keep their `<!DOCTYPE html>` instead of rendering in quirks mode ([#16](https://github.com/tamnd/kage/issues/16)).
kage serialises a rendered page as the outerHTML of `<html>`, and a doctype is a sibling of `<html>` rather than a child, so it was never in that string and every page kage has ever written came out without one.
A document with no doctype is quirks mode in every browser: the box model reverts to the pre-CSS2 IE one and `line-height`, table cell inheritance and `vertical-align` all change, so the saved copy laid out differently from the original, and the `<meta charset>` declaration lost its authority, leaving a reader free to fall back to its locale encoding and mojibake every multibyte character.
That is the encoding problem reported in #16, and a webview or e-reader with no encoding menu has no way back from it.
The doctype is now read from the DOM and reproduced exactly rather than replaced with `<!DOCTYPE html>`, because the string itself selects the rendering mode: HTML 4.01 Transitional is standards mode with its system identifier and quirks mode without it.
A page that genuinely had no doctype on the live web still gets none, so it keeps rendering the way its author saw it.
- The `cloned by kage` banner comment is written after the doctype rather than before it, so the doctype stays the first thing in the file.

## [0.3.11] - 2026-08-01

### Fixed
Expand Down
102 changes: 100 additions & 2 deletions browser/pool.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ package browser

import (
"context"
"encoding/json"
"fmt"
"os"
"runtime"
Expand All @@ -17,6 +18,7 @@ import (
"github.com/go-rod/rod/lib/proto"
"github.com/tamnd/kage/internal/rod"
"github.com/tamnd/kage/internal/stealth"
"golang.org/x/net/html"
)

// Options configure a Pool.
Expand Down Expand Up @@ -145,19 +147,115 @@ func (p *Pool) Render(ctx context.Context, rawURL string) (RenderResult, error)
settle(page, p.opts.Settle)
}

html, err := page.HTML()
doc, err := page.HTML()
if err != nil {
return RenderResult{}, fmt.Errorf("serialise %s: %w", rawURL, err)
}
// page.HTML() is the outerHTML of <html>, which cannot contain the doctype,
// so put it back (issue #16).
if dt := pageDoctype(page); dt != "" {
doc = dt + "\n" + doc
}

res := RenderResult{HTML: html, FinalURL: rawURL}
res := RenderResult{HTML: doc, FinalURL: rawURL}
if info, err := page.Info(); err == nil && info != nil {
res.FinalURL = info.URL
res.Title = info.Title
}
return res, nil
}

// doctypeJS reads the parts of document.doctype. It returns them as a JSON
// array rather than a ready-made string so the source form is assembled in Go,
// where a hostile page cannot influence it.
const doctypeJS = `() => {
const d = document.doctype;
return d ? JSON.stringify([d.name, d.publicId, d.systemId]) : "";
}`

// pageDoctype returns the document's doctype in source form, or "" when the
// page has none or Chrome will not say.
//
// Chrome's serialisation of a page is the outerHTML of <html>, and a doctype is
// a sibling of <html> rather than a child, so it is never in that string. Left
// alone, every page kage saves comes out with no doctype and every browser
// renders it in quirks mode: the box model reverts to the pre-CSS2 IE one, so
// the saved copy lays out differently from the original, and the <meta charset>
// declaration loses its authority, so a reader is free to fall back to its
// locale encoding and mojibake the text. A reader with no encoding menu, a
// webview or an e-reader, has no way back from that (issue #16).
//
// The doctype is reproduced exactly rather than replaced with <!DOCTYPE html>,
// because the string itself selects the rendering mode: HTML 4.01 Transitional
// is standards mode with its system identifier and quirks mode without it. A
// page that was genuinely quirks mode on the live web keeps no doctype and so
// keeps rendering the way its author saw it.
func pageDoctype(page *rod.Page) string {
obj, err := page.Eval(doctypeJS)
if err != nil || obj == nil {
return ""
}
var parts []string
if err := json.Unmarshal([]byte(obj.Value.Str()), &parts); err != nil || len(parts) != 3 {
return ""
}
return renderDoctype(parts[0], parts[1], parts[2])
}

// renderDoctype rebuilds the source form of a doctype from its DOM parts with
// the same renderer that writes the saved page, so the two always agree.
//
// The parts arrive from an untrusted page and land at the very top of a file we
// write, so anything that does not look like a doctype a parser produced is
// dropped rather than escaped. x/net/html quotes the identifiers but does not
// escape a quote inside one, and it writes the name verbatim.
func renderDoctype(name, publicID, systemID string) string {
if !validDoctypeName(name) || !validDoctypeID(publicID) || !validDoctypeID(systemID) {
return ""
}
n := &html.Node{Type: html.DoctypeNode, Data: name}
if publicID != "" {
n.Attr = append(n.Attr, html.Attribute{Key: "public", Val: publicID})
}
if systemID != "" {
n.Attr = append(n.Attr, html.Attribute{Key: "system", Val: systemID})
}
var b strings.Builder
if err := html.Render(&b, n); err != nil {
return ""
}
return b.String()
}

// validDoctypeName accepts the name of a doctype: ASCII letters only. In
// practice it is always "html", but "math" and "svg" are legal too.
func validDoctypeName(name string) bool {
if name == "" || len(name) > 32 {
return false
}
for _, r := range name {
if (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') {
return false
}
}
return true
}

// validDoctypeID accepts a public or system identifier: printable ASCII with no
// quote or angle bracket, which is every identifier any real doctype uses and
// nothing that could close the token early.
func validDoctypeID(id string) bool {
if len(id) > 256 {
return false
}
for _, r := range id {
if r < 0x20 || r > 0x7e || r == '"' || r == '\'' || r == '<' || r == '>' {
return false
}
}
return true
}

// getBrowser lazily connects to or launches Chrome.
func (p *Pool) getBrowser() (*rod.Browser, error) {
p.mu.Lock()
Expand Down
98 changes: 98 additions & 0 deletions browser/pool_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -247,3 +247,101 @@ func TestRenderRoutesNonHTML(t *testing.T) {
}
}
}

func TestRenderDoctype(t *testing.T) {
cases := []struct {
name, doctype, public, system, want string
}{
{"html5", "html", "", "", "<!DOCTYPE html>"},
{
"html401 transitional", "html",
"-//W3C//DTD HTML 4.01 Transitional//EN",
"http://www.w3.org/TR/html4/loose.dtd",
`<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">`,
},
{
// No system identifier: the difference between standards mode and
// quirks mode for this doctype, so it has to survive verbatim.
"html401 no system", "html",
"-//W3C//DTD HTML 4.01//EN", "",
`<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN">`,
},
{
"xhtml", "html",
"-//W3C//DTD XHTML 1.0 Strict//EN",
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd",
`<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">`,
},
{"legacy compat", "html", "", "about:legacy-compat", `<!DOCTYPE html SYSTEM "about:legacy-compat">`},

// A page can patch its own DOM, and whatever comes back is written to the
// top of a file we save, so anything that could close the token early or
// carry markup is dropped rather than escaped.
{"no name", "", "", "", ""},
{"name with markup", "html><script>alert(1)</script", "", "", ""},
{"quote in public id", "html", `x" "y><script>alert(1)</script`, "", ""},
{"angle bracket in system id", "html", "", "x><script>alert(1)</script", ""},
{"newline in public id", "html", "a\nb", "", ""},
}
for _, c := range cases {
if got := renderDoctype(c.doctype, c.public, c.system); got != c.want {
t.Errorf("%s: renderDoctype(%q, %q, %q) = %q, want %q",
c.name, c.doctype, c.public, c.system, got, c.want)
}
}
}

func TestRenderPreservesDoctype(t *testing.T) {
if testing.Short() {
t.Skip("render test drives Chrome; skipped under -short")
}
if _, ok := LookChrome(); !ok {
t.Skip("no Chrome/Chromium found; skipping render test")
}

pages := map[string]string{
"/html5": `<!DOCTYPE html><html><body><p>modern</p></body></html>`,
"/legacy": `<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd"><html><body><p>legacy</p></body></html>`,
"/quirks": `<html><body><p>quirks</p></body></html>`,
}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, ok := pages[r.URL.Path]
if !ok {
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", "text/html; charset=utf-8")
_, _ = w.Write([]byte(body))
}))
defer srv.Close()

p := New(Options{Headless: true, Workers: 1, Settle: 300 * time.Millisecond, RenderTimeout: 20 * time.Second})
defer func() { _ = p.Close() }()

ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()

cases := []struct{ path, want string }{
{"/html5", "<!DOCTYPE html>"},
{"/legacy", `<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">`},
// A page that was quirks mode on the live web stays quirks mode offline,
// so it keeps laying out the way its author saw it.
{"/quirks", ""},
}
for _, c := range cases {
res, err := p.Render(ctx, srv.URL+c.path)
if err != nil {
t.Errorf("render %s: %v", c.path, err)
continue
}
if c.want == "" {
if strings.Contains(strings.ToUpper(res.HTML), "<!DOCTYPE") {
t.Errorf("%s: got a doctype for a page that had none:\n%s", c.path, res.HTML)
}
continue
}
if !strings.HasPrefix(res.HTML, c.want) {
t.Errorf("%s: render should start with %s, got:\n%s", c.path, c.want, res.HTML)
}
}
}
14 changes: 11 additions & 3 deletions sanitize/sanitize.go
Original file line number Diff line number Diff line change
Expand Up @@ -380,11 +380,19 @@ func injectMobileCSS(root *html.Node) {
head.AppendChild(style)
}

// insertBanner prepends an HTML comment to the document.
// insertBanner prepends an HTML comment to the document, after the doctype so
// the doctype stays the first thing in the file. A comment ahead of it is legal
// and modern browsers still read the doctype that follows, but older ones and
// several offline readers take anything before it as a reason to drop into
// quirks mode, which is the whole thing the doctype is there to prevent.
func insertBanner(root *html.Node, text string) {
c := &html.Node{Type: html.CommentNode, Data: " " + text + " "}
if root.FirstChild != nil {
root.InsertBefore(c, root.FirstChild)
at := root.FirstChild
if at != nil && at.Type == html.DoctypeNode {
at = at.NextSibling
}
if at != nil {
root.InsertBefore(c, at)
} else {
root.AppendChild(c)
}
Expand Down
47 changes: 47 additions & 0 deletions sanitize/sanitize_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -284,3 +284,50 @@ func TestCharsetNotDuplicated(t *testing.T) {
}
}
}

func TestDoctypePreservedAndBannerFollowsIt(t *testing.T) {
// The browser hands the doctype back with the page, and everything sanitize
// does has to leave it at the top of the file. A doctype anywhere but first
// is what puts a saved page into quirks mode (issue #16).
cases := []struct {
name, in, want string
}{
{"html5", `<!doctype html><html><head></head><body><p>x</p></body></html>`, "<!DOCTYPE html>"},
{
"html401 transitional",
`<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">` +
`<html><head></head><body><p>x</p></body></html>`,
`<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">`,
},
}
for _, c := range cases {
out, _, err := Strip([]byte(c.in), Options{Banner: "cloned by kage"})
if err != nil {
t.Fatal(err)
}
s := string(out)
if !strings.HasPrefix(s, c.want) {
t.Errorf("%s: output should start with %s, got:\n%s", c.name, c.want, s)
}
if !strings.Contains(s, "<!-- cloned by kage -->") {
t.Errorf("%s: banner missing:\n%s", c.name, s)
}
if bannerIdx, dtIdx := strings.Index(s, "<!--"), strings.Index(s, "<!DOCTYPE"); bannerIdx < dtIdx {
t.Errorf("%s: banner must follow the doctype (banner=%d doctype=%d):\n%s", c.name, bannerIdx, dtIdx, s)
}
}
}

func TestNoDoctypeInvented(t *testing.T) {
// A page that carried no doctype was quirks mode on the live web. Adding one
// would switch it to standards mode and change its layout, so sanitize leaves
// that decision to the source.
in := `<html><head></head><body><p>x</p></body></html>`
out, _, err := Strip([]byte(in), Options{Banner: "cloned by kage"})
if err != nil {
t.Fatal(err)
}
if s := string(out); strings.Contains(strings.ToUpper(s), "<!DOCTYPE") {
t.Errorf("sanitize invented a doctype:\n%s", s)
}
}