diff --git a/.drun/spec.drun b/.drun/spec.drun index 88b6215..bcb90c6 100644 --- a/.drun/spec.drun +++ b/.drun/spec.drun @@ -5,6 +5,10 @@ version: 2.0 project "gopher-textmate" version "1.0": + requires tools: + go >= 1.25 + golangci-lint >= 2.12 + gosec task "default" means "Welcome to drun v2": info "Gopher Textmate task runner 🚀" @@ -12,6 +16,7 @@ task "default" means "Welcome to drun v2": task "test" means "Runs the test suite": step "Testing Gopher Textmate..." run "go test ./..." + success "Tests passed" task "test-full" means "Runs the extended test suite, including race": step "Test" @@ -23,13 +28,38 @@ task "test-full" means "Runs the extended test suite, including race": success "All tests succeeded" task "lint" means "Runs the linters": - if "golangci-lint" is available and version >= "2.12": - info "GolangCI Lint available and version satisfies requirements" - else: - fail "golangci-lint >= 2.12 is required" step "Running linters" run "golangci-lint run" + success "Lint passed" -task "ci" means "Runs the whole CI pipeline": +task "vet" means "Runs the vet": + step "Vet" + run "go vet ./..." + success "Vet passed" + +task "fuzz" means "Runs the fuzz tests": + step "Fuzz - Grammar" + run "go test ./grammar/ -fuzz=FuzzApplyTransforms -fuzztime=30s" + step "Fuzz - Oniglib" + run "go test ./oniglib/ -fuzz=FuzzScannerFindNextMatch -fuzztime=30s" + step "Fuzz - Theme" + run "go test ./theme/ -fuzz=FuzzParse -fuzztime=30s" + success "Fuzz passed" + +task "security" means "Gosec": + step "Gosec" + run "gosec -exclude=G115,G304 ./..." + success "Security check passed" + +task "ci" means "Runs the routine CI pipeline (not too time consuming, routine)": + call task vet call task test - call task lint \ No newline at end of file + call task lint + call task security + +task "ci-full" means "Runs the full CI pipeline including fuzzers and race tests": + call task vet + call task test-full + call task lint + call task security + call task fuzz \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..1a26015 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,57 @@ +name: CI + +on: + pull_request: + paths-ignore: + - ".drun/spec.drun" + - "*.md" + - "**/*.md" + +permissions: + contents: read + +jobs: + ci: + name: CI + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version: '1.26' + cache: true + cache-dependency-path: go.sum + + - name: Download dependencies + run: go mod download + + - name: golangci-lint + uses: golangci/golangci-lint-action@v8 + with: + version: v2.12 + + - name: Vet + run: go vet ./... + + - name: Test + run: go test ./... + + - name: Install gosec + run: go install github.com/securego/gosec/v2/cmd/gosec@latest + + # G115: hex color components are 0–255 by construction. + # G304: library/CLI APIs intentionally read caller-supplied paths. + - name: gosec + run: gosec -quiet -exclude=G115,G304 ./... + + - name: Race detector + run: go test -race ./... + + - name: Fuzz (smoke) + run: | + go test ./grammar/ -fuzz=. -fuzztime=10s + go test ./oniglib/ -fuzz=. -fuzztime=10s + go test ./theme/ -fuzz=. -fuzztime=10s diff --git a/README.md b/README.md index e6a461a..2b7819e 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,6 @@ It tokenizes source text into scoped tokens using TextMate grammars, resolves a php - ## Why pure Go? TextMate grammars rely on Oniguruma regular expressions (lookbehind, lookahead, `\G`, back-references, `\x{...}` codepoints) that Go's standard `regexp` (RE2) cannot handle. Instead of binding to Oniguruma via cgo, this library uses the pure-Go [`github.com/dlclark/regexp2/v2`](https://github.com/dlclark/regexp2) engine, so builds stay static and cross-compile cleanly. Oniguruma possessive quantifiers (`a++`) are rewritten as atomic groups (`(?>a+)`) to preserve their no-backtracking semantics. @@ -124,6 +123,7 @@ The facade is built on exported packages you can use directly: ```bash go run ./cmd/gtm -grammar grammars/php.tmLanguage.json -scope source.php examples/sample.php ``` + php Flags: @@ -136,20 +136,33 @@ Flags: ## Supported grammar features -`match`, `begin`/`end`, `begin`/`while`, `include` (`#repo`, `$self`, `$base`, cross-grammar `scope.name#sub`), `repository`, `captures` / `beginCaptures` / `endCaptures` with nested `patterns`, `contentName`, `applyEndPatternLast`, dynamic end patterns via back-references (`\1`…`\9`), scope-name templates (`$1`, `${1:/downcase|upcase|capitalize}`), and basic `injections`. +`match`, `begin`/`end`, `begin`/`while`, `include` (`#repo`, `$self`, `$base`, cross-grammar `scope.name#sub`), `repository`, `captures` / `beginCaptures` / `endCaptures` with nested `patterns`, `contentName`, `applyEndPatternLast`, dynamic end patterns via back-references (multi-digit and zero-padded, e.g. `\1`, `\12`, `\001`), `injections` (basic), and scope-name templates (`$1`, `${1}`) with the full set of TextMate transforms — `upcase`, `downcase`, `capitalize`/`titlecase`, `asciify`, `urlencode`, `shellescape`, `relative`, `number`, `duration`, `dirname`, `basename` — which may be chained, e.g. `${1:/downcase/capitalize}`. ## Known limitations - Themes: VSCode JSON format only (`.tmTheme` plist is not yet supported). - `$base` is treated as `$self` (identical for single-grammar tokenization). - Cross-grammar includes only resolve grammars that have been loaded; unresolved references are skipped, so mixed-language files highlight the languages whose grammars are present. -- Oniguruma possessive quantifiers (`a++`) are normalized to greedy; the rare `\g` subroutine call is unsupported and such a pattern simply never matches (graceful degradation). +- The rare `\g` subroutine call is unsupported and such a pattern simply never matches (graceful degradation). +- `asciify` and `urlencode` transforms approximate macOS/ICU behavior (NFD + combining-mark stripping; RFC 3986 unreserved set), and `(?x)` extended-mode `#` comments containing unbalanced parentheses are not parsed. - Injection selector matching is basic; advanced exclusion selectors degrade gracefully. ## Development It is advisable to use the [drun](https://github.com/phillarmonic/drun) task runner for development. It makes it easy to run routine tasks in a semantic way. Check the .drun/spec.drun to understand how the file works. +### Requirements + +- `Go >=1.25` + +- `golangci-lint >= 2.12` + +- `gosec >= 2.27` + +- `drun >= 2.0` + +Development lifecycle: + ```bash # For running only the tests: xdrun test @@ -157,8 +170,11 @@ xdrun test xdrun lint # For running the full test suite including the time consuming tests: xdrun test-full -# For running the whole CI lifecycle (test, lint) +# For running the whole CI lifecycle in fast mode (test, lint) xdrun ci +# For running CI after you're done coding, and run the expensive tests +# like race condition tests and fuzz +xdrun ci-full ``` ## License diff --git a/go.mod b/go.mod index 8aa345b..5a4ff73 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,7 @@ module github.com/andersonpem/gopher-textmate -go 1.25 +go 1.25.0 require github.com/dlclark/regexp2/v2 v2.1.1 + +require golang.org/x/text v0.37.0 diff --git a/go.sum b/go.sum index 2d8c647..d52e98d 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,4 @@ github.com/dlclark/regexp2/v2 v2.1.1 h1:LCUGyd9Wf+r+VVOl8Ny38JTpWJcAsdVnCIuhhtthmKw= github.com/dlclark/regexp2/v2 v2.1.1/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= diff --git a/grammar/fuzz_test.go b/grammar/fuzz_test.go new file mode 100644 index 0000000..ea49214 --- /dev/null +++ b/grammar/fuzz_test.go @@ -0,0 +1,15 @@ +package grammar + +import "testing" + +func FuzzApplyTransforms(f *testing.F) { + f.Add("hello world", "capitalize") + f.Add("1234567", "number") + f.Add("café déjà", "asciify") + f.Fuzz(func(t *testing.T, s, transform string) { + if len(s) > 4096 { + s = s[:4096] + } + _ = applyTransforms(s, []string{transform}) + }) +} diff --git a/grammar/grammar.go b/grammar/grammar.go index cea4dc0..d221ea7 100644 --- a/grammar/grammar.go +++ b/grammar/grammar.go @@ -388,8 +388,8 @@ func (g *Grammar) rootStack() *StateStack { // resolveScopeName expands a scope-name template that references match // captures, e.g. "keyword.control.$1.php" or "entity.name.tag.${1:/downcase}". -// Supported transforms are /downcase, /upcase and /capitalize. Templates with -// no "$" are returned unchanged. +// Transforms (see applyTransforms) may be chained, e.g. "${1:/downcase/capitalize}". +// Templates with no "$" are returned unchanged. func resolveScopeName(tmpl string, line []rune, groups []oniglib.Capture) string { if !strings.ContainsRune(tmpl, '$') { return tmpl @@ -400,7 +400,7 @@ func resolveScopeName(tmpl string, line []rune, groups []oniglib.Capture) string c := rs[i] if c == '$' && i+1 < len(rs) { if rs[i+1] >= '0' && rs[i+1] <= '9' { - b.WriteString(captureText(line, groups, int(rs[i+1]-'0'), "")) + b.WriteString(captureText(line, groups, int(rs[i+1]-'0'), nil)) i++ continue } @@ -410,8 +410,8 @@ func resolveScopeName(tmpl string, line []rune, groups []oniglib.Capture) string j++ } if j < len(rs) { - num, transform := parseGroupTemplate(string(rs[i+2 : j])) - b.WriteString(captureText(line, groups, num, transform)) + num, transforms := parseGroupTemplate(string(rs[i+2 : j])) + b.WriteString(captureText(line, groups, num, transforms)) i = j continue } @@ -422,21 +422,28 @@ func resolveScopeName(tmpl string, line []rune, groups []oniglib.Capture) string return b.String() } -func parseGroupTemplate(inner string) (int, string) { +// parseGroupTemplate parses the body of a ${...} capture reference, returning +// the group number and the list of transforms requested after the ':'. The +// transforms are written as "/name" segments, e.g. "1:/downcase/capitalize". +func parseGroupTemplate(inner string) (int, []string) { num := inner - transform := "" + var transforms []string if i := strings.IndexByte(inner, ':'); i >= 0 { num = inner[:i] - transform = strings.TrimPrefix(inner[i+1:], "/") + for _, t := range strings.Split(inner[i+1:], "/") { + if t = strings.TrimSpace(t); t != "" { + transforms = append(transforms, t) + } + } } n, err := strconv.Atoi(strings.TrimSpace(num)) if err != nil { - return -1, transform + return -1, transforms } - return n, transform + return n, transforms } -func captureText(line []rune, groups []oniglib.Capture, idx int, transform string) string { +func captureText(line []rune, groups []oniglib.Capture, idx int, transforms []string) string { if idx < 0 || idx >= len(groups) { return "" } @@ -444,21 +451,7 @@ func captureText(line []rune, groups []oniglib.Capture, idx int, transform strin if gp.Start < 0 || gp.End < 0 || gp.Start > gp.End || gp.End > len(line) { return "" } - s := string(line[gp.Start:gp.End]) - switch transform { - case "downcase": - return strings.ToLower(s) - case "upcase": - return strings.ToUpper(s) - case "capitalize": - if s == "" { - return s - } - r := []rune(s) - return strings.ToUpper(string(r[0])) + string(r[1:]) - default: - return s - } + return applyTransforms(string(line[gp.Start:gp.End]), transforms) } func pushScope(parent []string, names ...string) []string { diff --git a/grammar/transform.go b/grammar/transform.go new file mode 100644 index 0000000..6b44b5b --- /dev/null +++ b/grammar/transform.go @@ -0,0 +1,307 @@ +package grammar + +import ( + "fmt" + "math" + "path" + "strconv" + "strings" + "time" + "unicode" + + "github.com/dlclark/regexp2/v2" + "golang.org/x/text/runes" + "golang.org/x/text/transform" + "golang.org/x/text/unicode/norm" +) + +// transformOrder is the fixed order in which TextMate applies capture-format +// transforms, regardless of the order they are written in the template. This +// mirrors the sequence of bit-flag checks in TextMate's format_string.cc. +var transformOrder = []string{ + "upcase", + "downcase", + "capitalize", + "asciify", + "urlencode", + "shellescape", + "relative", + "number", + "duration", + "dirname", + "basename", +} + +// nowFunc is the clock used by the "relative" transform. It is a variable so +// tests can install a deterministic time. +var nowFunc = time.Now + +// applyTransforms applies the requested transforms to s in TextMate's fixed +// order. Unknown transform names are ignored. "titlecase" is an alias for +// "capitalize" (both map to TextMate's kCapitalize). +func applyTransforms(s string, transforms []string) string { + if len(transforms) == 0 { + return s + } + requested := make(map[string]bool, len(transforms)) + for _, t := range transforms { + t = strings.TrimSpace(t) + if t == "titlecase" { + t = "capitalize" + } + requested[t] = true + } + for _, name := range transformOrder { + if !requested[name] { + continue + } + s = applyTransform(name, s) + } + return s +} + +func applyTransform(name, s string) string { + switch name { + case "upcase": + return strings.ToUpper(s) + case "downcase": + return strings.ToLower(s) + case "capitalize": + return capitalize(s) + case "asciify": + return asciify(s) + case "urlencode": + return urlEncode(s) + case "shellescape": + return shellEscape(s) + case "relative": + return relativeTime(s) + case "number": + return formatNumber(s) + case "duration": + return formatDuration(s) + case "dirname": + return path.Dir(s) + case "basename": + return path.Base(s) + default: + return s + } +} + +// capitalize reproduces TextMate's English title-casing (kCapitalize). It first +// lowercases all-uppercase words, then uppercases the first letter of each +// significant word (skipping a small set of stop words and very short words +// unless they are at the start or end of the string). +var ( + capWordsRe = regexp2.MustCompile(`\A\P{Ll}+\z|\b\p{Lu}\P{Lu}+?\b`, regexp2.None) + capUpcaseRe = regexp2.MustCompile(`^([\W\d]*)(\w[-\w]*)|\b((?!(?:else|from|over|then|when)\b)\w[-\w]{3,}|\w[-\w]*[\W\d]*$)`, regexp2.None) +) + +func capitalize(s string) string { + if s == "" { + return s + } + lowered, err := capWordsRe.ReplaceFunc(s, func(m regexp2.Match) string { + return strings.ToLower(m.String()) + }, -1, -1) + if err != nil { + return s + } + out, err := capUpcaseRe.ReplaceFunc(lowered, func(m regexp2.Match) string { + if g := m.GroupByNumber(1); g != nil && len(g.Captures) > 0 { + return g.String() + upcaseFirst(m.GroupByNumber(2).String()) + } + return upcaseFirst(m.String()) + }, -1, -1) + if err != nil { + return lowered + } + return out +} + +// upcaseFirst uppercases only the first rune of s (TextMate's \u escape). +func upcaseFirst(s string) string { + if s == "" { + return s + } + r := []rune(s) + return strings.ToUpper(string(r[0])) + string(r[1:]) +} + +// asciify strips diacritics and combining marks, approximating TextMate's +// asciify (which additionally uses ICU's //TRANSLIT, unavailable without cgo). +func asciify(s string) string { + t := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC) + out, _, err := transform.String(t, s) + if err != nil { + return s + } + return out +} + +// urlEncode percent-encodes everything outside the RFC 3986 unreserved set. +func urlEncode(s string) string { + const upperhex = "0123456789ABCDEF" + var b strings.Builder + for _, c := range []byte(s) { + if isUnreserved(c) { + b.WriteByte(c) + continue + } + b.WriteByte('%') + b.WriteByte(upperhex[c>>4]) + b.WriteByte(upperhex[c&0x0f]) + } + return b.String() +} + +func isUnreserved(c byte) bool { + switch { + case c >= 'A' && c <= 'Z', c >= 'a' && c <= 'z', c >= '0' && c <= '9': + return true + case c == '-' || c == '_' || c == '.' || c == '~': + return true + } + return false +} + +// shellEscape ports TextMate's shell_escape: split the value on single quotes, +// single-quote any word containing a shell-special character, and rejoin the +// pieces with an escaped single quote. +func shellEscape(s string) string { + const special = "|&;<>()$`\\\" \t\n*?[#~=%" + var b strings.Builder + parts := strings.Split(s, "'") + for i, word := range parts { + if i > 0 { + b.WriteString(`\'`) + } + if strings.ContainsAny(word, special) { + b.WriteByte('\'') + b.WriteString(word) + b.WriteByte('\'') + } else { + b.WriteString(word) + } + } + return b.String() +} + +// relativeTime parses src as a timestamp and renders a human "... ago" string, +// porting the duration buckets from TextMate's relative_time. +func relativeTime(src string) string { + now := nowFunc() + layouts := []string{ + "2006-01-02 15:04:05 -0700", + "2006-01-02 15:04:05", + "2006-01-02", + "15:04:05", + } + for _, layout := range layouts { + t, err := time.Parse(layout, src) + if err != nil { + continue + } + d := math.Round(now.Sub(t).Seconds()) + switch { + case d < 0: + return "in the future" + case d < 2: + return "just now" + case d < 60: + return fmt.Sprintf("%.0f seconds ago", d) + case d < 90: + return "a minute ago" + case d < 3570: + return fmt.Sprintf("%.0f minutes ago", d/60) + case d < 5400: + return "an hour ago" + case d < 84600: + return fmt.Sprintf("%.0f hours ago", d/(60*60)) + case d < 129600: + return "a day ago" + case d < 561600: + return fmt.Sprintf("%.0f days ago", d/(24*60*60)) + case d < 1036800: + return "a week ago" + case d < 2419200: + return fmt.Sprintf("%.0f weeks ago", d/(7*24*60*60)) + case d < 3952800: + return "a month ago" + case d < 30304800: + return fmt.Sprintf("%.0f months ago", d/(30.5*24*60*60)) + case d < 47304000: + return "a year ago" + default: + return fmt.Sprintf("%.0f years ago", d/(365*24*60*60)) + } + } + return src +} + +// formatNumber inserts thousands separators into the integer part of each +// number in src, matching TextMate's format_number. +var numberRe = regexp2.MustCompile(`(\d+)(\.\d+)?`, regexp2.None) + +func formatNumber(src string) string { + out, err := numberRe.ReplaceFunc(src, func(m regexp2.Match) string { + intPart := m.GroupByNumber(1).String() + frac := m.GroupByNumber(2).String() + return groupThousands(intPart) + frac + }, -1, -1) + if err != nil { + return src + } + return out +} + +func groupThousands(digits string) string { + n := len(digits) + if n <= 3 { + return digits + } + var b strings.Builder + first := n % 3 + if first == 0 { + first = 3 + } + b.WriteString(digits[:first]) + for i := first; i < n; i += 3 { + b.WriteByte(',') + b.WriteString(digits[i : i+3]) + } + return b.String() +} + +// formatDuration renders a number of seconds as "d days, h hours, m minutes +// [, s seconds]", matching TextMate's format_duration (seconds are included +// only for durations under ten minutes). +func formatDuration(src string) string { + f, err := strconv.ParseFloat(strings.TrimSpace(src), 64) + if err != nil { + return src + } + seconds := int64(math.Round(f)) + units := []struct { + singular, plural string + amount int64 + include bool + }{ + {"day", "days", seconds / 60 / 60 / 24, true}, + {"hour", "hours", (seconds / 60 / 60) % 24, true}, + {"minute", "minutes", (seconds / 60) % 60, true}, + {"second", "seconds", seconds % 60, seconds < 10*60}, + } + var parts []string + for _, u := range units { + if u.amount != 0 && u.include { + name := u.plural + if u.amount == 1 { + name = u.singular + } + parts = append(parts, fmt.Sprintf("%d %s", u.amount, name)) + } + } + return strings.Join(parts, ", ") +} diff --git a/grammar/transform_test.go b/grammar/transform_test.go new file mode 100644 index 0000000..6ce81fb --- /dev/null +++ b/grammar/transform_test.go @@ -0,0 +1,110 @@ +package grammar + +import ( + "testing" + "time" +) + +func TestApplyTransformsSingle(t *testing.T) { + cases := []struct { + name string + transform string + in, want string + }{ + {"upcase", "upcase", "Hello World", "HELLO WORLD"}, + {"downcase", "downcase", "Hello World", "hello world"}, + {"capitalize", "capitalize", "hello world", "Hello World"}, + {"titlecase-alias", "titlecase", "hello world", "Hello World"}, + {"capitalize-allcaps", "capitalize", "HELLO", "Hello"}, + {"asciify", "asciify", "café déjà", "cafe deja"}, + {"asciify-plain", "asciify", "hello", "hello"}, + {"urlencode", "urlencode", "a b/c?d", "a%20b%2Fc%3Fd"}, + {"urlencode-unreserved", "urlencode", "A-z_0.9~", "A-z_0.9~"}, + {"shellescape-plain", "shellescape", "hello", "hello"}, + {"shellescape-special", "shellescape", "a b", "'a b'"}, + {"shellescape-quote", "shellescape", "it's", `it\'s`}, + {"number", "number", "1234567", "1,234,567"}, + {"number-decimal", "number", "1234.5678", "1,234.5678"}, + {"number-small", "number", "42", "42"}, + {"duration", "duration", "3661", "1 hour, 1 minute"}, // seconds dropped (>= 10 min) + {"duration-seconds", "duration", "61", "1 minute, 1 second"}, + {"duration-days", "duration", "90000", "1 day, 1 hour"}, + {"dirname", "dirname", "foo/bar/baz", "foo/bar"}, + {"basename", "basename", "foo/bar/baz", "baz"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := applyTransforms(c.in, []string{c.transform}); got != c.want { + t.Errorf("applyTransforms(%q, /%s) = %q, want %q", c.in, c.transform, got, c.want) + } + }) + } +} + +func TestApplyTransformsChainedFixedOrder(t *testing.T) { + // Transforms apply in TextMate's fixed order regardless of written order: + // downcase runs before capitalize, so both orderings yield the same result. + in := "HELLO WORLD" + want := "Hello World" + if got := applyTransforms(in, []string{"downcase", "capitalize"}); got != want { + t.Errorf("downcase/capitalize = %q, want %q", got, want) + } + if got := applyTransforms(in, []string{"capitalize", "downcase"}); got != want { + t.Errorf("capitalize/downcase (written reversed) = %q, want %q", got, want) + } +} + +func TestApplyTransformsUnknownIgnored(t *testing.T) { + if got := applyTransforms("hello", []string{"bogus"}); got != "hello" { + t.Errorf("unknown transform should be a no-op, got %q", got) + } +} + +func TestRelativeTimeDeterministic(t *testing.T) { + fixed := time.Date(2026, 6, 1, 12, 0, 0, 0, time.UTC) + orig := nowFunc + nowFunc = func() time.Time { return fixed } + defer func() { nowFunc = orig }() + + cases := []struct{ in, want string }{ + {"2026-06-01 11:59:00", "a minute ago"}, + {"2026-06-01 11:00:00", "an hour ago"}, + {"2026-05-31 12:00:00", "a day ago"}, + {"2026-06-01 13:00:00", "in the future"}, + } + for _, c := range cases { + if got := relativeTime(c.in); got != c.want { + t.Errorf("relativeTime(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +func TestParseGroupTemplate(t *testing.T) { + cases := []struct { + inner string + wantNum int + wantTransforms []string + }{ + {"1", 1, nil}, + {"1:/downcase", 1, []string{"downcase"}}, + {"1:/downcase/capitalize", 1, []string{"downcase", "capitalize"}}, + {"12:/upcase", 12, []string{"upcase"}}, + {"bogus", -1, nil}, + } + for _, c := range cases { + num, transforms := parseGroupTemplate(c.inner) + if num != c.wantNum { + t.Errorf("parseGroupTemplate(%q) num = %d, want %d", c.inner, num, c.wantNum) + } + if len(transforms) != len(c.wantTransforms) { + t.Errorf("parseGroupTemplate(%q) transforms = %v, want %v", c.inner, transforms, c.wantTransforms) + continue + } + for i := range transforms { + if transforms[i] != c.wantTransforms[i] { + t.Errorf("parseGroupTemplate(%q) transforms = %v, want %v", c.inner, transforms, c.wantTransforms) + break + } + } + } +} diff --git a/oniglib/fuzz_test.go b/oniglib/fuzz_test.go new file mode 100644 index 0000000..119caf1 --- /dev/null +++ b/oniglib/fuzz_test.go @@ -0,0 +1,18 @@ +package oniglib + +import "testing" + +func FuzzScannerFindNextMatch(f *testing.F) { + f.Add(`\w+`, "foo bar") + f.Add(`(?x)\bfoo\b`, "xx foo bar") + f.Fuzz(func(t *testing.T, pattern, text string) { + if len(pattern) > 2048 { + pattern = pattern[:2048] + } + if len(text) > 2048 { + text = text[:2048] + } + sc := NewScanner([]string{pattern}) + _, _ = sc.FindNextMatch([]rune(text), 0, true, true) + }) +} diff --git a/oniglib/oniglib.go b/oniglib/oniglib.go index ac95a6a..e57a8ad 100644 --- a/oniglib/oniglib.go +++ b/oniglib/oniglib.go @@ -28,10 +28,16 @@ import ( // per-match timeout bookkeeping in the hot path. var matchTimeout time.Duration = 0 -// neverMatch is a zero-width assertion that can never be satisfied (a position -// cannot be both a word boundary and a non-word boundary). It is used to -// neutralise \A or \G when the current scan position forbids them. -const neverMatch = `\b\B` +// neverMatch is a zero-width assertion that can never be satisfied (an empty +// negative look-ahead fails at every position). It is used to neutralise \A or +// \G when the current scan position forbids them. +const neverMatch = `(?!)` + +// neverMatchInClass neutralises \A or \G when they appear inside a character +// class, where an assertion like (?!) or \b\B is not valid. U+FFFF is a +// non-character that effectively never occurs in real text, mirroring +// vscode-textmate's use of \uFFFF as a never-matching sentinel. +const neverMatchInClass = `\x{FFFF}` // compileOptions are applied to every pattern. We deliberately do NOT use // regexp2.RE2 so that the engine keeps its Oniguruma/PCRE-compatible behaviour @@ -89,140 +95,264 @@ func NewRegex(source string) *Regex { } } -// normalizeOniguruma rewrites the Oniguruma-specific constructs that regexp2 -// rejects into equivalents: +// normalizeOniguruma rewrites the Oniguruma-specific quantifier constructs that +// regexp2 (.NET semantics) rejects or interprets differently into equivalents. +// It works as a single forward pass that parses each quantifiable atom (escape, +// character class, group/comment, or single character) followed by its optional +// quantifier, so nested classes, (?#...) comments and interval bodies never +// confuse the rewriter. The transformations applied (matching the default +// ONIG_SYNTAX_ONIGURUMA syntax) are: // -// - possessive quantifiers (a++, a*+, a?+, a{n,m}+) are converted into -// atomic groups, e.g. a++ -> (?>a+). This preserves the no-backtracking -// semantics of possessive matching, which is essential: rewriting them as -// plain greedy quantifiers can cause catastrophic backtracking on the large -// declaration/attribute patterns found in real grammars. +// - possessive single-char quantifiers a?+, a*+, a++ -> atomic groups +// (?>a?), (?>a*), (?>a+). This preserves the no-backtracking semantics, +// which is essential to avoid catastrophic backtracking on real grammars. +// - reversed interval a{n,m} with n>m, which Oniguruma defines as the +// possessive form of {m,n} -> (?>a{m,n}). +// - interval followed by '+' (a{n}+, a{n,m}+, a{n,}+) which is NOT possessive +// in the default syntax -> (?:a{n})+, so regexp2 accepts it. +// - the {,n} form (== {0,n}) which .NET does not accept -> {0,n}. // -// Char classes and escaped metacharacters are respected so literal quantifier -// characters are left untouched. +// Invalid braces such as a{abc}+ are left untouched: {abc} is a literal, so the +// trailing '+' is an ordinary quantifier applied to the literal '}'. func normalizeOniguruma(source string) string { rs := []rune(source) out := make([]rune, 0, len(rs)+8) - inClass := false - for i := 0; i < len(rs); i++ { - c := rs[i] - if c == '\\' && i+1 < len(rs) { - out = append(out, c, rs[i+1]) - i++ + i := 0 + for i < len(rs) { + atomStart := len(out) + ae := scanAtom(rs, i) + out = append(out, rs[i:ae]...) + i = ae + if i >= len(rs) { continue } - if inClass { - out = append(out, c) - if c == ']' { - inClass = false + + switch rs[i] { + case '?', '*', '+': + out = append(out, rs[i]) + i++ + if i < len(rs) && rs[i] == '+' { + // Possessive (a?+, a*+, a++): wrap atom+quantifier atomically. + out = wrapGroup(out, atomStart, "(?>") + i++ // consume the possessive '+' + } else if i < len(rs) && rs[i] == '?' { + // Reluctant (a*?, a+?, a??): regexp2 supports these natively. + out = append(out, rs[i]) + i++ } - continue - } - if c == '[' { - inClass = true - out = append(out, c) - continue - } - // A '+' immediately following a quantifier marks a possessive - // quantifier: wrap the already-emitted atom+quantifier atomically. - if c == '+' && len(out) > 0 { - switch out[len(out)-1] { - case '+', '*', '?', '}': - out = wrapAtomicTail(out) + case '{': + iv, ok := parseInterval(rs, i) + if !ok { + // Not a valid interval; '{' is a literal character. + out = append(out, rs[i]) + i++ continue } + switch { + case iv.reversed(): + // Oniguruma: {n,m} with n>m is possessive of {m,n}. + out = append(out, []rune("{"+iv.hi+","+iv.lo+"}")...) + out = wrapGroup(out, atomStart, "(?>") + i = iv.end + case iv.end < len(rs) && rs[iv.end] == '+': + // {n}+, {n,m}+, {n,}+ are NOT possessive in the default syntax: + // the '+' is an ordinary quantifier applied to the interval. + out = append(out, []rune(iv.text())...) + out = wrapGroup(out, atomStart, "(?:") + out = append(out, '+') + i = iv.end + 1 + default: + out = append(out, []rune(iv.text())...) + i = iv.end + } } - out = append(out, c) } return string(out) } -// wrapAtomicTail wraps the trailing "" already present in out -// with an atomic group: ...X -> ...(?>X). On any ambiguity it -// falls back to leaving out unchanged (which degrades a possessive quantifier -// to greedy rather than corrupting the pattern). -func wrapAtomicTail(out []rune) []rune { - end := len(out) - - // Locate the start of the quantifier token. - quantStart := end - 1 - if out[end-1] == '}' { - quantStart = scanBackTo(out, end-1, '{') - if quantStart < 0 { - return out // unmatched: leave as greedy +// wrapGroup rewrites out so that the run from atomStart to the end is enclosed +// in a group introduced by open (e.g. "(?>" or "(?:") and a closing ')'. +func wrapGroup(out []rune, atomStart int, open string) []rune { + res := make([]rune, 0, len(out)+len(open)+1) + res = append(res, out[:atomStart]...) + res = append(res, []rune(open)...) + res = append(res, out[atomStart:]...) + res = append(res, ')') + return res +} + +// scanAtom returns the index just past the quantifiable atom that begins at i. +// An atom is an escape (with any \x{...}/\p{...}/\o{...} brace body), a +// character class, a group (or (?#...) comment), or a single character. +func scanAtom(rs []rune, i int) int { + switch rs[i] { + case '\\': + if i+1 >= len(rs) { + return i + 1 + } + end := i + 2 + switch rs[i+1] { + case 'x', 'p', 'P', 'o': + if end < len(rs) && rs[end] == '{' { + for end < len(rs) && rs[end] != '}' { + end++ + } + if end < len(rs) { + end++ // include the closing '}' + } + } } + return end + case '[': + return scanClass(rs, i) + case '(': + return scanGroup(rs, i) + default: + return i + 1 } +} - // Locate the start of the atom the quantifier applies to. - atomEnd := quantStart - if atomEnd <= 0 { - return out +// scanClass returns the index just past the character class beginning at i +// (rs[i] == '['). A ']' immediately after '[' or '[^' is a literal member, not +// the terminator, mirroring Oniguruma ([]] == [\]]). Class scanning follows +// .NET semantics: '[' inside a class is literal and the first subsequent +// unescaped ']' closes it. +func scanClass(rs []rune, i int) int { + j := i + 1 + if j < len(rs) && rs[j] == '^' { + j++ } - prev := out[atomEnd-1] - var atomStart int - switch prev { - case ')': - atomStart = matchBackward(out, atomEnd-1, '(', ')') - case ']': - atomStart = matchBackward(out, atomEnd-1, '[', ']') - default: - atomStart = atomEnd - 1 - if atomStart > 0 && isEscaped(out, atomStart) { - atomStart-- // include the leading backslash of an escaped atom + if j < len(rs) && rs[j] == ']' { + j++ // leading ']' is a literal member + } + for j < len(rs) { + switch rs[j] { + case '\\': + j += 2 + continue + case ']': + return j + 1 + } + j++ + } + return j // unterminated; caller emits the remainder verbatim +} + +// scanGroup returns the index just past the group beginning at i (rs[i] == +// '('). A (?#...) comment ends at the first unescaped ')'. Nested groups and +// character classes are skipped so their parentheses do not unbalance the scan. +func scanGroup(rs []rune, i int) int { + if i+2 < len(rs) && rs[i+1] == '?' && rs[i+2] == '#' { + j := i + 3 + for j < len(rs) { + if rs[j] == '\\' { + j += 2 + continue + } + if rs[j] == ')' { + return j + 1 + } + j++ } + return j } - if atomStart < 0 { - return out + j := i + 1 + for j < len(rs) { + switch rs[j] { + case '\\': + j += 2 + continue + case '[': + j = scanClass(rs, j) + continue + case '(': + j = scanGroup(rs, j) + continue + case ')': + return j + 1 + } + j++ } + return j // unterminated +} - result := make([]rune, 0, len(out)+4) - result = append(result, out[:atomStart]...) - result = append(result, '(', '?', '>') - result = append(result, out[atomStart:end]...) - result = append(result, ')') - return result +// interval describes a parsed {..} quantifier body. +type interval struct { + lo, hi string // raw digit runs ("" when omitted) + hasComma bool + end int // index just past the closing '}' } -// scanBackTo returns the index of the nearest unescaped open rune at or before -// from, or -1 if none. -func scanBackTo(out []rune, from int, open rune) int { - for i := from; i >= 0; i-- { - if out[i] == open && !isEscaped(out, i) { - return i - } +// reversed reports whether the interval is a reversed range {n,m} with n>m, +// which Oniguruma treats as the possessive form of {m,n}. +func (iv interval) reversed() bool { + if !iv.hasComma || iv.lo == "" || iv.hi == "" { + return false } - return -1 + lo, hi := atoiClamp(iv.lo), atoiClamp(iv.hi) + return lo > hi } -// matchBackward finds the matching open rune for a close rune at closeIdx, -// honouring nesting and escapes, returning the open index or -1. -func matchBackward(out []rune, closeIdx int, open, close rune) int { - depth := 0 - for i := closeIdx; i >= 0; i-- { - if isEscaped(out, i) { - continue +// text returns the .NET-acceptable rendering of the interval. The {,n} form is +// rewritten to {0,n} because .NET does not accept an omitted minimum. +func (iv interval) text() string { + switch { + case !iv.hasComma: + return "{" + iv.lo + "}" + case iv.hi == "": + return "{" + iv.lo + ",}" + case iv.lo == "": + return "{0," + iv.hi + "}" + default: + return "{" + iv.lo + "," + iv.hi + "}" + } +} + +// parseInterval parses a quantifier interval beginning at rs[i] == '{'. It +// returns ok=false when the braces do not form a valid Oniguruma interval (so +// the '{' must be treated as a literal). Valid forms: {n}, {n,}, {,m}, {n,m}. +func parseInterval(rs []rune, i int) (interval, bool) { + j := i + 1 + var iv interval + for j < len(rs) && rs[j] >= '0' && rs[j] <= '9' { + iv.lo += string(rs[j]) + j++ + } + if j < len(rs) && rs[j] == ',' { + iv.hasComma = true + j++ + for j < len(rs) && rs[j] >= '0' && rs[j] <= '9' { + iv.hi += string(rs[j]) + j++ } - switch out[i] { - case close: - depth++ - case open: - depth-- - if depth == 0 { - return i - } + } + if j >= len(rs) || rs[j] != '}' { + return interval{}, false + } + // Reject empty bodies: {} and {,} are not quantifiers. + if iv.hasComma { + if iv.lo == "" && iv.hi == "" { + return interval{}, false } + } else if iv.lo == "" { + return interval{}, false } - return -1 + iv.end = j + 1 + return iv, true } -// isEscaped reports whether the rune at index i is preceded by an odd number of -// backslashes (and is therefore escaped). -func isEscaped(out []rune, i int) bool { +// atoiClamp parses a (possibly long) digit run, clamping overflow to a large +// value so that only the lo>hi ordering decision is affected. +func atoiClamp(s string) int { + const cap = 1 << 30 n := 0 - for j := i - 1; j >= 0 && out[j] == '\\'; j-- { - n++ + for _, r := range s { + n = n*10 + int(r-'0') + if n >= cap { + return cap + } } - return n%2 == 1 + return n } // Source returns the original pattern text. @@ -389,11 +519,17 @@ func SubstituteBackRefs(source string, captured []string) string { if c == '\\' && i+1 < len(rs) { n := rs[i+1] if n >= '0' && n <= '9' { - idx := int(n - '0') + // Consume the whole digit run: \12 is group 12 and \00001 is + // group 1 (leading zeros are allowed). \0 is the whole match. + j := i + 1 + for j < len(rs) && rs[j] >= '0' && rs[j] <= '9' { + j++ + } + idx := atoiClamp(string(rs[i+1 : j])) if idx < len(captured) { b.WriteString(escapeRegex(captured[idx])) } - i++ + i = j - 1 continue } // Preserve other escapes verbatim (e.g. \\ , \w). @@ -457,15 +593,22 @@ func containsAnchor(source string, letter rune) bool { } // neutralizeAnchor replaces every \ assertion with a never-matching -// assertion, leaving escaped backslashes untouched. +// construct, leaving escaped backslashes untouched. Inside a character class a +// zero-width assertion is not valid, so a never-occurring codepoint is used +// instead (see neverMatchInClass). func neutralizeAnchor(source string, letter rune) string { var b strings.Builder rs := []rune(source) + inClass := false for i := 0; i < len(rs); i++ { if rs[i] == '\\' && i+1 < len(rs) { n := rs[i+1] if n == letter { - b.WriteString(neverMatch) + if inClass { + b.WriteString(neverMatchInClass) + } else { + b.WriteString(neverMatch) + } i++ continue } @@ -474,6 +617,14 @@ func neutralizeAnchor(source string, letter rune) string { i++ continue } + switch rs[i] { + case '[': + if !inClass { + inClass = true + } + case ']': + inClass = false + } b.WriteRune(rs[i]) } return b.String() diff --git a/oniglib/oniglib_test.go b/oniglib/oniglib_test.go index 515b2e6..f84f7a2 100644 --- a/oniglib/oniglib_test.go +++ b/oniglib/oniglib_test.go @@ -84,6 +84,126 @@ func TestHasBackRefs(t *testing.T) { } } +func TestNormalizeOniguruma(t *testing.T) { + cases := []struct{ in, want string }{ + // Possessive single-char quantifiers become atomic groups. + {`a++`, `(?>a+)`}, + {`a*+`, `(?>a*)`}, + {`a?+`, `(?>a?)`}, + {`\.++`, `(?>\.+)`}, + {`(?:a)++`, `(?>(?:a)+)`}, + // Reluctant quantifiers are left for regexp2 to handle natively. + {`a*?`, `a*?`}, + {`a+?b`, `a+?b`}, + // Possessive wrapping must respect nested classes and comments. + {`([)])++`, `(?>([)])+)`}, + {`(a(?#comment [ comment))++`, `(?>(a(?#comment [ comment))+)`}, + // Interval validation. + {`a{abc}+`, `a{abc}+`}, // not a valid interval: left untouched + {`a{3,2}`, `(?>a{2,3})`}, // reversed range is possessive of {2,3} + {`a{3}+`, `(?:a{3})+`}, // {n}+ is not possessive in default syntax + {`a{2,3}+`, `(?:a{2,3})+`}, // {n,m}+ is not possessive either + {`a{2,}+`, `(?:a{2,})+`}, // {n,}+ is not possessive either + {`a{,3}`, `a{0,3}`}, // {,n} rewritten to {0,n} for regexp2 + {`a{,3}+`, `(?:a{0,3})+`}, // combined with the non-possessive '+' + {`a{2,3}`, `a{2,3}`}, // normal interval untouched + {`a{3}`, `a{3}`}, // fixed interval untouched + // Quantifier characters inside a class are literal, not quantifiers. + {`[a+]+`, `[a+]+`}, + {`[a*]++`, `(?>[a*]+)`}, + } + for _, c := range cases { + if got := normalizeOniguruma(c.in); got != c.want { + t.Errorf("normalizeOniguruma(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +func TestLeadingBracketIsLiteral(t *testing.T) { + cases := []struct { + pattern, text string + wantMatch bool + }{ + {`[]]`, `]`, true}, // []] == [\]] + {`[]]`, `a`, false}, + {`[^]]`, `a`, true}, // [^]] negates a class containing ']' + {`[^]]`, `]`, false}, + {`[]abc]`, `b`, true}, // leading ']' plus a,b,c + {`[]abc]`, `x`, false}, + } + for _, c := range cases { + r := NewRegex(c.pattern) + r.Warmup() + if r.Broken() { + t.Errorf("pattern %q failed to compile", c.pattern) + continue + } + caps, _ := r.match([]rune(c.text), 0, true, true) + if (caps != nil) != c.wantMatch { + t.Errorf("%q against %q: match=%v, want %v", c.pattern, c.text, caps != nil, c.wantMatch) + } + } +} + +func TestPossessiveWithNestedConstructsCompiles(t *testing.T) { + for _, p := range []string{`([)])++`, `(a(?#comment [ comment))++`} { + r := NewRegex(p) + r.Warmup() + if r.Broken() { + t.Errorf("pattern %q should compile after normalization", p) + } + } +} + +func TestReversedIntervalMatches(t *testing.T) { + // a{3,2} is the possessive form of a{2,3}: it matches 2 or 3 a's. + r := NewRegex(`a{3,2}`) + r.Warmup() + if r.Broken() { + t.Fatalf("a{3,2} should compile, got broken") + } + caps, _ := r.match([]rune("aaaa"), 0, true, true) + if caps == nil { + t.Fatal("expected a match") + } + if got := caps[0].End - caps[0].Start; got != 3 { + t.Errorf("expected to match 3 a's (max of range), matched %d", got) + } +} + +func TestNeutralizeAnchorClassAware(t *testing.T) { + if got := neutralizeAnchor(`\Afoo`, 'A'); got != `(?!)foo` { + t.Errorf("outside class: got %q, want %q", got, `(?!)foo`) + } + if got := neutralizeAnchor(`[\A]`, 'A'); got != `[\x{FFFF}]` { + t.Errorf("inside class: got %q, want %q", got, `[\x{FFFF}]`) + } + if got := neutralizeAnchor(`\G[\G]`, 'G'); got != `(?!)[\x{FFFF}]` { + t.Errorf("mixed: got %q, want %q", got, `(?!)[\x{FFFF}]`) + } +} + +func TestMultiDigitBackRefs(t *testing.T) { + captured := make([]string, 13) + captured[0] = "WHOLE" + captured[1] = "one" + captured[12] = "twelve" + cases := []struct{ in, want string }{ + {`\0`, `WHOLE`}, + {`\1`, `one`}, + {`\12`, `twelve`}, // two-digit group + {`\00001`, `one`}, // leading zeros + {`\999`, ``}, // out of range: dropped + {`\\1`, `\\1`}, // literal backslash then 1: not a back-reference + {`\\\1`, `\\one`}, // literal backslash then back-reference + } + for _, c := range cases { + if got := SubstituteBackRefs(c.in, captured); got != c.want { + t.Errorf("SubstituteBackRefs(%q) = %q, want %q", c.in, got, c.want) + } + } +} + func TestUnicodeCodepointEscapes(t *testing.T) { // \x{...} with codepoints beyond U+FFFF must compile and match. r := NewRegex(`[\x{7f}-\x{10ffff}]+`) diff --git a/theme/fuzz_test.go b/theme/fuzz_test.go new file mode 100644 index 0000000..8c02652 --- /dev/null +++ b/theme/fuzz_test.go @@ -0,0 +1,18 @@ +package theme + +import "testing" + +func FuzzParse(f *testing.F) { + f.Add([]byte(`{"name":"t","tokenColors":[]}`)) + f.Add([]byte(`{"colors":{"editor.foreground":"#aabbcc"},"tokenColors":[{"scope":"comment","settings":{"foreground":"#112233"}}]}`)) + f.Fuzz(func(t *testing.T, data []byte) { + if len(data) > 64*1024 { + data = data[:64*1024] + } + th, err := Parse(data) + if err != nil { + return + } + _ = th.Match([]string{"source", "comment.line"}) + }) +}