Feature/improve oniglib textmate capabilities

.drun/spec.drun

@@ -5,13 +5,18 @@
 version: 2.0
 
 project "gopher-textmate" version "1.0":
+	requires tools:
+		go >= 1.25
+		golangci-lint >= 2.12
+		gosec
 
 task "default" means "Welcome to drun v2":
 	info "Gopher Textmate task runner 🚀"
 
 task "test" means "Runs the test suite":
 	step "Testing Gopher Textmate..."
 	run "go test ./..."
+    success "Tests passed"
 
 task "test-full" means "Runs the extended test suite, including race":
     step "Test"
@@ -23,13 +28,38 @@ task "test-full" means "Runs the extended test suite, including race":
     success "All tests succeeded"
 
 task "lint" means "Runs the linters":
-    if "golangci-lint" is available and version >= "2.12":
-      info "GolangCI Lint available and version satisfies requirements"
-    else:
-      fail "golangci-lint >= 2.12 is required"
 	step "Running linters"
 	run "golangci-lint run"
+    success "Lint passed"
 
-task "ci" means "Runs the whole CI pipeline":
+task "vet" means "Runs the vet":
+    step "Vet"
+    run "go vet ./..."
+    success "Vet passed"
+
+task "fuzz" means "Runs the fuzz tests":
+    step "Fuzz - Grammar"
+    run "go test ./grammar/ -fuzz=FuzzApplyTransforms -fuzztime=30s"
+    step "Fuzz - Oniglib"
+    run "go test ./oniglib/ -fuzz=FuzzScannerFindNextMatch -fuzztime=30s"
+    step "Fuzz - Theme"
+    run "go test ./theme/ -fuzz=FuzzParse -fuzztime=30s"
+    success "Fuzz passed"
+
+task "security" means "Gosec":
+    step "Gosec"
+    run "gosec -exclude=G115,G304 ./..."
+    success "Security check passed"
+
+task "ci" means "Runs the routine CI pipeline (not too time consuming, routine)":
+	call task vet
 	call task test
-	call task lint
+	call task lint
+	call task security
+
+task "ci-full" means "Runs the full CI pipeline including fuzzers and race tests":
+	call task vet
+	call task test-full
+	call task lint
+	call task security
+	call task fuzz

.github/workflows/ci.yml

@@ -0,0 +1,57 @@
+name: CI
+
+on:
+  pull_request:
+    paths-ignore:
+      - ".drun/spec.drun"
+      - "*.md"
+      - "**/*.md"
+
+permissions:
+  contents: read
+
+jobs:
+  ci:
+    name: CI
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version: '1.26'
+          cache: true
+          cache-dependency-path: go.sum
+
+      - name: Download dependencies
+        run: go mod download
+
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v8
+        with:
+          version: v2.12
+
+      - name: Vet
+        run: go vet ./...
+
+      - name: Test
+        run: go test ./...
+
+      - name: Install gosec
+        run: go install github.com/securego/gosec/v2/cmd/gosec@latest
+
+      # G115: hex color components are 0–255 by construction.
+      # G304: library/CLI APIs intentionally read caller-supplied paths.
+      - name: gosec
+        run: gosec -quiet -exclude=G115,G304 ./...
+
+      - name: Race detector
+        run: go test -race ./...
+
+      - name: Fuzz (smoke)
+        run: |
+          go test ./grammar/ -fuzz=. -fuzztime=10s
+          go test ./oniglib/ -fuzz=. -fuzztime=10s
+          go test ./theme/ -fuzz=. -fuzztime=10s

README.md

@@ -6,7 +6,6 @@ It tokenizes source text into scoped tokens using TextMate grammars, resolves a
 
 <img width="1132" height="431" alt="php" src="https://github.com/user-attachments/assets/4b67b298-f9d1-453c-a139-6e9dca97730b" />
 
-
 ## Why pure Go?
 
 TextMate grammars rely on Oniguruma regular expressions (lookbehind, lookahead, `\G`, back-references, `\x{...}` codepoints) that Go's standard `regexp` (RE2) cannot handle. Instead of binding to Oniguruma via cgo, this library uses the pure-Go [`github.com/dlclark/regexp2/v2`](https://github.com/dlclark/regexp2) engine, so builds stay static and cross-compile cleanly. Oniguruma possessive quantifiers (`a++`) are rewritten as atomic groups (`(?>a+)`) to preserve their no-backtracking semantics.
@@ -124,6 +123,7 @@ The facade is built on exported packages you can use directly:
 ```bash
 go run ./cmd/gtm -grammar grammars/php.tmLanguage.json -scope source.php examples/sample.php
 ```
+
 <img width="1132" height="431" alt="php" src="https://github.com/user-attachments/assets/4b67b298-f9d1-453c-a139-6e9dca97730b" />
 
 Flags:
@@ -136,29 +136,45 @@ Flags:
 
 ## Supported grammar features
 
-`match`, `begin`/`end`, `begin`/`while`, `include` (`#repo`, `$self`, `$base`, cross-grammar `scope.name#sub`), `repository`, `captures` / `beginCaptures` / `endCaptures` with nested `patterns`, `contentName`, `applyEndPatternLast`, dynamic end patterns via back-references (`\1`…`\9`), scope-name templates (`$1`, `${1:/downcase|upcase|capitalize}`), and basic `injections`.
+`match`, `begin`/`end`, `begin`/`while`, `include` (`#repo`, `$self`, `$base`, cross-grammar `scope.name#sub`), `repository`, `captures` / `beginCaptures` / `endCaptures` with nested `patterns`, `contentName`, `applyEndPatternLast`, dynamic end patterns via back-references (multi-digit and zero-padded, e.g. `\1`, `\12`, `\001`), `injections` (basic), and scope-name templates (`$1`, `${1}`) with the full set of TextMate transforms — `upcase`, `downcase`, `capitalize`/`titlecase`, `asciify`, `urlencode`, `shellescape`, `relative`, `number`, `duration`, `dirname`, `basename` — which may be chained, e.g. `${1:/downcase/capitalize}`.
 
 ## Known limitations
 
 - Themes: VSCode JSON format only (`.tmTheme` plist is not yet supported).
 - `$base` is treated as `$self` (identical for single-grammar tokenization).
 - Cross-grammar includes only resolve grammars that have been loaded; unresolved references are skipped, so mixed-language files highlight the languages whose grammars are present.
-- Oniguruma possessive quantifiers (`a++`) are normalized to greedy; the rare `\g<name>` subroutine call is unsupported and such a pattern simply never matches (graceful degradation).
+- The rare `\g<name>` subroutine call is unsupported and such a pattern simply never matches (graceful degradation).
+- `asciify` and `urlencode` transforms approximate macOS/ICU behavior (NFD + combining-mark stripping; RFC 3986 unreserved set), and `(?x)` extended-mode `#` comments containing unbalanced parentheses are not parsed.
 - Injection selector matching is basic; advanced exclusion selectors degrade gracefully.
 
 ## Development
 
 It is advisable to use the [drun](https://github.com/phillarmonic/drun) task runner for development. It makes it easy to run routine tasks in a semantic way. Check the .drun/spec.drun to understand how the file works.
 
+### Requirements
+
+- `Go >=1.25`
+
+- `golangci-lint >= 2.12`
+
+- `gosec >= 2.27`
+
+- `drun >= 2.0`
+
+Development lifecycle:
+
 ```bash
 # For running only the tests:
 xdrun test
 # For running only the linter:
 xdrun lint
 # For running the full test suite including the time consuming tests:
 xdrun test-full
-# For running the whole CI lifecycle (test, lint)
+# For running the whole CI lifecycle in fast mode (test, lint)
 xdrun ci
+# For running CI after you're done coding, and run the expensive tests
+# like race condition tests and fuzz
+xdrun ci-full
 ```
 
 ## License

go.mod

@@ -1,5 +1,7 @@
 module github.com/andersonpem/gopher-textmate
 
-go 1.25
+go 1.25.0
 
 require github.com/dlclark/regexp2/v2 v2.1.1
+
+require golang.org/x/text v0.37.0

go.sum

@@ -1,2 +1,4 @@
 github.com/dlclark/regexp2/v2 v2.1.1 h1:LCUGyd9Wf+r+VVOl8Ny38JTpWJcAsdVnCIuhhtthmKw=
 github.com/dlclark/regexp2/v2 v2.1.1/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
+golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
+golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=

grammar/fuzz_test.go

@@ -0,0 +1,15 @@
+package grammar
+
+import "testing"
+
+func FuzzApplyTransforms(f *testing.F) {
+	f.Add("hello world", "capitalize")
+	f.Add("1234567", "number")
+	f.Add("café déjà", "asciify")
+	f.Fuzz(func(t *testing.T, s, transform string) {
+		if len(s) > 4096 {
+			s = s[:4096]
+		}
+		_ = applyTransforms(s, []string{transform})
+	})
+}

grammar/grammar.go

@@ -388,8 +388,8 @@ func (g *Grammar) rootStack() *StateStack {
 
 // resolveScopeName expands a scope-name template that references match
 // captures, e.g. "keyword.control.$1.php" or "entity.name.tag.${1:/downcase}".
-// Supported transforms are /downcase, /upcase and /capitalize. Templates with
-// no "$" are returned unchanged.
+// Transforms (see applyTransforms) may be chained, e.g. "${1:/downcase/capitalize}".
+// Templates with no "$" are returned unchanged.
 func resolveScopeName(tmpl string, line []rune, groups []oniglib.Capture) string {
 	if !strings.ContainsRune(tmpl, '$') {
 		return tmpl
@@ -400,7 +400,7 @@ func resolveScopeName(tmpl string, line []rune, groups []oniglib.Capture) string
 		c := rs[i]
 		if c == '$' && i+1 < len(rs) {
 			if rs[i+1] >= '0' && rs[i+1] <= '9' {
-				b.WriteString(captureText(line, groups, int(rs[i+1]-'0'), ""))
+				b.WriteString(captureText(line, groups, int(rs[i+1]-'0'), nil))
 				i++
 				continue
 			}
@@ -410,8 +410,8 @@ func resolveScopeName(tmpl string, line []rune, groups []oniglib.Capture) string
 					j++
 				}
 				if j < len(rs) {
-					num, transform := parseGroupTemplate(string(rs[i+2 : j]))
-					b.WriteString(captureText(line, groups, num, transform))
+					num, transforms := parseGroupTemplate(string(rs[i+2 : j]))
+					b.WriteString(captureText(line, groups, num, transforms))
 					i = j
 					continue
 				}
@@ -422,43 +422,36 @@ func resolveScopeName(tmpl string, line []rune, groups []oniglib.Capture) string
 	return b.String()
 }
 
-func parseGroupTemplate(inner string) (int, string) {
+// parseGroupTemplate parses the body of a ${...} capture reference, returning
+// the group number and the list of transforms requested after the ':'. The
+// transforms are written as "/name" segments, e.g. "1:/downcase/capitalize".
+func parseGroupTemplate(inner string) (int, []string) {
 	num := inner
-	transform := ""
+	var transforms []string
 	if i := strings.IndexByte(inner, ':'); i >= 0 {
 		num = inner[:i]
-		transform = strings.TrimPrefix(inner[i+1:], "/")
+		for _, t := range strings.Split(inner[i+1:], "/") {
+			if t = strings.TrimSpace(t); t != "" {
+				transforms = append(transforms, t)
+			}
+		}
 	}
 	n, err := strconv.Atoi(strings.TrimSpace(num))
 	if err != nil {
-		return -1, transform
+		return -1, transforms
 	}
-	return n, transform
+	return n, transforms
 }
 
-func captureText(line []rune, groups []oniglib.Capture, idx int, transform string) string {
+func captureText(line []rune, groups []oniglib.Capture, idx int, transforms []string) string {
 	if idx < 0 || idx >= len(groups) {
 		return ""
 	}
 	gp := groups[idx]
 	if gp.Start < 0 || gp.End < 0 || gp.Start > gp.End || gp.End > len(line) {
 		return ""
 	}
-	s := string(line[gp.Start:gp.End])
-	switch transform {
-	case "downcase":
-		return strings.ToLower(s)
-	case "upcase":
-		return strings.ToUpper(s)
-	case "capitalize":
-		if s == "" {
-			return s
-		}
-		r := []rune(s)
-		return strings.ToUpper(string(r[0])) + string(r[1:])
-	default:
-		return s
-	}
+	return applyTransforms(string(line[gp.Start:gp.End]), transforms)
 }
 
 func pushScope(parent []string, names ...string) []string {