fix(tokenizer): handle escaped chars in words

.changeset/curly-tools-shop.md

@@ -0,0 +1,5 @@
+---
+"@aliou/sh": patch
+---
+
+Fix top-level word scanning for escaped characters.

.gitignore

@@ -1,4 +1,5 @@
 node_modules
+.pi
 .pnpm-store
 dist
 .DS_Store

src/parser/positions.test.ts

@@ -60,4 +60,16 @@ describe("position tracking", () => {
     expect(expansion.pos).toEqual<Pos>({ offset: 5, line: 1, col: 6 });
     expect(expansion.end).toEqual<Pos>({ offset: 9, line: 1, col: 10 });
   });
+
+  it("keeps literal positions before escaped characters and continuations", () => {
+    const { ast } = parse("echo a\\ \\" + "\nb");
+    const cmd = expectDefined(ast.body[0]).command as SimpleCommand;
+    const word = expectDefined(expectDefined(cmd.words)[1]);
+    const part = expectDefined(word.parts[0]);
+    expect(part.type).toBe("Literal");
+    if (part.type !== "Literal") throw new Error("expected literal part");
+    expect(part.value).toBe("a b");
+    expect(part.pos).toEqual<Pos>({ offset: 5, line: 1, col: 6 });
+    expect(part.end).toEqual<Pos>({ offset: 11, line: 2, col: 2 });
+  });
 });

src/parser/words-quotes-comments.test.ts

@@ -54,4 +54,49 @@ describe("parse (phase 2: words, quotes, comments)", () => {
       ast: program(stmt(simple("foo", "bar"))),
     });
   });
+  it("treats backslash-CRLF as whitespace", () => {
+    expect(parse("foo \\\r\n bar")).toMatchAst({
+      ast: program(stmt(simple("foo", "bar"))),
+    });
+  });
+
+  it("keeps escaped parentheses as literal words", () => {
+    expect(parse('find . \\( -name "*.cs" \\)')).toMatchAst({
+      ast: program(
+        stmt({
+          type: "SimpleCommand",
+          words: [
+            wordParts(lit("find")),
+            wordParts(lit(".")),
+            wordParts(lit("(")),
+            wordParts(lit("-name")),
+            wordParts(dbl(lit("*.cs"))),
+            wordParts(lit(")")),
+          ],
+        }),
+      ),
+    });
+  });
+
+  it("keeps escaped spaces inside the same word", () => {
+    expect(parse("echo a\\ b")).toMatchAst({
+      ast: program(stmt(simple("echo", "a b"))),
+    });
+  });
+
+  it("keeps escaped operators inside literal words", () => {
+    expect(parse("echo foo\\;bar")).toMatchAst({
+      ast: program(stmt(simple("echo", "foo;bar"))),
+    });
+
+    expect(parse("echo a\\|b")).toMatchAst({
+      ast: program(stmt(simple("echo", "a|b"))),
+    });
+  });
+
+  it("keeps a trailing backslash literal", () => {
+    expect(parse("echo foo\\")).toMatchAst({
+      ast: program(stmt(simple("echo", "foo\\"))),
+    });
+  });
 });

src/tokenizer/tokenize.test.ts

@@ -0,0 +1,13 @@
+import { describe, expect, it } from "vitest";
+import { tokenize } from "./tokenize";
+
+describe("tokenize", () => {
+  it("does not treat an escaped question mark as an extglob opener", () => {
+    const tokens = tokenize("\\?(");
+
+    expect(tokens).toMatchObject([
+      { type: "word", parts: [{ type: "lit", value: "?" }] },
+      { type: "symbol", value: "(" },
+    ]);
+  });
+});

src/tokenizer/tokenize.ts

@@ -380,18 +380,28 @@ export function tokenize(source: string, options: ParseOptions = {}): Token[] {
 
       if (currentChar === "\\" && source.charAt(i + 1) === "\n") {
         i += 2;
-        litStart = i;
+        if (current.length === 0) {
+          litStart = i;
+        }
         continue;
       }
 
       if (currentChar === "\\" && source.charAt(i + 1) === "\r") {
         if (source.charAt(i + 2) === "\n") {
           i += 3;
-          litStart = i;
+          if (current.length === 0) {
+            litStart = i;
+          }
           continue;
         }
       }
 
+      if (currentChar === "\\" && i + 1 < source.length) {
+        current += source.charAt(i + 1);
+        i += 2;
+        continue;
+      }
+
       // Try to recognize an extended glob (`?(`, `*(`, `+(`, `@(`, `!(`)
       // before the break check, since `(` is otherwise a word terminator.
       if (