diff --git a/.changeset/curly-tools-shop.md b/.changeset/curly-tools-shop.md new file mode 100644 index 0000000..efab384 --- /dev/null +++ b/.changeset/curly-tools-shop.md @@ -0,0 +1,5 @@ +--- +"@aliou/sh": patch +--- + +Fix top-level word scanning for escaped characters. diff --git a/.gitignore b/.gitignore index cfe7e3f..9fbe48d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ node_modules +.pi .pnpm-store dist .DS_Store diff --git a/src/parser/positions.test.ts b/src/parser/positions.test.ts index c4407ae..cfe85cd 100644 --- a/src/parser/positions.test.ts +++ b/src/parser/positions.test.ts @@ -60,4 +60,16 @@ describe("position tracking", () => { expect(expansion.pos).toEqual({ offset: 5, line: 1, col: 6 }); expect(expansion.end).toEqual({ offset: 9, line: 1, col: 10 }); }); + + it("keeps literal positions before escaped characters and continuations", () => { + const { ast } = parse("echo a\\ \\" + "\nb"); + const cmd = expectDefined(ast.body[0]).command as SimpleCommand; + const word = expectDefined(expectDefined(cmd.words)[1]); + const part = expectDefined(word.parts[0]); + expect(part.type).toBe("Literal"); + if (part.type !== "Literal") throw new Error("expected literal part"); + expect(part.value).toBe("a b"); + expect(part.pos).toEqual({ offset: 5, line: 1, col: 6 }); + expect(part.end).toEqual({ offset: 11, line: 2, col: 2 }); + }); }); diff --git a/src/parser/words-quotes-comments.test.ts b/src/parser/words-quotes-comments.test.ts index 0295e38..fa90e69 100644 --- a/src/parser/words-quotes-comments.test.ts +++ b/src/parser/words-quotes-comments.test.ts @@ -54,4 +54,49 @@ describe("parse (phase 2: words, quotes, comments)", () => { ast: program(stmt(simple("foo", "bar"))), }); }); + it("treats backslash-CRLF as whitespace", () => { + expect(parse("foo \\\r\n bar")).toMatchAst({ + ast: program(stmt(simple("foo", "bar"))), + }); + }); + + it("keeps escaped parentheses as literal words", () => { + expect(parse('find . \\( -name "*.cs" \\)')).toMatchAst({ + ast: program( + stmt({ + type: "SimpleCommand", + words: [ + wordParts(lit("find")), + wordParts(lit(".")), + wordParts(lit("(")), + wordParts(lit("-name")), + wordParts(dbl(lit("*.cs"))), + wordParts(lit(")")), + ], + }), + ), + }); + }); + + it("keeps escaped spaces inside the same word", () => { + expect(parse("echo a\\ b")).toMatchAst({ + ast: program(stmt(simple("echo", "a b"))), + }); + }); + + it("keeps escaped operators inside literal words", () => { + expect(parse("echo foo\\;bar")).toMatchAst({ + ast: program(stmt(simple("echo", "foo;bar"))), + }); + + expect(parse("echo a\\|b")).toMatchAst({ + ast: program(stmt(simple("echo", "a|b"))), + }); + }); + + it("keeps a trailing backslash literal", () => { + expect(parse("echo foo\\")).toMatchAst({ + ast: program(stmt(simple("echo", "foo\\"))), + }); + }); }); diff --git a/src/tokenizer/tokenize.test.ts b/src/tokenizer/tokenize.test.ts new file mode 100644 index 0000000..b0a6c76 --- /dev/null +++ b/src/tokenizer/tokenize.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from "vitest"; +import { tokenize } from "./tokenize"; + +describe("tokenize", () => { + it("does not treat an escaped question mark as an extglob opener", () => { + const tokens = tokenize("\\?("); + + expect(tokens).toMatchObject([ + { type: "word", parts: [{ type: "lit", value: "?" }] }, + { type: "symbol", value: "(" }, + ]); + }); +}); diff --git a/src/tokenizer/tokenize.ts b/src/tokenizer/tokenize.ts index e30f69a..02361cf 100644 --- a/src/tokenizer/tokenize.ts +++ b/src/tokenizer/tokenize.ts @@ -380,18 +380,28 @@ export function tokenize(source: string, options: ParseOptions = {}): Token[] { if (currentChar === "\\" && source.charAt(i + 1) === "\n") { i += 2; - litStart = i; + if (current.length === 0) { + litStart = i; + } continue; } if (currentChar === "\\" && source.charAt(i + 1) === "\r") { if (source.charAt(i + 2) === "\n") { i += 3; - litStart = i; + if (current.length === 0) { + litStart = i; + } continue; } } + if (currentChar === "\\" && i + 1 < source.length) { + current += source.charAt(i + 1); + i += 2; + continue; + } + // Try to recognize an extended glob (`?(`, `*(`, `+(`, `@(`, `!(`) // before the break check, since `(` is otherwise a word terminator. if (