diff --git a/src/scanner.c b/src/scanner.c index d5835856..47f231e2 100644 --- a/src/scanner.c +++ b/src/scanner.c @@ -67,6 +67,9 @@ typedef struct { Array(Heredoc) open_heredocs; } Scanner; +static const int32_t ASCII_C0_CONTROL_MAX = 0x1f; +static const int32_t ASCII_MAX = 0x7f; + const char NON_IDENTIFIER_CHARS[] = { '\0', '\n', '\r', '\t', ' ', ':', ';', '`', '"', '\'', '@', '$', '#', '.', ',', '|', '^', '&', '<', '=', '>', '+', '-', '*', '/', '\\', '%', '?', '!', '~', '(', ')', '[', ']', '{', '}', @@ -336,8 +339,14 @@ static inline bool scan_operator(TSLexer *lexer) { } } -static inline bool is_iden_char(char c) { - return memchr(&NON_IDENTIFIER_CHARS, c, sizeof(NON_IDENTIFIER_CHARS)) == NULL; +static inline bool is_iden_char(int32_t c) { + if (c <= ASCII_C0_CONTROL_MAX || iswspace(c)) { + return false; + } + if (c > ASCII_MAX) { + return true; + } + return memchr(&NON_IDENTIFIER_CHARS, (char)c, sizeof(NON_IDENTIFIER_CHARS)) == NULL; } static inline bool scan_symbol_identifier(TSLexer *lexer) { @@ -350,13 +359,13 @@ static inline bool scan_symbol_identifier(TSLexer *lexer) { advance(lexer); } - if (is_iden_char((char)lexer->lookahead)) { + if (is_iden_char(lexer->lookahead)) { advance(lexer); } else if (!scan_operator(lexer)) { return false; } - while (is_iden_char((char)lexer->lookahead)) { + while (is_iden_char(lexer->lookahead)) { advance(lexer); } @@ -656,7 +665,7 @@ static inline bool scan_short_interpolation(TSLexer *lexer, const bool has_conte if (lexer->lookahead == '@') { advance(lexer); } - is_short_interpolation = is_iden_char((char)lexer->lookahead) && !iswdigit(lexer->lookahead); + is_short_interpolation = is_iden_char(lexer->lookahead) && !iswdigit(lexer->lookahead); } if (is_short_interpolation) { diff --git a/test/corpus/literals.txt b/test/corpus/literals.txt index b4f59a34..a469c809 100644 --- a/test/corpus/literals.txt +++ b/test/corpus/literals.txt @@ -673,7 +673,9 @@ interpolation "#{foo}" "foo#@foo" +"foo#@言" "#@@foo" +"#@@言" "#$foo" "#$%" "#$1" @@ -694,6 +696,13 @@ interpolation (string_content) (interpolation (instance_variable))) + (string + (string_content) + (interpolation + (instance_variable))) + (string + (interpolation + (class_variable))) (string (interpolation (class_variable))) @@ -2290,6 +2299,9 @@ Cß @äö @@äö :äö +:言 +:你 +:일 äö --- @@ -2299,4 +2311,7 @@ Cß (instance_variable) (class_variable) (simple_symbol) + (simple_symbol) + (simple_symbol) + (simple_symbol) (identifier))