@@ -64,7 +64,15 @@ func (l *Lexer) NextToken() token.Token {
6464
6565 switch l .ch {
6666 case 0 :
67- return l .NewToken (token .EOF , "EOF" )
67+ level := l .whitespaceStack .Find ("" )
68+ if level > 0 {
69+ for i := 0 ; i < level ; i ++ {
70+ l .whitespaceStack .Pop ()
71+ }
72+ return l .MakeToken (token .END , fmt .Sprint (level ))
73+ } else {
74+ return l .NewToken (token .EOF , "EOF" )
75+ }
6876 case '\n' :
6977 return l .NewToken (token .NEWLINE , ";" )
7078 case '\\' :
@@ -198,7 +206,7 @@ func (l *Lexer) NextToken() token.Token {
198206 l .readChar ()
199207 return l .NewToken (tType , text )
200208 case token .EMDASH :
201- return l .MakeToken (tType , l .readSnippet ())
209+ return l .MakeToken (tType , strings . TrimSpace ( l .readSnippet () ))
202210 default :
203211 return l .NewToken (tType , lit )
204212 }
@@ -393,8 +401,10 @@ func (l *Lexer) readSnippet() string {
393401 }
394402 // There are two possibilities. Either we found a non-whitespace character, and the whole snippet is on the same line as the
395403 // `---` token, or we found a newline and the snipped is an indent on the succeeding lines. Just like with a colon.
396- if l .peekChar () == '\n' { // --- then we have to mess with whitespace.
397- l .readChar ()
404+ if l .peekChar () == '\n' || l .peekChar () == '\r' { // --- then we have to mess with whitespace.
405+ for l .peekChar () == '\n' || l .peekChar () == '\r' {
406+ l .readChar ()
407+ }
398408 langIndent := ""
399409 stackTop , ok := l .whitespaceStack .HeadValue ()
400410 if ! ok {
@@ -417,8 +427,8 @@ func (l *Lexer) readSnippet() string {
417427 return result
418428 }
419429 }
420- if strings .HasPrefix (stackTop , currentWhitespace ) || currentWhitespace == "\n " { // Then we've unindented. Dobby is free!
421- if currentWhitespace != "\n " {
430+ if strings .HasPrefix (stackTop , currentWhitespace ) || currentWhitespace == "\n " || currentWhitespace == " \r " || currentWhitespace == string ( 0 ) { // Then we've unindented. Dobby is free!
431+ if currentWhitespace != "\n " && currentWhitespace != " \r " && currentWhitespace == string ( 0 ) {
422432 l .snippetWhitespace = currentWhitespace
423433 }
424434 return result
@@ -427,18 +437,19 @@ func (l *Lexer) readSnippet() string {
427437 l .Throw ("lex/emdash/indent/c" , l .NewToken (token .ILLEGAL , "bad emdash" ))
428438 return result
429439 }
430- for l .peekChar () != '\n' && l .peekChar () != 0 {
440+ for l .peekChar () != '\n' && l .peekChar () != '\r' && l . peekChar () != 0 {
431441 l .readChar ()
432442 result = result + string (l .ch )
433443 }
434444 if l .peekChar () == 0 {
445+ l .readChar ()
435446 return result
436447 }
437448 l .readChar ()
438449 result = result + "\n "
439450 }
440451 } else {
441- for l .peekChar () != '\n' && l .peekChar () != 0 {
452+ for l .peekChar () != '\n' && l .peekChar () != '\r' && l . peekChar () != 0 {
442453 l .readChar ()
443454 result = result + string (l .ch )
444455 }
@@ -621,7 +632,7 @@ func isHexDigit(ch rune) bool {
621632
622633func isProtectedPunctuationOrWhitespace (ch rune ) bool {
623634 return ch == '(' || ch == ')' || ch == '[' || ch == ']' || ch == '{' || ch == '}' || ch == ' ' || ch == ',' ||
624- ch == ':' || ch == ';' || ch == '\t' || ch == '\n' || ch == 0
635+ ch == ':' || ch == ';' || ch == '\t' || ch == '\n' || ch == '\r' || ch == 0
625636}
626637
627638func isSymbol (ch rune ) bool {
@@ -646,7 +657,7 @@ func (l *Lexer) NewToken(tokenType token.TokenType, st string) token.Token {
646657
647658func (l * Lexer ) MakeToken (tokenType token.TokenType , st string ) token.Token {
648659 if settings .SHOW_LEXER && ! (settings .IGNORE_BOILERPLATE && settings .ThingsToIgnore .Contains (l .source )) {
649- fmt .Println (tokenType , st , l . line , l . tstart )
660+ fmt .Println (tokenType , st )
650661 }
651662 return token.Token {Type : tokenType , Literal : st , Source : l .source , Line : l .line , ChStart : l .tstart , ChEnd : l .char }
652663}
0 commit comments