ref: 6655e04ef01bd78acd01f5d9efca4d50340b0a0f
parent: 40c6b840f8078f6a4ac494d30c638696a115f3e2
author: Rangi <[email protected]>
date: Tue Mar 2 14:35:24 EST 2021
Restore the "EOF-newline" lexer hack This was removed in b3c0db218d564bbc9a801d8635591fd969c36e82 (along with two unrelated changes). Removing this hack introduced issue #742, whereby INCLUDing a file without a trailing newline can cause a syntax error. A more proper fix would involve Bison's tracking locations, but for now the EOF-newline hack fixes the issue while only affecting some reported errors (expecting "newline" instead of "end of file"). Fixes #742
--- a/src/asm/lexer.c
+++ b/src/asm/lexer.c
@@ -349,6 +349,7 @@
bool atLineStart;
uint32_t lineNo;
uint32_t colNo;
+ int lastToken;
struct IfStack *ifStack;
@@ -372,6 +373,7 @@
{
state->mode = LEXER_NORMAL;
state->atLineStart = true; /* yylex() will init colNo due to this */
+ state->lastToken = T_EOF;
state->ifStack = NULL;
@@ -2235,17 +2237,15 @@
return T_STRING;
lexer_SetMode(LEXER_NORMAL);
- // If a macro is invoked on the last line of a file, with no blank
- // line afterwards, returning EOF afterwards will cause Bison to
- // stop parsing, despite the lexer being ready to output more.
- // To avoid this, return T_NEWLINE for EOF as well.
if (c == '\r' || c == '\n') {
shiftChars(1);
/* Handle CRLF */
if (c == '\r' && peek(0) == '\n')
shiftChars(1);
+ return T_NEWLINE;
}
- return T_NEWLINE;
+
+ return T_EOF;
}
#undef append_yylval_tzString
@@ -2456,16 +2456,22 @@
int token = lexerModeFuncs[lexerState->mode]();
if (token == T_EOF) {
- /* Try to switch to new buffer; if it succeeds, scan again */
- dbgPrint("Reached EOB!\n");
- /* Captures end at their buffer's boundary no matter what */
- if (!lexerState->capturing) {
- if (!yywrap())
- goto restart;
- dbgPrint("Reached end of input.\n");
- return T_EOF;
+ if (lexerState->lastToken != T_NEWLINE) {
+ dbgPrint("Forcing EOL at EOF\n");
+ token = T_NEWLINE;
+ } else {
+ /* Try to switch to new buffer; if it succeeds, scan again */
+ dbgPrint("Reached EOF!\n");
+ /* Captures end at their buffer's boundary no matter what */
+ if (!lexerState->capturing) {
+ if (!yywrap())
+ goto restart;
+ dbgPrint("Reached end of input.\n");
+ return T_EOF;
+ }
}
}
+ lexerState->lastToken = token;
lexerState->atLineStart = token == T_NEWLINE;
@@ -2524,6 +2530,7 @@
* We know we have read exactly "ENDR", not e.g. an EQUS
*/
lexerState->captureSize -= strlen("ENDR");
+ lexerState->lastToken = T_POP_ENDR; // Force EOL at EOF
goto finish;
}
level--;
@@ -2586,6 +2593,7 @@
* We know we have read exactly "ENDM", not e.g. an EQUS
*/
lexerState->captureSize -= strlen("ENDM");
+ lexerState->lastToken = T_POP_ENDM; // Force EOL at EOF
goto finish;
}
}
--- a/src/asm/parser.y
+++ b/src/asm/parser.y
@@ -603,20 +603,19 @@
%%
-asmfile : lines last_line
+asmfile : lines
;
+/* Note: The lexer adds T_NEWLINE at the end of the input */
lines : %empty
| lines line
;
-last_line : label
- | label cpu_command
- | label macro
- | label directive
- | assignment_directive
-;
-line : last_line T_NEWLINE
+line : label T_NEWLINE
+ | label cpu_command T_NEWLINE
+ | label macro T_NEWLINE
+ | label directive T_NEWLINE
+ | assignment_directive T_NEWLINE
| line_directive /* Directives that manage newlines themselves */
| error T_NEWLINE { /* Continue parsing the next line on a syntax error */
fstk_StopRept();
--- a/test/asm/block-comment-termination-error.err
+++ b/test/asm/block-comment-termination-error.err
@@ -1,5 +1,5 @@
ERROR: block-comment-termination-error.asm(1):
Unterminated block comment
ERROR: block-comment-termination-error.asm(1):
- syntax error, unexpected end of file
+ syntax error, unexpected newline
error: Assembly aborted (2 errors)!
--- a/test/asm/code-after-endm-endr-endc.err
+++ b/test/asm/code-after-endm-endr-endc.err
@@ -7,9 +7,9 @@
ERROR: code-after-endm-endr-endc.asm(17):
syntax error, unexpected PRINTLN, expecting newline
ERROR: code-after-endm-endr-endc.asm(19):
- syntax error, unexpected PRINTLN, expecting end of file or newline
+ syntax error, unexpected PRINTLN, expecting newline
ERROR: code-after-endm-endr-endc.asm(23):
syntax error, unexpected PRINTLN, expecting newline
ERROR: code-after-endm-endr-endc.asm(25):
- syntax error, unexpected PRINTLN, expecting end of file or newline
+ syntax error, unexpected PRINTLN, expecting newline
error: Assembly aborted (7 errors)!