From bd3694a706bbb964bc32c30d9390123c3dff476d Mon Sep 17 00:00:00 2001 From: cassowarii Date: Sat, 27 Jun 2026 14:34:09 -0700 Subject: [PATCH] more parser stuff + much more friendly syntax error reporting --- src/parse/ast.h | 1 + src/parse/filereader.c | 123 +++++++++++-- src/parse/filereader.h | 6 + src/parse/lexer.c | 36 ++-- src/parse/parser.c | 481 ++++++++++++++++++++++++++++--------------------- src/parse/parser.h | 1 + src/parse/scanner.c | 57 +++--- src/parse/token.h | 5 + 8 files changed, 443 insertions(+), 267 deletions(-) diff --git a/src/parse/ast.h b/src/parse/ast.h index 2ed4424..33238ca 100644 --- a/src/parse/ast.h +++ b/src/parse/ast.h @@ -5,6 +5,7 @@ typedef enum sbAstType { AST_NULL, + AST_ERROR, AST_VAL_NIL, AST_VAL_INT, AST_VAL_STRING, diff --git a/src/parse/filereader.c b/src/parse/filereader.c index b30f260..fce211c 100644 --- a/src/parse/filereader.c +++ b/src/parse/filereader.c @@ -1,5 +1,9 @@ #include "filereader.h" +#include "mem/mem.h" + +#define N_READBACK_LINES 3 + /* this seems pointless right now as a wrapper for FILE* but later, we can make * this work to read from stdin in interactive mode as well, which will be easier * if we just have these few functions separate from the actual lexer code. @@ -7,20 +11,34 @@ struct sbFileReader { flag valid; + i32 line; + i32 col; + i32 char_index; + sbBuffer current_line; + sbBuffer readback_lines[N_READBACK_LINES]; FILE *file; }; +static void read_in_new_line(hFileReader r); + hFileReader sbFileReader_open(const char *filename) { - hFileReader r = malloc(sizeof(struct sbFileReader)); - r->file = fopen(filename, "r"); - if (!r->file) { - perror("Error opening file for reading"); - r->valid = 0; - } else { - r->valid = 1; + hFileReader r = calloc(1, sizeof(struct sbFileReader)); + r->file = fopen(filename, "r"); + r->line = 1; + r->char_index = 0; + r->col = 1; + if (!r->file) { + perror("Error opening file for reading"); + } else { + r->valid = 1; + for (usize i = 0; i < N_READBACK_LINES; i++) { + sbBuffer_initialize(&r->readback_lines[i], 128); } + sbBuffer_initialize(&r->current_line, 128); + read_in_new_line(r); + } - return r; + return r; } flag sbFileReader_ok(hFileReader r) { @@ -28,27 +46,96 @@ flag sbFileReader_ok(hFileReader r) { } int sbFileReader_peek(hFileReader r) { - if (r->valid) { - char c = fgetc(r->file); - ungetc(c, r->file); - return c; + if (r->valid) { + int c = ((int*)r->current_line.data)[r->char_index]; + return c; + } else { + return EOF; + } +} + +int sbFileReader_next(hFileReader r) { + if (r->valid) { + r->char_index ++; + int c = sbFileReader_peek(r); + if (c == '\t') { + r->col += 4; + } else if (c == 0) { + read_in_new_line(r); + r->line ++; + r->char_index = 0; + r->col = 1; + c = sbFileReader_peek(r); } else { - return EOF; + r->col ++; } + return c; + } else { + return EOF; + } } -int sbFileReader_next(hFileReader r) { - if (r->valid) { - fgetc(r->file); - return sbFileReader_peek(r); +int sbFileReader_get_line(hFileReader r) { + return r->line; +} + +int sbFileReader_get_col(hFileReader r) { + return r->col; +} + +void sbFileReader_fprint_line_at(FILE *out, hFileReader r, int line_num) { + isize readback_amount = r->line - line_num; + sbBuffer *buffer; + if (readback_amount == 0) { + buffer = &r->current_line; + } else if (readback_amount > 0 && readback_amount <= N_READBACK_LINES) { + buffer = &r->readback_lines[N_READBACK_LINES - readback_amount]; + } else { + fprintf(out, "\n"); + return; + } + int *to_read = (int*)buffer->data; + for (int i = 0; i < buffer->size / sizeof(int); i++) { + if (to_read[i] == 0 || to_read[i] == '\n') break; + if (to_read[i] == '\t') { + fputs(" ", out); } else { - return EOF; + fputc(to_read[i], out); } + } + fputc('\n', out); } void sbFileReader_close(hFileReader r) { if (r->valid) { fclose(r->file); } + sbBuffer_deinitialize(&r->current_line); + for (usize i = 0; i < N_READBACK_LINES; i++) { + sbBuffer_deinitialize(&r->readback_lines[i]); + } free(r); } + +/* --- */ + +static void read_in_new_line(hFileReader r) { + if (!r->valid) return; + + /* recycle line buffers */ + sbBuffer oldest_line_buffer = r->readback_lines[0]; + for (int i = 0; i < N_READBACK_LINES - 1; i++) { + r->readback_lines[i] = r->readback_lines[i + 1]; + } + r->readback_lines[N_READBACK_LINES - 1] = r->current_line; + r->current_line = oldest_line_buffer; + sbBuffer_reset(&r->current_line); + + int c; + do { + c = fgetc(r->file); + sbBuffer_append(&r->current_line, &c, sizeof(int)); + } while (c != '\n' && c != EOF); + int zero = 0; + sbBuffer_append(&r->current_line, &zero, sizeof(int)); +} diff --git a/src/parse/filereader.h b/src/parse/filereader.h index 06ba22b..fa90516 100644 --- a/src/parse/filereader.h +++ b/src/parse/filereader.h @@ -10,4 +10,10 @@ int sbFileReader_next(hFileReader r); int sbFileReader_peek(hFileReader r); +int sbFileReader_get_line(hFileReader r); + +int sbFileReader_get_col(hFileReader r); + +void sbFileReader_fprint_line_at(FILE *out, hFileReader r, int line_num); + void sbFileReader_close(hFileReader r); diff --git a/src/parse/lexer.c b/src/parse/lexer.c index d6101c0..c19eaa7 100644 --- a/src/parse/lexer.c +++ b/src/parse/lexer.c @@ -75,6 +75,14 @@ static void enqueue_output_token(hLexer lx, sbLexToken token) { unstack_visible_bracket(lx, token); } + if (token.line == 0) { + /* fake tokens that are inserted get their location set as just after the + * last token seen */ + token.line = lx->last_token_seen.line; + token.start_col = lx->last_token_seen.end_col; + token.end_col = lx->last_token_seen.end_col + 1; + } + sbTokenQueue_enqueue(&lx->output_queue, token); if (is_closing_bracket(token.type) && !token.invisible) { @@ -203,7 +211,7 @@ static flag begins_brace_terminated_state(sbTokenType type) { /* things we can potentially insert a ( after because they may be a call */ static flag can_end_expression(sbTokenType type) { - return is_literal(type) + return type == T_IDENTIFIER || type == T_RPAREN || type == T_RBRACKET || type == T_RBRACE; @@ -213,6 +221,7 @@ static flag can_end_expression(sbTokenType type) { * in order to exit brace-terminated state */ static flag block_header_can_end_after(sbTokenType type) { return can_end_expression(type) + || is_literal(type) || type == T_FATARROW || type == T_SQUIGARROW || type == T_rCASE @@ -353,14 +362,7 @@ static void unstack_visible_bracket(hLexer lx, sbLexToken closing_token) { } } - if (lx->brackets_stack.size != 0) { - fprintf(stderr, "syntax error: mismatched bracket ('%c' for '%c')\n", - lx->brackets_stack.data[lx->brackets_stack.size - 1], closing_token.type); - } else { - fprintf(stderr, "syntax error: mismatched bracket\n"); - } - - enqueue_output_token(lx, (sbLexToken) { .type = T_ERROR }); + enqueue_output_token(lx, (sbLexToken) { .type = T_WRONGBRACKET }); } static void compute_next_token(hLexer lx) { @@ -397,8 +399,6 @@ static void compute_next_token(hLexer lx) { unstack_all_invisible_parentheses(lx); } - //printf("\nstack: %s\n", lx->brackets_stack.data); - /* --- HERE IS WHERE THE TOKEN ACTUALLY GETS OUTPUT TO THE STREAM --- */ /* Everything before this gets put into the stream ahead of this token. */ /* Everything after this comes after the token. */ @@ -485,6 +485,12 @@ static void compute_next_token(hLexer lx) { } } + if (token.line == 0) { + token.line = lx->last_token_seen.line; + token.start_col = lx->last_token_seen.start_col; + token.end_col = lx->last_token_seen.end_col; + } + if (token.type != T_SPACE && token.type != T_NEWLINE) { lx->last_token_seen = token; } @@ -501,7 +507,13 @@ static void compute_next_token(hLexer lx) { sbLexToken after_newline = input_peek_ahead(lx, 0); if (!cancel_semicolon_before(after_newline.type)) { - sbLexToken invisible_semicolon = { .type = T_SEMICOLON, .invisible = 1 }; + sbLexToken invisible_semicolon = { + .type = T_SEMICOLON, + .invisible = 1, + .line = lx->last_token_seen.line, + .start_col = lx->last_token_seen.start_col, + .end_col = lx->last_token_seen.end_col, + }; enqueue_output_token(lx, invisible_semicolon); lx->last_token_seen = invisible_semicolon; } diff --git a/src/parse/parser.c b/src/parse/parser.c index 83c09ee..667f8ea 100644 --- a/src/parse/parser.c +++ b/src/parse/parser.c @@ -6,6 +6,7 @@ static sbAst do_parse(hParser pr); void sbParser_initialize(hParser pr) { + *pr = (sbParser) {0}; sbArena_initialize(&pr->node_arena, 32768); sbTokenQueue_initialize(&pr->input_queue, 8); } @@ -13,6 +14,7 @@ void sbParser_initialize(hParser pr) { void sbParser_deinitialize(hParser pr) { sbTokenQueue_deinitialize(&pr->input_queue); sbArena_deinitialize(&pr->node_arena); + *pr = (sbParser) {0}; } sbAst sbParser_parse_file(hParser pr, const char *filename) { @@ -47,13 +49,129 @@ typedef struct unop { sbAstOp ast_op; } unop; -typedef struct opspelling { - sbAstOp ast_op; +typedef struct tokenspelling { + sbTokenType type; const char *name; -} opspelling; +} tokenspelling; + +static sbAstNode NONE_SENTINEL_VALUE = {0}; +static sbAst NO_NODE = &NONE_SENTINEL_VALUE; +static sbAstNode ERROR_SENTINEL_VALUE = { .type = AST_ERROR }; +static sbAst ERROR_NODE = &ERROR_SENTINEL_VALUE; + +static tokenspelling token_spellings[] = { + { T_NEWLINE, "newline" }, + { T_EOF, "end-of-file" }, + { T_IDENTIFIER, "identifier" }, + { T_SYMBOL, "symbol" }, + { T_INTEGER, "integer" }, + { T_FLOAT, "floating-point literal" }, + { T_STRING, "string literal" }, + { T_rAND, "'and'" }, + { T_rAS, "'as'" }, + { T_rCASE, "'case'" }, + { T_rDEF, "'def'" }, + { T_rDO, "'do'" }, + { T_rELSE, "'else'" }, + { T_rFALSE, "'false'" }, + { T_rIF, "'if'" }, + { T_rIN, "'in'" }, + { T_rLET, "'let'" }, + { T_rMATCH, "'match'" }, + { T_rNIL, "'nil'" }, + { T_rNOT, "'not'" }, + { T_rOR, "'or'" }, + { T_rREPEAT, "'repeat'" }, + { T_rRETURN, "'return'" }, + { T_rTRUE, "'true'" }, + { T_rUNLESS, "'unless'" }, + { T_rUNTIL, "'until'" }, + { T_rWHEN, "'when'" }, + { T_rWHILE, "'while'" }, + { T_LPAREN, "'('" }, + { T_RPAREN, "')'" }, + { T_LBRACKET, "'['" }, + { T_RBRACKET, "']'" }, + { T_LBRACE, "'{'" }, + { T_RBRACE, "'}'" }, + { T_ASTERISK, "'*'" }, + { T_SLASH, "'/'" }, + { T_PLUS, "'+'" }, + { T_MINUS, "'-'" }, + { T_PERCENT, "'%'" }, + { T_PIPE, "'|'" }, + { T_DOT, "'.'" }, + { T_COMMA, "','" }, + { T_EQUALS, "'='" }, + { T_LESS, "'<'" }, + { T_GREATER, "'>'" }, + { T_COLON, "':'" }, + { T_SEMICOLON, "';'" }, + { T_BACKSLASH, "'\\'" }, + { T_SPACE, "' '" }, + { T_ARROW, "'->'" }, + { T_FATARROW, "'=>'" }, + { T_SQUIGARROW, "'~>'" }, + { T_BACKSQUIGARROW, "'<~'" }, + { T_COLONBRACE, "':{'" }, + { T_PAAMAYIM_NEKUDOTAYIM, "'::'" }, + { T_DOUBLEEQUALS, "'=='" }, + { T_DOUBLEMINUS, "'--'" }, + { T_DOUBLEPLUS, "'++'" }, + { T_DOUBLEASTERISK, "'**'" }, + { T_DOUBLESLASH, "'//'" }, + { T_DOUBLEPERCENT, "'%%'" }, + { T_DOUBLEGREATER, "'>>'" }, + { T_DOUBLELESS, "'<<'" }, + { T_MINUSEQUALS, "'-='" }, + { T_PLUSEQUALS, "'+='" }, + { T_ASTERISKEQUALS, "'*='" }, + { T_SLASHEQUALS, "'/='" }, + { T_PERCENTEQUALS, "'%='" }, + { T_LESSEQUALS, "'<='" }, + { T_GREATEREQUALS, "'>='" }, + { T_NOTEQUALS, "'!='" }, + { T_TWODOT, "'..'" }, + { T_ELLIPSIS, "'...'" }, +}; -static sbAstNode SENTINEL_VALUE = {0}; -static sbAst NO_NODE = &SENTINEL_VALUE; +const char *const TOKEN_SPELLING_UNKNOWN = ""; + +static binop binops[] = { + { T_PIPE, 6, 7, AST_OP_PIPE }, + { T_rOR, 10, 11, AST_OP_OR }, + { T_rAND, 20, 21, AST_OP_AND }, + { T_rIN, 25, 26, AST_OP_IN }, + { T_DOUBLEEQUALS, 30, 31, AST_OP_EQ }, + { T_GREATER, 30, 31, AST_OP_GT }, + { T_LESS, 30, 31, AST_OP_LT }, + { T_GREATEREQUALS, 30, 31, AST_OP_GE }, + { T_LESSEQUALS, 30, 31, AST_OP_LE }, + { T_NOTEQUALS, 30, 31, AST_OP_NE }, + { T_DOUBLEPERCENT, 30, 31, AST_OP_DIVBY }, + { T_PLUS, 45, 46, AST_OP_ADD }, + { T_MINUS, 45, 46, AST_OP_SUB }, + { T_ASTERISK, 60, 61, AST_OP_MUL }, + { T_SLASH, 60, 61, AST_OP_DIV }, + { T_PERCENT, 60, 61, AST_OP_MOD }, + { T_DOUBLESLASH, 60, 61, AST_OP_FLDIV }, + { T_DOUBLEASTERISK, 71, 70, AST_OP_POW }, + { T_TWODOT, 80, 81, AST_OP_RANGE }, + { T_DOT, 90, 91, AST_OP_NULL }, + { T_LPAREN, 90, 91, AST_OP_NULL }, + { T_LBRACKET, 90, 91, AST_OP_INDEX }, + { T_PAAMAYIM_NEKUDOTAYIM, 90, 91, AST_OP_SCOPE }, +}; + +static unop unops[] = { + { T_rNOT, 25, AST_OP_NOT }, + { T_PLUS, 85, AST_OP_UNPLUS }, + { T_MINUS, 85, AST_OP_UNMINUS }, +}; + +const usize NUM_BINOPS = sizeof(binops) / sizeof(binops[0]); +const usize NUM_UNOPS = sizeof(unops) / sizeof(unops[0]); +const usize NUM_TOKEN_SPELLINGS = sizeof(token_spellings) / sizeof(token_spellings[0]); static sbLexToken peek_ahead(hParser pr, usize count) { while (sbTokenQueue_size(&pr->input_queue) < count + 1) { @@ -63,19 +181,103 @@ static sbLexToken peek_ahead(hParser pr, usize count) { return sbTokenQueue_at(&pr->input_queue, count); } +static void fprint_token(FILE *out, sbLexToken t); static sbLexToken next_token(hParser pr) { if (sbTokenQueue_size(&pr->input_queue) > 0) { sbTokenQueue_shift(&pr->input_queue); } - return sbTokenQueue_at(&pr->input_queue, 0); + pr->error_state = 0; + + sbLexToken t = peek_ahead(pr, 0); + + return t; +} + +flag is_literal_token(sbTokenType type) { + return type == T_IDENTIFIER + || type == T_SYMBOL + || type == T_INTEGER + || type == T_FLOAT + || type == T_STRING; +} + +const char *token_spelling(sbTokenType type) { + for (usize i = 0; i < NUM_TOKEN_SPELLINGS; i++) { + if (token_spellings[i].type == type) { + return token_spellings[i].name; + } + } + return TOKEN_SPELLING_UNKNOWN; +} + +static void fprint_token(FILE *out, sbLexToken t) { + fprintf(out, "%s", token_spelling(t.type)); + + if (t.type == T_IDENTIFIER) { + fprintf(out, " '%s'", sbSymbol_name(t.symb)); + } else if (t.type == T_SYMBOL) { + fprintf(out, " ':%s'", sbSymbol_name(t.symb)); + } else if (t.type == T_INTEGER) { + fprintf(out, " '%d'", t.i); + } else if (t.type == T_FLOAT) { + fprintf(out, " '%g'", t.fl); + } +} + +static void fprint_repeat_char(FILE *out, int ch, usize length) { + for (int i = 0; i < length; i++) { + fputc(ch, out); + } } -static flag expect(hParser pr, sbTokenType type, sbLexToken *tok_out) { +static void fprint_left_error_margin(FILE *out, usize line_num) { + /* indentation */ + if (line_num) { + fprintf(out, "%6zu | ", line_num); + } else { + fprintf(out, " | "); + } +} + +static sbAst syntax_error(hParser pr) { + if (pr->error_state) return ERROR_NODE; + + sbLexToken unexpected_token = peek_ahead(pr, 0); + if (unexpected_token.type == T_ERROR) { + fprintf(stderr, "syntax error: invalid character"); + } else if (unexpected_token.type == T_WRONGBRACKET) { + fprintf(stderr, "syntax error: mismatched bracket"); + } else if (unexpected_token.type == T_BADNUMBER) { + fprintf(stderr, "syntax error: invalid number"); + } else if (unexpected_token.type == T_SEMICOLON && unexpected_token.invisible) { + fprintf(stderr, "syntax error: unexpected end-of-line"); + } else if (unexpected_token.invisible) { + fprintf(stderr, "syntax error"); + } else { + fprintf(stderr, "syntax error: unexpected "); + fprint_token(stderr, unexpected_token); + } + fprintf(stderr, " at line %d\n", unexpected_token.line); + fprint_left_error_margin(stderr, 0); + fprintf(stderr, "\n"); + fprint_left_error_margin(stderr, unexpected_token.line); + sbFileReader_fprint_line_at(stderr, pr->lexer.scanner.file_reader, unexpected_token.line); + fprint_left_error_margin(stderr, 0); + fprint_repeat_char(stderr, ' ', unexpected_token.start_col - 1); + fprint_repeat_char(stderr, '^', unexpected_token.end_col - unexpected_token.start_col + 1); + fprintf(stderr, "\n\n"); + + /* set error flag until we move to the next token so that syntax error + * isn't printed multiple times */ + pr->error_state = TRUE; + return ERROR_NODE; +} + +static flag expect(hParser pr, sbTokenType type) { sbLexToken up_next = peek_ahead(pr, 0); if (up_next.type == type) { next_token(pr); - if (tok_out) *tok_out = up_next; return TRUE; } else { return FALSE; @@ -143,7 +345,7 @@ static sbAst atomic_node(hParser pr, sbAstType type) { static sbAst name_node(hParser pr, sbLexToken token) { if (token.type != T_IDENTIFIER) { - PANIC("can't create name node with token of type %d\n", token.type); + PANIC("can't create name node with token of type %d", token.type); } sbAstNode n = (sbAstNode) { @@ -153,68 +355,6 @@ static sbAst name_node(hParser pr, sbLexToken token) { return new_node(pr, &n); } -static binop binops[] = { - { T_PIPE, 6, 7, AST_OP_PIPE }, - { T_rOR, 10, 11, AST_OP_OR }, - { T_rAND, 20, 21, AST_OP_AND }, - { T_rIN, 25, 26, AST_OP_IN }, - { T_DOUBLEEQUALS, 30, 31, AST_OP_EQ }, - { T_GREATER, 30, 31, AST_OP_GT }, - { T_LESS, 30, 31, AST_OP_LT }, - { T_GREATEREQUALS, 30, 31, AST_OP_GE }, - { T_LESSEQUALS, 30, 31, AST_OP_LE }, - { T_NOTEQUALS, 30, 31, AST_OP_NE }, - { T_DOUBLEPERCENT, 30, 31, AST_OP_DIVBY }, - { T_PLUS, 45, 46, AST_OP_ADD }, - { T_MINUS, 45, 46, AST_OP_SUB }, - { T_ASTERISK, 60, 61, AST_OP_MUL }, - { T_SLASH, 60, 61, AST_OP_DIV }, - { T_PERCENT, 60, 61, AST_OP_MOD }, - { T_DOUBLESLASH, 60, 61, AST_OP_FLDIV }, - { T_DOUBLEASTERISK, 71, 70, AST_OP_POW }, - { T_TWODOT, 80, 81, AST_OP_RANGE }, - { T_DOT, 90, 91, AST_OP_NULL }, - { T_LPAREN, 90, 91, AST_OP_NULL }, - { T_LBRACKET, 90, 91, AST_OP_INDEX }, - { T_PAAMAYIM_NEKUDOTAYIM, 100, 101, AST_OP_SCOPE }, -}; - -static unop unops[] = { - { T_rNOT, 25, AST_OP_NOT }, - { T_PLUS, 85, AST_OP_UNPLUS }, - { T_MINUS, 85, AST_OP_UNMINUS }, -}; - -static opspelling op_spellings[] = { - { AST_OP_OR, "or" }, - { AST_OP_AND, "and" }, - { AST_OP_NOT, "not" }, - { AST_OP_IN, "in" }, - { AST_OP_EQ, "==" }, - { AST_OP_NE, "!=" }, - { AST_OP_GT, ">" }, - { AST_OP_LT, "<" }, - { AST_OP_GE, ">=" }, - { AST_OP_LE, "<=" }, - { AST_OP_DIVBY, "%%" }, - { AST_OP_ADD, "+" }, - { AST_OP_UNPLUS, "+@" }, - { AST_OP_SUB, "-" }, - { AST_OP_UNMINUS, "-@" }, - { AST_OP_MUL, "*" }, - { AST_OP_DIV, "/" }, - { AST_OP_MOD, "%" }, - { AST_OP_FLDIV, "//" }, - { AST_OP_POW, "**" }, - { AST_OP_RANGE, ".." }, - { AST_OP_INDEX, "[]" }, - { AST_OP_SCOPE, "::" }, -}; - -const int NUM_BINOPS = sizeof(binops) / sizeof(binops[0]); -const int NUM_UNOPS = sizeof(unops) / sizeof(unops[0]); -const int NUM_SPELLINGS = sizeof(op_spellings) / sizeof(op_spellings[0]); - static sbAst parse_expr(hParser pr, u8 min_precedence); static sbAst parse_comma_exprs(hParser pr, sbAst after) { sbAst result = NO_NODE; @@ -227,7 +367,7 @@ static sbAst parse_comma_exprs(hParser pr, sbAst after) { } do { - if (expect(pr, T_ELLIPSIS, NULL)) { + if (expect(pr, T_ELLIPSIS)) { sbAst splatted_expr = parse_expr(pr, 0); if (splatted_expr != NO_NODE) { /* "...something" */ @@ -245,7 +385,7 @@ static sbAst parse_comma_exprs(hParser pr, sbAst after) { *put_here = seq_node(pr, AST_NODE_MULTIVAL, expr, NO_NODE); put_here = &(*put_here)->seq.right; - } while (expect(pr, T_COMMA, NULL)); + } while (expect(pr, T_COMMA)); return result; } @@ -253,7 +393,8 @@ static sbAst parse_comma_exprs(hParser pr, sbAst after) { static sbAst parse_name(hParser pr) { sbLexToken t = peek_ahead(pr, 0); if (t.type == T_IDENTIFIER) { - return name_node(pr, next_token(pr)); + next_token(pr); + return name_node(pr, t); } return NO_NODE; } @@ -265,62 +406,53 @@ static sbAst parse_expr(hParser pr, u8 min_precedence) { sbLexToken t = peek_ahead(pr, 0); sbAst lhs = NO_NODE; if (t.type == T_rNIL) { + next_token(pr); sbAstNode n = { .type = AST_VAL_NIL }; lhs = new_node(pr, &n); } else if (t.type == T_rTRUE || t.type == T_rFALSE) { + next_token(pr); sbAstNode n = { .type = AST_VAL_BOOLEAN, .i = (t.type == T_rTRUE) }; lhs = new_node(pr, &n); } else if (t.type == T_INTEGER) { + next_token(pr); sbAstNode n = { .type = AST_VAL_INT, .i = t.i }; lhs = new_node(pr, &n); } else if (t.type == T_FLOAT) { + next_token(pr); sbAstNode n = { .type = AST_VAL_FLOAT, .fl = t.fl }; lhs = new_node(pr, &n); } else if (t.type == T_SYMBOL) { + next_token(pr); sbAstNode n = { .type = AST_VAL_SYMBOL, .symb = t.symb }; lhs = new_node(pr, &n); } else if (t.type == T_STRING) { + next_token(pr); sbAstNode n = { .type = AST_VAL_STRING, .str = t.hstr }; lhs = new_node(pr, &n); } else if (t.type == T_IDENTIFIER) { lhs = parse_name(pr); } else if (t.type == T_FATARROW) { next_token(pr); - if (!expect(pr, T_LPAREN, NULL)) { - fprintf(stderr, "expected '(' after '=>' token\n"); - return NO_NODE; - } + if (!expect(pr, T_LPAREN)) return syntax_error(pr); sbAst params = parse_comma_exprs(pr, NULL); - if (!expect(pr, T_RPAREN, NULL)) { - fprintf(stderr, "expected ')' after function parameters\n"); - return NO_NODE; - } + if (!expect(pr, T_RPAREN)) return syntax_error(pr); sbAst body = parse_block(pr); return seq_node(pr, AST_VAL_FUNC, params, body); } else if (t.type == T_SQUIGARROW) { next_token(pr); - if (!expect(pr, T_LPAREN, NULL)) { - fprintf(stderr, "expected '(' after '~>' token\n"); - return NO_NODE; - } + if (!expect(pr, T_LPAREN)) return syntax_error(pr); sbAst params = parse_comma_exprs(pr, NULL); - if (!expect(pr, T_RPAREN, NULL)) { - fprintf(stderr, "expected ')' after function parameters\n"); - return NO_NODE; - } + if (!expect(pr, T_RPAREN)) return syntax_error(pr); sbAst body = parse_block(pr); return seq_node(pr, AST_VAL_OBJ, params, body); } else if (t.type == T_LPAREN) { next_token(pr); lhs = parse_expr(pr, 0); - if (!expect(pr, T_RPAREN, NULL)) { - fprintf(stderr, "expected closing parenthesis!\n"); - return NO_NODE; - } + if (!expect(pr, T_RPAREN)) return syntax_error(pr); } else { sbLexToken op = peek_ahead(pr, 0); unop *prefix = NULL; - for (int i = 0; i < NUM_UNOPS; i++) { + for (usize i = 0; i < NUM_UNOPS; i++) { if (unops[i].type == op.type) { prefix = &unops[i]; break; @@ -343,7 +475,7 @@ static sbAst parse_expr(hParser pr, u8 min_precedence) { sbLexToken op = peek_ahead(pr, 0); sbAstType ast_type = AST_NODE_OP; binop *infix = NULL; - for (int i = 0; i < NUM_BINOPS; i++) { + for (usize i = 0; i < NUM_BINOPS; i++) { if (binops[i].type == op.type) { infix = &binops[i]; break; @@ -367,35 +499,18 @@ static sbAst parse_expr(hParser pr, u8 min_precedence) { /* function call */ rhs = parse_comma_exprs(pr, NULL); ast_type = AST_NODE_FUNCCALL; - if (!expect(pr, T_RPAREN, NULL)) { - fprintf(stderr, "expected ')' after function parameter list\n"); - return NO_NODE; - } + if (!expect(pr, T_RPAREN)) return syntax_error(pr); } else if (op.type == T_LBRACKET) { /* indexing */ rhs = parse_expr(pr, 0); - if (!expect(pr, T_RBRACKET, NULL)) { - fprintf(stderr, "expected ']'\n"); - return NO_NODE; - } + if (!expect(pr, T_RBRACKET)) return syntax_error(pr); } else if (op.type == T_DOT) { - sbLexToken method_name = {0}; - if (!expect(pr, T_IDENTIFIER, &method_name)) { - fprintf(stderr, "expected identifier after '.' token\n"); - return NO_NODE; - } - sbAst name = name_node(pr, method_name); - if (!expect(pr, T_LPAREN, NULL)) { - fprintf(stderr, "expected opening-parenthesis after '.%s'\n", sbSymbol_name(method_name.symb)); - return NO_NODE; - } + sbAst method_name = parse_name(pr); + if (!expect(pr, T_LPAREN)) return syntax_error(pr); sbAst params = parse_comma_exprs(pr, NULL); - if (!expect(pr, T_RPAREN, NULL)) { - fprintf(stderr, "expected closing-parenthesis after '.%s(...'\n", sbSymbol_name(method_name.symb)); - return NO_NODE; - } + if (!expect(pr, T_RPAREN)) return syntax_error(pr); ast_type = AST_NODE_METHODCALL; - rhs = seq_node(pr, AST_NODE_NEXT, name, params); + rhs = seq_node(pr, AST_NODE_NEXT, method_name, params); } else { rhs = parse_expr(pr, infix->right_precedence); } @@ -414,17 +529,11 @@ static sbAst parse_stmt(hParser pr); static sbAst parse_stmtseq(hParser pr); static sbAst parse_block(hParser pr) { - if (!expect(pr, '{', NULL)) { - fprintf(stderr, "syntax error! expecting '{'\n"); - return NO_NODE; - } + if (!expect(pr, '{')) return syntax_error(pr); sbAst result = parse_stmtseq(pr); - if (!expect(pr, '}', NULL)) { - fprintf(stderr, "syntax error! expecting '}'\n"); - return NO_NODE; - } + if (!expect(pr, '}')) return syntax_error(pr); return result; } @@ -438,12 +547,12 @@ static sbAst parse_stmtseq(hParser pr) { if (stmt == NO_NODE) { /* if failed to parse a statement, try eating an * additional semicolon first */ - if (expect(pr, ';', NULL)) continue; + if (expect(pr, ';')) continue; break; } *put_here = seq_node(pr, AST_NODE_SEQ, stmt, NO_NODE); put_here = &(*put_here)->seq.right; - } while (expect(pr, ';', NULL)); + } while (expect(pr, ';')); return result; } @@ -451,10 +560,10 @@ static sbAst parse_stmtseq(hParser pr) { static sbAst with_trailing_conditional(hParser pr, sbAst stmt) { sbAst result = stmt; - if (expect(pr, T_rIF, NULL)) { + if (expect(pr, T_rIF)) { sbAst condition = parse_expr(pr, 0); result = tri_node(pr, AST_NODE_IF, condition, stmt, NO_NODE); - } else if (expect(pr, T_rUNLESS, NULL)) { + } else if (expect(pr, T_rUNLESS)) { sbAst condition = parse_expr(pr, 0); result = tri_node(pr, AST_NODE_IF, unop_node(pr, AST_OP_NOT, condition), stmt, NO_NODE); } @@ -465,11 +574,12 @@ static sbAst with_trailing_conditional(hParser pr, sbAst stmt) { static sbAst parse_stmt(hParser pr) { sbLexToken t = peek_ahead(pr, 0); if (t.type == T_rIF) { + /* if condition1 { ... } else if condition2 { ... } else { ... } */ next_token(pr); sbAst condition = parse_expr(pr, 0); sbAst then_body = parse_block(pr); sbAst else_body = NO_NODE; - if (expect(pr, T_rELSE, NULL)) { + if (expect(pr, T_rELSE)) { sbLexToken t = peek_ahead(pr, 0); if (t.type == T_rIF) { /* else if ...as above... */ @@ -480,17 +590,20 @@ static sbAst parse_stmt(hParser pr) { } return tri_node(pr, AST_NODE_IF, condition, then_body, else_body); } else if (t.type == T_rUNLESS) { + /* unless condition { ... } */ next_token(pr); sbAst condition = parse_expr(pr, 0); /* won't permit unless..else. it is too confusing. unless can only have 1 block */ sbAst unless_body = parse_block(pr); return tri_node(pr, AST_NODE_IF, unop_node(pr, AST_OP_NOT, condition), unless_body, NO_NODE); } else if (t.type == T_rWHILE) { + /* while condition { ... } */ next_token(pr); sbAst condition = parse_expr(pr, 0); sbAst body = parse_block(pr); return seq_node(pr, AST_NODE_WHILE, condition, body); } else if (t.type == T_rUNTIL) { + /* until condition { ... } */ next_token(pr); sbAst condition = parse_expr(pr, 0); sbAst body = parse_block(pr); @@ -498,31 +611,32 @@ static sbAst parse_stmt(hParser pr) { } else if (t.type == T_rREPEAT) { next_token(pr); sbAst body = parse_block(pr); - if (expect(pr, T_rWHILE, NULL)) { - /* repeat..while */ + if (expect(pr, T_rWHILE)) { + /* repeat { ... } while */ sbAst condition = parse_expr(pr, 0); return seq_node(pr, AST_NODE_REPEAT, body, condition); - } else if (expect(pr, T_rUNTIL, NULL)) { - /* repeat..until */ + } else if (expect(pr, T_rUNTIL)) { + /* repeat { ... } until */ sbAst condition = parse_expr(pr, 0); return seq_node(pr, AST_NODE_REPEAT, body, condition); } else { - fprintf(stderr, "syntax error! 'while' or 'until' required after 'repeat { ... }'\n"); - return NO_NODE; + return syntax_error(pr); } } else if (t.type == T_rCASE) { + /* case x, y { ... } */ next_token(pr); sbAst values = parse_comma_exprs(pr, NULL); sbAst body = parse_block(pr); return seq_node(pr, AST_NODE_CASE, values, body); } else if (t.type == T_rMATCH) { + /* match :x if j { ... } */ next_token(pr); sbAst pattern = parse_comma_exprs(pr, NULL); sbAst guard_clause = NO_NODE; - if (expect(pr, T_rIF, NULL)) { + if (expect(pr, T_rIF)) { next_token(pr); guard_clause = parse_expr(pr, 0); - } else if (expect(pr, T_rUNLESS, NULL)) { + } else if (expect(pr, T_rUNLESS)) { next_token(pr); guard_clause = parse_expr(pr, 0); guard_clause = unop_node(pr, AST_OP_NOT, guard_clause); @@ -530,111 +644,58 @@ static sbAst parse_stmt(hParser pr) { sbAst body = parse_block(pr); return tri_node(pr, AST_NODE_MATCH, pattern, guard_clause, body); } else if (t.type == T_rLET) { + /* let a, b = ... */ next_token(pr); sbAst bindings = parse_comma_exprs(pr, NULL); - if (!expect(pr, T_EQUALS, NULL)) { - fprintf(stderr, "expected '=' after let ...\n"); - return NO_NODE; - } + if (!expect(pr, T_EQUALS)) return syntax_error(pr); sbAst values = parse_comma_exprs(pr, NULL); return seq_node(pr, AST_NODE_LET, bindings, values); } else if (t.type == T_rDEF) { + /* def a b, c { ... } */ next_token(pr); sbAst name = parse_name(pr); - if (!expect(pr, T_LPAREN, NULL)) { - fprintf(stderr, "expected '(' after 'def' and name\n"); - return NO_NODE; - } + if (!expect(pr, T_LPAREN)) return syntax_error(pr); sbAst params = parse_comma_exprs(pr, NULL); - if (!expect(pr, T_RPAREN, NULL)) { - fprintf(stderr, "expected ')' after function parameters\n"); - return NO_NODE; - } + if (!expect(pr, T_RPAREN)) return syntax_error(pr); sbAst body = parse_block(pr); sbAst func_node = seq_node(pr, AST_VAL_FUNC, params, body); return seq_node(pr, AST_NODE_DEF, name, func_node); } else if (t.type == T_rRETURN) { + /* return ... */ next_token(pr); sbAst returned_val = parse_comma_exprs(pr, 0); sbAst return_node = wrap_node(pr, AST_NODE_RETURN, returned_val); + /* return ... unless condition */ return_node = with_trailing_conditional(pr, return_node); return return_node; } else { + /* (statement that is just an expression) */ sbAst expr = parse_expr(pr, 0); + if (expr == NO_NODE) return NO_NODE; + if (peek_ahead(pr, 0).type == ',') { + /* (implicit return) a, b, c */ next_token(pr); expr = parse_comma_exprs(pr, expr); } + if (peek_ahead(pr, 0).type == '=') { + /* a, b, c = 1, 2, 3 */ next_token(pr); sbAst assigned_values = parse_comma_exprs(pr, 0); expr = seq_node(pr, AST_NODE_ASSIGN, expr, assigned_values); } + if (expr != NO_NODE) { + /* a, b, c = 1, 2, 3 (or other expr) if condition */ expr = with_trailing_conditional(pr, expr); - return expr; - } - } - - return NO_NODE; -} - -static void print_ast_node(sbAst n, int indent) { - if (n == NULL) return; - - printf("\n"); - for (int i = 0; i < indent; i++) { - printf(" "); - } - - if (n->type == AST_NODE_NAME) { - printf("%s", sbSymbol_name(n->symb)); - } else if (n->type == AST_NODE_MULTIVAL) { - print_ast_node(n->seq.left, indent + 1); - printf(","); - print_ast_node(n->seq.right, indent + 1); - } else { - printf("("); - if (n->type == AST_NODE_OP) { - flag found = 0; - for (int i = 0; i < NUM_SPELLINGS; i++) { - if (op_spellings[i].ast_op == n->op.type) { - printf("%s", op_spellings[i].name); - found = 1; - break; - } - } - if (!found) printf("??"); - print_ast_node(n->op.left, indent + 1); - if (n->op.right) { - print_ast_node(n->op.right, indent + 1); - } - } else if (n->type == AST_NODE_FUNCCALL) { - print_ast_node(n->seq.left, indent + 1); - printf("("); - print_ast_node(n->seq.right, indent + 1); - printf(")"); - } else if (n->type == AST_NODE_METHODCALL) { - print_ast_node(n->seq.left, indent + 1); - printf("."); - print_ast_node(n->seq.right->seq.left, indent + 1); - printf("("); - print_ast_node(n->seq.right->seq.right, indent + 1); - printf(")"); - } else { - printf("?<%d>", n->type); } - printf("\n"); - for (int i = 0; i < indent; i++) { - printf(" "); - } - printf(")"); + return expr; } } static sbAst do_parse(hParser pr) { - sbAst program = parse_expr(pr, 0); - print_ast_node(program, 0); + sbAst program = parse_stmtseq(pr); return program; } diff --git a/src/parse/parser.h b/src/parse/parser.h index 3847ef9..cbfe3d2 100644 --- a/src/parse/parser.h +++ b/src/parse/parser.h @@ -7,6 +7,7 @@ typedef struct sbParser { sbLexer lexer; sbTokenQueue input_queue; sbArena node_arena; + flag error_state; } sbParser; typedef sbParser *hParser; diff --git a/src/parse/scanner.c b/src/parse/scanner.c index f444e10..fa45aab 100644 --- a/src/parse/scanner.c +++ b/src/parse/scanner.c @@ -204,6 +204,8 @@ static usize read_symbol(hScanner sc) { static sbLexToken compute_next_token(hScanner sc) { int ch_int = PEEK; unsigned char ch = (unsigned char)ch_int; + u32 token_line = sbFileReader_get_line(sc->file_reader); + u32 token_start_col = sbFileReader_get_col(sc->file_reader); sbBuffer_reset(&sc->dynamic_buffer); @@ -245,8 +247,7 @@ static sbLexToken compute_next_token(hScanner sc) { new_token.type = T_NOTEQUALS; NEXT; } else { - /* ! on its own is not an operator i guess. not sure how to handle this elegantly */ - fprintf(stderr, "don't know how to handle character: '!'\n"); + /* ! on its own is not an operator */ new_token.type = T_ERROR; } } else if (ch == '~') { @@ -256,7 +257,6 @@ static sbLexToken compute_next_token(hScanner sc) { NEXT; } else { /* ~ on its own is not an operator */ - fprintf(stderr, "don't know how to handle character: '~'\n"); new_token.type = T_ERROR; } } else if (ch == '>') { @@ -379,7 +379,6 @@ static sbLexToken compute_next_token(hScanner sc) { token_size = read_symbol(sc); new_token.symb = sbSymbol_from_bytes(sc->dynamic_buffer.data, sc->dynamic_buffer.size - 1); - printf("%s -> %p\n", (char*)new_token.symb, (void*)new_token.symb); new_token.size = token_size; } else { /* : */ @@ -448,13 +447,13 @@ static sbLexToken compute_next_token(hScanner sc) { token_size = read_identifier(sc); new_token.symb = sbSymbol_from_bytes(sc->dynamic_buffer.data, sc->dynamic_buffer.size - 1); - printf("%s -> %p\n", (char*)new_token.symb, (void*)new_token.symb); new_token.size = token_size; } else if (is_digit(ch)) { new_token.type = T_INTEGER; i64 intval = 0; int base = 10; + int num_done = FALSE; if (ch == '0') { /* skip a leading zero, and see if it has some base indicator */ @@ -468,37 +467,36 @@ static sbLexToken compute_next_token(hScanner sc) { } else if (ch == 'x') { ch = NEXT; base = 16; + } else if (is_space(ch) || ch == '\n') { + /* it was literally just the number 0 (probably) */ + num_done = TRUE; + } else { + new_token.type = T_BADNUMBER; + num_done = TRUE; } } - do { - /* TODO: promote to bigint, don't allow overflow */ - intval *= base; - intval += base_digit_value(ch); + if (!num_done) { do { - /* skip over underscores in numeric literals */ - ch = NEXT; - } while (ch == '_'); - } while (is_base_digit(ch, base)); - - /* if we are now looking at still a character that's a letter or digit, throw error */ - if (is_digit(ch) || is_alpha(ch)) { - if (base == 10) { - fprintf(stderr, "unexpected character in numeric literal: '%c'\n", ch); + /* TODO: promote to bigint, don't allow overflow */ + intval *= base; + intval += base_digit_value(ch); + do { + /* skip over underscores in numeric literals */ + ch = NEXT; + } while (ch == '_'); + } while (is_base_digit(ch, base)); + + /* if we are now looking at still a character that's a letter or digit, throw error */ + if (is_digit(ch) || is_alpha(ch)) { + new_token.type = T_BADNUMBER; + NEXT; } else { - fprintf(stderr, "unexpected character in base %d numeric literal: '%c'\n", base, ch); + new_token.i = intval; } - - new_token.type = T_ERROR; - NEXT; - } else { - new_token.i = intval; } } else { new_token.type = T_ERROR; - - fprintf(stderr, "don't know how to process character: '%c'\n", ch); - NEXT; } @@ -512,6 +510,11 @@ static sbLexToken compute_next_token(hScanner sc) { } } + u32 token_end_col = sbFileReader_get_col(sc->file_reader) - 1; + + new_token.line = token_line; + new_token.start_col = token_start_col; + new_token.end_col = token_end_col; return new_token; } diff --git a/src/parse/token.h b/src/parse/token.h index 7fa61ed..e001be4 100644 --- a/src/parse/token.h +++ b/src/parse/token.h @@ -8,6 +8,8 @@ typedef enum sbTokenType { T_NULL = 0, T_ERROR = 1, + T_WRONGBRACKET = 2, + T_BADNUMBER = 3, T_LPAREN = '(', // op::call T_RPAREN = ')', T_LBRACKET = '[', // op::index @@ -90,6 +92,9 @@ typedef struct sbLexToken { sbTokenType type; usize size; flag invisible; + i32 line; + i32 start_col; + i32 end_col; union { char *cstr; hString hstr; -- 1.8.3.1