From: cassowarii <2374677+cassowarii@users.noreply.github.com> Date: Sun, 21 Jun 2026 04:38:21 +0000 (-0700) Subject: dynamically sized token queue X-Git-Url: https://www.git.cassowary.me/gitweb.cgi?a=commitdiff_plain;h=c13937c181f2e0045e63df8d765fe4b707f6d046;p=sarabande.git dynamically sized token queue --- diff --git a/src/mem/buffer.c b/src/mem/buffer.c index f3580a4..c474aea 100644 --- a/src/mem/buffer.c +++ b/src/mem/buffer.c @@ -48,6 +48,22 @@ void *sbBuffer_expand(hBuffer buf, usize expand_size) { return result; } +void *sbBuffer_shrink(hBuffer buf, usize shrink_size) { + if (shrink_size > buf->size) { + fprintf(stderr, "internal warning: can't shrink buffer past zero"); + shrink_size = buf->size; + } + + buf->size -= shrink_size; + + /* ok to return this even though it's "outside" the buffer, + * because we never shrink the actual allocation of the buffer. + * however, bear in mind not to keep this pointer around very + * long, because if we append to the buffer it could change + * again. */ + return &buf->data[buf->size]; +} + void sbBuffer_append(hBuffer buf, const char *data, usize data_length) { if (data_length == 0) return; diff --git a/src/mem/buffer.h b/src/mem/buffer.h index 3767a75..f04347c 100644 --- a/src/mem/buffer.h +++ b/src/mem/buffer.h @@ -19,6 +19,8 @@ void sbBuffer_initialize(hBuffer buf, usize initial_size); void *sbBuffer_expand(hBuffer buf, usize expand_size); +void *sbBuffer_shrink(hBuffer buf, usize shrink_size); + void sbBuffer_append(hBuffer buf, const char *data, usize data_length); void sbBuffer_reset(hBuffer buf); diff --git a/src/parse/lexer.c b/src/parse/lexer.c index af6eb2b..5e2cb9f 100644 --- a/src/parse/lexer.c +++ b/src/parse/lexer.c @@ -34,25 +34,21 @@ static sbLexToken advance_output_queue(hLexer lx); void sbLexer_initialize(hLexer lx, hFileReader fr) { sbBuffer_initialize(&lx->brackets_stack, 64); sbScanner_initialize(&lx->scanner, fr); - memset(lx->input_queue, 0, LEXER_QUEUE_LENGTH * sizeof(sbLexToken)); - memset(lx->output_queue, 0, LEXER_QUEUE_LENGTH * sizeof(sbLexToken)); - lx->input_queue_length = 0; - lx->output_queue_length = 0; + sbTokenQueue_initialize(&lx->input_queue, 8); + sbTokenQueue_initialize(&lx->output_queue, 8); lx->last_token_seen = (sbLexToken) {0}; - lx->n_brackets = 0; - lx->brace_terminated_state = 0; } sbLexToken sbLexer_peek(hLexer lx) { - if (lx->output_queue_length == 0) { + if (sbTokenQueue_size(&lx->output_queue) == 0) { compute_next_token(lx); } - return lx->output_queue[0]; + return sbTokenQueue_at(&lx->output_queue, 0); } sbLexToken sbLexer_next(hLexer lx) { - if (lx->output_queue_length == 0) { + if (sbTokenQueue_size(&lx->output_queue) == 0) { compute_next_token(lx); } @@ -62,8 +58,8 @@ sbLexToken sbLexer_next(hLexer lx) { void sbLexer_deinitialize(hLexer lx) { sbScanner_deinitialize(&lx->scanner); sbBuffer_deinitialize(&lx->brackets_stack); - lx->input_queue_length = 0; - lx->output_queue_length = 0; + sbTokenQueue_deinitialize(&lx->input_queue); + sbTokenQueue_deinitialize(&lx->output_queue); } struct ReservedWord { @@ -78,13 +74,16 @@ static struct ReservedWord reserved_words[] = { { "def", T_rDEF }, { "do", T_rDO }, { "else", T_rELSE }, + { "false", T_rFALSE }, { "if", T_rIF }, { "let", T_rLET }, { "in", T_rIN }, + { "match", T_rMATCH }, { "not", T_rNOT }, { "or", T_rOR }, { "repeat", T_rREPEAT }, { "return", T_rRETURN }, + { "true", T_rTRUE}, { "unless", T_rUNLESS }, { "until", T_rUNTIL }, { "when", T_rWHEN }, @@ -93,44 +92,6 @@ static struct ReservedWord reserved_words[] = { #define N_RESERVED_WORDS ((sizeof(reserved_words))/(sizeof(reserved_words[0]))) -static sbLexToken advance_token_queue(sbLexToken *q, int *length, char version) { - if (*length == 0) { - if (version == 'o') { - PANIC("can't advance empty lexer output queue!"); - } else { - PANIC("can't advance empty lexer input queue!"); - } - } - - sbLexToken result = q[0]; - - (*length)--; - for (int i = 0; i < *length; i++) { - if (i < LEXER_QUEUE_LENGTH - 1) { - q[i] = q[i + 1]; - } else { - q[i] = (sbLexToken) {0}; - } - } - - q[*length] = (sbLexToken) {0}; - - return result; -} - -static void enqueue_token(sbLexToken *q, int *length, sbLexToken token, char version) { - if (*length == LEXER_QUEUE_LENGTH) { - if (version == 'o') { - PANIC("output token queue is full!"); - } else { - PANIC("input token queue is full!"); - } - } - - q[*length] = token; - (*length)++; -} - static flag is_stackable(sbTokenType type); static void stack_token(hLexer lx, sbLexToken token); static flag is_closing_bracket(sbTokenType type); @@ -144,7 +105,7 @@ static void enqueue_output_token(hLexer lx, sbLexToken token) { unstack_visible_bracket(lx, token); } - enqueue_token(lx->output_queue, &lx->output_queue_length, token, 'o'); + sbTokenQueue_enqueue(&lx->output_queue, token); if (is_closing_bracket(token.type) && !token.invisible) { unstack_sticky_invisible_parentheses(lx); @@ -169,31 +130,33 @@ static void enqueue_input_token(hLexer lx, sbLexToken token) { } } - enqueue_token(lx->input_queue, &lx->input_queue_length, token, 'i'); + sbTokenQueue_enqueue(&lx->input_queue, token); } static sbLexToken advance_output_queue(hLexer lx) { - while (lx->output_queue_length == 0) { + while (sbTokenQueue_size(&lx->output_queue) == 0) { compute_next_token(lx); } - return advance_token_queue(lx->output_queue, &lx->output_queue_length, 'o'); + + return sbTokenQueue_shift(&lx->output_queue); } static sbLexToken advance_input_queue(hLexer lx) { - if (lx->input_queue_length == 0) { + if (sbTokenQueue_size(&lx->input_queue) == 0) { enqueue_input_token(lx, sbScanner_next(&lx->scanner)); } - return advance_token_queue(lx->input_queue, &lx->input_queue_length, 'i'); + + return sbTokenQueue_shift(&lx->input_queue); } static sbLexToken input_peek_ahead(hLexer lx, int count) { /* 0 = next token; 1 = token after that; etc. * so if count is 0, length must be at least 1 */ - while (lx->input_queue_length <= count) { + while (sbTokenQueue_size(&lx->input_queue) <= count) { enqueue_input_token(lx, sbScanner_next(&lx->scanner)); } - return lx->input_queue[count]; + return sbTokenQueue_at(&lx->input_queue, count); } static flag is_literal(sbTokenType type) { @@ -201,7 +164,9 @@ static flag is_literal(sbTokenType type) { || type == T_STRING || type == T_INTEGER || type == T_FLOAT - || type == T_SYMBOL; + || type == T_SYMBOL + || type == T_rTRUE + || type == T_rFALSE; } static flag insert_semicolon_after(sbTokenType type) { @@ -312,10 +277,9 @@ static char brackets_stack_top(hLexer lx) { static char brackets_stack_pop(hLexer lx) { if (lx->brackets_stack.size == 0) return 0; - char result = brackets_stack_top(lx); - lx->brackets_stack.data[lx->brackets_stack.size - 1] = 0; - lx->brackets_stack.size --; - return result; + char *result = sbBuffer_shrink(&lx->brackets_stack, 1); + + return *result; } static void brackets_stack_push(hLexer lx, char c) { diff --git a/src/parse/lexer.h b/src/parse/lexer.h index 8eaac99..98bd427 100644 --- a/src/parse/lexer.h +++ b/src/parse/lexer.h @@ -4,19 +4,12 @@ #include "token.h" #include "scanner.h" -#define LEXER_QUEUE_LENGTH 32 - -/* TODO: input_queue and output_queue should be sbBuffer */ typedef struct sbLexer { sbScanner scanner; - sbLexToken input_queue[LEXER_QUEUE_LENGTH]; - sbLexToken output_queue[LEXER_QUEUE_LENGTH]; - int input_queue_length; - int output_queue_length; - sbLexToken last_token_seen; - flag brace_terminated_state; + sbTokenQueue input_queue; + sbTokenQueue output_queue; sbBuffer brackets_stack; - int n_brackets; + sbLexToken last_token_seen; } sbLexer; typedef struct sbLexer *hLexer; diff --git a/src/parse/scanner.c b/src/parse/scanner.c index 608c714..6229cf6 100644 --- a/src/parse/scanner.c +++ b/src/parse/scanner.c @@ -9,7 +9,7 @@ * after this stage so we can differentiate between things like 'a.b (1)' and 'a.b(1)', * which have different semantics. */ -#define MEM_BLOCK_SIZE 8 +#define MEM_BLOCK_SIZE 65536 #define TOKEN_BUFFER_SIZE 64 static void check_if_start(hScanner sc); @@ -206,11 +206,10 @@ static sbLexToken compute_next_token(hScanner sc) { || ch == '[' || ch == ']' || ch == '{' || ch == '}' || ch == ';' || ch == '|' - || ch == '.' || ch == ',' - || ch == '\\') { + || ch == ',' || ch == '\\') { /* unambiguously single-character tokens */ - new_token.type = ch; NEXT; + new_token.type = ch; } else if (ch == '!') { NEXT; ch = PEEK; @@ -245,6 +244,20 @@ static sbLexToken compute_next_token(hScanner sc) { } else { new_token.type = T_GREATER; } + } else if (ch == '.') { + new_token.type = T_DOT; + NEXT; + ch = PEEK; + if (ch == '.') { + new_token.type = T_TWODOT; + NEXT; + ch = PEEK; + + if (ch == '.') { + new_token.type = T_ELLIPSIS; + NEXT; + } + } } else if (ch == '<') { NEXT; ch = PEEK; diff --git a/src/parse/token.h b/src/parse/token.h index 0158ffd..8ea30b4 100644 --- a/src/parse/token.h +++ b/src/parse/token.h @@ -1,26 +1,28 @@ #ifndef __SB_TOKEN_H__ #define __SB_TOKEN_H__ +#include "common.h" + typedef enum sbTokenType { T_NULL = 0, T_ERROR = 1, - T_LPAREN = '(', + T_LPAREN = '(', // op::call T_RPAREN = ')', - T_LBRACKET = '[', + T_LBRACKET = '[', // op::index T_RBRACKET = ']', T_LBRACE = '{', T_RBRACE = '}', - T_ASTERISK = '*', - T_SLASH = '/', - T_PLUS = '+', - T_MINUS = '-', - T_PERCENT = '%', + T_ASTERISK = '*', // op::mul + T_SLASH = '/', // op::div + T_PLUS = '+', // op::add + T_MINUS = '-', // op::sub + T_PERCENT = '%', // op::mod T_PIPE = '|', T_DOT = '.', T_COMMA = ',', T_EQUALS = '=', - T_LESS = '<', - T_GREATER = '>', + T_LESS = '<', // op::lt + T_GREATER = '>', // op::gt T_COLON = ':', T_SEMICOLON = ';', T_BACKSLASH = '\\', @@ -36,22 +38,25 @@ typedef enum sbTokenType { T_SQUIGARROW, // ~> T_BACKSQUIGARROW, // <~ T_COLONBRACE, // :{ - T_DOUBLEEQUALS, // == - T_DOUBLEMINUS, // -- - T_DOUBLEPLUS, // ++ - T_DOUBLEASTERISK, // ** - T_DOUBLESLASH, // // - T_DOUBLEGREATER, // >> - T_DOUBLELESS, // << + T_PAAMAYIM_NEKUDOTAYIM, // :: op::scope + T_DOUBLEEQUALS, // == op::eq + T_DOUBLEMINUS, // -- op::decr + T_DOUBLEPLUS, // ++ op::incr + T_DOUBLEASTERISK, // ** op::pow + T_DOUBLESLASH, // // op::floordiv + T_DOUBLEPERCENT, // %% op::divisible + T_DOUBLEGREATER, // >> op::rshift + T_DOUBLELESS, // << op::lshift T_MINUSEQUALS, // -= T_PLUSEQUALS, // += T_ASTERISKEQUALS, // *= T_SLASHEQUALS, // /= T_PERCENTEQUALS, // %= - T_LESSEQUALS, // <= - T_GREATEREQUALS, // >= - T_NOTEQUALS, // != - T_PAAMAYIM_NEKUDOTAYIM, // :: + T_LESSEQUALS, // <= op::le + T_GREATEREQUALS, // >= op::ge + T_NOTEQUALS, // != op::ne + T_TWODOT, // .. + T_ELLIPSIS, // ... /* reserved words */ T_rAND, // and @@ -60,13 +65,16 @@ typedef enum sbTokenType { T_rDEF, // def T_rDO, // do T_rELSE, // else + T_rFALSE, // false T_rIF, // if T_rIN, // in T_rLET, // let + T_rMATCH, // match T_rNOT, // not T_rOR, // or T_rREPEAT, // repeat T_rRETURN, // return + T_rTRUE, // true T_rUNLESS, // unless T_rUNTIL, // until T_rWHEN, // when @@ -86,4 +94,22 @@ typedef struct sbLexToken { }; } sbLexToken; +typedef struct sbTokenQueue { + sbBuffer buffer; +} sbTokenQueue; + +typedef sbTokenQueue *hTokenQueue; + +void sbTokenQueue_initialize(hTokenQueue q, i16 initial_size); + +void sbTokenQueue_enqueue(hTokenQueue q, sbLexToken t); + +sbLexToken sbTokenQueue_shift(hTokenQueue q); + +i32 sbTokenQueue_size(hTokenQueue q); + +sbLexToken sbTokenQueue_at(hTokenQueue q, i32 index); + +void sbTokenQueue_deinitialize(hTokenQueue q); + #endif