From: cassowarii Date: Wed, 24 Jun 2026 02:41:35 +0000 (-0700) Subject: use hString to store string literal data; fix bugs X-Git-Url: https://www.git.cassowary.me/gitweb.cgi?a=commitdiff_plain;h=a08b588b2865010777b2ceb087b44162f120e4ec;p=sarabande.git use hString to store string literal data; fix bugs --- diff --git a/src/data/string.c b/src/data/string.c index 18f5c4d..8041e5d 100644 --- a/src/data/string.c +++ b/src/data/string.c @@ -7,7 +7,7 @@ #define STRINGS_PER_BLOCK 256 /* highest bit set on handle value means it is a tinystr */ -#define FLAG_TINY (1L << 63) +#define FLAG_TINY (1ULL << 63) /* if string length < REQ_ALLOC_LENGTH, we can just put the * string in the handle itself. this should be 64 / 8 = 8, @@ -18,20 +18,18 @@ /* TODO: Move these globals into some kind of "execution context" struct * that we can pass around to be reentrant */ /* also TODO a better (slightly more annoying to code) way to structure - * this would be SoA style: instead of storing free_count and allocated + * this would be SoA style: instead of storing used_count and allocated * inside the block and entry respectively, store them in a separate * array next to the pointers that we can scan quickly. less important - * for allocated since it's already in the strblk anyway, but for free_count + * for allocated since it's already in the strblk anyway, but for used_count * this could be important because the counts will otherwise be scattered * to the four winds */ sbBuffer g_string_blocks; usize g_free_block_index; -typedef u64 hString_mutable; - typedef struct strentry { usize length; - hString_mutable handle; + hString handle; flag is_external; flag allocated; union { @@ -42,7 +40,7 @@ typedef struct strentry { typedef struct strblk { usize id; - usize free_count; + usize used_count; usize next_free_index; strentry entries[STRINGS_PER_BLOCK]; } strblk; @@ -53,14 +51,21 @@ static strentry *get_entry(strblk *blk, usize index); static strentry *new_entry(usize length); static void place_str_at_offset(strentry *e, usize offset, const char *str, usize length); static flag is_tinystr(hString handle); +static strblk *new_strblk(); void sbString_sys_init() { g_free_block_index = 0; sbBuffer_initialize(&g_string_blocks, sizeof(strblk*) * 8); + + /* we need to have at least one strblk to start */ + new_strblk(); } void sbString_sys_deinit() { - sbBuffer_deinitialize(&g_string_blocks); + /* TODO free these i guess, or don't deinitialize this buffer, right now this + * would just drop the pointers to no benefit. look, i'm going to write a + * garbage collector, ok? give me a second */ + //sbBuffer_deinitialize(&g_string_blocks); } hString sbString_new(const char *value, usize length) { @@ -70,7 +75,7 @@ hString sbString_new(const char *value, usize length) { return e->handle; } else { /* tinystr */ - hString_mutable new_handle = 0; + hString new_handle = 0; for (int i = 0; i < length; i++) { new_handle <<= 8; new_handle |= value[i]; @@ -88,7 +93,7 @@ const char *sbString_get_value(hString handle, char *buffer_i_might_use, usize * if (length_out) *length_out = tinylength; if (buffer_i_might_use) { usize index = 0; - hString_mutable handle_to_eat = handle; + hString handle_to_eat = handle; while (index < tinylength) { buffer_i_might_use[index] = (handle_to_eat & 0xFF); /* get lowest byte and move down */ handle_to_eat >>= 8; @@ -143,13 +148,13 @@ static char *get_mutable_ptr_of_entry(strentry *e) { static strblk *get_strblk(usize index) { usize nblocks = g_string_blocks.size / sizeof(strblk*); - if (index >= nblocks) PANIC("attempt to get a string from a block (block #%zu) that does not exist!", index); + if (index >= nblocks) PANIC("attempt to get a string from a block (#%zu) that does not exist!", index); return ((strblk**)g_string_blocks.data)[index]; } static strentry *get_entry(strblk *blk, usize index) { - if (index >= STRINGS_PER_BLOCK) PANIC("attempt to get a string (string #%zu) that does not exist in block at %zu!", index, blk->id); + if (index >= STRINGS_PER_BLOCK) PANIC("attempt to get a string (#%zu) that does not exist in block (#%zu)!", index, blk->id); return &blk->entries[index]; } @@ -164,13 +169,13 @@ static strblk *new_strblk() { } static strentry *alloc_entry(strblk *blk) { - if (blk->free_count == 0) PANIC("attempt to allocate from a full strblk (%zu)!", blk->id); + if (blk->used_count == STRINGS_PER_BLOCK) PANIC("attempt to allocate from a full strblk (#%zu)!", blk->id); - blk->free_count --; + blk->used_count ++; usize free_index = blk->next_free_index; - if (blk->free_count > 0) { + if (blk->used_count < STRINGS_PER_BLOCK) { usize next_free_index = blk->next_free_index; do { next_free_index ++; @@ -178,7 +183,7 @@ static strentry *alloc_entry(strblk *blk) { } while (blk->entries[free_index].allocated && next_free_index != blk->next_free_index); if (next_free_index == blk->next_free_index) { - PANIC("free count lied! for strblk #%zu", blk->id); + PANIC("free count lied! for strblk (#%zu)", blk->id); } blk->next_free_index = next_free_index; @@ -192,7 +197,7 @@ static strentry *alloc_entry(strblk *blk) { static strblk *find_free_block() { strblk *block_with_free = get_strblk(g_free_block_index); - if (block_with_free->free_count == 0) { + if (block_with_free->used_count == STRINGS_PER_BLOCK) { usize nblocks = g_string_blocks.size / sizeof(strblk*); usize start_index = g_free_block_index; @@ -200,7 +205,7 @@ static strblk *find_free_block() { g_free_block_index ++; g_free_block_index %= nblocks; block_with_free = get_strblk(g_free_block_index); - } while (g_free_block_index != start_index && block_with_free->free_count == 0); + } while (g_free_block_index != start_index && block_with_free->used_count == STRINGS_PER_BLOCK); if (g_free_block_index == start_index) { /* Where did that bring you? Back to me. */ @@ -216,6 +221,7 @@ static void set_entry_size(strentry *e, usize length) { if (e->is_external) { sbBuffer_set_size(&e->somewhere_else_value, length + 1); /* if we shortened string, need to replace NUL at end */ + e->length = length; e->somewhere_else_value.data[length] = '\0'; } else { if (length > INLINE_BUFFER_SIZE - 1) { @@ -227,11 +233,13 @@ static void set_entry_size(strentry *e, usize length) { if (e->length > 0) { sbBuffer_append(&new_buf, e->inline_value, e->length + 1); } + e->length = length; e->is_external = TRUE; e->somewhere_else_value = new_buf; e->somewhere_else_value.data[length] = '\0'; } else { /* we can remain inline, but need to set NUL in the right place */ + e->length = length; e->inline_value[length] = '\0'; } } diff --git a/src/data/string.h b/src/data/string.h index a933198..467dc5f 100644 --- a/src/data/string.h +++ b/src/data/string.h @@ -1,6 +1,6 @@ #include "common.h" -typedef const u64 hString; +typedef u64 hString; hString sbString_new(const char *value, usize length); diff --git a/src/global.h b/src/global.h index 8bfc379..5e5f976 100644 --- a/src/global.h +++ b/src/global.h @@ -2,7 +2,7 @@ #define __SB_GLOBAL_H__ #ifdef DEBUG -#define PANIC(...) do { fprintf(stderr, "PANIC: " __VA_ARGS__); fprintf(stderr, "\n at " __FILE__ ":%d\n", __LINE__); abort(); } while (0) +#define PANIC(...) do { fprintf(stderr, "PANIC: " __VA_ARGS__); fprintf(stderr, "\n at " __FILE__ ":%d\n", __LINE__); abort(); } while (0) #else #define PANIC(...) do { fprintf(stderr, "PANIC: " __VA_ARGS__); fprintf(stderr, "\n"); abort(); } while (0) #endif diff --git a/src/main.c b/src/main.c index 1e328b5..3663642 100644 --- a/src/main.c +++ b/src/main.c @@ -2,6 +2,7 @@ #include "parse/filereader.h" #include "parse/lexer.h" +#include "data/string.h" int main(int argc, char **argv) { if (argc == 2) { @@ -15,7 +16,10 @@ int main(int argc, char **argv) { sbLexer lx; sbLexer_initialize(&lx, fr); + sbString_sys_init(); + sbLexToken t; + char scratch[8]; do { t = sbLexer_next(&lx); @@ -27,15 +31,15 @@ int main(int argc, char **argv) { } else if (t.type == T_NEWLINE) { printf("\n"); } else if (t.type == T_IDENTIFIER) { - printf("%s", t.str); + printf("%s", t.cstr); } else if (t.type == T_INTEGER) { printf("", t.i); } else if (t.type == T_STRING) { - printf("", t.str); + printf("", sbString_get_value(t.hstr, scratch, NULL)); } else if (t.type == T_SYMBOL) { - printf("", t.str); + printf("", t.cstr); } else if (t.type >= T_rAND && t.type <= T_rWHILE) { - printf("<%s>", t.str); + printf("<%s>", t.cstr); } else if (t.type == T_SEMICOLON) { printf(";\n"); } else { @@ -54,6 +58,8 @@ int main(int argc, char **argv) { sbLexer_deinitialize(&lx); sbFileReader_close(fr); + + sbString_sys_deinit(); } else { fprintf(stderr, "please provide a file as input\n"); } diff --git a/src/parse/lexer.c b/src/parse/lexer.c index 5e2cb9f..fb5ab94 100644 --- a/src/parse/lexer.c +++ b/src/parse/lexer.c @@ -62,36 +62,6 @@ void sbLexer_deinitialize(hLexer lx) { sbTokenQueue_deinitialize(&lx->output_queue); } -struct ReservedWord { - const char *name; - sbTokenType token_type; -}; - -static struct ReservedWord reserved_words[] = { - { "and", T_rAND }, - { "as", T_rAS }, - { "case", T_rCASE }, - { "def", T_rDEF }, - { "do", T_rDO }, - { "else", T_rELSE }, - { "false", T_rFALSE }, - { "if", T_rIF }, - { "let", T_rLET }, - { "in", T_rIN }, - { "match", T_rMATCH }, - { "not", T_rNOT }, - { "or", T_rOR }, - { "repeat", T_rREPEAT }, - { "return", T_rRETURN }, - { "true", T_rTRUE}, - { "unless", T_rUNLESS }, - { "until", T_rUNTIL }, - { "when", T_rWHEN }, - { "while", T_rWHILE }, -}; - -#define N_RESERVED_WORDS ((sizeof(reserved_words))/(sizeof(reserved_words[0]))) - static flag is_stackable(sbTokenType type); static void stack_token(hLexer lx, sbLexToken token); static flag is_closing_bracket(sbTokenType type); @@ -118,18 +88,6 @@ static void enqueue_output_token(hLexer lx, sbLexToken token) { } static void enqueue_input_token(hLexer lx, sbLexToken token) { - /* if we receive an identifier, check if it is a reserved word or not. - * (we need to do this as soon as tokens enter the input queue, because we - * may be peeking ahead to see whether something is an identifier or not.) */ - if (token.type == T_IDENTIFIER) { - for (int i = 0; i < N_RESERVED_WORDS; i++) { - if (!sbstrncmp(reserved_words[i].name, token.str, token.size + 1)) { - token.type = reserved_words[i].token_type; - break; - } - } - } - sbTokenQueue_enqueue(&lx->input_queue, token); } diff --git a/src/parse/scanner.c b/src/parse/scanner.c index e61653f..6b0c391 100644 --- a/src/parse/scanner.c +++ b/src/parse/scanner.c @@ -12,6 +12,36 @@ #define MEM_BLOCK_SIZE 65536 #define TOKEN_BUFFER_SIZE 64 +struct ReservedWord { + const char *name; + sbTokenType token_type; +}; + +static struct ReservedWord reserved_words[] = { + { "and", T_rAND }, + { "as", T_rAS }, + { "case", T_rCASE }, + { "def", T_rDEF }, + { "do", T_rDO }, + { "else", T_rELSE }, + { "false", T_rFALSE }, + { "if", T_rIF }, + { "let", T_rLET }, + { "in", T_rIN }, + { "match", T_rMATCH }, + { "not", T_rNOT }, + { "or", T_rOR }, + { "repeat", T_rREPEAT }, + { "return", T_rRETURN }, + { "true", T_rTRUE}, + { "unless", T_rUNLESS }, + { "until", T_rUNTIL }, + { "when", T_rWHEN }, + { "while", T_rWHILE }, +}; + +#define N_RESERVED_WORDS ((sizeof(reserved_words))/(sizeof(reserved_words[0]))) + static void check_if_start(hScanner sc); static sbLexToken compute_next_token(hScanner sc); @@ -352,7 +382,7 @@ static sbLexToken compute_next_token(hScanner sc) { token_size = read_symbol(sc); char *storage = save_buffer(sc); - new_token.str = storage; + new_token.cstr = storage; new_token.size = token_size; } else { /* : */ @@ -412,9 +442,9 @@ static sbLexToken compute_next_token(hScanner sc) { read_char_into_buffer(sc, '\0'); finalize_char_buffer(sc); - char *storage = save_buffer(sc); + hString hstr = sbString_new(sc->dynamic_buffer.data, sc->dynamic_buffer.size); - new_token.str = storage; + new_token.hstr = hstr; new_token.size = sc->dynamic_buffer.size; } else if (is_start_identifier(ch)) { new_token.type = T_IDENTIFIER; @@ -422,7 +452,7 @@ static sbLexToken compute_next_token(hScanner sc) { token_size = read_identifier(sc); char *storage = save_buffer(sc); - new_token.str = storage; + new_token.cstr = storage; new_token.size = token_size; } else if (is_digit(ch)) { new_token.type = T_INTEGER; @@ -476,6 +506,17 @@ static sbLexToken compute_next_token(hScanner sc) { NEXT; } + /* if we receive an identifier, check if it is a reserved word or not. */ + if (new_token.type == T_IDENTIFIER) { + for (int i = 0; i < N_RESERVED_WORDS; i++) { + if (!sbstrncmp(reserved_words[i].name, new_token.cstr, new_token.size + 1)) { + new_token.type = reserved_words[i].token_type; + break; + } + } + } + + return new_token; } diff --git a/src/parse/token.h b/src/parse/token.h index 8ea30b4..66c8864 100644 --- a/src/parse/token.h +++ b/src/parse/token.h @@ -3,6 +3,8 @@ #include "common.h" +#include "data/string.h" + typedef enum sbTokenType { T_NULL = 0, T_ERROR = 1, @@ -88,7 +90,8 @@ typedef struct sbLexToken { usize size; flag invisible; union { - char *str; + char *cstr; + hString hstr; float fl; int i; };