basic Pratt expression parser

diff --git a/src/data/symbol.c b/src/data/symbol.c
index ab29e8c..666a954 100644 (file)
--- a/src/data/symbol.c
+++ b/src/data/symbol.c
@@ -53,6 +53,10 @@ void sbSymbol_sys_init() {
   sbSymbol_from_bytes("", 0);
 }
 
+void sbSymbol_sys_deinit() {
+  // nothing
+}
+
 char *sbSymbol_name(hSymbol sym) {
   return (char*)sym;
 }

diff --git a/src/data/symbol.h b/src/data/symbol.h
index 091b66e..c341687 100644 (file)
--- a/src/data/symbol.h
+++ b/src/data/symbol.h
@@ -3,6 +3,7 @@
 #include "data/value.h"
 
 void sbSymbol_sys_init();
+void sbSymbol_sys_deinit();
 
 hSymbol sbSymbol_from_string(hString str);
 

diff --git a/src/main.c b/src/main.c
index e1c0d78..867aff4 100644 (file)
--- a/src/main.c
+++ b/src/main.c
@@ -1,82 +1,25 @@
 #include "common.h"
 
-#include "parse/filereader.h"
-#include "parse/lexer.h"
+#include "parse/parser.h"
 #include "data/string.h"
 #include "data/hashtable.h"
 #include "data/symbol.h"
 
 int main(int argc, char **argv) {
     if (argc == 2) {
-        hFileReader fr = sbFileReader_open(argv[1]);
-
-        if (!sbFileReader_ok(fr)) {
-            fprintf(stderr, "Error getting input stream.\n");
-            return -1;
-        }
-
-        sbLexer lx;
-        sbLexer_initialize(&lx, fr);
-
         sbString_sys_init();
         sbHash_sys_init();
         sbSymbol_sys_init();
 
-        sbLexToken t;
-        char scratch[8];
-
-        printf("ok here i go\n");
-        hString hello_str = OBJSL("hello! ");
-        hString long_str = OBJSL("");
-        hString test_str = OBJSL("hello! hello! hello! hello! hello! ");
-        for (int i = 0; i < 300; i++) {
-          sbString_release(long_str);
-          long_str = sbString_concat(long_str, hello_str);
-          if (sbString_eq(test_str, long_str)) {
-            printf("BREAK!\n");
-            break;
-          }
-        }
-        printf("%s\n", sbString_get_value(long_str, scratch, NULL));
-
-        do {
-            t = sbLexer_next(&lx);
-
-            if (t.type == T_EOF) {
-                printf("<EOF>");
-            } else if (t.type == T_SPACE) {
-                printf("<SPACE>");
-            } else if (t.type == T_NEWLINE) {
-                printf("\n");
-            } else if (t.type == T_IDENTIFIER) {
-                printf("%s", t.cstr);
-            } else if (t.type == T_INTEGER) {
-                printf("<INT %d>", t.i);
-            } else if (t.type == T_STRING) {
-                printf("<STRING '%s'>", sbString_get_value(t.hstr, scratch, NULL));
-            } else if (t.type == T_SYMBOL) {
-                printf("<SYMBOL :%s>", t.cstr);
-            } else if (t.type >= T_rAND && t.type <= T_rWHILE) {
-                printf("<%s>", t.cstr);
-            } else if (t.type == T_SEMICOLON) {
-                printf(";\n");
-            } else {
-                switch (t.type) {
-                    case T_COLONBRACE:
-                        printf(":{"); break;
-                    case T_DOUBLEEQUALS:
-                        printf("=="); break;
-                    default:
-                        printf("%c", t.type);
-                }
-            }
-        } while (t.type != T_EOF);
-        printf("\n");
+        sbParser pr;
+        sbParser_initialize(&pr);
 
-        sbLexer_deinitialize(&lx);
+        sbParser_parse_file(&pr, argv[1]);
 
-        sbFileReader_close(fr);
+        sbParser_deinitialize(&pr);
 
+        sbSymbol_sys_deinit();
+        sbHash_sys_deinit();
         sbString_sys_deinit();
     } else {
         fprintf(stderr, "please provide a file as input\n");

diff --git a/src/parse/ast.h b/src/parse/ast.h
new file mode 100644 (file)
index 0000000..c552c4e
--- /dev/null
+++ b/src/parse/ast.h
@@ -0,0 +1,71 @@
+#include "common.h"
+
+#include "data/value.h"
+#include "parse/token.h"
+
+typedef enum sbAstType {
+  AST_NULL,
+  AST_VAL_NIL,
+  AST_VAL_INT,
+  AST_VAL_STRING,
+  AST_VAL_FLOAT,
+  AST_VAL_SYMBOL,
+  AST_VAL_BOOLEAN,
+  AST_NODE_NAME,
+  AST_NODE_SEQ,
+  AST_NODE_OP,
+  AST_NODE_DEF,
+  AST_NODE_LET,
+  AST_NODE_IF,
+  AST_NODE_THENELSE,
+  AST_NODE_WHILE,
+  AST_NODE_LIST,
+  AST_NODE_HASH,
+  AST_NODE_HASHENTRY,
+  AST_NODE_NEXT,
+  AST_NODE_GROUPING,
+  AST_NODE_MULTIVAL,
+  AST_NODE_FUNCCALL,
+  AST_NODE_METHODCALL,
+  AST_NODE_SEND,
+  AST_NODE_ELLIPSIS,
+} sbAstType;
+
+typedef enum sbAstOp {
+  AST_OP_NULL,
+  AST_OP_PLUS = '+',
+  AST_OP_MINUS = '-',
+  AST_OP_TIMES = '*',
+  AST_OP_DIV = '/',
+  AST_OP_MOD = '%',
+  AST_OP_EQ = '=',
+  AST_OP_LT = '<',
+  AST_OP_GT = '>',
+  AST_OP_LE = 128,
+  AST_OP_GE,
+  AST_OP_NE,
+  AST_OP_FLDIV,
+  AST_OP_DIVBY,
+  AST_OP_PN,
+} sbAstOp;
+
+typedef struct sbAstNode {
+  sbAstType type;
+  union {
+    hString str;
+    hSymbol symb;
+    hInteger i;
+    double fl;
+    struct {
+      const struct sbAstNode *this;
+      const struct sbAstNode *next;
+    } seq;
+    struct {
+      sbTokenType type;
+      const struct sbAstNode *left;
+      const struct sbAstNode *right;
+    } op;
+  };
+} sbAstNode;
+
+typedef const sbAstNode *sbAst;

diff --git a/src/parse/parser.c b/src/parse/parser.c
new file mode 100644 (file)
index 0000000..11fe90b
--- /dev/null
+++ b/src/parse/parser.c
@@ -0,0 +1,208 @@
+#include "parser.h"
+
+#include "parse/filereader.h"
+#include "data/symbol.h"
+
+static sbAst do_parse(hParser pr);
+
+void sbParser_initialize(hParser pr) {
+  sbArena_initialize(&pr->node_arena, 32768);
+  sbTokenQueue_initialize(&pr->input_queue, 8);
+}
+
+void sbParser_deinitialize(hParser pr) {
+  sbTokenQueue_deinitialize(&pr->input_queue);
+  sbArena_deinitialize(&pr->node_arena);
+}
+
+sbAst sbParser_parse_file(hParser pr, const char *filename) {
+  hFileReader fr = sbFileReader_open(filename);
+
+  if (!sbFileReader_ok(fr)) {
+    fprintf(stderr, "Error getting input stream.\n");
+    return NULL;
+  }
+
+  sbLexer_initialize(&pr->lexer, fr);
+
+  sbAst result = do_parse(pr);
+
+  sbFileReader_close(fr);
+
+  return result;
+}
+
+/* --- */
+
+typedef struct binop {
+  sbTokenType type;
+  u8 left_precedence;
+  u8 right_precedence;
+} binop;
+
+static const sbAstNode SENTINEL_VALUE = {0};
+static sbAst NO_NODE = &SENTINEL_VALUE;
+
+static sbLexToken peek_ahead(hParser pr, usize count) {
+  while (sbTokenQueue_size(&pr->input_queue) < count + 1) {
+    sbTokenQueue_enqueue(&pr->input_queue, sbLexer_next(&pr->lexer));
+  }
+
+  return sbTokenQueue_at(&pr->input_queue, count);
+}
+
+static sbLexToken next_token(hParser pr) {
+  if (sbTokenQueue_size(&pr->input_queue) > 0) {
+    sbTokenQueue_shift(&pr->input_queue);
+  }
+
+  return sbTokenQueue_at(&pr->input_queue, 0);
+}
+
+static flag accept(hParser pr, sbTokenType type, sbLexToken *tok_out) {
+  sbLexToken up_next = peek_ahead(pr, 0);
+  if (up_next.type == type) {
+    next_token(pr);
+    if (tok_out) *tok_out = up_next;
+    return TRUE;
+  } else {
+    return FALSE;
+  }
+}
+
+static flag expect(hParser pr, sbTokenType type, sbLexToken *tok_out) {
+  sbLexToken up_next = peek_ahead(pr, 0);
+  if (up_next.type == type) {
+    next_token(pr);
+    if (tok_out) *tok_out = up_next;
+    return TRUE;
+  } else {
+    fprintf(stderr, "syntax error ! unexpected type %d\n", up_next.type); // TODO add line information, etc.
+    return FALSE;
+  }
+}
+
+static sbAst new_node(hParser pr, sbAstNode *n) {
+  sbAstNode *new_node = sbArena_alloc(&pr->node_arena, sizeof(sbAstNode));
+  *new_node = *n;
+  return new_node;
+}
+
+static sbAst unop_node(hParser pr, sbTokenType operation, sbAstNode *child) {
+  sbAstNode n = (sbAstNode) {
+    .type = AST_NODE_OP,
+    .op.type = operation,
+    .op.left = child,
+  };
+  return new_node(pr, &n);
+}
+
+static sbAst binop_node(hParser pr, sbTokenType operation, sbAst left, sbAst right) {
+  sbAstNode n = (sbAstNode) {
+    .type = AST_NODE_OP,
+    .op.type = operation,
+    .op.left = left,
+    .op.right = right,
+  };
+  return new_node(pr, &n);
+}
+
+static binop binops[] = {
+  { T_rOR, 3, 4 },
+  { T_rAND, 6, 7 },
+  { T_DOUBLEEQUALS, 10, 11 },
+  { T_GREATER, 10, 11 },
+  { T_LESS, 10, 11 },
+  { T_GREATEREQUALS, 10, 11 },
+  { T_LESSEQUALS, 10, 11 },
+  { T_NOTEQUALS, 10, 11 },
+  { T_PLUS, 15, 16 },
+  { T_MINUS, 15, 16 },
+  { T_ASTERISK, 30, 31 },
+  { T_SLASH, 30, 31 },
+  { T_PERCENT, 30, 31 },
+  { T_DOUBLESLASH, 30, 31 },
+  { T_DOUBLEPERCENT, 30, 31 },
+  { T_DOUBLEASTERISK, 51, 50 },
+  { T_TWODOT, 60, 61 },
+  { T_PAAMAYIM_NEKUDOTAYIM, 70, 71 },
+};
+
+const int NUM_BINOPS = sizeof(binops) / sizeof(binops[0]);
+
+/* https://matklad.github.io/2020/04/13/simple-but-powerful-pratt-parsing.html */
+static sbAst parse_expr(hParser pr, u8 min_precedence) {
+  sbLexToken t = peek_ahead(pr, 0);
+  sbAst lhs = NO_NODE;
+  if (t.type == T_IDENTIFIER) {
+    expect(pr, T_IDENTIFIER, NULL);
+    sbAstNode n = { .type = AST_NODE_NAME, .symb = t.symb };
+    lhs = new_node(pr, &n);
+  } else if (t.type == T_LPAREN) {
+    next_token(pr);
+    lhs = parse_expr(pr, 0);
+    if (!expect(pr, T_RPAREN, NULL)) {
+      fprintf(stderr, "expected closing parenthesis!\n");
+      return NO_NODE;
+    }
+  } else {
+    fprintf(stderr, "expected identifier!\n");
+    return NO_NODE;
+  }
+
+  while (1) {
+    sbLexToken op = peek_ahead(pr, 0);
+    if (op.type == T_EOF || op.type == T_SEMICOLON) {
+      break;
+    } else {
+      binop *infix = NULL;
+      for (int i = 0; i < NUM_BINOPS; i++) {
+        if (binops[i].type == op.type) {
+          infix = &binops[i];
+          break;
+        }
+      }
+
+      if (!infix) {
+        /* something that isn't an infix operator: means end of expr */
+        break;
+      }
+
+      if (infix->left_precedence < min_precedence) {
+        /* end of this expr because something with lower precedence */
+        break;
+      }
+
+      next_token(pr); /* consume operator token */
+
+      sbAst rhs = parse_expr(pr, infix->right_precedence);
+
+      lhs = binop_node(pr, infix->type, lhs, rhs);
+    }
+  }
+
+  return lhs;
+}
+
+static void print_ast_node(sbAst n) {
+  if (n->type == AST_NODE_NAME) {
+    printf("%s", sbSymbol_name(n->symb));
+  } else {
+    printf("(");
+    if (n->type == AST_NODE_OP) {
+      printf("%c ", n->op.type);
+      print_ast_node(n->op.left);
+      printf(" ");
+      print_ast_node(n->op.right);
+    } else {
+      printf("?<%d>", n->type);
+    }
+    printf(")");
+  }
+}
+
+static sbAst do_parse(hParser pr) {
+  sbAst program = parse_expr(pr, 0);
+  print_ast_node(program);
+  return program;
+}

diff --git a/src/parse/parser.h b/src/parse/parser.h
new file mode 100644 (file)
index 0000000..5cbf389
--- /dev/null
+++ b/src/parse/parser.h
@@ -0,0 +1,18 @@
+#include "common.h"
+
+#include "parse/ast.h"
+#include "parse/lexer.h"
+
+typedef struct sbParser {
+  sbLexer lexer;
+  sbTokenQueue input_queue;
+  sbArena node_arena;
+} sbParser;
+
+typedef sbParser *hParser;
+
+const sbAstNode *sbParser_parse_file(hParser pr, const char *filename);
+
+void sbParser_initialize(hParser pr);
+
+void sbParser_deinitialize(hParser pr);

diff --git a/src/parse/scanner.c b/src/parse/scanner.c
index c297bed..fb6e50f 100644 (file)
--- a/src/parse/scanner.c
+++ b/src/parse/scanner.c
@@ -69,7 +69,7 @@ sbLexToken sbScanner_next(hScanner sc) {
 }
 
 void sbScanner_deinitialize(hScanner sc) {
-    sbArena_destroy(&sc->arena);
+    sbArena_deinitialize(&sc->arena);
     sbBuffer_deinitialize(&sc->dynamic_buffer);
 }
 

diff --git a/src/parse/scanner.h b/src/parse/scanner.h
index e3aabd3..e2a8bd7 100644 (file)
--- a/src/parse/scanner.h
+++ b/src/parse/scanner.h
@@ -1,3 +1,6 @@
+#ifndef __SARABANDE_SCANNER_H__
+#define __SARABANDE_SCANNER_H__
+
 #include "common.h"
 
 #include "filereader.h"
@@ -20,3 +23,5 @@ sbLexToken sbScanner_next(hScanner sc);
 sbLexToken sbScanner_peek(hScanner sc);
 
 void sbScanner_deinitialize(hScanner sc);
+
+#endif