From f5c96a17eac78f0eaff004dfcaf6e63c71dfc546 Mon Sep 17 00:00:00 2001 From: cassowarii <2374677+cassowarii@users.noreply.github.com> Date: Thu, 18 Jun 2026 23:03:19 -0700 Subject: [PATCH 1/1] world's worst lexer --- Makefile | 22 ++++++ src/.main.c.swp | Bin 0 -> 12288 bytes src/common.h | 23 ++++++ src/global.h | 5 ++ src/main.c | 38 ++++++++++ src/mem/.arena.c.swp | Bin 0 -> 12288 bytes src/mem/.arena.h.swp | Bin 0 -> 12288 bytes src/mem/.string.c.swp | Bin 0 -> 12288 bytes src/mem/arena.c | 88 ++++++++++++++++++++++ src/mem/arena.h | 11 +++ src/mem/mem.h | 4 + src/mem/string.c | 14 ++++ src/mem/string.h | 3 + src/parse/.filereader.h.swp | Bin 0 -> 12288 bytes src/parse/.scanner.c.swp | Bin 0 -> 28672 bytes src/parse/.scanner.h.swp | Bin 0 -> 12288 bytes src/parse/filereader.c | 38 ++++++++++ src/parse/filereader.h | 11 +++ src/parse/scanner.c | 175 ++++++++++++++++++++++++++++++++++++++++++++ src/parse/scanner.h | 55 ++++++++++++++ 20 files changed, 487 insertions(+) create mode 100644 Makefile create mode 100644 src/.main.c.swp create mode 100644 src/common.h create mode 100644 src/global.h create mode 100644 src/main.c create mode 100644 src/mem/.arena.c.swp create mode 100644 src/mem/.arena.h.swp create mode 100644 src/mem/.string.c.swp create mode 100644 src/mem/arena.c create mode 100644 src/mem/arena.h create mode 100644 src/mem/mem.h create mode 100644 src/mem/string.c create mode 100644 src/mem/string.h create mode 100644 src/parse/.filereader.h.swp create mode 100644 src/parse/.scanner.c.swp create mode 100644 src/parse/.scanner.h.swp create mode 100644 src/parse/filereader.c create mode 100644 src/parse/filereader.h create mode 100644 src/parse/scanner.c create mode 100644 src/parse/scanner.h diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..579f8a0 --- /dev/null +++ b/Makefile @@ -0,0 +1,22 @@ +CC=gcc +CFLAGS=-Wall -g -DDEBUG -fsanitize=undefined +#CFLAGS=-Wall -O3 -flto -DGLEW_STATIC -DDEBUG -fsanitize=undefined +LIBS= + +SRC := src +OBJ := obj +BUILDDIR := build + +SOURCES := $(shell find $(SRC) -type f -name '*.c' -not -path '$(SRC)/$(RES)/*') +OBJECTS := $(patsubst $(SRC)/%.c, $(OBJ)/%.o, $(SOURCES)) + +build/a.out: $(OBJECTS) + @mkdir -p $(dir $@) + $(CC) $(CFLAGS) -I$(SRC) $(OBJECTS) $(LIBS) -o $@ + +$(OBJ)/%.o: $(SRC)/%.c + @mkdir -p $(dir $@) + $(CC) $(CFLAGS) $(LIBS) -I$(SRC) -c $< -o $@ + +clean: + rm -rf $(OBJ) $(BUILDDIR) diff --git a/src/.main.c.swp b/src/.main.c.swp new file mode 100644 index 0000000000000000000000000000000000000000..db6743f7bd8f6afec5d8ffb69c270d7626c3f2b3 GIT binary patch literal 12288 zcmeI2Pfrs;7{&*$ii#f8+shKnmS79$#gJ+Y6lsFhK-GgNneI;8)pmBXyC7&#&l=;& zZ{W>nq8E>T0`Tg0(1XSgpwF~hsjDf3Tg^Mk)9KFpXWseE%%SO4CT8!I`61Fn|Ca_NegW~MeNfC=;?fjAft zk5Tx6r1;ZGKalE^NA8#a6JP>NfC(@GCcp%k025#WOn?b6fj%T4T_K(w6k_@iipT%| zo!|doj|lMv+JfFg@1PgZ1E>JaLMNdU&;TT$ABTnb1Z_jFp{LLt=pr-(eH#?wBlHrg zK^LG5^b>P`hqj?D=q>aHdJa8-W}rOOs<96lbIDAA2`~XBzyz286JP>N;6EU+DG06o zX}(p1(9?0%j$&7Z;W$}AL&-={5c+GLt4LDSYoLqB)4^)ItgXznFoBz-8t{AAdN2b~ zWe%Q;N+QvU)9@o@S3@{=IV?F+YgkMARU)3I2~>mOJy2$Gpz*PzoWhC@AC4dJcJZ37Zx*J z9R5x_OQTUcE0(>-yt9Q;xj0wcCH3;M&U7}UGjFmn=}Jq5n+wHKp=YJE*JMlAF3t2# zW%eyyS-K?S18|~Kx#?rd<0N$sidO4!`;>HiT)#Uasw}AWvVTu$in~r)J&C8t`6HbZ zDjBDbWqs$^QZ;P(?~JVYfzq}NYiqYAZzo%4mVptZ&eN1x08Ss7_uy^w#aPD4smqYY u#!O&qi6fqN8mqWZRv^O&ug +#include +#include +#include + +typedef uint64_t u64; +typedef int64_t i64; +typedef uint32_t u32; +typedef int32_t i32; +typedef uint16_t u16; +typedef int16_t i16; +typedef uint8_t u8; +typedef int8_t i8; +typedef uint8_t flag; +typedef size_t usize; +typedef ptrdiff_t isize; + +#include "global.h" + +#endif diff --git a/src/global.h b/src/global.h new file mode 100644 index 0000000..4abd545 --- /dev/null +++ b/src/global.h @@ -0,0 +1,5 @@ +#ifdef DEBUG +#define PANIC(msg) do { fprintf(stderr, "panic: " __FILE__ ":%d: " msg "\n", __LINE__); abort(); } while (0) +#else +#define PANIC(x) 0 +#endif diff --git a/src/main.c b/src/main.c new file mode 100644 index 0000000..edcc69f --- /dev/null +++ b/src/main.c @@ -0,0 +1,38 @@ +#include "common.h" + +#include "parse/filereader.h" +#include "parse/scanner.h" + +int main(int argc, char **argv) { + if (argc == 2) { + hFileReader fr = sbFileReader_open(argv[1]); + + hScanner sc = sbScanner_create(fr); + + sbLexToken t; + + do { + t = sbScanner_next(sc); + + if (t.type == T_EOF) { + printf("(EOF)"); + } else if (t.type == T_SPACE) { + printf("( )"); + } else if (t.type == T_NEWLINE) { + printf("(\\n)"); + } else if (t.type == T_IDENTIFIER) { + printf("(ID %s)", t.str); + } else if (t.type == T_INTEGER) { + printf("(INT %d)", t.i); + } else { + printf(" %c ", t.type); + } + } while (t.type != T_EOF); + + sbScanner_destroy(sc); + + sbFileReader_close(fr); + } else { + fprintf(stderr, "please provide a file as input\n"); + } +} diff --git a/src/mem/.arena.c.swp b/src/mem/.arena.c.swp new file mode 100644 index 0000000000000000000000000000000000000000..dda4630ab06ef2dc93c3a63ebd568737bbc2704f GIT binary patch literal 12288 zcmeI2&u`pB6vwAX1q!sKEk`6K%Ma~tvOiKd1k$V&K%$Yts+aJqLd(RSUAy(#E8E); zqQDWv5AhH1L!g2X5*H5T!WFm!5>#A!q#TR5@qOd*u4lb%)nlZw^oiq{H*em2=gqUC zc)R`5#*6&v<}$1h>oo{Q0t)U;b z27cN~L@c&N=*m_SJFTG{wnQvL(R6O-$@W_XtO9qWz&PzUR*teKPcOG=+@qBRe(L1b z9l5Z^RspMkRlq7>6|f3e1*`&A0jt2lRv;Z6V((xHcj;xkqVI=h?)uT*tO8a6tAJI& zDqt0`3RnfK0#*U5fK|XMU==uk3J906-w!jkc@Kie|Nry9|F0is>~HWV_yha`eg;2) z&%wuF4_p8P&<9)K3|Ih1z}?_5xO$ASufbO!0wL&t8aM&2AH`hYS8xS<3O)hvgZIF@ z;AOB3+TfoD82b?Pz%$?^`0IYe2Zo>ry5JFT?LNl71?NEv90$k1wIhsu5556kf-k^p zUl^gtIVtq)+X z*pRIPRspMkRbajX`;6o2$5K|+cWkv)rdsy-j^_uGSGR+UXd@%K@Wj{XXD^g*rqg!i zpk%6yGW$knI>3$i9wTo_8jqb6?jq+RujgJdK!wui^!+$Viz7<|W^7m4rO0=Avb|2< z&M0nu*NQ;(U~8s=u%s`V_+IEHmva-~8n>cAH!k?Ap}GL6nSkuLnSGhuGN&eB{) z*s)Axdb6NonU3R-ivXgDRJy%U8tXm7$A~o0=vw0KC7$?iNgb-WW*zb3nlj0vR-x1l zCkXe6*@3P|#rfGrTL0^L1}Xh$n=`fPH@eI*QzZhC&>pB{m?l>vobQfMixzo!ZP21UIqagp=+^Rx_vH#MCy|{X`FQDE$7y| z8_!iML4@5(V*yQ~K1b$7@vipiBRlE%@gy2bFWOnqu9%* z)(`#E7eQ}&ShS7W$~8Vc`MTY#%*A6%FCOD)7kwj7RRmhSViz}6EOwEaYFJS|tZ8oT z(1O=6l{U|}z=f4R(2RQvC)5)!m`vu)WA@h(w%fctzY-^n7V%gox_&#l+kS0RdxI!ap)^`$J2OWw zw*9&<-OM81e9Rkqkv3jmJ1>#3>`D_TFNu9GQjsTPWn^RUUnlHC00MIfj9jmNyh4Wu zt!8$vQr;0;n>TZ{KrsX$009U<00Izz00bbg*aB`?q9=ZYrTj6L^SShE&X<@V009U< z00Izz00bZa0SG_<0uWe00T~c|t`hC8@#Xpd|Kj)mH?O}qADnm23+I_*Ifiq|x!`2y zDjW<5KmY;|fB*y_009U<00Izzz=8{;X=+oe1Ceykve(l)fl6F#@2i9HQjFhO-#aCm zDvDL2+}|a&qLBrd6}i&3Hc`0ZM4%e>=d=`ey9S~6MVUm b?~2=s+N1|Jt7zkgqdfe9I}C*FOWzJnpme7y{<6Y62!OyO1R~S6 zc4yf3R=pON6$@*8Y4Kzd2jqeP2!H?xfB*=900@8p2uwKv)6cOxIzv7?$86T;e)ZW9 z9SDE`2!H?xfB*=900@8p2!H?xfWQPu#KO!EP#GWHZdh(tP z(k!>fZ_=fO=LDP<7_AzG>RM)|NiYrg19~p6MSnoMp^8niuTWk5N#+%vW~F + +#define ALIGN 8 + +struct block { + struct block *next; + usize used; + usize capacity; + char data[]; +}; + +struct sbArena { + struct block *first; + struct block *current; + struct block *last; +}; + +hArena sbArena_create(usize initial_size) { + hArena arena = malloc(sizeof(struct sbArena)); + + while (initial_size % ALIGN != 0) initial_size++; + + struct block *block = malloc(initial_size); + block->used = 0; + block->next = NULL; + block->capacity = initial_size; + + arena->first = block; + arena->current = block; + arena->last = block; + + return arena; +} + +void *sbArena_alloc(hArena arena, usize size) { + while (size % ALIGN != 0) size++; + + if (size > arena->current->capacity - arena->current->used) { + /* not enough space in current block. need to move to next block or allocate a new one */ + if (arena->current != arena->last) { + /* move to next block */ + if (arena->current->next) { + arena->current = arena->current->next; + } else { + PANIC("lost track of memory block in arena somehow; should not happen"); + } + } else { + /* need to allocate a new block */ + usize new_capacity = arena->current->capacity; + if (size > new_capacity) new_capacity = size; + struct block *block = malloc(sizeof(struct block) + new_capacity); + block->capacity = new_capacity; + block->next = NULL; + + arena->current->next = block; + arena->current = arena->last = block; + } + } + + void *allocated_ptr = &arena->current->data[arena->current->used]; + arena->current->used += size; + memset(allocated_ptr, 0, size); + + return allocated_ptr; +} + +void sbArena_reset(hArena arena) { + struct block *blk = arena->first; + do { + blk->used = 0; + } while (blk != arena->current); + + arena->current = arena->first; +} + +void sbArena_destroy(hArena arena) { + struct block *blk = arena->first; + struct block *blk_next = arena->first->next; + do { + blk_next = blk->next; + free(blk); + blk = blk_next; + } while (blk); + + free(arena); +} diff --git a/src/mem/arena.h b/src/mem/arena.h new file mode 100644 index 0000000..6eb5836 --- /dev/null +++ b/src/mem/arena.h @@ -0,0 +1,11 @@ +#include "common.h" + +typedef struct sbArena *hArena; + +hArena sbArena_create(usize initial_size); + +void *sbArena_alloc(hArena arena, usize size); + +void sbArena_reset(hArena arena); + +void sbArena_destroy(hArena arena); diff --git a/src/mem/mem.h b/src/mem/mem.h new file mode 100644 index 0000000..fdd935a --- /dev/null +++ b/src/mem/mem.h @@ -0,0 +1,4 @@ +#include "common.h" + +#include "arena.h" +#include "string.h" diff --git a/src/mem/string.c b/src/mem/string.c new file mode 100644 index 0000000..b9f01fe --- /dev/null +++ b/src/mem/string.c @@ -0,0 +1,14 @@ +#include "string.h" + +usize mystrncpy(char *dst, char *src, usize limit) { + usize count = 0; + char ch = 0; + + do { + ch = dst[count] = src[count]; + count ++; + } while (ch && count < limit); + dst[--count] = 0; + + return count; +} diff --git a/src/mem/string.h b/src/mem/string.h new file mode 100644 index 0000000..c73b83a --- /dev/null +++ b/src/mem/string.h @@ -0,0 +1,3 @@ +#include "common.h" + +usize mystrncpy(char *dst, char *src, usize limit); diff --git a/src/parse/.filereader.h.swp b/src/parse/.filereader.h.swp new file mode 100644 index 0000000000000000000000000000000000000000..fc6b1d5b20c2a34dd2f960b911c203217c3ee9b5 GIT binary patch literal 12288 zcmeI&Jx;?g6bEpZ74cOrVA`crqG^Ezx}!3%Ai;{9kxpz90E&X zXS>zRH&iPda=m;q$6lmD00Izz00bZa0SG_<0{>kgOG@IJ53x`jWU-h_U*_Tw69gat z0SG_<0uX=z1Rwwb2tWV=|4=~rLOd@EQC{KZ`Tzfq@BcS`e&u-Nc;L9@xZ&_PT#h!! zK1aUqh4*7X00Izz00bZa0SG_<0uX=z1pZoJFfgGn5BOsF@#tdtnNMkE&9D~azl|@O zOS{?TuiHS`QF7`|q|L+zRj9tu7XmS{g2WhtNgIQSe^C4fQK2!uAXXCtiqVL^5k!y|nrHyOzvs`+%--zn z-L)Ex&m^DQoq3+;H_!8(XP$X}&-`v{Yi^&om2If3wXnU@vYh9yyYsRyZ@0F;%d%SB zw*!w;%7R-X3a74i>}0`ANIa?xjHc2MVIapa7(?I^}K#|{D>Rh&WA zOuA9PD6ljIF1D_{X6+g(^Zub#?9zAdUs`dK+bCcZFbWt2i~>dhqkvJsC}0%$pH?7g zpKpB}Em)yi;i}JbdOqKx_TQsE{hsvm)xQ3`yC?lTwLhXh>72Cb^PDSpEuHUx+Slo7 z*L)ZSi~>dhqkvJsC}0#Y3K#{90!9I&fKk9GunYz4x@Fx9=X)DD?s)&N{r{IPv8)%t zFTfMv2jEU{DSdwjzJp7^V;5W2Bv=pDfh)l4Z?~+Mz-jOl_%S#RJg^m%!8yPJf4#`E zUIBjrr@&XhNpJ$(18lGt+y-`obHNJm;@d3iC*Tx#8axUdV1ti=?O-Li9K09&@j}b` zKKLRy3Vcunw}M;1IJg-w@Y-7~>*wGb;5hg+m;rZz5Cq`E;7ah%3oPq5;D_Lw;9>9( zcn~}QnqUL?Ah;5o2hIh5z=FyP;2H2$a6j-t6Knz%@CL5Qli*8W4jch{0bQ$=;1cj# zESP*9d=0e0?VtkQz%_jxoC3$dPH-bw4=x9Wyc+6M~*P;*FD5=(EjkIZhk7}xNI|vYuiw654Y<)sl-*$ zjl{v8LP^-QZX`TniOU&U?t1}eBV(x=R4{k0Mbx!G!JXqD z*)g$mJiUCR9d(DztTY*vvQ{{4KM4eS5vQ^HD zT~v8$l-|U}*Er~iIE#`N1c#!C+`D_rO-jgoKa?GV z?z(J0TvpFXuM@SPNWy-1Z2xvrw;j~m3YweiIhh6*cb)N?he#t!Hf{$$ zxR1sd>SglI$4!5D&Dg-8wj7 ztKCU>m2NXxnXpPSm5qO&vG`Pyr0d=+xTiDC^rI)u|jYfU^=k_vQD= zB)b!pHd*2vm=x&Y2`9+*V`ZsavPY3|(FtW^9nA9*US7X$-L>nL4w>2@V++%h&?(cLFKpn9N&=crrWy6osxnv9S5x^^ zJ-e01V6O*QndFMPF;Q1_Bs)l@)6-2B%?snOKb{GaOH+1RZTEX1>?ttYlYe zth{!(BxZE9M?0g!E~a=h9`!@+a}ySGnwO#;|kODx|euiVEb%n$_?XWNIw5!IZdM}BqZT>3)A9Y+Kas!`~$>pqx~+d z#-pSHZXrLUi5*Tu#)d_$aU==bxT)l%PFkTodcMXCvpqZ&(W$}~=@7G4Snt5N7qBpb zA3-&j79nGcQ)Q_(UVu^+whv!@eh}G^U(}L2ApzmDa}RaX@PT9;kwgz~8V2@H_B5a1UsL{on>L z49egF@DHpDyb4Z(Z-aZlR9-3j+ni>LY1Z&==#->L!0@PiiL)WNp2w*dE~yH0dA+IWU%=Kag}PmvCr z_J9_f4?c)!`{Gqq1$2j4>WOqY;bBV8Z8;|LWzxO$+cLYC=n{D~2H4E_=7s%8uS8O3 zU%=D?Nx~J^S5{v!A>>jE-ZEXmYlO_2qh7(w*QjUdko^BA;2+PC4-Nl+L``I#gl|vp z{XYSY0UK-u?*q@lpMMs72RsfAf|~&O`xk;Y;mf}U$k+cF_%3(^d%mH}0=x|0 z{y}gbxEIvGBsc)Z!A*c-23CSsQO9qAPlBU>>O2k1W)v_A7zK<1MggOMQNSo*6!Y@r%`u+}s@Q!$R|WD$ot0yR#GH<#S?32of1(2uka4)@h3uh7B)5nsi&} zoisKI`pNqeDhQIIAb0NHv4f&Z(g=X`)whUO>JzD?mL)T*X5w6#)@36?Y3-sQR#OzA zDyGF~kG4iTLW~yT1QNUnjz-x~Jq$_-YX;#AbAfoP9U}NFMgS^)R}Aqo(MDGKfP=`x zp-~)DF$mHcLiKpV?)b?@OCQjmHM}xyO0Q2JqR(!_bikA^KDoD z5pQz6m}(n&Q*I(c12U!^8?U8Hl~TSc^5}5k(Q@W!JmVz@L=bVg2(Z$2=T&!Kb$rjB zUHQ<~!`ETGCJ;SAO9M$5As|FW;82d%EK&lejrbs$MsWoc z6)gh~^`4B8W^2)(EhkohH^dZ2GTR{QH25e?kc8e+Q;{)bPU@`sWY)(}7%1 z5DOe4{{IVjw@&Zk$^X~C?{^wLKmAX@eP9>347`kY`+o+Hfk!|HR)bYw1$Z7dehr=h z$G{Z029WK);~Xb}*^B~40i%FXz$jo8FbWt2i~>dhqre~qwEuIKevnF=%Y2?C%l{`o zb#)egaEP?ez3v|nDnR*jKRiSS;14hF=dRNC@!cN)%l!xu85e;v*|3$ufP{a~&s&`Z Jffpn0zX1MlAu9j? literal 0 HcmV?d00001 diff --git a/src/parse/.scanner.h.swp b/src/parse/.scanner.h.swp new file mode 100644 index 0000000000000000000000000000000000000000..dab6bd2e71602bdc6e2628e5adc3f5b8358cec3c GIT binary patch literal 12288 zcmeI2Pixa)9LJw|6Pq(Z2mA~4PLiBmq$H29@W6VHR{;mjz#Fa*S%r7;d<#?ZkTS_fnBz} z3QxBJC*8Dz7OouI^LQ{;`_tX1pA3+Je>2d}&P`5?M`H#DPD)3Ql>TivRg(cSKnBPF z86X2>fDDiUGO()+gv~y7A3NC7-NnW3x$l>Aw@D{5KnBPF86X2>fDDiUGC&5%02v?y zWMCH>uq%vB?qlrqek>k;|4)AZe?7?97w`#u1TVo0@Ce)k%fJFxKnffK417Dl*k|w- zyavy}L$D5(feB_n3LFP}!H<571DoIjcmrO6r{DnyfDH_A4NQR(;3#0=JJ!AlHo$xE z4m<&mK^ufX133`ya~FP3v}7_s2FL&zAOmE843Ggb@V^YK`m4TMky>T5^S`-E72XO1 z|ISdY8%Ql@c$%?RS>v}Ye}#KnYMY#|{8gdHZ->7Y_9hT>I{~*tK2*!Pb#8FG!UL%q zM6<7B(q#T=CgE7`oT5wD3njPLtk!)yl&bY!;?(RwN?}`_sO`DFC#|t)Y^xma$-2|z zQizXf7JXKDRYG~&3AfCP5%D_2fX>~lC8dy+W~9lfsgd}W)>2WH^YWsaU(%IaX|`ZR z*GXTdpcVA1hMZB7BwNxmvSAb!l3-gyomF&8&8mu#bmx@Cg@Q4gG)>D;^|_>#)e5qe zG*sPE=6VAaOk!ZA6G^{b(loTkleSsJvJjk+&LokpENH48hc`Wuno3@cHxd<3i&#XL zw?z58994~o2tuM@Oaw)>7*BAwmncRC+m6aki7>C~B{PafMO4%}$yt$@nrzNTMJW-< zrllCFncJQk@plkq>!MN<*_z1Wx||ZP<%US(%HL9>C>x3%-E+8CDY7AC$-~9Hc4OPK uhi5D46AxbU0;ZBT1uw#&>pAsyg-Zifyd!vB@Q#et2DWxMexu=gQTP+j^hY-U literal 0 HcmV?d00001 diff --git a/src/parse/filereader.c b/src/parse/filereader.c new file mode 100644 index 0000000..4f18799 --- /dev/null +++ b/src/parse/filereader.c @@ -0,0 +1,38 @@ +#include "filereader.h" + +/* this seems pointless right now as a wrapper for FILE* but later, we can make + * this work to read from stdin in interactive mode as well, which will be easier + * if we just have these few functions separate from the actual lexer code. + * we may also want to have multiple files open for imports and stuff... */ + +struct sbFileReader { + FILE *file; +}; + +hFileReader sbFileReader_open(const char *filename) { + hFileReader fr = malloc(sizeof(struct sbFileReader)); + fr->file = fopen(filename, "r"); + if (!fr->file) { + perror("Error opening file for reading:"); + } else { + return fr; + } + + return fr; +} + +char sbFileReader_next(hFileReader r) { + char c = fgetc(r->file); + return c; +} + +char sbFileReader_peek(hFileReader r) { + char c = fgetc(r->file); + ungetc(c, r->file); + return c; +} + +void sbFileReader_close(hFileReader r) { + fclose(r->file); + free(r); +} diff --git a/src/parse/filereader.h b/src/parse/filereader.h new file mode 100644 index 0000000..145ac6d --- /dev/null +++ b/src/parse/filereader.h @@ -0,0 +1,11 @@ +#include "common.h" + +typedef struct sbFileReader *hFileReader; + +hFileReader sbFileReader_open(const char *filename); + +char sbFileReader_next(hFileReader r); + +char sbFileReader_peek(hFileReader r); + +void sbFileReader_close(hFileReader r); diff --git a/src/parse/scanner.c b/src/parse/scanner.c new file mode 100644 index 0000000..a84651a --- /dev/null +++ b/src/parse/scanner.c @@ -0,0 +1,175 @@ +#include "scanner.h" + +#include "filereader.h" +#include "mem/mem.h" + +/* This is like, the pre lexing stage. I mean, this does a lot of the tokenizing, + * but it includes stuff like spaces and newlines, too. Think of this as 'normalizing' + * the input. The lexer in the second stage will use some additional context and state + * on top of the stream of tokens output from this module in order to insert some + * additional "ghost" tokens into the stream (invisible parentheses, semicolon insertion), + * and will delete stuff like spaces and newlines. But we need to preserve spaces and such + * after this stage so we can differentiate between things like 'a.b (1)' and 'a.b(1)', + * which have different semantics. */ + +typedef struct sbScanner { + hFileReader file_reader; + hArena arena; + sbLexToken next_token; +} sbScanner; + +hScanner sbScanner_create(hFileReader fr) { + hScanner sc = malloc(sizeof(sbScanner)); + sc->arena = sbArena_create(65536); + sc->file_reader = fr; + sc->next_token = (sbLexToken) {0}; + + return sc; +} + +sbLexToken sbScanner_peek(hScanner sc) { + return sc->next_token; +} + +void sbScanner_destroy(hScanner sc) { + sbArena_destroy(sc->arena); + free(sc); +} + +/* yeah, yeah, unicode! get off my achin' back! i'll do it later! (maybe) */ + +flag is_digit(char c) { + return ('0' <= c && c <= '9'); +} + +flag is_alpha(char c) { + return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || c == '_'; +} + +flag is_space(char c) { + return c == ' ' || c == '\t' || c == '\r'; +} + +flag accept_char(char expected, hFileReader fr) { + char actual = sbFileReader_peek(fr); + if (expected == actual) { + sbFileReader_next(fr); + return 1; + } else { + return 0; + } +} + +char accept_digit(hFileReader fr) { + char actual_char = sbFileReader_peek(fr); + if (is_digit(actual_char)) { + sbFileReader_next(fr); + return actual_char; + } else { + return 0; + } +} + +char accept_alpha(hFileReader fr) { + char actual_char = sbFileReader_peek(fr); + if (is_alpha(actual_char)) { + sbFileReader_next(fr); + return actual_char; + } else { + return 0; + } +} + +sbLexToken compute_next_token(hScanner sc) { + char ch = sbFileReader_peek(sc->file_reader); + + char token_buffer[256]; + int token_size = 0; + + sbLexToken new_token = {0}; + + if (ch == EOF) { + new_token.type = T_EOF; + } else if (ch == '\n') { + new_token.type = T_NEWLINE; + + /* skip all spaces after the newline */ + do { + sbFileReader_next(sc->file_reader); + } while (is_space(sbFileReader_peek(sc->file_reader))); + } else if (is_space(ch)) { + new_token.type = T_SPACE; + + /* skip all subsequent spaces */ + do { + sbFileReader_next(sc->file_reader); + } while (is_space(sbFileReader_peek(sc->file_reader))); + + /* if we run into a newline, forget about the spaces */ + if (sbFileReader_peek(sc->file_reader) == '\n') { + new_token.type = T_NEWLINE; + + /* and skip all spaces after the newline */ + do { + sbFileReader_next(sc->file_reader); + } while (is_space(sbFileReader_peek(sc->file_reader))); + } + } else if (is_alpha(ch)) { + new_token.type = T_IDENTIFIER; + + do { + token_buffer[token_size] = sbFileReader_next(sc->file_reader); + token_size ++; + ch = sbFileReader_peek(sc->file_reader); + } while (is_alpha(ch) || is_digit(ch)); + + /* TODO: I am sleepy. Please check that this has not overflowed the buffer. */ + + token_buffer[token_size] = '\0'; + + char *storage = sbArena_alloc(sc->arena, token_size + 1); + + mystrncpy(storage, token_buffer, token_size + 1); + + new_token.str = storage; + } else if (is_digit(ch)) { + new_token.type = T_INTEGER; + /* TODO: yes, yes. this isn't correct. listen. i am tired. */ + + i64 intval = ch - '0'; + + while (is_digit(sbFileReader_peek(sc->file_reader))) { + ch = sbFileReader_next(sc->file_reader); + intval *= 10; + intval += ch - '0'; + } + } else { + new_token.type = T_ERROR; + + switch (ch) { + case '(': + case ')': + case '.': + /* TODO ok blah blah this isn't right */ + new_token.type = ch; + break; + default: + fprintf(stderr, "don't know how to process character: '%c'\n", ch); + } + + sbFileReader_next(sc->file_reader); + } + + return new_token; +} + +sbLexToken sbScanner_next(hScanner sc) { + if (sc->next_token.type == T_NULL) { + /* if we just started, queue up the first token */ + sc->next_token = compute_next_token(sc); + } + + sbLexToken to_return = sc->next_token; + sc->next_token = compute_next_token(sc); + return to_return; +} diff --git a/src/parse/scanner.h b/src/parse/scanner.h new file mode 100644 index 0000000..608f5f9 --- /dev/null +++ b/src/parse/scanner.h @@ -0,0 +1,55 @@ +#include "common.h" + +#include "filereader.h" + +typedef struct sbScanner *hScanner; + +typedef enum sbTokenType { + T_LPAREN = '(', + T_RPAREN = ')', + T_LBRACKET = '[', + T_RBRACKET = '[', + T_LBRACE = '{', + T_RBRACE = '}', + T_ASTERISK = '*', + T_SLASH = '*', + T_PLUS = '+', + T_MINUS = '-', + T_PERCENT = '%', + T_PIPE = '|', + T_DOT = '.', + T_COMMA = ',', + T_COLON = ':', + T_SEMICOLON = ';', + T_NEWLINE = '\n', + T_SPACE = ' ', + T_NULL = 0, + T_ERROR = 1, + T_INTEGER, + T_FLOAT, + T_STRING, + T_KEYWORD, + T_IDENTIFIER, + T_ARROW, + T_FUNCARROW, + T_COLONBRACE, + T_PAAMAYIM_NEKUDOTAYIM, + T_EOF = 255, +} sbTokenType; + +typedef struct sbLexToken { + sbTokenType type; + union { + char *str; + float fl; + int i; + }; +} sbLexToken; + +hScanner sbScanner_create(hFileReader fr); + +sbLexToken sbScanner_next(hScanner sc); + +sbLexToken sbScanner_peek(hScanner sc); + +void sbScanner_destroy(hScanner sc); -- 1.8.3.1