Preparation for better tokenizer

src/Main.c

@@ -49,12 +49,25 @@ int main(int argCount, const char** args) {
     Tokenizer t;
     Error e = tokenizerInit(&t, args[1]);
     memoryDump();
+    if(hasError(&e)) {
+        puts(e.text);
+        tokenizerDestroy(&t);
+        return 0;
+    }
+    Code code;
+    codeInit(&code);
+
+    e = compileFile(&t, &code);
+    tokenizerDestroy(&t);
     if(hasError(&e)) {
         puts(e.text);
     } else {
-        compileAndRun(&t);
+        codeRun(&code);
     }
 
+    codeDestroy(&code);
+    compileAndRun(&t);
+
     // while(true) {
     //     Token token = tokenizerNext(&t);
     //     if(tokenizerHasError(&t)) {
@@ -68,7 +81,6 @@ int main(int argCount, const char** args) {
     //         break;
     //     }
     // }
-    tokenizerDestroy(&t);
 
     //  char line[256];
     //  while(true) {

src/Memory.c

@@ -114,6 +114,7 @@ void memoryDump() {
     //     }
     // }
     // fputc('\n', stderr);
+    fputs("-------------------\n", stderr);
     Slot* current = memoryNextSlot(slots);
     while(current != nullptr) {
         fprintf(stderr, "Free slot %u\n", current->units);

src/Tokenizer.c

@@ -39,18 +39,6 @@ static void tAddToken(TState* t, TokenType type) {
     }
 }
 
-static void tAddSize(TState* t, size_t s) {
-    if(bufferWrite(&t->tokenizer->buffer, &s, sizeof(s))) {
-        tTooMuchTokens(t);
-    }
-}
-
-static void tAddChar(TState* t, char c) {
-    if(bufferWriteI8(&t->tokenizer->buffer, c)) {
-        tTooMuchTokens(t);
-    }
-}
-
 static bool isLetter(char c) {
     return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
 }
@@ -69,65 +57,58 @@ static bool isTokenEnd(char c) {
 
 static const char* tokenizerAddLiteral(TState* t, const char* s) {
     tAddToken(t, LITERAL);
-    size_t l = bufferGetWriteIndex(&t->tokenizer->buffer);
-    tAddSize(t, 0);
-    size_t total = 1;
-    tAddChar(t, *s);
+    size_t index = 0;
+    char buffer[256] = {};
+    buffer[index++] = *s;
     while(true) {
         char c = *(++s);
-        if(isAlphaNumeric(c)) {
-            total++;
-            tAddChar(t, c);
-        } else if(isTokenEnd(c)) {
+        if(isTokenEnd(c)) {
             break;
-        } else {
+        } else if(!isAlphaNumeric(c)) {
             tInvalidToken(t, c);
+        } else if(index >= sizeof(buffer) - 1) {
+            tTooMuchTokens(t);
         }
+        buffer[index++] = c;
+    }
+    if(bufferWriteString(&t->tokenizer->buffer, buffer)) {
+        tTooMuchTokens(t);
     }
-    total++;
-    tAddChar(t, '\0');
-    size_t o = bufferGetWriteIndex(&t->tokenizer->buffer);
-    bufferSetWriteIndex(&t->tokenizer->buffer, l);
-    tAddSize(t, total);
-    bufferSetWriteIndex(&t->tokenizer->buffer, o);
     return s;
 }
 
 static const char* tokenizerAddNumber(TState* t, const char* s) {
-    size_t nIndex = 0;
+    size_t index = 0;
     char number[64] = {};
-    number[nIndex++] = *s;
+    number[index++] = *s;
     while(true) {
         char c = *(++s);
         if(isTokenEnd(c)) {
             break;
         } else if(!isNumber(c)) {
             tInvalidToken(t, c);
-        } else if(nIndex >= sizeof(number) - 1) {
+        } else if(index >= sizeof(number) - 1) {
             tInvalidNumber(t);
         }
-        number[nIndex++] = c;
+        number[index++] = c;
     }
     char* end = nullptr;
     errno = 0;
     i64 i = strtoll(number, &end, 10);
-    if(errno != 0) {
+    if(errno != 0 || *end != '\0') {
         tInvalidNumber(t);
-    } else if(*end == '\0') {
-        tAddToken(t, INT64);
-        if(bufferWriteI64(&t->tokenizer->buffer, i)) {
-            tTooMuchTokens(t);
-        }
-        return s;
+    }
+    tAddToken(t, INT64);
+    if(bufferWriteI64(&t->tokenizer->buffer, i)) {
+        tTooMuchTokens(t);
     }
     return s;
 }
 
 static const char* tokenizerAddString(TState* t, const char* s) {
     tAddToken(t, STRING);
-    size_t l = bufferGetWriteIndex(&t->tokenizer->buffer);
-    tAddSize(t, 0);
-    size_t total = 0;
+    size_t index = 0;
+    char buffer[256] = {};
     while(true) {
         char c = *(++s);
         if(c == '\0') {
@@ -135,16 +116,14 @@ static const char* tokenizerAddString(TState* t, const char* s) {
         } else if(c == '"') {
             s++;
             break;
+        } else if(index >= sizeof(buffer) - 1) {
+            tTooMuchTokens(t);
         }
-        tAddChar(t, c);
-        total++;
+        buffer[index++] = c;
+    }
+    if(bufferWriteString(&t->tokenizer->buffer, buffer)) {
+        tTooMuchTokens(t);
     }
-    tAddChar(t, '\0');
-    total++;
-    size_t o = bufferGetWriteIndex(&t->tokenizer->buffer);
-    bufferSetWriteIndex(&t->tokenizer->buffer, l);
-    tAddSize(t, total);
-    bufferSetWriteIndex(&t->tokenizer->buffer, o);
     return s;
 }