preprocessor works with primitive tokens, adapted tokenizer

meson.build

@@ -3,7 +3,7 @@ project('lonelytiger', 'c')
 src = [
     'tokenizer/Tokenizer.c', 
     'tokenizer/Token.c', 
-    'tokenizer/FileReader.c', 
+    'tokenizer/FileTokens.c', 
     'utils/SnuviUtils.c', 
     'utils/Variables.c',
     'utils/Functions.c',

tests/strings/chars

@@ -1,4 +1,5 @@
 void main() {
+    test('a');
     test('!');
     test('¡');
     test('ࠠ');

tests/strings/chars.out

@@ -1,3 +1,4 @@
+97
 33
 161
 2080

tests/system/math

@@ -1,4 +1,4 @@
-float PI = 3.14159265358979323846;
+float PI = 3.1415926535897932384;
 
 float abs(float f) {
     if(f < 0.0) {

tokenizer/FileReader.c

@@ -1,684 +0,0 @@
-#include <errno.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "tokenizer/FileReader.h"
-
-typedef char String[64];
-#define STRING_LENGTH ((int)sizeof(String))
-
-#define END_IF "#endif"
-#define END_IF_LENGTH ((int)sizeof(END_IF) - 1)
-
-static Error* error;
-
-typedef struct {
-    String name;
-} Define;
-
-#define DEFINES 256
-static Define defines[DEFINES];
-static int defineIndex = 0;
-
-static bool frHasError() {
-    return error->message[0] != '\0';
-}
-
-static bool frStringCompare(const String a, const char* b) {
-    return strcmp(a, b) == 0;
-}
-
-static void frInitFileReader(FileReader* fr, const char* path) {
-    fr->data = NULL;
-    fr->path = path;
-    fr->length = 0;
-    fr->readIndex = 0;
-    fr->line = 1;
-}
-
-static void frSystemError(const char* msg, FileReader* fr) {
-    eInitError(error, fr->line, "%s '%s': %s", msg, fr->path, strerror(errno));
-}
-
-static void frFullRead(FILE* file, FileReader* fr) {
-    if(fseek(file, 0, SEEK_END)) {
-        frSystemError("cannot seek end of file", fr);
-        return;
-    }
-    long length = ftell(file);
-    if(length < 0) {
-        frSystemError("cannot tell end of file", fr);
-        return;
-    }
-    rewind(file);
-    fr->data = malloc(length + 1);
-    fr->length = length;
-    int readValues = fread(fr->data, 1, length, file);
-    if(readValues != length) {
-        frSystemError("cannot read file", fr);
-        fr->data[0] = '\0';
-        fr->length = 0;
-        return;
-    }
-    fr->data[length] = '\0';
-}
-
-static void frFullReadFile(FileReader* fr) {
-    FILE* file = fopen(fr->path, "r");
-    if(file == NULL) {
-        frSystemError("cannot open file", fr);
-        return;
-    }
-    frFullRead(file, fr);
-}
-
-static int frReadUntil(int from, char end, String s, FileReader* fr) {
-    int index = 0;
-    while(from < fr->length && index < STRING_LENGTH - 1 &&
-          fr->data[from] != end) {
-        s[index++] = fr->data[from];
-        from++;
-    }
-    s[index] = '\0';
-    return index;
-}
-
-static int frReadUntilOrLine(int from, char end, String s, FileReader* fr) {
-    int index = 0;
-    while(from < fr->length && index < STRING_LENGTH - 1 &&
-          fr->data[from] != end && fr->data[from] != '\n') {
-        s[index++] = fr->data[from];
-        from++;
-    }
-    s[index] = '\0';
-    return index;
-}
-
-static void frMakePath(String fullPath, const char* parent, String name) {
-    int i = 0;
-    int lastSlash = 0;
-    while(i < STRING_LENGTH - 1 && parent[i] != '\0') {
-        fullPath[i] = parent[i];
-        lastSlash = parent[i] == '/' ? i : lastSlash;
-        i++;
-    }
-    i = lastSlash + 1;
-    int nameI = 0;
-    while(i < STRING_LENGTH - 1 && name[nameI] != '\0') {
-        fullPath[i] = name[nameI];
-        i++;
-        nameI++;
-    }
-    fullPath[i] = '\0';
-}
-
-static void frReplace(int from, int to, FileReader* fr,
-                      FileReader* replacement) {
-    int replaceLength = replacement->length - 1;
-    replaceLength = replaceLength < 0 ? 0 : replaceLength;
-    int newLength = fr->length + replaceLength - (to - from);
-    unsigned char* newData = malloc(newLength + 1);
-    memcpy(newData, fr->data, from);
-    memcpy(newData + from, replacement->data, replaceLength);
-    memcpy(newData + from + replaceLength, fr->data + to, fr->length - to);
-    newData[newLength] = '\0';
-
-    free(fr->data);
-    fr->data = newData;
-    fr->length = newLength;
-}
-
-static int skipSpaces(int index, FileReader* fr) {
-    while(index < fr->length && fr->data[index] == ' ') {
-        index++;
-    }
-    return index;
-}
-
-static void frHandleInclude(int from, int length, FileReader* fr) {
-    int index = from + length;
-    index = skipSpaces(index, fr);
-    if(index >= fr->length || fr->data[index] != '"') {
-        eInitError(error, fr->line, "expected '\"' and not '%c' in '%s'",
-                   fr->data[index], fr->path);
-        return;
-    }
-    index++;
-    String s;
-    int pathLength = frReadUntil(index, '"', s, fr) + 1;
-
-    String fullPath;
-    frMakePath(fullPath, fr->path, s);
-
-    FileReader include;
-    frInitFileReader(&include, fullPath);
-    frFullReadFile(&include);
-    if(frHasError()) {
-        frDelete(&include);
-        return;
-    }
-    frReplace(from, index + pathLength, fr, &include);
-    frDelete(&include);
-}
-
-static int frReadWord(String s, int from, FileReader* fr) {
-    int index = skipSpaces(from, fr);
-    int wordLength = frReadUntilOrLine(index, ' ', s, fr);
-    return index + wordLength;
-}
-
-static bool frIsDefined(String s) {
-    for(int i = 0; i < defineIndex; i++) {
-        if(frStringCompare(defines[i].name, s)) {
-            return true;
-        }
-    }
-    return false;
-}
-
-static bool frIsMatching(int from, FileReader* fr, const char* s) {
-    int i = 0;
-    while(true) {
-        if(s[i] == '\0') {
-            return true;
-        } else if(from >= fr->length || fr->data[from] != s[i]) {
-            return false;
-        }
-        from++;
-        i++;
-    }
-}
-
-static int frFindEndIf(int from, FileReader* fr) {
-    int layer = 0;
-    for(int i = from; i < fr->length; i++) {
-        if(frIsMatching(i, fr, END_IF)) {
-            if(layer == 0) {
-                return i;
-            }
-            layer--;
-        } else if(frIsMatching(i, fr, "#ifndef")) {
-            layer++;
-        }
-    }
-    return -1;
-}
-
-static void frHandleIfNotDefined(int from, int length, FileReader* fr) {
-    String s;
-    int end = frReadWord(s, from + length, fr);
-    int endIf = frFindEndIf(end, fr);
-    if(endIf == -1) {
-        eInitError(error, fr->line, "cannot find #endif in '%s'", fr->path);
-        return;
-    }
-    if(frIsDefined(s)) {
-        memset(fr->data + from, ' ', endIf - from + END_IF_LENGTH);
-    } else {
-        memset(fr->data + from, ' ', end - from);
-        memset(fr->data + endIf, ' ', END_IF_LENGTH);
-    }
-}
-
-static void frHandleDefine(int from, int length, FileReader* fr) {
-    if(defineIndex >= DEFINES) {
-        eInitError(error, fr->line, "too much defines in '%s'", fr->path);
-        return;
-    }
-    String s;
-    int end = frReadWord(s, from + length, fr);
-    if(frIsDefined(s)) {
-        puts("already defined");
-        return;
-    }
-    strncpy(defines[defineIndex].name, s, STRING_LENGTH);
-    defineIndex++;
-    printf("defined '%s'\n", s);
-    memset(fr->data + from, ' ', end - from);
-}
-
-static void frParseInstruction(int from, FileReader* fr) {
-    String s;
-    int length = frReadUntil(from, ' ', s, fr);
-    if(frStringCompare(s, "#include")) {
-        frHandleInclude(from, length, fr);
-    } else if(frStringCompare(s, "#ifndef")) {
-        frHandleIfNotDefined(from, length, fr);
-    } else if(frStringCompare(s, "#define")) {
-        frHandleDefine(from, length, fr);
-    } else {
-        eInitError(error, fr->line, "unknown preprocessor token '%s' in '%s'",
-                   s, fr->path);
-    }
-}
-
-static void frSearchInstruction(FileReader* fr) {
-    if(fr->data == NULL) {
-        return;
-    }
-    for(int i = 0; i < fr->length && !frHasError(); i++) {
-        unsigned char c = fr->data[i];
-        if(c == '#') {
-            frParseInstruction(i, fr);
-            i--;
-        } else if(c == '\n') {
-            fr->line++;
-        }
-    }
-}
-
-void frInit(const char* path, FileReader* fr, Error* e) {
-    defineIndex = 0;
-    error = e;
-    eInitSuccess(error);
-    frInitFileReader(fr, path);
-    frFullReadFile(fr);
-    frSearchInstruction(fr);
-    if(fr->data != NULL) {
-        printf((char*)fr->data);
-    }
-}
-
-void frDelete(FileReader* fr) {
-    free(fr->data);
-    fr->data = NULL;
-    fr->path = NULL;
-    fr->length = 0;
-    fr->readIndex = 0;
-}
-
-int frRead(FileReader* fr) {
-    if(fr->readIndex < fr->length) {
-        return fr->data[fr->readIndex++];
-    }
-    return -1;
-}
-
-int frPeek(FileReader* fr) {
-    if(fr->readIndex < fr->length) {
-        return fr->data[fr->readIndex];
-    }
-    return -1;
-}
-
-bool frReadIf(FileReader* fr, int c) {
-    if(frPeek(fr) == c) {
-        frRead(fr);
-        return true;
-    }
-    return false;
-}
-
-/*#define MAX_INDEX 50
-
-typedef struct {
-    char* content;
-    int size;
-    int index;
-    int arrayIndex;
-    bool newLines;
-    char* path;
-    FILE* file;
-} OpenFile;
-
-typedef struct {
-    int fileIndex;
-    int index;
-} FilePointer;
-
-typedef struct {
-    FilePointer name;
-    FilePointer code;
-} Define;
-
-typedef struct {
-    int fileIndex;
-    int defineIndex;
-    OpenFile files[MAX_INDEX];
-    Define defines[MAX_INDEX];
-    FilePointer readIndex;
-    FileError error;
-} OpenFiles;
-
-static OpenFiles files;
-static int lastChar = 0;
-static int stackIndex = 0;
-static FilePointer stack[MAX_INDEX];
-
-static void fAdd(OpenFile* of, const void* data, int size) {
-    while(of->index + size > of->size) {
-        of->size *= 2;
-        of->content = realloc(of->content, of->size);
-    }
-    memcpy(of->content + of->index, data, size);
-    of->index += size;
-}
-
-static void fAddChar(OpenFile* of, int ic) {
-    char c = ic;
-    fAdd(of, &c, 1);
-}
-
-static int fReserverInt(OpenFile* of) {
-    int address = of->index;
-    int empty = 0;
-    fAdd(of, &empty, sizeof(int));
-    return address;
-}
-
-static void fSetInt(OpenFile* of, int address, int i) {
-    memcpy(of->content + address, &i, sizeof(int));
-}
-
-static int fReadInt(OpenFile* of, int address) {
-    int i = 0;
-    memcpy(&i, of->content + address, sizeof(int));
-    of->index += sizeof(int);
-    return i;
-}
-
-static int fGet(OpenFile* of) {
-    int c = fgetc(of->file);
-    if(!of->newLines && c == '\n') {
-        return ' ';
-    }
-    return c;
-}
-
-static bool fReadCommandString(OpenFile* of, char* buffer, int size) {
-    int index = 0;
-    int c = fGet(of);
-    while(c == ' ') {
-        c = fGet(of);
-    }
-    int text = 0;
-    while(true) {
-        if(c == EOF) {
-            files.error("unexpected end of file");
-        } else if(index >= size - 1) {
-            files.error("unknown too long command");
-        } else if(c == '"' && text < 2) {
-            text++;
-            buffer[index++] = c;
-        } else if((isLetter(c) || text == 1) && text < 2) {
-            buffer[index++] = c;
-        } else if(c == ' ' || c == '\n') {
-            buffer[index] = '\0';
-            return c == '\n';
-        } else {
-            files.error("unexpected character '%c'", (char)c);
-        }
-        c = fGet(of);
-    }
-}
-
-static const char* fGetDefineName(Define* d) {
-    return files.files[d->name.fileIndex].content + d->name.index;
-}
-
-static int fStartDefine(OpenFile* of, bool* newLine) {
-    fAddChar(of, '#');
-    int end = fReserverInt(of);
-    int nameFileIndex = fReserverInt(of);
-    int nameIndex = fReserverInt(of);
-    int codeFileIndex = fReserverInt(of);
-    int codeIndex = fReserverInt(of);
-
-    char command[64];
-    *newLine = fReadCommandString(of, command, 64);
-    fSetInt(of, nameFileIndex, of->arrayIndex);
-    fSetInt(of, nameIndex, of->index);
-    fAdd(of, command, strlen(command) + 1);
-
-    fSetInt(of, codeFileIndex, of->arrayIndex);
-    fSetInt(of, codeIndex, of->index);
-    return end;
-}
-
-static void fFinishDefine(OpenFile* of, int end, int newLines) {
-    fAddChar(of, '\0');
-    fSetInt(of, end, of->index);
-    for(int i = 0; i < newLines; i++) {
-        fAddChar(of, '\n');
-    }
-}
-
-static void fDefine(OpenFile* of) {
-    bool newLine = false;
-    int end = fStartDefine(of, &newLine);
-    int newLines = newLine;
-    while(true) {
-        int c = fGet(of);
-        if(c == '#') {
-            char end[64];
-            newLines += fReadCommandString(of, end, 64);
-            if(strcmp(end, "end") == 0) {
-                break;
-            }
-            files.error("invalid command '%s' inside define region", end);
-        } else if(c == EOF) {
-            files.error("unclosed #define");
-        }
-        if(c == '\n') {
-            newLines++;
-            c = ' ';
-        }
-        fAddChar(of, c);
-    }
-    fFinishDefine(of, end, newLines);
-}
-
-static void fReadFile(const char* path, bool newLines);
-static void fEnter(FilePointer* fp);
-
-static int fConcat(char* buffer, int size, const char* s) {
-    int l = strlen(s) + 1;
-    if(l > size) {
-        files.error("too long include path");
-    }
-    memcpy(buffer, s, l);
-    return l;
-}
-
-static void fInclude(OpenFile* of) {
-    char path[64];
-    if(fReadCommandString(of, path, 64)) {
-        fAddChar(of, '\n');
-    }
-    int l = strlen(path);
-    if(l < 2 || path[0] != '"' || path[l - 1] != '"') {
-        files.error("invalid include path '%s'", path);
-    }
-    path[l - 1] = '\0';
-    const char* cutPath = path + 1;
-
-    char fullName[256];
-    int index = fConcat(fullName, 256, of->path);
-    while(index > 0 && fullName[index - 1] != '/') {
-        index--;
-    }
-    fConcat(fullName + index, 256 - index, cutPath);
-
-    FilePointer fp = {files.fileIndex, 0};
-    fEnter(&fp);
-    fReadFile(fullName, false);
-}
-
-static void fReadCommand(OpenFile* of) {
-    char command[64];
-    fReadCommandString(of, command, 64);
-    if(strcmp(command, "define") == 0) {
-        fDefine(of);
-    } else if(strcmp(command, "include") == 0) {
-        fInclude(of);
-    } else {
-        files.error("invalid command '%s'", command);
-    }
-}
-
-static void fReadFile(const char* path, bool newLines) {
-    if(files.fileIndex >= MAX_INDEX) {
-        files.error("cannot read file '%s': too many open files", path);
-    }
-    OpenFile* of = files.files + files.fileIndex;
-    of->arrayIndex = files.fileIndex++;
-    of->index = 0;
-    of->size = 16;
-    of->newLines = newLines;
-    of->content = malloc(of->size);
-    of->file = fopen(path, "r");
-    of->path = strdup(path);
-    if(of->file == NULL) {
-        files.error("cannot read file '%s'", path);
-    }
-    while(true) {
-        int c = fGet(of);
-        if(c == '#') {
-            fReadCommand(of);
-            continue;
-        }
-        fAddChar(of, c);
-        if(c == EOF) {
-            break;
-        }
-    }
-    fAddChar(of, '\0');
-}
-
-void fOpen(const char* path, FileError fe) {
-    files.fileIndex = 0;
-    files.defineIndex = 0;
-    files.readIndex.fileIndex = 0;
-    files.readIndex.index = 0;
-    files.error = fe;
-    lastChar = 0;
-    stackIndex = 0;
-    fReadFile(path, true);
-}
-
-void fClose() {
-    for(int i = 0; i < files.fileIndex; i++) {
-        if(files.files[i].file != NULL) {
-            fclose(files.files[i].file);
-        }
-        free(files.files[i].path);
-        free(files.files[i].content);
-    }
-}
-
-static OpenFile* currentFile() {
-    return files.files + files.readIndex.fileIndex;
-}
-
-static int fReadChar() {
-    return currentFile()->content[files.readIndex.index];
-}
-
-static bool fCompare(const char* a, const char* b, unsigned int bLength) {
-    return strncmp(a, b, bLength) == 0 && strlen(a) == bLength;
-}
-
-static void fEnter(FilePointer* fp) {
-    if(stackIndex >= MAX_INDEX) {
-        files.error("define stack overflow");
-    }
-    stack[stackIndex++] = files.readIndex;
-    files.readIndex = *fp;
-}
-
-static bool fCheckForReplacement() {
-    OpenFile* of = currentFile();
-    int start = files.readIndex.index;
-    int index = start;
-    while(index < of->index && isLetter(of->content[index])) {
-        index++;
-    }
-    for(int i = 0; i < files.defineIndex; i++) {
-        Define* d = files.defines + i;
-        const char* name = fGetDefineName(d);
-        if(fCompare(name, of->content + start, index - start)) {
-            files.readIndex.index = index;
-            fEnter(&d->code);
-            return true;
-        }
-    }
-    return false;
-}
-
-static void fReadDefine() {
-    int base = files.readIndex.index + 1;
-    int end = fReadInt(currentFile(), base);
-    int nameFileIndex = fReadInt(currentFile(), base + sizeof(int));
-    int nameIndex = fReadInt(currentFile(), base + sizeof(int) * 2);
-    int codeFileIndex = fReadInt(currentFile(), base + sizeof(int) * 3);
-    int codeIndex = fReadInt(currentFile(), base + sizeof(int) * 4);
-
-    const char* name = files.files[nameFileIndex].content + nameIndex;
-
-    if(files.defineIndex >= MAX_INDEX) {
-        files.error("too many defines");
-    }
-    Define* d = files.defines + files.defineIndex++;
-    for(int i = 0; i < files.defineIndex - 1; i++) {
-        if(strcmp(name, fGetDefineName(files.defines + i)) == 0) {
-            d = files.defines + i;
-            files.defineIndex--;
-            break;
-        }
-    }
-    d->name.fileIndex = nameFileIndex;
-    d->name.index = nameIndex;
-    d->code.fileIndex = codeFileIndex;
-    d->code.index = codeIndex;
-
-    files.readIndex.index = end;
-}
-
-static void fPrepareChar() {
-    while(true) {
-        int c = fReadChar();
-        if(c == '\0') {
-            if(stackIndex <= 0) {
-                files.error("define stack underflow");
-            }
-            files.readIndex = stack[--stackIndex];
-        } else if(c == EOF && stackIndex > 0) {
-            files.readIndex = stack[--stackIndex];
-        } else if(c == '#') {
-            fReadDefine();
-        } else if(isLetter(c) && !isLetter(lastChar)) {
-            if(fCheckForReplacement()) {
-                continue;
-            }
-            break;
-        } else {
-            break;
-        }
-    }
-}
-
-static int fReadI() {
-    fPrepareChar();
-    lastChar = fReadChar();
-    return currentFile()->content[files.readIndex.index++];
-}
-
-int fRead() {
-    int i = fReadI();
-    // putchar(i);
-    return i;
-}
-
-int fPeek() {
-    fPrepareChar();
-    return fReadChar();
-}
-
-bool fReadIf(int c) {
-    if(fPeek() == c) {
-        fRead();
-        return true;
-    }
-    return false;
-}*/

tokenizer/FileReader.h

@@ -1,29 +0,0 @@
-#ifndef FILE_READER_H
-#define FILE_READER_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "Error.h"
-
-typedef struct FileReader {
-    unsigned char* data;
-    const char* path;
-    int length;
-    int readIndex;
-    int line;
-} FileReader;
-
-void frInit(const char* path, FileReader* fr, Error* e);
-void frDelete(FileReader* fr);
-
-int frRead(FileReader* fr);
-int frPeek(FileReader* fr);
-bool frReadIf(FileReader* fr, int c);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

tokenizer/FileTokens.c

@@ -0,0 +1,430 @@
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "tokenizer/FileTokens.h"
+#include "utils/SnuviUtils.h"
+
+/*typedef char String[64];
+#define STRING_LENGTH ((int)sizeof(String))
+
+#define END_IF "#endif"
+#define END_IF_LENGTH ((int)sizeof(END_IF) - 1)*/
+
+static Error* error;
+
+static const char* ftGetPath(const FileTokens* ft, int line) {
+    if(ft->capacity < (2 + (int)sizeof(int)) || ft->data[0] != FT_PATH) {
+        eInitError(error, line, "path not set");
+        return "";
+    }
+    return (const char*)(ft->data + 1 + sizeof(int));
+}
+
+static void ftSystemError(const char* msg, int line, const FileTokens* ft) {
+    eInitError(error, line, "%s '%s': %s", msg, ftGetPath(ft, line),
+               strerror(errno));
+}
+
+static void ftAddData(FileTokens* ft, const void* data, int dataSize) {
+    if(ft->writeIndex + dataSize > ft->capacity) {
+        int newSize = ft->capacity;
+        while(ft->writeIndex + dataSize > newSize) {
+            newSize += 1024;
+        }
+        ft->data = realloc(ft->data, newSize);
+        ft->capacity = newSize;
+    }
+    memcpy(ft->data + ft->writeIndex, data, dataSize);
+    ft->writeIndex += dataSize;
+}
+
+static void ftAddToken(FileTokens* ft, FileToken t) {
+    unsigned char c = t;
+    ftAddData(ft, &c, 1);
+}
+
+static void ftAddInt(FileTokens* ft, int i) {
+    ftAddData(ft, &i, sizeof(int));
+}
+
+static void ftAddString(FileTokens* ft, const void* s, int length) {
+    ftAddInt(ft, length + 1);
+    ftAddData(ft, s, length);
+    const char c = '\0';
+    ftAddData(ft, &c, 1);
+}
+
+static void ftAddTerminatedString(FileTokens* ft, const char* s) {
+    ftAddString(ft, s, strlen(s));
+}
+
+/*typedef struct {
+    String name;
+} Define;
+
+#define DEFINES 256
+static Define defines[DEFINES];
+static int defineIndex = 0;
+
+static bool frHasError() {
+    return error->message[0] != '\0';
+}
+
+static bool frStringCompare(const String a, const char* b) {
+    return strcmp(a, b) == 0;
+}*/
+
+static void ftInitFileTokens(FileTokens* ft, const char* path) {
+    ft->data = NULL;
+    ft->capacity = 0;
+    ft->writeIndex = 0;
+    ft->readIndex = 0;
+
+    ftAddToken(ft, FT_PATH);
+    ftAddTerminatedString(ft, path);
+}
+
+static void ftTokenize(FileTokens* ft, unsigned char* data) {
+    int index = 0;
+    int oldIndex = 0;
+    bool inString = false;
+    int line = 1;
+    while(true) {
+        unsigned char c = data[index];
+        if(isAllowedInName(c) || (inString && c != '"' && c != '\0')) {
+            index++;
+            continue;
+        }
+        if(oldIndex < index) {
+            ftAddToken(ft, FT_LITERAL);
+            ftAddString(ft, data + oldIndex, index - oldIndex);
+        }
+        if(c == '"') {
+            inString = !inString;
+        }
+        if(c == '\0') {
+            break;
+        } else if(c == '\n') {
+            line++;
+            ftAddToken(ft, FT_NEWLINE);
+        } else if(c == ' ') {
+            ftAddToken(ft, FT_SPACE);
+        } else if(c == '/' && data[index + 1] == '/') {
+            while(data[index] != '\0' && data[index] != '\n') {
+                index++;
+            }
+            ftAddToken(ft, FT_NEWLINE);
+        } else if(c == '/' && data[index + 1] == '*') {
+            int startLine = line;
+            index += 2;
+            while(true) {
+                if(data[index] == '\0') {
+                    eInitError(error, startLine, "unclosed multiline comment");
+                    return;
+                } else if(data[index] == '\n') {
+                    ftAddToken(ft, FT_NEWLINE);
+                    line++;
+                } else if(data[index] == '*' && data[index + 1] == '/') {
+                    index++;
+                    break;
+                }
+                index++;
+            }
+        } else {
+            ftAddToken(ft, FT_SINGLE);
+            ftAddData(ft, &c, 1);
+        }
+        index++;
+        oldIndex = index;
+    }
+}
+
+static void ftReadFull(FILE* file, FileTokens* ft, int line) {
+    if(fseek(file, 0, SEEK_END)) {
+        ftSystemError("cannot seek end of file", line, ft);
+        return;
+    }
+    long length = ftell(file);
+    if(length < 0) {
+        ftSystemError("cannot tell end of file", line, ft);
+        return;
+    }
+    rewind(file);
+
+    unsigned char* data = malloc(length + 1);
+    int readValues = fread(data, 1, length, file);
+    if(readValues != length) {
+        ftSystemError("cannot read file", line, ft);
+        free(data);
+        return;
+    }
+    data[length] = '\0';
+    ftTokenize(ft, data);
+    free(data);
+}
+
+static void ftReadFullFile(FileTokens* ft, int line) {
+    FILE* file = fopen(ftGetPath(ft, line), "r");
+    if(file == NULL) {
+        ftSystemError("cannot open file", 1, ft);
+        return;
+    }
+    ftReadFull(file, ft, line);
+    if(fclose(file) < 0) {
+        ftSystemError("cannot close file", 1, ft);
+    }
+}
+
+/*static int frReadUntil(int from, char end, String s, FileReader* fr) {
+    int index = 0;
+    while(from < fr->length && index < STRING_LENGTH - 1 &&
+          fr->data[from] != end) {
+        s[index++] = fr->data[from];
+        from++;
+    }
+    s[index] = '\0';
+    return index;
+}
+
+static int frReadUntilOrLine(int from, char end, String s, FileReader* fr) {
+    int index = 0;
+    while(from < fr->length && index < STRING_LENGTH - 1 &&
+          fr->data[from] != end && fr->data[from] != '\n') {
+        s[index++] = fr->data[from];
+        from++;
+    }
+    s[index] = '\0';
+    return index;
+}
+
+static void frMakePath(String fullPath, const char* parent, String name) {
+    int i = 0;
+    int lastSlash = 0;
+    while(i < STRING_LENGTH - 1 && parent[i] != '\0') {
+        fullPath[i] = parent[i];
+        lastSlash = parent[i] == '/' ? i : lastSlash;
+        i++;
+    }
+    i = lastSlash + 1;
+    int nameI = 0;
+    while(i < STRING_LENGTH - 1 && name[nameI] != '\0') {
+        fullPath[i] = name[nameI];
+        i++;
+        nameI++;
+    }
+    fullPath[i] = '\0';
+}
+
+static void frReplace(int from, int to, FileReader* fr,
+                      FileReader* replacement) {
+    int replaceLength = replacement->length - 1;
+    replaceLength = replaceLength < 0 ? 0 : replaceLength;
+    int newLength = fr->length + replaceLength - (to - from);
+    unsigned char* newData = malloc(newLength + 1);
+    memcpy(newData, fr->data, from);
+    memcpy(newData + from, replacement->data, replaceLength);
+    memcpy(newData + from + replaceLength, fr->data + to, fr->length - to);
+    newData[newLength] = '\0';
+
+    free(fr->data);
+    fr->data = newData;
+    fr->length = newLength;
+}
+
+static int skipSpaces(int index, FileReader* fr) {
+    while(index < fr->length && fr->data[index] == ' ') {
+        index++;
+    }
+    return index;
+}
+
+static void frHandleInclude(int from, int length, FileReader* fr) {
+    int index = from + length;
+    index = skipSpaces(index, fr);
+    if(index >= fr->length || fr->data[index] != '"') {
+        eInitError(error, fr->line, "expected '\"' and not '%c' in '%s'",
+                   fr->data[index], fr->path);
+        return;
+    }
+    index++;
+    String s;
+    int pathLength = frReadUntil(index, '"', s, fr) + 1;
+
+    String fullPath;
+    frMakePath(fullPath, fr->path, s);
+
+    FileReader include;
+    ftInitFileTokens(&include, fullPath);
+    ftReadFullFile(&include);
+    if(frHasError()) {
+        frDelete(&include);
+        return;
+    }
+    frReplace(from, index + pathLength, fr, &include);
+    frDelete(&include);
+}
+
+static int frReadWord(String s, int from, FileReader* fr) {
+    int index = skipSpaces(from, fr);
+    int wordLength = frReadUntilOrLine(index, ' ', s, fr);
+    return index + wordLength;
+}
+
+static bool frIsDefined(String s) {
+    for(int i = 0; i < defineIndex; i++) {
+        if(frStringCompare(defines[i].name, s)) {
+            return true;
+        }
+    }
+    return false;
+}
+
+static bool frIsMatching(int from, FileReader* fr, const char* s) {
+    int i = 0;
+    while(true) {
+        if(s[i] == '\0') {
+            return true;
+        } else if(from >= fr->length || fr->data[from] != s[i]) {
+            return false;
+        }
+        from++;
+        i++;
+    }
+}
+
+static int frFindEndIf(int from, FileReader* fr) {
+    int layer = 0;
+    for(int i = from; i < fr->length; i++) {
+        if(frIsMatching(i, fr, END_IF)) {
+            if(layer == 0) {
+                return i;
+            }
+            layer--;
+        } else if(frIsMatching(i, fr, "#ifndef")) {
+            layer++;
+        }
+    }
+    return -1;
+}
+
+static void frHandleIfNotDefined(int from, int length, FileReader* fr) {
+    String s;
+    int end = frReadWord(s, from + length, fr);
+    int endIf = frFindEndIf(end, fr);
+    if(endIf == -1) {
+        eInitError(error, fr->line, "cannot find #endif in '%s'", fr->path);
+        return;
+    }
+    if(frIsDefined(s)) {
+        memset(fr->data + from, ' ', endIf - from + END_IF_LENGTH);
+    } else {
+        memset(fr->data + from, ' ', end - from);
+        memset(fr->data + endIf, ' ', END_IF_LENGTH);
+    }
+}
+
+static void frHandleDefine(int from, int length, FileReader* fr) {
+    if(defineIndex >= DEFINES) {
+        eInitError(error, fr->line, "too much defines in '%s'", fr->path);
+        return;
+    }
+    String s;
+    int end = frReadWord(s, from + length, fr);
+    if(frIsDefined(s)) {
+        puts("already defined");
+        return;
+    }
+    strncpy(defines[defineIndex].name, s, STRING_LENGTH);
+    defineIndex++;
+    printf("defined '%s'\n", s);
+    memset(fr->data + from, ' ', end - from);
+}
+
+static void frParseInstruction(int from, FileReader* fr) {
+    String s;
+    int length = frReadUntil(from, ' ', s, fr);
+    if(frStringCompare(s, "#include")) {
+        frHandleInclude(from, length, fr);
+    } else if(frStringCompare(s, "#ifndef")) {
+        frHandleIfNotDefined(from, length, fr);
+    } else if(frStringCompare(s, "#define")) {
+        frHandleDefine(from, length, fr);
+    } else {
+        eInitError(error, fr->line, "unknown preprocessor token '%s' in '%s'",
+                   s, fr->path);
+    }
+}
+
+static void frSearchInstruction(FileReader* fr) {
+    if(fr->data == NULL) {
+        return;
+    }
+    for(int i = 0; i < fr->length && !frHasError(); i++) {
+        unsigned char c = fr->data[i];
+        if(c == '#') {
+            frParseInstruction(i, fr);
+            i--;
+        } else if(c == '\n') {
+            fr->line++;
+        }
+    }
+}*/
+
+void ftInit(const char* path, FileTokens* ft, Error* e) {
+    // defineIndex = 0;
+    error = e;
+    eInitSuccess(error);
+    ftInitFileTokens(ft, path);
+    ftReadFullFile(ft, 1);
+    // frSearchInstruction(fr);
+    // if(fr->data != NULL) {
+    //     printf((char*)fr->data);
+    // }
+}
+
+void ftDelete(FileTokens* ft) {
+    free(ft->data);
+    ft->data = NULL;
+    ft->capacity = 0;
+    ft->writeIndex = 0;
+    ft->readIndex = 0;
+}
+
+FileToken ftRead(FileTokens* ft) {
+    if(ft->readIndex < ft->writeIndex) {
+        return ft->data[ft->readIndex++];
+    }
+    return FT_END;
+}
+
+bool ftReadSingleIf(FileTokens* ft, char c) {
+    if(ft->readIndex + 1 >= ft->writeIndex) {
+        return false;
+    } else if(ft->data[ft->readIndex] != FT_SINGLE) {
+        return false;
+    } else if(ft->data[ft->readIndex + 1] != c) {
+        return false;
+    }
+    ft->readIndex += 2;
+    return true;
+}
+
+char ftReadSingle(FileTokens* ft) {
+    if(ft->readIndex < ft->writeIndex) {
+        return ft->data[ft->readIndex++];
+    }
+    return -1;
+}
+
+const char* ftReadString(FileTokens* ft, int* length) {
+    *length = 0;
+    if(ft->readIndex + ((int)sizeof(int)) + 1 >= ft->writeIndex) {
+        return NULL;
+    }
+    memcpy(length, ft->data + ft->readIndex, sizeof(int));
+    const char* s = (const char*)(ft->data + ft->readIndex + sizeof(int));
+    ft->readIndex += *length + sizeof(int);
+    return s;
+}

tokenizer/FileTokens.h

@@ -0,0 +1,46 @@
+#ifndef FILE_READER_H
+#define FILE_READER_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "Error.h"
+
+typedef enum {
+    FT_PATH,
+    FT_NEWLINE,
+    FT_LITERAL,
+    FT_SINGLE,
+    FT_SPACE,
+    FT_END
+} FileToken;
+
+typedef struct {
+    unsigned char* data;
+    int capacity;
+    int writeIndex;
+    int readIndex;
+} FileTokens;
+
+typedef struct {
+    unsigned char* data;
+    const char* path;
+    int length;
+    int readIndex;
+    int line;
+} FileReader;
+
+void ftInit(const char* path, FileTokens* ft, Error* e);
+void ftDelete(FileTokens* ft);
+
+FileToken ftRead(FileTokens* ft);
+bool ftReadSingleIf(FileTokens* ft, char c);
+char ftReadSingle(FileTokens* ft);
+const char* ftReadString(FileTokens* ft, int* length);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

tokenizer/Tokenizer.c

@@ -6,14 +6,14 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "tokenizer/FileReader.h"
+#include "tokenizer/FileTokens.h"
 #include "tokenizer/Tokenizer.h"
 #include "utils/SnuviUtils.h"
 
 #define TOKEN_BUFFER_LENGTH (1024 * 1024)
 #define ERROR_LENGTH 256
 
-static FileReader fileReader;
+static FileTokens fileTokens;
 static jmp_buf errorJump;
 static Error* error;
 
@@ -53,17 +53,7 @@ static bool tReadTokens(void* dest, int length) {
     return false;
 }
 
-static void tParseLiteral(int c) {
-    int index = 1;
-    char buffer[64];
-    buffer[0] = c;
-    while(isAllowedInName(frPeek(&fileReader))) {
-        if(index >= 63) {
-            tError("literal is too long");
-        }
-        buffer[index++] = frRead(&fileReader);
-    }
-    buffer[index] = '\0';
+static void tParseLiteral(const char* buffer, int length) {
     if(strcmp(buffer, "return") == 0) {
         tAddToken(T_RETURN);
     } else if(strcmp(buffer, "if") == 0) {
@@ -100,54 +90,57 @@ static void tParseLiteral(int c) {
         tAdd(&i, sizeof(int32));
     } else {
         tAddToken(T_LITERAL);
-        tAdd(buffer, index + 1);
+        tAdd(buffer, length);
+    }
+}
+
+static const char* tReadFileString(int* length) {
+    const char* buffer = ftReadString(&fileTokens, length);
+    if(buffer == NULL) {
+        tError("cannot read literal string");
     }
+    return buffer;
 }
 
-static bool tParseInt(char* s, int length, long long* l) {
-    *l = 0;
+static long long tParseInt(const char* s, int length) {
+    long long l = 0;
     for(int i = 0; i < length; i++) {
-        if(*l > (LLONG_MAX / 10)) {
-            return true;
+        if(l > (LLONG_MAX / 10)) {
+            tError("invalid number on line %d", line);
+        }
+        l *= 10;
+        if(!isNumber(s[i])) {
+            tError("invalid character in number '%c' on line %d", s[i], line);
         }
-        *l *= 10;
         int digit = s[i] - '0';
-        if(*l > LLONG_MAX - digit) {
-            return true;
+        if(l > LLONG_MAX - digit) {
+            tError("invalid number on line %d", line);
         }
-        *l += digit;
+        l += digit;
     }
-    return false;
+    return l;
 }
 
-static void tParseNumber(int c) {
-    int index = 1;
-    char buffer[64];
-    buffer[0] = c;
-    bool point = false;
-    while(true) {
-        int c = frPeek(&fileReader);
-        if(c == '.') {
-            point = true;
-        } else if(!isNumber(c)) {
-            break;
-        } else if(index >= 63) {
-            tError("number is too long");
+static void tParseNumber(const char* buffer, int length) {
+    long long l = tParseInt(buffer, length);
+    if(ftReadSingleIf(&fileTokens, '.')) {
+        FileToken t = ftRead(&fileTokens);
+        if(t != FT_LITERAL) {
+            tError("expected literal after comma of number on line %d", line);
         }
-        buffer[index++] = frRead(&fileReader);
-    }
-    buffer[index] = '\0';
-    if(point) {
-        char* end = NULL;
-        float f = strtof(buffer, &end);
-        if(end[0] != '\0') {
-            tError("invalid float on line %d", line);
+        int commaLength = 0;
+        const char* commaBuffer = tReadFileString(&commaLength);
+        long long comma = tParseInt(commaBuffer, commaLength - 1);
+
+        float f = comma;
+        while(f > 1.0f) {
+            f /= 10.0f;
         }
+        f += l;
         tAddToken(T_FLOAT_VALUE);
         tAdd(&f, sizeof(float));
     } else {
-        long long l;
-        if(tParseInt(buffer, index, &l) || l > INT32_MAX) {
+        if(l > INT32_MAX) {
             tError("invalid int on line %d", line);
         }
         int32 i = l;
@@ -156,9 +149,58 @@ static void tParseNumber(int c) {
     }
 }
 
+static int32 tNextUnicodePart() {
+    FileToken t = ftRead(&fileTokens);
+    if(t == FT_SINGLE) {
+        return ftReadSingle(&fileTokens);
+    } else if(t == FT_LITERAL) {
+        int length = 0;
+        const char* s = tReadFileString(&length);
+        if(length != 2) {
+            tError("unicode literal has wrong length %d", length);
+        }
+        return s[0];
+    } else {
+        tError("cannot read next unicode character part on line %d", line);
+        return 0;
+    }
+}
+
 static int32 tUnicode(int32 c) {
     if(c == '\\') {
-        switch(frRead(&fileReader)) {
+        switch(tNextUnicodePart()) {
+            case '"': c = '"'; break;
+            case '\\': c = '\\'; break;
+            case 'n': c = '\n'; break;
+            case 'r': c = '\r'; break;
+            case 't': c = '\t'; break;
+            default: tError("unknown escaped character at line %d", line);
+        }
+    }
+    if((c & 0xE0) == 0xC0) {
+        c = ((c & 0x1F) << 6) | (tNextUnicodePart() & 0x3F);
+    } else if((c & 0xF0) == 0xE0) {
+        c = ((c & 0xF) << 12) | ((tNextUnicodePart() & 0x3F) << 6);
+        c |= tNextUnicodePart(&fileTokens) & 0x3F;
+    } else if((c & 0xF8) == 0xF0) {
+        c = ((c & 0x7) << 18) | ((tNextUnicodePart() & 0x3F) << 12);
+        c |= (tNextUnicodePart() & 0x3F) << 6;
+        c |= tNextUnicodePart() & 0x3F;
+    }
+    return c;
+}
+
+static int32 tReadNextPart(const char* s, int* index, int length) {
+    if(*index >= length) {
+        tError("missing escape character");
+    }
+    return s[(*index)++];
+}
+
+static int32 tStringUnicode(const char* s, int* index, int length) {
+    int32 c = s[(*index)++];
+    if(c == '\\') {
+        switch(tReadNextPart(s, index, length)) {
             case '"': c = '"'; break;
             case '\\': c = '\\'; break;
             case 'n': c = '\n'; break;
@@ -168,46 +210,74 @@ static int32 tUnicode(int32 c) {
         }
     }
     if((c & 0xE0) == 0xC0) {
-        c = ((c & 0x1F) << 6) | (frRead(&fileReader) & 0x3F);
+        c = ((c & 0x1F) << 6) | (tReadNextPart(s, index, length) & 0x3F);
     } else if((c & 0xF0) == 0xE0) {
-        c = ((c & 0xF) << 12) | ((frRead(&fileReader) & 0x3F) << 6);
-        c |= frRead(&fileReader) & 0x3F;
+        c = ((c & 0xF) << 12) | ((tReadNextPart(s, index, length) & 0x3F) << 6);
+        c |= tReadNextPart(s, index, length) & 0x3F;
     } else if((c & 0xF8) == 0xF0) {
-        c = ((c & 0x7) << 18) | ((frRead(&fileReader) & 0x3F) << 12);
-        c |= (frRead(&fileReader) & 0x3F) << 6;
-        c |= frRead(&fileReader) & 0x3F;
+        c = ((c & 0x7) << 18) |
+            ((tReadNextPart(s, index, length) & 0x3F) << 12);
+        c |= (tReadNextPart(s, index, length) & 0x3F) << 6;
+        c |= tReadNextPart(s, index, length) & 0x3F;
     }
     return c;
 }
 
 static void tAddString() {
     tAddToken(T_TEXT);
-    while(true) {
-        int32 c = frRead(&fileReader);
-        if(c == '"') {
-            break;
-        } else if(c == EOF) {
-            tError("unclosed string starting at line %d", line);
+
+    FileToken t = ftRead(&fileTokens);
+    if(t == FT_SINGLE) {
+        int32 c = ftReadSingle(&fileTokens);
+        if(c != '"') {
+            tError("unexpected single %d", c);
         }
-        c = tUnicode(c);
-        tAdd(&c, sizeof(int32));
+    } else if(t == FT_LITERAL) {
+        int length = 0;
+        const char* s = tReadFileString(&length);
+        length--;
+        int index = 0;
+        while(index < length) {
+            int32 c = tStringUnicode(s, &index, length);
+            tAdd(&c, sizeof(int32));
+        }
+        if(!ftReadSingleIf(&fileTokens, '"')) {
+            tError("unclosed string");
+        }
+    } else {
+        tError("unexpected string file token %d", t);
     }
+
     int32 c = 0;
     tAdd(&c, sizeof(int32));
 }
 
 static void tAddUnicode() {
-    int32 c = frRead(&fileReader);
-    c = tUnicode(c);
-    tAddToken(T_INT_VALUE);
-    tAdd(&c, sizeof(int32));
-    if(frRead(&fileReader) != '\'') {
+    FileToken t = ftRead(&fileTokens);
+    if(t == FT_LITERAL) {
+        int length = 0;
+        const char* s = tReadFileString(&length);
+        if(length != 2) {
+            tError("invalid character on line %d", line);
+        }
+        int32 c = s[0];
+        tAddToken(T_INT_VALUE);
+        tAdd(&c, sizeof(int32));
+    } else if(t == FT_SINGLE) {
+        int32 c = ftReadSingle(&fileTokens);
+        c = tUnicode(c);
+        tAddToken(T_INT_VALUE);
+        tAdd(&c, sizeof(int32));
+    } else {
+        tError("invalid character on line %d", line);
+    }
+    if(!ftReadSingleIf(&fileTokens, '\'')) {
         tError("expecting unicode end");
     }
 }
 
 static void tAddToken2(Token te, Token t) {
-    if(frReadIf(&fileReader, '=')) {
+    if(ftReadSingleIf(&fileTokens, '=')) {
         tAddToken(te);
     } else {
         tAddToken(t);
@@ -215,7 +285,7 @@ static void tAddToken2(Token te, Token t) {
 }
 
 static void tAddToken3(int c, Token tc, Token te, Token t) {
-    if(frReadIf(&fileReader, c)) {
+    if(ftReadSingleIf(&fileTokens, c)) {
         tAddToken(tc);
     } else {
         tAddToken2(te, t);
@@ -223,61 +293,44 @@ static void tAddToken3(int c, Token tc, Token te, Token t) {
 }
 
 static void tAddToken4(int c, Token tce, Token tc, Token te, Token t) {
-    if(frReadIf(&fileReader, c)) {
+    if(ftReadSingleIf(&fileTokens, c)) {
         tAddToken2(tce, tc);
     } else {
         tAddToken2(te, t);
     }
 }
 
-static void tLineComment() {
-    while(true) {
-        int c = frRead(&fileReader);
-        if(c == EOF || c == '\n') {
-            line++;
-            return;
-        }
-    }
-}
-
-static void tMultipleLineComment() {
-    while(true) {
-        int c = frRead(&fileReader);
-        if(c == EOF) {
-            tError("unclosed comment at line %d", line);
-        } else if(c == '\n') {
-            line++;
-        } else if(c == '*' && frReadIf(&fileReader, '/')) {
-            return;
+static void tParseToken(FileToken t) {
+    if(t == FT_PATH) {
+        int length = 0;
+        const char* s = ftReadString(&fileTokens, &length);
+        (void)s;
+        // TODO: do something useful with the path
+        return;
+    } else if(t == FT_NEWLINE) {
+        line++;
+        return;
+    } else if(t == FT_LITERAL) {
+        int length = 0;
+        const char* buffer = tReadFileString(&length);
+        if(isLetter(buffer[0])) {
+            tParseLiteral(buffer, length);
+        } else if(isNumber(buffer[0])) {
+            tParseNumber(buffer, length - 1);
+        } else {
+            tError("invalid literal string '%s'", buffer);
         }
-    }
-}
-
-static void tSlash() {
-    if(frReadIf(&fileReader, '/')) {
-        tLineComment();
-    } else if(frReadIf(&fileReader, '*')) {
-        tMultipleLineComment();
-    } else {
-        tAddToken2(T_DIV_SET, T_DIV);
-    }
-}
-
-static void tParseToken(int c) {
-    if(isLetter(c)) {
-        tParseLiteral(c);
         return;
-    } else if(isNumber(c)) {
-        tParseNumber(c);
+    } else if(t == FT_END || t == FT_SPACE) {
         return;
     }
+    char c = ftReadSingle(&fileTokens);
     switch(c) {
         case ' ': return;
-        case '\n': line++; return;
         case '+': tAddToken3('+', T_INCREMENT, T_ADD_SET, T_ADD); return;
         case '-': tAddToken3('-', T_DECREMENT, T_SUB_SET, T_SUB); return;
         case '*': tAddToken2(T_MUL_SET, T_MUL); return;
-        case '/': tSlash(); return;
+        case '/': tAddToken2(T_DIV_SET, T_DIV); return;
         case '%': tAddToken2(T_MOD_SET, T_MOD); return;
         case '<':
             tAddToken4('<', T_LEFT_SHIFT_SET, T_LEFT_SHIFT, T_LESS_EQUAL,
@@ -317,21 +370,21 @@ static void tParseFile() {
     writeIndex = 0;
     line = 1;
     while(true) {
-        int c = frRead(&fileReader);
-        if(c == EOF) {
+        FileToken t = ftRead(&fileTokens);
+        if(t == FT_END) {
             return;
         }
-        tParseToken(c);
+        tParseToken(t);
     }
 }
 
 void tTokenize(const char* path, Error* e) {
     error = e;
-    frInit(path, &fileReader, e);
+    ftInit(path, &fileTokens, e);
     if(!eHasError(e) && !setjmp(errorJump)) {
         tParseFile();
     }
-    frDelete(&fileReader);
+    ftDelete(&fileTokens);
 }
 
 Token tPeekToken() {