complete renewed preprocessor

tests/pre/pre

@@ -5,4 +5,4 @@
 void main() {
     WUSI WUSI BAUM WUSI
     IF BAUM }
-}
+}

tokenizer/File.c

@@ -5,157 +5,246 @@
 #include "tokenizer/File.h"
 #include "utils/Utils.h"
 
-#define MAX_DEFINES 50
+#define MAX_INDEX 50
 
-static FileError fileError = NULL;
-static FILE* file = NULL;
-static int readIndex = 0;
-static int writeIndex = 0;
-static int fileSize = 0;
-static char* fileContent = NULL;
+typedef struct {
+    char* content;
+    int size;
+    int index;
+    int arrayIndex;
+    FILE* file;
+} OpenFile;
+
+typedef struct {
+    int fileIndex;
+    int index;
+} FilePointer;
+
+typedef struct {
+    FilePointer name;
+    FilePointer code;
+} Define;
 
 typedef struct {
-    int nameLength;
-    const char* name;
-    const char* code;
-} Defines;
-
-static int defineIndex = 0;
-static Defines defines[MAX_DEFINES];
-static int defineReadIndex = 0;
-static const char* defineCode = NULL;
-
-static void fAdd(int c) {
-    if(writeIndex >= fileSize) {
-        fileSize *= 2;
-        fileContent = realloc(fileContent, fileSize);
+    int fileIndex;
+    int defineIndex;
+    OpenFile files[MAX_INDEX];
+    Define defines[MAX_INDEX];
+    FilePointer readIndex;
+    FileError error;
+} OpenFiles;
+
+static OpenFiles files;
+static int lastChar = 0;
+static int stackIndex = 0;
+static FilePointer stack[MAX_INDEX];
+
+static void fAdd(OpenFile* of, const void* data, int size) {
+    while(of->index + size > of->size) {
+        of->size *= 2;
+        of->content = realloc(of->content, of->size);
     }
-    fileContent[writeIndex++] = c;
+    memcpy(of->content + of->index, data, size);
+    of->index += size;
 }
 
-static void fReset() {
-    readIndex = 0;
-    writeIndex = 0;
-    fileSize = 16;
-    fileContent = malloc(fileSize);
-    defineIndex = 0;
-    defineReadIndex = 0;
-    defineCode = NULL;
+static void fAddChar(OpenFile* of, int ic) {
+    char c = ic;
+    fAdd(of, &c, 1);
 }
 
-void fOpen(const char* path, FileError fe) {
-    fileError = fe;
-    file = fopen(path, "r");
-    if(file == NULL) {
-        fileError("cannot read file '%s'", path);
-        return;
+static int fReserverInt(OpenFile* of) {
+    int address = of->index;
+    int empty = 0x40404040;
+    fAdd(of, &empty, sizeof(int));
+    return address;
+}
+
+static void fSetInt(OpenFile* of, int address, int i) {
+    memcpy(of->content + address, &i, sizeof(int));
+}
+
+static int fReadInt(OpenFile* of, int address) {
+    int i = 0;
+    memcpy(&i, of->content + address, sizeof(int));
+    of->index += sizeof(int);
+    return i;
+}
+
+static int fGet(OpenFile* of) {
+    return fgetc(of->file);
+}
+
+static void fReadCommandString(OpenFile* of, char* buffer, int size) {
+    int index = 0;
+    int c = fGet(of);
+    while(c == ' ') {
+        c = fGet(of);
     }
-    fReset();
     while(true) {
-        int c = fgetc(file);
-        if(c == '#') {
-            while(c != EOF && c != '\n') {
-                fAdd(c);
-                c = fgetc(file);
-            }
-            fAdd('\0');
-        }
-        fAdd(c);
         if(c == EOF) {
+            files.error("unexpected end of file");
+        } else if(index >= size - 1) {
+            files.error("unknown too long command");
+        } else if(isLetter(c)) {
+            buffer[index++] = c;
+        } else if(c == ' ') {
             break;
+        } else {
+            files.error("unexpected character '%c'", (char)c);
         }
+        c = fGet(of);
     }
-    fAdd('\0');
-    fclose(file);
+    buffer[index] = '\0';
 }
 
-void fClose() {
-    free(fileContent);
+static void fDefine(OpenFile* of) {
+    if(files.defineIndex >= MAX_INDEX) {
+        files.error("too many defines");
+    }
+    Define* d = files.defines + files.defineIndex++;
+
+    fAddChar(of, '#');
+    int end = fReserverInt(of);
+
+    char command[64];
+    fReadCommandString(of, command, 64);
+    d->name.fileIndex = of->arrayIndex;
+    d->name.index = of->index;
+    fAdd(of, command, strlen(command) + 1);
+
+    d->code.fileIndex = of->arrayIndex;
+    d->code.index = of->index;
+    while(true) {
+        int c = fGet(of);
+        if(c == EOF || c == '\n') {
+            break;
+        }
+        fAddChar(of, c);
+    }
+    fAddChar(of, '\0');
+    fSetInt(of, end, of->index);
+    fAddChar(of, '\n');
 }
 
-static void fSkipString(int skip) {
-    readIndex += skip;
-    while(fileContent[readIndex] == ' ') {
-        readIndex++;
+static void fReadCommand(OpenFile* of) {
+    char command[64];
+    fReadCommandString(of, command, 64);
+    if(strcmp(command, "define") == 0) {
+        fDefine(of);
+    } else {
+        files.error("invalid command '%s'", command);
     }
 }
 
-static void fDefine() {
-    fSkipString(7);
-    const char* name = fileContent + readIndex;
+static void fReadFile(const char* path) {
+    if(files.fileIndex >= MAX_INDEX) {
+        files.error("cannot read file '%s': too many open files", path);
+    }
+    OpenFile* of = files.files + files.fileIndex;
+    of->arrayIndex = files.fileIndex++;
+    of->index = 0;
+    of->size = 16;
+    of->content = malloc(of->size);
+    of->file = fopen(path, "r");
+    if(of->file == NULL) {
+        files.error("cannot read file '%s'", path);
+    }
     while(true) {
-        if(fileContent[readIndex] == ' ') {
-            fileContent[readIndex] = '\0';
+        int c = fGet(of);
+        if(c == '#') {
+            fReadCommand(of);
+            continue;
+        }
+        fAddChar(of, c);
+        if(c == EOF) {
             break;
-        } else if(!isLetter(fileContent[readIndex])) {
-            fileError("invalid define name '%s'", name);
         }
-        readIndex++;
     }
-    readIndex++;
-    if(defineIndex >= MAX_DEFINES) {
-        fileError("too many defines");
+    fAddChar(of, '\0');
+}
+
+void fOpen(const char* path, FileError fe) {
+    files.fileIndex = 0;
+    files.defineIndex = 0;
+    files.readIndex.fileIndex = 0;
+    files.readIndex.index = 0;
+    files.error = fe;
+    lastChar = 0;
+    stackIndex = 0;
+    fReadFile(path);
+}
+
+void fClose() {
+    for(int i = 0; i < files.fileIndex; i++) {
+        fclose(files.files[i].file);
+        free(files.files[i].content);
     }
-    defines[defineIndex].nameLength = strlen(name);
-    defines[defineIndex].name = name;
-    defines[defineIndex].code = fileContent + readIndex;
-    defineIndex++;
-    fSkipString(strlen(fileContent + readIndex));
 }
 
-static bool fMatch(Defines* def, const char* code) {
-    return strncmp(def->name, code, def->nameLength) == 0;
+static OpenFile* currentFile() {
+    return files.files + files.readIndex.fileIndex;
 }
 
-static bool fPrepareChar() {
-    if(defineCode != NULL) {
-        if(defineCode[defineReadIndex] == '\0') {
-            defineCode = NULL;
-            defineReadIndex = 0;
-        } else {
-            return true;
+static int fReadChar() {
+    return currentFile()->content[files.readIndex.index];
+}
+
+static const char* fGetDefineName(Define* d) {
+    return files.files[d->name.fileIndex].content + d->name.index;
+}
+
+static bool fCompare(const char* a, const char* b, unsigned int bLength) {
+    return strncmp(a, b, bLength) == 0 && strlen(a) == bLength;
+}
+
+static void fEnter(Define* d) {
+    if(stackIndex >= MAX_INDEX) {
+        files.error("define stack overflow");
+    }
+    stack[stackIndex++] = files.readIndex;
+    files.readIndex = d->code;
+}
+
+static void fPrepareChar() {
+    int c = fReadChar();
+    if(c == '\0') {
+        if(stackIndex <= 0) {
+            files.error("define stack underflow");
         }
+        files.readIndex = stack[--stackIndex];
     }
-    if(!isLetter(fileContent[readIndex])) {
-        defineReadIndex = 0;
+    if(c == '#') {
+        int end = fReadInt(currentFile(), files.readIndex.index + 1);
+        files.readIndex.index = end;
     }
-    if(isLetter(fileContent[readIndex]) && defineReadIndex == 0) {
-        const char* replace = fileContent + readIndex;
-        for(int i = 0; i < defineIndex; i++) {
-            if(fMatch(defines + i, replace)) {
-                defineCode = defines[i].code;
-                readIndex += defines[i].nameLength;
-                return true;
-            }
+    if(isLetter(c) && !isLetter(lastChar)) {
+        OpenFile* of = currentFile();
+        int start = files.readIndex.index;
+        int index = start;
+        while(index < of->index && isLetter(of->content[index])) {
+            index++;
         }
-        defineReadIndex = -1;
-    }
-    if(fileContent[readIndex] == '#') {
-        const char* command = fileContent + readIndex + 1;
-        if(strncmp(command, "define", 6) == 0) {
-            fDefine();
-        } else {
-            fileError("unknown preprocessor command '%s'", command);
+        for(int i = 0; i < files.defineIndex; i++) {
+            Define* d = files.defines + i;
+            const char* name = fGetDefineName(d);
+            if(fCompare(name, of->content + start, index - start)) {
+                files.readIndex.index = index;
+                fEnter(d);
+            }
         }
     }
-    if(fileContent[readIndex] == '\0') {
-        readIndex++;
-    }
-    return false;
 }
 
 int fRead() {
-    if(fPrepareChar()) {
-        return defineCode[defineReadIndex++];
-    }
-    return fileContent[readIndex++];
+    fPrepareChar();
+    lastChar = fReadChar();
+    return currentFile()->content[files.readIndex.index++];
 }
 
 int fPeek() {
-    if(fPrepareChar()) {
-        return defineCode[defineReadIndex];
-    }
-    return fileContent[readIndex];
+    fPrepareChar();
+    return fReadChar();
 }
 
 bool fReadIf(int c) {