file include, defines always need an end

tests/pre/include

@@ -0,0 +1,7 @@
+#define WUSI
+    print 5;
+#end
+
+int test() {
+    return 5;
+}

tests/pre/pre

@@ -1,16 +1,27 @@
-#define WUSI print 5;
-#define BAUM print 6;
-#define GUSI WUSI WUSI
-#define IF if(true) {
+#include "include"
 
-#start HUSI
+#define BAUM 
+    print 6; 
+#end 
+
+#define GUSI 
+    WUSI WUSI 
+#end
+
+#define IF 
+    if(true) { 
+#end
+
+#define HUSI
     print 1;
     print 2;
 #end
 
+
 void main() {
     WUSI GUSI BAUM WUSI
     IF BAUM }
-    HUSI 
     HUSI
+    HUSI
+    print test(); a
 }

tests/pre/pre.out

@@ -8,3 +8,4 @@
 2
 1
 2
+5

tokenizer/File.c

@@ -12,6 +12,8 @@ typedef struct {
     int size;
     int index;
     int arrayIndex;
+    bool newLines;
+    char* path;
     FILE* file;
 } OpenFile;
 
@@ -55,7 +57,7 @@ static void fAddChar(OpenFile* of, int ic) {
 
 static int fReserverInt(OpenFile* of) {
     int address = of->index;
-    int empty = 0x40404040;
+    int empty = 0;
     fAdd(of, &empty, sizeof(int));
     return address;
 }
@@ -72,37 +74,45 @@ static int fReadInt(OpenFile* of, int address) {
 }
 
 static int fGet(OpenFile* of) {
-    return fgetc(of->file);
+    int c = fgetc(of->file);
+    if(!of->newLines && c == '\n') {
+        return ' ';
+    }
+    return c;
 }
 
-static void fReadCommandString(OpenFile* of, char* buffer, int size) {
+static bool fReadCommandString(OpenFile* of, char* buffer, int size) {
     int index = 0;
     int c = fGet(of);
     while(c == ' ') {
         c = fGet(of);
     }
+    int text = 0;
     while(true) {
         if(c == EOF) {
             files.error("unexpected end of file");
         } else if(index >= size - 1) {
             files.error("unknown too long command");
-        } else if(isLetter(c)) {
+        } else if(c == '"' && text < 2) {
+            text++;
+            buffer[index++] = c;
+        } else if((isLetter(c) || text == 1) && text < 2) {
             buffer[index++] = c;
         } else if(c == ' ' || c == '\n') {
-            break;
+            buffer[index] = '\0';
+            return c == '\n';
         } else {
             files.error("unexpected character '%c'", (char)c);
         }
         c = fGet(of);
     }
-    buffer[index] = '\0';
 }
 
 static const char* fGetDefineName(Define* d) {
     return files.files[d->name.fileIndex].content + d->name.index;
 }
 
-static int fStartDefine(OpenFile* of) {
+static int fStartDefine(OpenFile* of, bool* newLine) {
     if(files.defineIndex >= MAX_INDEX) {
         files.error("too many defines");
     }
@@ -112,7 +122,7 @@ static int fStartDefine(OpenFile* of) {
     int end = fReserverInt(of);
 
     char command[64];
-    fReadCommandString(of, command, 64);
+    *newLine = fReadCommandString(of, command, 64);
     d->name.fileIndex = of->arrayIndex;
     d->name.index = of->index;
     fAdd(of, command, strlen(command) + 1);
@@ -137,33 +147,20 @@ static void fFinishDefine(OpenFile* of, int end, int newLines) {
 }
 
 static void fDefine(OpenFile* of) {
-    int end = fStartDefine(of);
-    while(true) {
-        int c = fGet(of);
-        if(c == '#') {
-            files.error("# inside define");
-        } else if(c == EOF || c == '\n') {
-            break;
-        }
-        fAddChar(of, c);
-    }
-    fFinishDefine(of, end, 1);
-}
-
-static void fStart(OpenFile* of) {
-    int end = fStartDefine(of);
-    int newLines = 2;
+    bool newLine = false;
+    int end = fStartDefine(of, &newLine);
+    int newLines = newLine;
     while(true) {
         int c = fGet(of);
         if(c == '#') {
             char end[64];
-            fReadCommandString(of, end, 64);
+            newLines += fReadCommandString(of, end, 64);
             if(strcmp(end, "end") == 0) {
                 break;
             }
             files.error("invalid command '%s' inside define region", end);
         } else if(c == EOF) {
-            files.error("unclosed #start");
+            files.error("unclosed #define");
         }
         if(c == '\n') {
             newLines++;
@@ -174,19 +171,55 @@ static void fStart(OpenFile* of) {
     fFinishDefine(of, end, newLines);
 }
 
+static void fReadFile(const char* path, bool newLines);
+static void fEnter(FilePointer* fp);
+
+static int fConcat(char* buffer, int size, const char* s) {
+    int l = strlen(s) + 1;
+    if(l > size) {
+        files.error("too long include path");
+    }
+    memcpy(buffer, s, l);
+    return l;
+}
+
+static void fInclude(OpenFile* of) {
+    char path[64];
+    if(fReadCommandString(of, path, 64)) {
+        fAddChar(of, '\n');
+    }
+    int l = strlen(path);
+    if(l < 2 || path[0] != '"' || path[l - 1] != '"') {
+        files.error("invalid include path '%s'", path);
+    }
+    path[l - 1] = '\0';
+    const char* cutPath = path + 1;
+
+    char fullName[256];
+    int index = fConcat(fullName, 256, of->path);
+    while(index > 0 && fullName[index - 1] != '/') {
+        index--;
+    }
+    fConcat(fullName + index, 256 - index, cutPath);
+
+    FilePointer fp = {files.fileIndex, 0};
+    fEnter(&fp);
+    fReadFile(fullName, false);
+}
+
 static void fReadCommand(OpenFile* of) {
     char command[64];
     fReadCommandString(of, command, 64);
     if(strcmp(command, "define") == 0) {
         fDefine(of);
-    } else if(strcmp(command, "start") == 0) {
-        fStart(of);
+    } else if(strcmp(command, "include") == 0) {
+        fInclude(of);
     } else {
         files.error("invalid command '%s'", command);
     }
 }
 
-static void fReadFile(const char* path) {
+static void fReadFile(const char* path, bool newLines) {
     if(files.fileIndex >= MAX_INDEX) {
         files.error("cannot read file '%s': too many open files", path);
     }
@@ -194,8 +227,10 @@ static void fReadFile(const char* path) {
     of->arrayIndex = files.fileIndex++;
     of->index = 0;
     of->size = 16;
+    of->newLines = newLines;
     of->content = malloc(of->size);
     of->file = fopen(path, "r");
+    of->path = strdup(path);
     if(of->file == NULL) {
         files.error("cannot read file '%s'", path);
     }
@@ -221,7 +256,7 @@ void fOpen(const char* path, FileError fe) {
     files.error = fe;
     lastChar = 0;
     stackIndex = 0;
-    fReadFile(path);
+    fReadFile(path, true);
 }
 
 void fClose() {
@@ -229,6 +264,7 @@ void fClose() {
         if(files.files[i].file != NULL) {
             fclose(files.files[i].file);
         }
+        free(files.files[i].path);
         free(files.files[i].content);
     }
 }
@@ -245,12 +281,12 @@ static bool fCompare(const char* a, const char* b, unsigned int bLength) {
     return strncmp(a, b, bLength) == 0 && strlen(a) == bLength;
 }
 
-static void fEnter(Define* d) {
+static void fEnter(FilePointer* fp) {
     if(stackIndex >= MAX_INDEX) {
         files.error("define stack overflow");
     }
     stack[stackIndex++] = files.readIndex;
-    files.readIndex = d->code;
+    files.readIndex = *fp;
 }
 
 static bool fCheckForReplacement() {
@@ -265,7 +301,7 @@ static bool fCheckForReplacement() {
         const char* name = fGetDefineName(d);
         if(fCompare(name, of->content + start, index - start)) {
             files.readIndex.index = index;
-            fEnter(d);
+            fEnter(&d->code);
             return true;
         }
     }
@@ -280,6 +316,8 @@ static void fPrepareChar() {
                 files.error("define stack underflow");
             }
             files.readIndex = stack[--stackIndex];
+        } else if(c == EOF && stackIndex > 0) {
+            files.readIndex = stack[--stackIndex];
         } else if(c == '#') {
             files.readIndex.index =
                 fReadInt(currentFile(), files.readIndex.index + 1);