multi line defines, replace defines, all old tests implemented

old_tests/pre/include

@@ -1,7 +0,0 @@
-#define WUSI
-    test(5);
-#end
-
-int test() {
-    return 5;
-}

old_tests/pre/list

@@ -1,50 +0,0 @@
-#define LIST 
-    struct LISTNAME {
-        int size;
-        LISTTYPE* data;     
-    };
-
-    LISTNAME* LISTNEW() {
-        LISTNAME* list = new LISTNAME[1];
-        list->size = 0;
-        list->data = new LISTTYPE[4];
-        return list;
-    }
-
-    void listAdd(LISTNAME* list, LISTTYPE i) {
-        int l = length(list->data);
-        if(list->size >= l) {
-            LISTTYPE* newData = new LISTTYPE[l * 2];
-            for(int i = 0; i < l; i++) {
-                newData[i] = list->data[i];
-            }
-            delete list->data;
-            list->data = newData;
-        }
-        list->data[list->size] = i;
-        list->size++;
-    }
-
-    int listGetLength(LISTNAME* list) {
-        return list->size;
-    }
-
-    LISTTYPE listGetIndex(LISTNAME* list, int index) {
-        return list->data[index];
-    }
-
-    void listDelete(LISTNAME* list) {
-        delete list->data;
-        delete list;
-    }
-#end
-
-#define LISTNAME IntList #end
-#define LISTTYPE int #end
-#define LISTNEW listNewInt #end
-LIST
-
-#define LISTNAME FloatList #end
-#define LISTTYPE float #end
-#define LISTNEW listNewFloat #end
-LIST

old_tests/pre/pre

@@ -1,27 +0,0 @@
-#include "include"
-
-#define BAUM 
-    test(6); 
-#end 
-
-#define GUSI 
-    WUSI WUSI 
-#end
-
-#define IF 
-    if(true) { 
-#end
-
-#define HUSI
-    test(1);
-    test(2);
-#end
-
-
-void main() {
-    WUSI GUSI BAUM WUSI
-    IF BAUM }
-    HUSI
-    HUSI
-    test(test());
-}

tests/includes/other

@@ -4,7 +4,7 @@
 #include "another"
 
 void wusi(int a) {
-    //test(a);
+    test(a);
 }
 
 #ifndef WUSI

tests/pre/include

@@ -0,0 +1,5 @@
+#define WUSI test(5);
+
+int test() {
+    return 5;
+}

tests/pre/list

@@ -0,0 +1,41 @@
+#define LIST                                                                   \
+    struct LISTNAME {                                                          \
+        int size;                                                              \
+        LISTTYPE[] data;                                                       \
+    };                                                                         \
+                                                                               \
+    void listNew(LISTNAME list) {                                              \
+        list.size = 0;                                                         \
+        list.data = new LISTTYPE[4];                                           \
+    }                                                                          \
+                                                                               \
+    void listAdd(LISTNAME list, LISTTYPE i) {                                  \
+        int l = length(list.data);                                             \
+        if(list.size >= l) {                                                   \
+            LISTTYPE[] newData = new LISTTYPE[l * 2];                          \
+            for(int k = 0; k < l; k++) {                                       \
+                newData[k] = list.data[k];                                     \
+            }                                                                  \
+            list.data = newData;                                               \
+        }                                                                      \
+        list.data[list.size] = i;                                              \
+        list.size++;                                                           \
+    }                                                                          \
+                                                                               \
+    int listGetLength(LISTNAME list) {                                         \
+        return list.size;                                                      \
+    }                                                                          \
+                                                                               \
+    LISTTYPE listGetIndex(LISTNAME list, int index) {                          \
+        return list.data[index];                                               \
+    }
+
+#define LISTNAME IntList
+#define LISTTYPE int
+LIST
+
+#undef LISTNAME
+#undef LISTTYPE
+#define LISTNAME FloatList
+#define LISTTYPE float
+LIST

old_tests/pre/list_test → tests/pre/list_test

@@ -1,8 +1,10 @@
 #include "list"
 
 void main() {
-    IntList* intList = listNewInt();
-    FloatList* floatList = listNewFloat();
+    IntList intList; 
+    listNew(intList);
+    FloatList floatList;
+    listNew(floatList);
 
     for(int i = 0; i < 5; i++) {
         listAdd(intList, i * 50 + 3);
@@ -16,8 +18,5 @@ void main() {
     for(int i = 0; i < listGetLength(floatList); i++) {
         test(listGetIndex(floatList, i));
     }
-    
-    listDelete(intList);
-    listDelete(floatList);
 }
 

tests/pre/pre

@@ -0,0 +1,17 @@
+#include "include"
+
+#define BAUM test(6);
+
+#define GUSI WUSI WUSI
+
+#define IF if(true) {
+
+#define HUSI test(1); test(2);
+
+void main() {
+    WUSI GUSI BAUM WUSI
+    IF BAUM }
+    HUSI
+    HUSI
+    test(test());
+}

tests/pre/replace

@@ -0,0 +1,17 @@
+#define MAIN main wusi
+#undef MAIN
+#define MAIN main
+#define V void
+#define N_W 3
+#define HMM V MAIN
+
+#define A B
+#define B A
+
+HMM() {
+    test(N_W);
+#undef N_W
+#define N_W 5
+    test(N_W);
+}
+

tests/pre/replace.out

@@ -0,0 +1,2 @@
+3
+5

tokenizer/FileTokens.c

@@ -13,6 +13,7 @@ typedef char String[64];
 
 typedef struct {
     String name;
+    FileTokens tokens;
 } Define;
 
 #define DEFINES 256
@@ -103,6 +104,13 @@ static FileToken* ftAddToken(FileTokens* ft, FileTokenType type) {
     return t;
 }
 
+static FileToken* ftAddCopyToken(FileTokens* ft, const FileToken* t) {
+    FileToken* added = ftAddToken(ft, t->type);
+    added->literal = t->literal == NULL ? NULL : strdup(t->literal);
+    added->single = t->single;
+    return added;
+}
+
 static void ftAddPath(FileTokens* ft, const char* path) {
     ftAddToken(ft, FT_PATH)->literal = strdup(path);
 }
@@ -155,10 +163,14 @@ static bool ftStringCompare(const char* a, const char* b) {
     return strcmp(a, b) == 0;
 }
 
-static void ftInitFileTokens(FileTokens* ft, const char* path) {
+static void ftInitFileTokens(FileTokens* ft) {
     ft->tokens = NULL;
     ft->capacity = 0;
     ft->length = 0;
+}
+
+static void ftInitFileTokensWithPath(FileTokens* ft, const char* path) {
+    ftInitFileTokens(ft);
     ftAddPath(ft, path);
 }
 
@@ -250,32 +262,32 @@ static void ftReadFullFile(FileTokens* parent, FileTokens* ft, int index) {
     }
 }
 
-static void ftReplace(int from, int to, FileTokens* ft,
-                      FileTokens* replacement) {
+static void ftReplace(int from, int toEx, FileTokens* ft,
+                      const FileTokens* replacement) {
     int newCapacity = ft->capacity + replacement->capacity;
     FileToken* newTokens = malloc(newCapacity * sizeof(FileToken));
 
     // these will be overwritten
-    for(int i = from; i < to; i++) {
+    for(int i = from; i < toEx; i++) {
         free(ft->tokens[i].literal);
         ft->tokens[i].literal = NULL;
     }
 
     memcpy(newTokens, ft->tokens, from * sizeof(FileToken));
-    memcpy(newTokens + from, replacement->tokens,
-           replacement->length * sizeof(FileToken));
-    memcpy(newTokens + from + replacement->length, ft->tokens + to,
-           (ft->length - to) * sizeof(FileToken));
+    for(int i = 0; i < replacement->length; i++) {
+        FileToken* src = replacement->tokens + i;
+        FileToken* dst = newTokens + from + i;
+        dst->literal = src->literal == NULL ? NULL : strdup(src->literal);
+        dst->single = src->single;
+        dst->type = src->type;
+    }
+    memcpy(newTokens + from + replacement->length, ft->tokens + toEx,
+           (ft->length - toEx) * sizeof(FileToken));
 
     free(ft->tokens);
     ft->tokens = newTokens;
     ft->capacity = newCapacity;
-    ft->length = from + replacement->length + ft->length - to;
-
-    // ownership moved
-    for(int i = 0; i < replacement->length; i++) {
-        replacement->tokens[i].literal = NULL;
-    }
+    ft->length = from + replacement->length + ft->length - toEx;
 }
 
 static void ftSkipSpaces(int* index, FileTokens* ft) {
@@ -326,7 +338,7 @@ static void ftHandleInclude(int* index, int start, FileTokens* ft) {
     memcpy(fullPath + afterLastSlash, path, pathLength);
 
     FileTokens include;
-    ftInitFileTokens(&include, fullPath);
+    ftInitFileTokensWithPath(&include, fullPath);
     free(fullPath);
 
     ftReadFullFile(ft, &include, *index - 1);
@@ -337,13 +349,17 @@ static void ftHandleInclude(int* index, int start, FileTokens* ft) {
     ftDelete(&include);
 }
 
-static bool ftIsDefined(const char* s) {
+static int ftGetDefineIndex(const char* s) {
     for(int i = 0; i < defineIndex; i++) {
         if(ftStringCompare(defines[i].name, s)) {
-            return true;
+            return i;
         }
     }
-    return false;
+    return -1;
+}
+
+static bool ftIsDefined(const char* s) {
+    return ftGetDefineIndex(s) != -1;
 }
 
 static int ftFindEndIf(int from, FileTokens* ft) {
@@ -402,16 +418,79 @@ static void ftHandleDefine(int* index, int start, FileTokens* ft) {
         ftError(ft, *index - 1, "define expects a literal");
         return;
     }
-    const char* define = ft->tokens[(*index)++].literal;
-    if(ftIsDefined(define)) {
+    const char* defineName = ft->tokens[(*index)++].literal;
+
+    ftSkipSpaces(index, ft);
+    int defineStart = *index;
+    bool skip = false;
+    while(*index < ft->length) {
+        FileToken* t = ft->tokens + *index;
+        if(!skip && t->type == FT_SINGLE && t->single == '\\') {
+            skip = true;
+            ftToSpace(t);
+        } else if(t->type == FT_NEWLINE) {
+            if(skip) {
+                skip = false;
+            } else {
+                break;
+            }
+        } else if(skip) {
+            ftError(ft, *index - 1, "expected newline after \\");
+            return;
+        }
+        (*index)++;
+    }
+
+    int foundIndex = ftGetDefineIndex(defineName);
+    if(foundIndex != -1) {
+        ftError(ft, *index - 1, "'%s' redefined", defineName);
+        return;
+    }
+    Define* define = NULL;
+    if(foundIndex == -1) {
+        define = defines + defineIndex++;
+        strncpy(define->name, defineName, STRING_LENGTH - 1);
+        define->name[STRING_LENGTH - 1] = '\0';
+    } else {
+        define = defines + foundIndex;
+        ftDelete(&define->tokens);
+    }
+
+    ftInitFileTokens(&define->tokens);
+    for(int i = defineStart; i < *index; i++) {
+        ftAddCopyToken(&define->tokens, ft->tokens + i);
+        if(ft->tokens[i].type == FT_NEWLINE) {
+            FileToken* t = define->tokens.tokens + (define->tokens.length - 1);
+            ftToSpace(t);
+            t->type = FT_SPACE;
+        }
+    }
+
+    for(int i = start; i < *index; i++) {
+        ftToSpace(ft->tokens + i);
+    }
+}
+
+static void ftHandleUndefine(int* index, int start, FileTokens* ft) {
+    ftSkipSpaces(index, ft);
+    if(*index >= ft->length || ft->tokens[*index].type != FT_LITERAL) {
+        ftError(ft, *index - 1, "undefine expects a literal");
         return;
     }
-    strncpy(defines[defineIndex].name, define, STRING_LENGTH);
-    defineIndex++;
+    const char* defineName = ft->tokens[(*index)++].literal;
 
-    ftToSpace(ft->tokens + start);        // #
-    ftToSpace(ft->tokens + start + 1);    // define
-    ftToSpace(ft->tokens + (*index - 1)); // DEFINITION
+    int foundIndex = ftGetDefineIndex(defineName);
+    if(foundIndex != -1) {
+        Define* define = defines + foundIndex;
+        ftDelete(&define->tokens);
+        defineIndex--;
+        if(defineIndex != foundIndex) {
+            memcpy(define, defines + defineIndex, sizeof(Define));
+        }
+    }
+    for(int i = start; i < *index; i++) {
+        ftToSpace(ft->tokens + i);
+    }
 }
 
 static void ftParseInstruction(int* index, FileTokens* ft) {
@@ -428,6 +507,8 @@ static void ftParseInstruction(int* index, FileTokens* ft) {
         ftHandleIfNotDefined(index, start, ft);
     } else if(ftStringCompare(name, "define")) {
         ftHandleDefine(index, start, ft);
+    } else if(ftStringCompare(name, "undef")) {
+        ftHandleUndefine(index, start, ft);
     } else if(ftStringCompare(name, "endif")) {
         ftError(ft, *index - 1, "endif without if");
     } else {
@@ -435,10 +516,24 @@ static void ftParseInstruction(int* index, FileTokens* ft) {
     }
 }
 
+static bool ftReplaceMakros(FileTokens* ft, int index) {
+    FileToken* t = ft->tokens + index;
+    int foundIndex = ftGetDefineIndex(t->literal);
+    if(foundIndex != -1) {
+        ftReplace(index, index + 1, ft, &(defines[foundIndex].tokens));
+        return true;
+    }
+    return false;
+}
+
 static void ftSearchInstruction(FileTokens* ft) {
     for(int i = 0; i < ft->length && !ftHasError(); i++) {
         if(ft->tokens[i].type == FT_SINGLE && ft->tokens[i].single == '#') {
             ftParseInstruction(&i, ft);
+        } else if(ft->tokens[i].type == FT_LITERAL) {
+            if(ftReplaceMakros(ft, i)) {
+                i--;
+            }
         }
     }
 }
@@ -447,11 +542,14 @@ void ftInit(const char* path, FileTokens* ft, Error* e) {
     defineIndex = 0;
     error = e;
     eInitSuccess(error);
-    ftInitFileTokens(ft, path);
+    ftInitFileTokensWithPath(ft, path);
     ftReadFullFile(ft, ft, 0);
     if(!ftHasError()) {
         ftSearchInstruction(ft);
     }
+    for(int i = 0; i < defineIndex; i++) {
+        ftDelete(&(defines[i].tokens));
+    }
 }
 
 void ftDelete(FileTokens* ft) {

tokenizer/Tokenizer.c

@@ -390,6 +390,9 @@ void tTokenize(const char* path, Error* e) {
     if(!eHasError(e) && !setjmp(errorJump)) {
         tParseFile();
     }
+    if(eHasError(e)) {
+        ftPrint(&fileTokens);
+    }
     ftDelete(&fileTokens);
 }