lonely-tiger/tokenizer/FileTokens.c

#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "tokenizer/FileTokens.h"
#include "utils/SnuviUtils.h"

static Error* error;

typedef char String[64];
#define STRING_LENGTH ((int)sizeof(String))

typedef struct {
    String name;
    FileTokens tokens;
} Define;

#define DEFINES 256
static Define defines[DEFINES];
static int defineIndex = 0;

static int ftGetLine(const FileTokens* ft, int index) {
    int fileLayer = 0;
    int lines = 1;
    for(int i = index; i >= 0; i--) {
        FileToken* t = ft->tokens + i;
        if(t->type == FT_PATH) {
            if(fileLayer == 0) {
                return lines;
            }
            fileLayer--;
        } else if(t->type == FT_END_PATH) {
            fileLayer++;
        } else if(t->type == FT_NEWLINE) {
            lines += fileLayer == 0;
        }
    }
    return 0;
}

static const char* ftGetPath(const FileTokens* ft, int index) {
    int fileLayer = 0;
    for(int i = index; i >= 0; i--) {
        FileToken* t = ft->tokens + i;
        if(t->type == FT_PATH) {
            if(fileLayer == 0) {
                return t->literal;
            }
            fileLayer--;
        } else if(t->type == FT_END_PATH) {
            fileLayer++;
        }
    }
    return "unknown path";
}

static void ftError(const FileTokens* ft, int index, const char* format, ...) {
    va_list args;
    va_start(args, format);
    eInitErrorV(error, "", ftGetLine(ft, index), format, args);

    int fileLayer = 0;
    for(int i = index; i >= 0; i--) {
        FileToken* t = ft->tokens + i;
        if(t->type == FT_PATH) {
            if(fileLayer == 0) {
                eAddPath(error, t->literal);
            } else {
                fileLayer--;
            }
        } else if(t->type == FT_END_PATH) {
            fileLayer++;
        }
    }

    va_end(args);
}

static void ftSystemError(const char* msg, const FileTokens* parent,
                          const FileTokens* ft, int index) {
    ftError(parent, index, "%s '%s': %s", msg, ftGetPath(ft, 0),
            strerror(errno));
}

static void ftToSpace(FileToken* t) {
    free(t->literal);
    if(t->type != FT_NEWLINE) {
        t->type = FT_SPACE;
    }
    t->literal = NULL;
    t->single = 0;
}

static FileToken* ftAddToken(FileTokens* ft, FileTokenType type) {
    if(ft->length >= ft->capacity) {
        ft->capacity = (ft->capacity * 5) / 4 + 8;
        ft->tokens = realloc(ft->tokens, ft->capacity * sizeof(FileToken));
    }
    FileToken* t = ft->tokens + ft->length++;
    t->type = type;
    t->literal = NULL;
    t->single = 0;
    return t;
}

static FileToken* ftAddCopyToken(FileTokens* ft, const FileToken* t) {
    FileToken* added = ftAddToken(ft, t->type);
    added->literal = t->literal == NULL ? NULL : strdup(t->literal);
    added->single = t->single;
    return added;
}

static void ftAddPath(FileTokens* ft, const char* path) {
    ftAddToken(ft, FT_PATH)->literal = strdup(path);
}

static void ftAddEndPath(FileTokens* ft) {
    ftAddToken(ft, FT_END_PATH);
}

static void ftAddNewline(FileTokens* ft) {
    ftAddToken(ft, FT_NEWLINE);
}

static void ftAddLiteral(FileTokens* ft, const char* path, int length) {
    ftAddToken(ft, FT_LITERAL)->literal = strndup(path, length);
}

static void ftAddSingle(FileTokens* ft, int single) {
    ftAddToken(ft, FT_SINGLE)->single = single;
}

static void ftAddSpace(FileTokens* ft) {
    ftAddToken(ft, FT_SPACE);
}

void ftPrint(FileTokens* ft) {
    for(int i = 0; i < ft->length; i++) {
        switch(ft->tokens[i].type) {
            case FT_PATH:
                printf("\033[0;34mPath: %s\n", ft->tokens[i].literal);
                break;
            case FT_END_PATH: puts("\033[0;34mEnd Path"); break;
            case FT_NEWLINE: putchar('\n'); break;
            case FT_LITERAL:
                printf("\033[0;33m%s", ft->tokens[i].literal);
                break;
            case FT_SINGLE:
                printf("\033[0;31m%c", (char)ft->tokens[i].single);
                break;
            case FT_SPACE: putchar(' '); break;
        }
    }
    printf("\033[0m");
}

static bool ftHasError() {
    return eHasError(error);
}

static bool ftStringCompare(const char* a, const char* b) {
    return strcmp(a, b) == 0;
}

static void ftInitFileTokens(FileTokens* ft) {
    ft->tokens = NULL;
    ft->capacity = 0;
    ft->length = 0;
}

static void ftInitFileTokensWithPath(FileTokens* ft, const char* path) {
    ftInitFileTokens(ft);
    ftAddPath(ft, path);
}

static void ftTokenize(FileTokens* ft, unsigned char* data) {
    int index = 0;
    int oldIndex = 0;
    bool inString = false;
    while(true) {
        unsigned char c = data[index];
        if(isAllowedInName(c) || (inString && c != '"' && c != '\0')) {
            index++;
            continue;
        }
        if(oldIndex < index) {
            ftAddLiteral(ft, (const char*)(data + oldIndex), index - oldIndex);
        }
        if(c == '"') {
            inString = !inString;
        }
        if(c == '\0') {
            break;
        } else if(c == '\n') {
            ftAddNewline(ft);
        } else if(c == ' ') {
            ftAddSpace(ft);
        } else if(c == '/' && data[index + 1] == '/') {
            while(data[index] != '\0' && data[index] != '\n') {
                index++;
            }
            ftAddNewline(ft);
        } else if(c == '/' && data[index + 1] == '*') {
            int startIndex = ft->length - 1;
            index += 2;
            while(true) {
                if(data[index] == '\0') {
                    ftError(ft, startIndex, "test");
                    return;
                } else if(data[index] == '\n') {
                    ftAddNewline(ft);
                } else if(data[index] == '*' && data[index + 1] == '/') {
                    index++;
                    break;
                }
                index++;
            }
        } else {
            ftAddSingle(ft, c);
        }
        index++;
        oldIndex = index;
    }
}

static void ftReadFull(FILE* file, FileTokens* parent, FileTokens* ft,
                       int index) {
    if(fseek(file, 0, SEEK_END)) {
        ftSystemError("cannot seek end of file", parent, ft, index);
        return;
    }
    long length = ftell(file);
    if(length < 0) {
        ftSystemError("cannot tell end of file", parent, ft, index);
        return;
    }
    rewind(file);

    unsigned char* data = malloc(length + 1);
    int readValues = fread(data, 1, length, file);
    if(readValues != length) {
        ftSystemError("cannot read file", parent, ft, index);
        free(data);
        return;
    }
    data[length] = '\0';
    ftTokenize(ft, data);
    ftAddEndPath(ft);
    free(data);
}

static void ftReadFullFile(FileTokens* parent, FileTokens* ft, int index) {
    FILE* file = fopen(ftGetPath(ft, 0), "r");
    if(file == NULL) {
        ftSystemError("cannot open file", parent, ft, index);
        return;
    }
    ftReadFull(file, parent, ft, index);
    if(fclose(file) < 0) {
        ftSystemError("cannot close file", parent, ft, index);
    }
}

static void ftReplace(int from, int toEx, FileTokens* ft,
                      const FileTokens* replacement) {
    int newCapacity = ft->capacity + replacement->capacity;
    FileToken* newTokens = malloc(newCapacity * sizeof(FileToken));

    // these will be overwritten
    for(int i = from; i < toEx; i++) {
        free(ft->tokens[i].literal);
        ft->tokens[i].literal = NULL;
    }

    memcpy(newTokens, ft->tokens, from * sizeof(FileToken));
    for(int i = 0; i < replacement->length; i++) {
        FileToken* src = replacement->tokens + i;
        FileToken* dst = newTokens + from + i;
        dst->literal = src->literal == NULL ? NULL : strdup(src->literal);
        dst->single = src->single;
        dst->type = src->type;
    }
    memcpy(newTokens + from + replacement->length, ft->tokens + toEx,
           (ft->length - toEx) * sizeof(FileToken));

    free(ft->tokens);
    ft->tokens = newTokens;
    ft->capacity = newCapacity;
    ft->length = from + replacement->length + ft->length - toEx;
}

static void ftSkipSpaces(int* index, FileTokens* ft) {
    while(*index < ft->length) {
        FileToken* t = ft->tokens + *index;
        if(t->type != FT_SPACE) {
            return;
        }
        (*index)++;
    }
}

const char* INCLUDE_ERROR = "include path must be a string";

static bool ftIncludeDoubleQuote(int* index, FileTokens* ft) {
    if(*index >= ft->length || ft->tokens[*index].type != FT_SINGLE ||
       ft->tokens[*index].single != '"') {
        ftError(ft, *index - 1, INCLUDE_ERROR);
        return true;
    }
    (*index)++;
    return false;
}

static void ftHandleInclude(int* index, int start, FileTokens* ft) {
    ftSkipSpaces(index, ft);
    if(ftIncludeDoubleQuote(index, ft)) {
        return;
    }
    if(*index >= ft->length || ft->tokens[*index].type != FT_LITERAL) {
        ftError(ft, *index - 1, INCLUDE_ERROR);
        return;
    }
    const char* path = ft->tokens[(*index)++].literal;
    if(ftIncludeDoubleQuote(index, ft)) {
        return;
    }
    const char* parentPath = ftGetPath(ft, *index - 1);

    int afterLastSlash = 0;
    for(int i = 0; parentPath[i] != '\0'; i++) {
        afterLastSlash = parentPath[i] == '/' ? i + 1 : afterLastSlash;
    }

    int pathLength = strlen(path) + 1;
    char* fullPath = malloc(afterLastSlash + pathLength);
    memcpy(fullPath, parentPath, afterLastSlash);
    memcpy(fullPath + afterLastSlash, path, pathLength);

    FileTokens include;
    ftInitFileTokensWithPath(&include, fullPath);
    free(fullPath);

    ftReadFullFile(ft, &include, *index - 1);
    if(!ftHasError()) {
        ftReplace(start, *index, ft, &include);
        *index = start;
    }
    ftDelete(&include);
}

static int ftGetDefineIndex(const char* s) {
    for(int i = 0; i < defineIndex; i++) {
        if(ftStringCompare(defines[i].name, s)) {
            return i;
        }
    }
    return -1;
}

static bool ftIsDefined(const char* s) {
    return ftGetDefineIndex(s) != -1;
}

static int ftFindEndIf(int from, FileTokens* ft) {
    int layer = 0;
    for(int i = from; i < ft->length - 1; i++) {
        if(ft->tokens[i].type != FT_SINGLE || ft->tokens[i].single != '#') {
            continue;
        } else if(ft->tokens[i + 1].type != FT_LITERAL) {
            continue;
        }
        if(ftStringCompare(ft->tokens[i + 1].literal, "ifndef")) {
            layer++;
        } else if(ftStringCompare(ft->tokens[i + 1].literal, "endif")) {
            if(layer == 0) {
                return i;
            }
            layer--;
        }
    }
    return -1;
}

static void ftHandleIfNotDefined(int* index, int start, FileTokens* ft) {
    ftSkipSpaces(index, ft);
    if(*index >= ft->length || ft->tokens[*index].type != FT_LITERAL) {
        ftError(ft, start, "ifndef expects a literal");
        return;
    }
    const char* check = ft->tokens[(*index)++].literal;
    int endIf = ftFindEndIf(*index, ft);
    if(endIf == -1) {
        ftError(ft, start, "cannot find #endif");
        return;
    }
    if(ftIsDefined(check)) {
        for(int i = start; i < endIf + 2; i++) {
            ftToSpace(ft->tokens + i);
        }
    } else {
        ftToSpace(ft->tokens + start);        // #
        ftToSpace(ft->tokens + start + 1);    // ifndef
        ftToSpace(ft->tokens + (*index - 1)); // DEFINITION

        ftToSpace(ft->tokens + endIf);     // #
        ftToSpace(ft->tokens + endIf + 1); // endif
    }
}

static void ftHandleDefine(int* index, int start, FileTokens* ft) {
    if(defineIndex >= DEFINES) {
        ftError(ft, *index - 1, "too much defines");
        return;
    }
    ftSkipSpaces(index, ft);
    if(*index >= ft->length || ft->tokens[*index].type != FT_LITERAL) {
        ftError(ft, *index - 1, "define expects a literal");
        return;
    }
    const char* defineName = ft->tokens[(*index)++].literal;

    ftSkipSpaces(index, ft);
    int defineStart = *index;
    bool skip = false;
    while(*index < ft->length) {
        FileToken* t = ft->tokens + *index;
        if(!skip && t->type == FT_SINGLE && t->single == '\\') {
            skip = true;
            ftToSpace(t);
        } else if(t->type == FT_NEWLINE) {
            if(skip) {
                skip = false;
            } else {
                break;
            }
        } else if(skip) {
            ftError(ft, *index - 1, "expected newline after \\");
            return;
        }
        (*index)++;
    }

    int foundIndex = ftGetDefineIndex(defineName);
    if(foundIndex != -1) {
        ftError(ft, *index - 1, "'%s' redefined", defineName);
        return;
    }
    Define* define = NULL;
    if(foundIndex == -1) {
        define = defines + defineIndex++;
        strncpy(define->name, defineName, STRING_LENGTH - 1);
        define->name[STRING_LENGTH - 1] = '\0';
    } else {
        define = defines + foundIndex;
        ftDelete(&define->tokens);
    }

    ftInitFileTokens(&define->tokens);
    for(int i = defineStart; i < *index; i++) {
        ftAddCopyToken(&define->tokens, ft->tokens + i);
        if(ft->tokens[i].type == FT_NEWLINE) {
            FileToken* t = define->tokens.tokens + (define->tokens.length - 1);
            ftToSpace(t);
            t->type = FT_SPACE;
        }
    }

    for(int i = start; i < *index; i++) {
        ftToSpace(ft->tokens + i);
    }
}

static void ftHandleUndefine(int* index, int start, FileTokens* ft) {
    ftSkipSpaces(index, ft);
    if(*index >= ft->length || ft->tokens[*index].type != FT_LITERAL) {
        ftError(ft, *index - 1, "undefine expects a literal");
        return;
    }
    const char* defineName = ft->tokens[(*index)++].literal;

    int foundIndex = ftGetDefineIndex(defineName);
    if(foundIndex != -1) {
        Define* define = defines + foundIndex;
        ftDelete(&define->tokens);
        defineIndex--;
        if(defineIndex != foundIndex) {
            memcpy(define, defines + defineIndex, sizeof(Define));
        }
    }
    for(int i = start; i < *index; i++) {
        ftToSpace(ft->tokens + i);
    }
}

static void ftParseInstruction(int* index, FileTokens* ft) {
    int start = *index;
    (*index)++;
    if(*index >= ft->length || ft->tokens[*index].type != FT_LITERAL) {
        ftError(ft, *index - 1, "expected literal after #");
        return;
    }
    const char* name = ft->tokens[(*index)++].literal;
    if(ftStringCompare(name, "include")) {
        ftHandleInclude(index, start, ft);
    } else if(ftStringCompare(name, "ifndef")) {
        ftHandleIfNotDefined(index, start, ft);
    } else if(ftStringCompare(name, "define")) {
        ftHandleDefine(index, start, ft);
    } else if(ftStringCompare(name, "undef")) {
        ftHandleUndefine(index, start, ft);
    } else if(ftStringCompare(name, "endif")) {
        ftError(ft, *index - 1, "endif without if");
    } else {
        ftError(ft, *index - 1, "unknown preprocessor literal '%s'", name);
    }
}

static bool ftReplaceMakros(FileTokens* ft, int index) {
    FileToken* t = ft->tokens + index;
    int foundIndex = ftGetDefineIndex(t->literal);
    if(foundIndex != -1) {
        ftReplace(index, index + 1, ft, &(defines[foundIndex].tokens));
        return true;
    }
    return false;
}

static void ftSearchInstruction(FileTokens* ft) {
    for(int i = 0; i < ft->length && !ftHasError(); i++) {
        if(ft->tokens[i].type == FT_SINGLE && ft->tokens[i].single == '#') {
            ftParseInstruction(&i, ft);
        } else if(ft->tokens[i].type == FT_LITERAL) {
            if(ftReplaceMakros(ft, i)) {
                i--;
            }
        }
    }
}

void ftInit(const char* path, FileTokens* ft, Error* e) {
    defineIndex = 0;
    error = e;
    eInitSuccess(error);
    ftInitFileTokensWithPath(ft, path);
    ftReadFullFile(ft, ft, 0);
    if(!ftHasError()) {
        ftSearchInstruction(ft);
    }
    for(int i = 0; i < defineIndex; i++) {
        ftDelete(&(defines[i].tokens));
    }
}

void ftDelete(FileTokens* ft) {
    for(int i = 0; i < ft->length; i++) {
        free(ft->tokens[i].literal);
    }
    free(ft->tokens);
    ft->tokens = NULL;
    ft->capacity = 0;
    ft->length = 0;
}