lonely-tiger/tokenizer/FileTokens.c

#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "tokenizer/FileTokens.h"
#include "utils/SnuviUtils.h"

static Error* error;
static int line = 0;

typedef char String[64];
#define STRING_LENGTH ((int)sizeof(String))

typedef struct {
    String name;
} Define;

#define DEFINES 256
static Define defines[DEFINES];
static int defineIndex = 0;

static const char* ftGetPath(const FileTokens* ft) {
    if(ft->length < 1 || ft->tokens[0].type != FT_PATH) {
        eInitError(error, line, "path not set");
        return "";
    }
    return ft->tokens[0].literal;
}

static void ftSystemError(const char* msg, const FileTokens* ft) {
    eInitError(error, line, "%s '%s': %s", msg, ftGetPath(ft), strerror(errno));
}

static void ftToSpace(FileToken* t) {
    free(t->literal);
    if(t->type != FT_NEWLINE) {
        t->type = FT_SPACE;
    }
    t->literal = NULL;
    t->single = 0;
}

static FileToken* ftAddToken(FileTokens* ft, FileTokenType type) {
    if(ft->length >= ft->capacity) {
        ft->capacity = (ft->capacity * 5) / 4 + 8;
        ft->tokens = realloc(ft->tokens, ft->capacity * sizeof(FileToken));
    }
    FileToken* t = ft->tokens + ft->length++;
    t->type = type;
    t->literal = NULL;
    t->single = 0;
    return t;
}

static void ftAddPath(FileTokens* ft, const char* path) {
    ftAddToken(ft, FT_PATH)->literal = strdup(path);
}

static void ftAddEndPath(FileTokens* ft) {
    ftAddToken(ft, FT_END_PATH);
}

static void ftAddNewline(FileTokens* ft) {
    ftAddToken(ft, FT_NEWLINE);
}

static void ftAddLiteral(FileTokens* ft, const char* path, int length) {
    ftAddToken(ft, FT_LITERAL)->literal = strndup(path, length);
}

static void ftAddSingle(FileTokens* ft, int single) {
    ftAddToken(ft, FT_SINGLE)->single = single;
}

static void ftAddSpace(FileTokens* ft) {
    ftAddToken(ft, FT_SPACE);
}

static void ftPrint(FileTokens* ft) {
    for(int i = 0; i < ft->length; i++) {
        switch(ft->tokens[i].type) {
            case FT_PATH:
                printf("\033[0;34mPath: %s\n", ft->tokens[i].literal);
                break;
            case FT_END_PATH: puts("\033[0;34mEnd Path"); break;
            case FT_NEWLINE: putchar('\n'); break;
            case FT_LITERAL:
                printf("\033[0;33m%s", ft->tokens[i].literal);
                break;
            case FT_SINGLE:
                printf("\033[0;31m%c", (char)ft->tokens[i].single);
                break;
            case FT_SPACE: putchar(' '); break;
        }
    }
    printf("\033[0m");
}

static bool ftHasError() {
    return eHasError(error);
}

static bool ftStringCompare(const char* a, const char* b) {
    return strcmp(a, b) == 0;
}

static void ftInitFileTokens(FileTokens* ft, const char* path) {
    ft->tokens = NULL;
    ft->capacity = 0;
    ft->length = 0;
    ftAddPath(ft, path);
}

static void ftTokenize(FileTokens* ft, unsigned char* data) {
    int index = 0;
    int oldIndex = 0;
    bool inString = false;
    int line = 1;
    while(true) {
        unsigned char c = data[index];
        if(isAllowedInName(c) || (inString && c != '"' && c != '\0')) {
            index++;
            continue;
        }
        if(oldIndex < index) {
            ftAddLiteral(ft, (const char*)(data + oldIndex), index - oldIndex);
        }
        if(c == '"') {
            inString = !inString;
        }
        if(c == '\0') {
            break;
        } else if(c == '\n') {
            line++;
            ftAddNewline(ft);
        } else if(c == ' ') {
            ftAddSpace(ft);
        } else if(c == '/' && data[index + 1] == '/') {
            while(data[index] != '\0' && data[index] != '\n') {
                index++;
            }
            ftAddNewline(ft);
        } else if(c == '/' && data[index + 1] == '*') {
            int startLine = line;
            index += 2;
            while(true) {
                if(data[index] == '\0') {
                    eInitError(error, startLine, "unclosed multiline comment");
                    return;
                } else if(data[index] == '\n') {
                    ftAddNewline(ft);
                    line++;
                } else if(data[index] == '*' && data[index + 1] == '/') {
                    index++;
                    break;
                }
                index++;
            }
        } else {
            ftAddSingle(ft, c);
        }
        index++;
        oldIndex = index;
    }
}

static void ftReadFull(FILE* file, FileTokens* ft) {
    if(fseek(file, 0, SEEK_END)) {
        ftSystemError("cannot seek end of file", ft);
        return;
    }
    long length = ftell(file);
    if(length < 0) {
        ftSystemError("cannot tell end of file", ft);
        return;
    }
    rewind(file);

    unsigned char* data = malloc(length + 1);
    int readValues = fread(data, 1, length, file);
    if(readValues != length) {
        ftSystemError("cannot read file", ft);
        free(data);
        return;
    }
    data[length] = '\0';
    ftTokenize(ft, data);
    ftAddEndPath(ft);
    free(data);
}

static void ftReadFullFile(FileTokens* ft) {
    FILE* file = fopen(ftGetPath(ft), "r");
    if(file == NULL) {
        ftSystemError("cannot open file", ft);
        return;
    }
    ftReadFull(file, ft);
    if(fclose(file) < 0) {
        ftSystemError("cannot close file", ft);
    }
}

static void ftReplace(int from, int to, FileTokens* ft,
                      FileTokens* replacement) {
    int newCapacity = ft->capacity + replacement->capacity;
    FileToken* newTokens = malloc(newCapacity * sizeof(FileToken));

    // these will be overwritten
    for(int i = from; i < to; i++) {
        free(ft->tokens[i].literal);
        ft->tokens[i].literal = NULL;
    }

    memcpy(newTokens, ft->tokens, from * sizeof(FileToken));
    memcpy(newTokens + from, replacement->tokens,
           replacement->length * sizeof(FileToken));
    memcpy(newTokens + from + replacement->length, ft->tokens + to,
           (ft->length - to) * sizeof(FileToken));

    free(ft->tokens);
    ft->tokens = newTokens;
    ft->capacity = newCapacity;
    ft->length = from + replacement->length + ft->length - to;

    // ownership moved
    for(int i = 0; i < replacement->length; i++) {
        replacement->tokens[i].literal = NULL;
    }
}

static void ftSkipSpaces(int* index, FileTokens* ft) {
    while(*index < ft->length) {
        FileToken* t = ft->tokens + *index;
        if(t->type != FT_SPACE) {
            return;
        }
        (*index)++;
    }
}

const char* INCLUDE_ERROR = "include path must be a string";

static bool ftIncludeDoubleQuote(int* index, FileTokens* ft) {
    if(*index >= ft->length || ft->tokens[*index].type != FT_SINGLE ||
       ft->tokens[*index].single != '"') {
        eInitError(error, line, INCLUDE_ERROR);
        return true;
    }
    (*index)++;
    return false;
}

static void ftHandleInclude(int* index, int start, FileTokens* ft) {
    ftSkipSpaces(index, ft);
    if(ftIncludeDoubleQuote(index, ft)) {
        return;
    }
    if(*index >= ft->length || ft->tokens[*index].type != FT_LITERAL) {
        eInitError(error, line, INCLUDE_ERROR);
        return;
    }
    const char* path = ft->tokens[(*index)++].literal;
    if(ftIncludeDoubleQuote(index, ft)) {
        return;
    }
    const char* parentPath = ftGetPath(ft);

    int afterLastSlash = 0;
    for(int i = 0; parentPath[i] != '\0'; i++) {
        afterLastSlash = parentPath[i] == '/' ? i + 1 : afterLastSlash;
    }

    int pathLength = strlen(path) + 1;
    char* fullPath = malloc(afterLastSlash + pathLength);
    memcpy(fullPath, parentPath, afterLastSlash);
    memcpy(fullPath + afterLastSlash, path, pathLength);

    FileTokens include;
    ftInitFileTokens(&include, fullPath);
    free(fullPath);

    ftReadFullFile(&include);
    if(!ftHasError()) {
        ftReplace(start, *index, ft, &include);
        *index = start;
    }
    ftDelete(&include);
}

static bool ftIsDefined(const char* s) {
    for(int i = 0; i < defineIndex; i++) {
        if(ftStringCompare(defines[i].name, s)) {
            return true;
        }
    }
    return false;
}

static int ftFindEndIf(int from, FileTokens* ft) {
    int layer = 0;
    for(int i = from; i < ft->length - 1; i++) {
        if(ft->tokens[i].type != FT_SINGLE || ft->tokens[i].single != '#') {
            continue;
        } else if(ft->tokens[i + 1].type != FT_LITERAL) {
            continue;
        }
        if(ftStringCompare(ft->tokens[i + 1].literal, "ifndef")) {
            layer++;
        } else if(ftStringCompare(ft->tokens[i + 1].literal, "endif")) {
            if(layer == 0) {
                return i;
            }
            layer--;
        }
    }
    return -1;
}

static void ftHandleIfNotDefined(int* index, int start, FileTokens* ft) {
    (void)start;
    ftSkipSpaces(index, ft);
    if(*index >= ft->length || ft->tokens[*index].type != FT_LITERAL) {
        eInitError(error, line, "ifndef expects a literal");
        return;
    }
    const char* check = ft->tokens[(*index)++].literal;
    int endIf = ftFindEndIf(*index, ft);
    if(endIf == -1) {
        eInitError(error, line, "cannot find #endif");
        return;
    }
    if(ftIsDefined(check)) {
        for(int i = start; i < endIf + 2; i++) {
            ftToSpace(ft->tokens + i);
        }
    } else {
        ftToSpace(ft->tokens + start);        // #
        ftToSpace(ft->tokens + start + 1);    // ifndef
        ftToSpace(ft->tokens + (*index - 1)); // DEFINITION

        ftToSpace(ft->tokens + endIf);     // #
        ftToSpace(ft->tokens + endIf + 1); // endif
    }
}

static void frHandleDefine(int* index, int start, FileTokens* ft) {
    if(defineIndex >= DEFINES) {
        eInitError(error, line, "too much defines");
        return;
    }
    ftSkipSpaces(index, ft);
    if(*index >= ft->length || ft->tokens[*index].type != FT_LITERAL) {
        eInitError(error, line, "define expects a literal");
        return;
    }
    const char* define = ft->tokens[(*index)++].literal;
    if(ftIsDefined(define)) {
        return;
    }
    strncpy(defines[defineIndex].name, define, STRING_LENGTH);
    defineIndex++;

    ftToSpace(ft->tokens + start);        // #
    ftToSpace(ft->tokens + start + 1);    // define
    ftToSpace(ft->tokens + (*index - 1)); // DEFINITION
}

static void ftParseInstruction(int* index, FileTokens* ft) {
    int start = *index;
    (*index)++;
    if(*index >= ft->length || ft->tokens[*index].type != FT_LITERAL) {
        eInitError(error, line, "expected literal after #");
        return;
    }
    const char* name = ft->tokens[(*index)++].literal;
    if(ftStringCompare(name, "include")) {
        ftHandleInclude(index, start, ft);
    } else if(ftStringCompare(name, "ifndef")) {
        ftHandleIfNotDefined(index, start, ft);
    } else if(ftStringCompare(name, "define")) {
        frHandleDefine(index, start, ft);
    } else if(ftStringCompare(name, "endif")) {
        eInitError(error, line, "endif without if");
    } else {
        eInitError(error, line, "unknown preprocessor literal '%s'", name);
    }
}

static void ftSearchInstruction(FileTokens* ft) {
    for(int i = 0; i < ft->length && !ftHasError(); i++) {
        if(ft->tokens[i].type == FT_SINGLE && ft->tokens[i].single == '#') {
            ftParseInstruction(&i, ft);
        }
    }
}

void ftInit(const char* path, FileTokens* ft, Error* e) {
    defineIndex = 0;
    error = e;
    eInitSuccess(error);
    ftInitFileTokens(ft, path);
    ftReadFullFile(ft);
    if(!ftHasError()) {
        ftSearchInstruction(ft);
    }
    (void)ftPrint;
    // ftPrint(ft);
}

void ftDelete(FileTokens* ft) {
    for(int i = 0; i < ft->length; i++) {
        free(ft->tokens[i].literal);
    }
    free(ft->tokens);
    ft->tokens = NULL;
    ft->capacity = 0;
    ft->length = 0;
}