lonely-tiger/tokenizer/FileTokens.c

#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "tokenizer/FileTokens.h"
#include "utils/SnuviUtils.h"

/*typedef char String[64];
#define STRING_LENGTH ((int)sizeof(String))

#define END_IF "#endif"
#define END_IF_LENGTH ((int)sizeof(END_IF) - 1)*/

static Error* error;

static const char* ftGetPath(const FileTokens* ft, int line) {
    if(ft->capacity < (2 + (int)sizeof(int)) || ft->data[0] != FT_PATH) {
        eInitError(error, line, "path not set");
        return "";
    }
    return (const char*)(ft->data + 1 + sizeof(int));
}

static void ftSystemError(const char* msg, int line, const FileTokens* ft) {
    eInitError(error, line, "%s '%s': %s", msg, ftGetPath(ft, line),
               strerror(errno));
}

static void ftAddData(FileTokens* ft, const void* data, int dataSize) {
    if(ft->writeIndex + dataSize > ft->capacity) {
        int newSize = ft->capacity;
        while(ft->writeIndex + dataSize > newSize) {
            newSize += 1024;
        }
        ft->data = realloc(ft->data, newSize);
        ft->capacity = newSize;
    }
    memcpy(ft->data + ft->writeIndex, data, dataSize);
    ft->writeIndex += dataSize;
}

static void ftAddToken(FileTokens* ft, FileToken t) {
    unsigned char c = t;
    ftAddData(ft, &c, 1);
}

static void ftAddInt(FileTokens* ft, int i) {
    ftAddData(ft, &i, sizeof(int));
}

static void ftAddString(FileTokens* ft, const void* s, int length) {
    ftAddInt(ft, length + 1);
    ftAddData(ft, s, length);
    const char c = '\0';
    ftAddData(ft, &c, 1);
}

static void ftAddTerminatedString(FileTokens* ft, const char* s) {
    ftAddString(ft, s, strlen(s));
}

/*typedef struct {
    String name;
} Define;

#define DEFINES 256
static Define defines[DEFINES];
static int defineIndex = 0;

static bool frHasError() {
    return error->message[0] != '\0';
}

static bool frStringCompare(const String a, const char* b) {
    return strcmp(a, b) == 0;
}*/

static void ftInitFileTokens(FileTokens* ft, const char* path) {
    ft->data = NULL;
    ft->capacity = 0;
    ft->writeIndex = 0;
    ft->readIndex = 0;

    ftAddToken(ft, FT_PATH);
    ftAddTerminatedString(ft, path);
}

static void ftTokenize(FileTokens* ft, unsigned char* data) {
    int index = 0;
    int oldIndex = 0;
    bool inString = false;
    int line = 1;
    while(true) {
        unsigned char c = data[index];
        if(isAllowedInName(c) || (inString && c != '"' && c != '\0')) {
            index++;
            continue;
        }
        if(oldIndex < index) {
            ftAddToken(ft, FT_LITERAL);
            ftAddString(ft, data + oldIndex, index - oldIndex);
        }
        if(c == '"') {
            inString = !inString;
        }
        if(c == '\0') {
            break;
        } else if(c == '\n') {
            line++;
            ftAddToken(ft, FT_NEWLINE);
        } else if(c == ' ') {
            ftAddToken(ft, FT_SPACE);
        } else if(c == '/' && data[index + 1] == '/') {
            while(data[index] != '\0' && data[index] != '\n') {
                index++;
            }
            ftAddToken(ft, FT_NEWLINE);
        } else if(c == '/' && data[index + 1] == '*') {
            int startLine = line;
            index += 2;
            while(true) {
                if(data[index] == '\0') {
                    eInitError(error, startLine, "unclosed multiline comment");
                    return;
                } else if(data[index] == '\n') {
                    ftAddToken(ft, FT_NEWLINE);
                    line++;
                } else if(data[index] == '*' && data[index + 1] == '/') {
                    index++;
                    break;
                }
                index++;
            }
        } else {
            ftAddToken(ft, FT_SINGLE);
            ftAddData(ft, &c, 1);
        }
        index++;
        oldIndex = index;
    }
}

static void ftReadFull(FILE* file, FileTokens* ft, int line) {
    if(fseek(file, 0, SEEK_END)) {
        ftSystemError("cannot seek end of file", line, ft);
        return;
    }
    long length = ftell(file);
    if(length < 0) {
        ftSystemError("cannot tell end of file", line, ft);
        return;
    }
    rewind(file);

    unsigned char* data = malloc(length + 1);
    int readValues = fread(data, 1, length, file);
    if(readValues != length) {
        ftSystemError("cannot read file", line, ft);
        free(data);
        return;
    }
    data[length] = '\0';
    ftTokenize(ft, data);
    free(data);
}

static void ftReadFullFile(FileTokens* ft, int line) {
    FILE* file = fopen(ftGetPath(ft, line), "r");
    if(file == NULL) {
        ftSystemError("cannot open file", 1, ft);
        return;
    }
    ftReadFull(file, ft, line);
    if(fclose(file) < 0) {
        ftSystemError("cannot close file", 1, ft);
    }
}

/*static int frReadUntil(int from, char end, String s, FileReader* fr) {
    int index = 0;
    while(from < fr->length && index < STRING_LENGTH - 1 &&
          fr->data[from] != end) {
        s[index++] = fr->data[from];
        from++;
    }
    s[index] = '\0';
    return index;
}

static int frReadUntilOrLine(int from, char end, String s, FileReader* fr) {
    int index = 0;
    while(from < fr->length && index < STRING_LENGTH - 1 &&
          fr->data[from] != end && fr->data[from] != '\n') {
        s[index++] = fr->data[from];
        from++;
    }
    s[index] = '\0';
    return index;
}

static void frMakePath(String fullPath, const char* parent, String name) {
    int i = 0;
    int lastSlash = 0;
    while(i < STRING_LENGTH - 1 && parent[i] != '\0') {
        fullPath[i] = parent[i];
        lastSlash = parent[i] == '/' ? i : lastSlash;
        i++;
    }
    i = lastSlash + 1;
    int nameI = 0;
    while(i < STRING_LENGTH - 1 && name[nameI] != '\0') {
        fullPath[i] = name[nameI];
        i++;
        nameI++;
    }
    fullPath[i] = '\0';
}

static void frReplace(int from, int to, FileReader* fr,
                      FileReader* replacement) {
    int replaceLength = replacement->length - 1;
    replaceLength = replaceLength < 0 ? 0 : replaceLength;
    int newLength = fr->length + replaceLength - (to - from);
    unsigned char* newData = malloc(newLength + 1);
    memcpy(newData, fr->data, from);
    memcpy(newData + from, replacement->data, replaceLength);
    memcpy(newData + from + replaceLength, fr->data + to, fr->length - to);
    newData[newLength] = '\0';

    free(fr->data);
    fr->data = newData;
    fr->length = newLength;
}

static int skipSpaces(int index, FileReader* fr) {
    while(index < fr->length && fr->data[index] == ' ') {
        index++;
    }
    return index;
}

static void frHandleInclude(int from, int length, FileReader* fr) {
    int index = from + length;
    index = skipSpaces(index, fr);
    if(index >= fr->length || fr->data[index] != '"') {
        eInitError(error, fr->line, "expected '\"' and not '%c' in '%s'",
                   fr->data[index], fr->path);
        return;
    }
    index++;
    String s;
    int pathLength = frReadUntil(index, '"', s, fr) + 1;

    String fullPath;
    frMakePath(fullPath, fr->path, s);

    FileReader include;
    ftInitFileTokens(&include, fullPath);
    ftReadFullFile(&include);
    if(frHasError()) {
        frDelete(&include);
        return;
    }
    frReplace(from, index + pathLength, fr, &include);
    frDelete(&include);
}

static int frReadWord(String s, int from, FileReader* fr) {
    int index = skipSpaces(from, fr);
    int wordLength = frReadUntilOrLine(index, ' ', s, fr);
    return index + wordLength;
}

static bool frIsDefined(String s) {
    for(int i = 0; i < defineIndex; i++) {
        if(frStringCompare(defines[i].name, s)) {
            return true;
        }
    }
    return false;
}

static bool frIsMatching(int from, FileReader* fr, const char* s) {
    int i = 0;
    while(true) {
        if(s[i] == '\0') {
            return true;
        } else if(from >= fr->length || fr->data[from] != s[i]) {
            return false;
        }
        from++;
        i++;
    }
}

static int frFindEndIf(int from, FileReader* fr) {
    int layer = 0;
    for(int i = from; i < fr->length; i++) {
        if(frIsMatching(i, fr, END_IF)) {
            if(layer == 0) {
                return i;
            }
            layer--;
        } else if(frIsMatching(i, fr, "#ifndef")) {
            layer++;
        }
    }
    return -1;
}

static void frHandleIfNotDefined(int from, int length, FileReader* fr) {
    String s;
    int end = frReadWord(s, from + length, fr);
    int endIf = frFindEndIf(end, fr);
    if(endIf == -1) {
        eInitError(error, fr->line, "cannot find #endif in '%s'", fr->path);
        return;
    }
    if(frIsDefined(s)) {
        memset(fr->data + from, ' ', endIf - from + END_IF_LENGTH);
    } else {
        memset(fr->data + from, ' ', end - from);
        memset(fr->data + endIf, ' ', END_IF_LENGTH);
    }
}

static void frHandleDefine(int from, int length, FileReader* fr) {
    if(defineIndex >= DEFINES) {
        eInitError(error, fr->line, "too much defines in '%s'", fr->path);
        return;
    }
    String s;
    int end = frReadWord(s, from + length, fr);
    if(frIsDefined(s)) {
        puts("already defined");
        return;
    }
    strncpy(defines[defineIndex].name, s, STRING_LENGTH);
    defineIndex++;
    printf("defined '%s'\n", s);
    memset(fr->data + from, ' ', end - from);
}

static void frParseInstruction(int from, FileReader* fr) {
    String s;
    int length = frReadUntil(from, ' ', s, fr);
    if(frStringCompare(s, "#include")) {
        frHandleInclude(from, length, fr);
    } else if(frStringCompare(s, "#ifndef")) {
        frHandleIfNotDefined(from, length, fr);
    } else if(frStringCompare(s, "#define")) {
        frHandleDefine(from, length, fr);
    } else {
        eInitError(error, fr->line, "unknown preprocessor token '%s' in '%s'",
                   s, fr->path);
    }
}

static void frSearchInstruction(FileReader* fr) {
    if(fr->data == NULL) {
        return;
    }
    for(int i = 0; i < fr->length && !frHasError(); i++) {
        unsigned char c = fr->data[i];
        if(c == '#') {
            frParseInstruction(i, fr);
            i--;
        } else if(c == '\n') {
            fr->line++;
        }
    }
}*/

void ftInit(const char* path, FileTokens* ft, Error* e) {
    // defineIndex = 0;
    error = e;
    eInitSuccess(error);
    ftInitFileTokens(ft, path);
    ftReadFullFile(ft, 1);
    // frSearchInstruction(fr);
    // if(fr->data != NULL) {
    //     printf((char*)fr->data);
    // }
}

void ftDelete(FileTokens* ft) {
    free(ft->data);
    ft->data = NULL;
    ft->capacity = 0;
    ft->writeIndex = 0;
    ft->readIndex = 0;
}

FileToken ftRead(FileTokens* ft) {
    if(ft->readIndex < ft->writeIndex) {
        return ft->data[ft->readIndex++];
    }
    return FT_END;
}

bool ftReadSingleIf(FileTokens* ft, char c) {
    if(ft->readIndex + 1 >= ft->writeIndex) {
        return false;
    } else if(ft->data[ft->readIndex] != FT_SINGLE) {
        return false;
    } else if(ft->data[ft->readIndex + 1] != c) {
        return false;
    }
    ft->readIndex += 2;
    return true;
}

char ftReadSingle(FileTokens* ft) {
    if(ft->readIndex < ft->writeIndex) {
        return ft->data[ft->readIndex++];
    }
    return -1;
}

const char* ftReadString(FileTokens* ft, int* length) {
    *length = 0;
    if(ft->readIndex + ((int)sizeof(int)) + 1 >= ft->writeIndex) {
        return NULL;
    }
    memcpy(length, ft->data + ft->readIndex, sizeof(int));
    const char* s = (const char*)(ft->data + ft->readIndex + sizeof(int));
    ft->readIndex += *length + sizeof(int);
    return s;
}