lonely-tiger/Tokenizer.c

#include <stdarg.h>
#include <stdio.h>
#include <string.h>

#include "Tokenizer.h"
#include "Utils.h"

#define TOKEN_BUFFER_LENGTH (1024 * 1024)
#define ERROR_LENGTH 256

static char tokenBuffer[TOKEN_BUFFER_LENGTH];
static int writeIndex = 0;
static int readIndex = 0;
static int line = 1;
static FILE* file = NULL;
static char error[ERROR_LENGTH] = {'\0'};

static void tError(const char* format, ...) {
    va_list args;
    va_start(args, format);
    vsnprintf(error, ERROR_LENGTH, format, args);
    va_end(args);
}

static bool tAdd(const void* data, int length) {
    if(writeIndex + length > TOKEN_BUFFER_LENGTH) {
        return false;
    }
    memcpy(tokenBuffer + writeIndex, data, length);
    writeIndex += length;
    return true;
}

static bool tAddToken(Token token) {
    unsigned char c = token;
    return tAdd(&c, 1) && tAdd(&line, sizeof(int));
}

static bool tReadTokens(void* dest, int length) {
    if(readIndex + length > writeIndex) {
        return false;
    }
    memcpy(dest, tokenBuffer + readIndex, length);
    readIndex += length;
    return true;
}

static int tRead() {
    return fgetc(file);
}

static int tPeek() {
    int c = tRead();
    ungetc(c, file);
    return c;
}

static bool tParseLiteral(int c) {
    int index = 1;
    char buffer[64];
    buffer[0] = c;
    while(isLetter(tPeek())) {
        if(index >= 63) {
            tError("literal is too long");
            return false;
        }
        buffer[index++] = tRead();
    }
    buffer[index] = '\0';
    if(strcmp(buffer, "print") == 0) {
        return tAddToken(T_PRINT);
    } else if(strcmp(buffer, "null") == 0) {
        return tAddToken(T_NULL);
    } else if(strcmp(buffer, "true") == 0) {
        return tAddToken(T_TRUE);
    } else if(strcmp(buffer, "false") == 0) {
        return tAddToken(T_FALSE);
    }
    return true;
}

static bool tParseNumber(int c) {
    int sum = c - '0';
    while(isNumber(tPeek())) {
        sum = sum * 10 + (tRead() - '0');
    }
    return tAddToken(T_INT) && tAdd(&sum, sizeof(int));
}

static bool tParseToken() {
    int c = tRead();
    if(c == EOF) {
        return false;
    } else if(isLetter(c)) {
        return tParseLiteral(c);
    } else if(isNumber(c)) {
        return tParseNumber(c);
    }
    switch(c) {
        case ' ': return true;
        case '\n': line++; return true;
        case '+': return tAddToken(T_ADD);
        case '*': return tAddToken(T_MUL);
        case ';': return tAddToken(T_SEMICOLON);
        case '(': return tAddToken(T_OPEN_BRACKET);
        case ')': return tAddToken(T_CLOSE_BRACKET);
    }
    tError("unknown character on line %d: %c", line, c);
    return false;
}

static void tParseFile() {
    readIndex = 0;
    writeIndex = 0;
    line = 1;
    error[0] = '\0';
    while(tParseToken()) {
    }
}

bool tTokenize(const char* path) {
    file = fopen(path, "r");
    if(file == NULL) {
        tError("cannot read file '%s'", path);
        return true;
    }
    tParseFile();
    fclose(file);
    return error[0] != '\0';
}

const char* tGetError() {
    return error;
}

void tResetReader() {
    readIndex = 0;
}

Token tPeekToken() {
    if(readIndex >= writeIndex) {
        return T_END;
    }
    return tokenBuffer[readIndex];
}

Token tReadToken() {
    if(readIndex >= writeIndex) {
        return T_END;
    }
    return tokenBuffer[readIndex++];
}

bool tReadInt(int* i) {
    if(tReadTokens(i, sizeof(int))) {
        return true;
    }
    return false;
}

const char* tGetTokenName(Token token) {
    switch(token) {
        case T_INT: return "int";
        case T_NULL: return "null";
        case T_TRUE: return "true";
        case T_FALSE: return "false";
        case T_ADD: return "+";
        case T_MUL: return "*";
        case T_PRINT: return "print";
        case T_SEMICOLON: return ";";
        case T_OPEN_BRACKET: return "(";
        case T_CLOSE_BRACKET: return ")";
        case T_END: return "end";
    }
    return "Unknown";
}

void tPrint() {
    puts("----------------");
    while(true) {
        Token t = tReadToken();
        if(t == T_END) {
            break;
        }
        int line;
        tReadInt(&line);
        printf("%d: %s\n", line, tGetTokenName(t));
        if(t == T_INT) {
            tReadInt(&line);
        }
    }
    tResetReader();
}