lonely-tiger/tokenizer/TokenStream.c

#include <stdlib.h>
#include <string.h>
#include <stdio.h>

#include "tokenizer/TokenStream.h"

TokenStream* newTokenStream() {
    TokenStream* tokenStream = malloc(sizeof (TokenStream));
    tokenStream->readIndex = 0;
    tokenStream->index = 0;
    tokenStream->length = 1024;
    tokenStream->buffer = malloc(sizeof (char) * tokenStream->length);
    return tokenStream;
}

void deleteTokenStream(TokenStream** tokenStream) {
    if(tokenStream == NULL || (*tokenStream) == NULL) {
        return;
    }
    free((*tokenStream)->buffer);
    free(*tokenStream);
    *tokenStream = NULL;
}

bool hasToken(TokenStream* tokenStream) {
    return tokenStream->readIndex < tokenStream->index;
}

static size_t tokenstream_concat(char* dst, const char* src, size_t bufferLength) {
    size_t length = strlen(src);
    strncat(dst, src, bufferLength);
    if(length > bufferLength) {
        return 0;
    }
    return bufferLength - length;
}

void nextTokenString(TokenStream* tokenStream, char* buffer, size_t bufferLength) {
    if(bufferLength == 0) {
        return;
    }
    buffer[0] = '\0';
    bufferLength--;

    bufferLength = tokenstream_concat(buffer, "(", bufferLength);
    // line
    char lineBuffer[32];
    unsigned int line = nextLine(tokenStream);
    snprintf(lineBuffer, 32, "%u, ", line);
    bufferLength = tokenstream_concat(buffer, lineBuffer, bufferLength);

    // tokentype
    Token token = nextToken(tokenStream);
    bufferLength = tokenstream_concat(buffer, getTokenEnumName(token), bufferLength);

    if(token == STRING || token == LITERAL || token == LABEL) {
        bufferLength = tokenstream_concat(buffer, ", \"", bufferLength);
        bufferLength = tokenstream_concat(buffer, nextString(tokenStream), bufferLength);
        bufferLength = tokenstream_concat(buffer, "\"", bufferLength);
    }

    if(token == NUMBER) {
        char doubleBuffer[32];
        double d = nextDouble(tokenStream);
        snprintf(doubleBuffer, 32, (d == (long) d) ? ", %lg.0" : ", %lg", d);
        bufferLength = tokenstream_concat(buffer, doubleBuffer, bufferLength);
    }

    bufferLength = tokenstream_concat(buffer, ")", bufferLength);
}

void tokenstream_read(TokenStream* tokenStream, void* data, size_t length) {
    memcpy(data, tokenStream->buffer + tokenStream->readIndex, length);
    tokenStream->readIndex += length;
}

Token nextToken(TokenStream* tokenStream) {
    Token token = EOF_TOKEN;
    tokenstream_read(tokenStream, &token, sizeof (Token));
    return token;
}

unsigned int nextLine(TokenStream* tokenStream) {
    unsigned int line = 0;
    tokenstream_read(tokenStream, &line, sizeof (unsigned int));
    return line;
}

const char* nextString(TokenStream* tokenStream) {
    size_t offset = tokenStream->readIndex;
    tokenStream->readIndex += strlen(tokenStream->buffer + tokenStream->readIndex) + 1;
    return tokenStream->buffer + offset;
}

double nextDouble(TokenStream* tokenStream) {
    double d;
    tokenstream_read(tokenStream, &d, sizeof (double));
    return d;
}

static void tokenstream_write(TokenStream* tokenStream, const void* data, size_t length) {
    if(tokenStream->index + length >= tokenStream->length) {
        size_t newSize = tokenStream->length;
        while(newSize < tokenStream->index + length) {
            newSize *= 2;
        }
        tokenStream->buffer = realloc(tokenStream->buffer, sizeof (char) * newSize);
        tokenStream->length = newSize;
    }
    const char* chars = (const char*) data;
    for(size_t i = 0; i < length; i++) {
        tokenStream->buffer[tokenStream->index++] = chars[i];
    }
}

void addToken(TokenStream* tokenStream, Token token, unsigned int line) {
    tokenstream_write(tokenStream, &line, sizeof (unsigned int));
    tokenstream_write(tokenStream, &token, sizeof (Token));
}

void addDoubleToken(TokenStream* tokenStream, Token token, unsigned int line, double d) {
    addToken(tokenStream, token, line);
    tokenstream_write(tokenStream, &d, sizeof (double));
}

void addStringToken(TokenStream* tokenStream, Token token, unsigned int line, const char* text) {
    addToken(tokenStream, token, line);
    tokenstream_write(tokenStream, text, strlen(text) + 1);
}