lonely-tiger/tokenizer/TokenStream.cpp

#include <cstring>

#include "TokenStream.h"

TokenStream::TokenStream() : nextToken(0) {
}

bool TokenStream::hasToken() const {
    return nextToken < bytes.size();
}

std::string TokenStream::nextTokenString() {
    std::string t = "(";
    // line
    unsigned int line = nextLine();
    t += std::to_string(line);
    t += ", ";
    // tokentype
    TokenType type = nextTokenType();
    t += TokenTypeUtils::getEnumName(type);

    if(type == TokenType::STRING || type == TokenType::LITERAL || type == TokenType::LABEL) {
        t += ", \"";
        t += nextString();
        t += "\"";
    }

    if(type == TokenType::NUMBER) {
        t += ", ";
        double d = nextDouble();
        char buffer[32];
        snprintf(buffer, 32, (d == (long) d) ? "%lg.0" : "%lg", d);
        t += buffer;
    }

    t += ")";
    return t;
}

TokenType TokenStream::nextTokenType() {
    TokenType type = TokenType::EOF_TOKEN;
    read(&type, sizeof (TokenType));
    return type;
}

unsigned int TokenStream::nextLine() {
    unsigned int line = 0;
    read(&line, sizeof (unsigned int));
    return line;
}

std::string TokenStream::nextString() {
    std::string text;
    char c;
    while(true) {
        read(&c, 1);
        if(c == '\0') {
            break;
        }
        text += c;
    }
    return text;
}

double TokenStream::nextDouble() {
    double d;
    read(&d, sizeof (double));
    return d;
}

void TokenStream::write(const void* data, size_t length) {
    const char* chars = reinterpret_cast<const char*> (data);
    for(size_t i = 0; i < length; i++) {
        bytes.push_back(chars[i]);
    }
}

void TokenStream::read(void* data, size_t length) {
    memcpy(data, bytes.data() + nextToken, length);
    nextToken += length;
}

void TokenStream::add(TokenType type, unsigned int line) {
    write(&line, sizeof (unsigned int));
    write(&type, sizeof (TokenType));
}

void TokenStream::add(TokenType type, unsigned int line, double d) {
    add(type, line);
    write(&d, sizeof (double));
}

void TokenStream::add(TokenType type, unsigned int line, const std::string& text) {
    add(type, line);
    write(text.data(), text.length());
    write("\0", 1);
}