#include #include "TokenStream.h" TokenStream::TokenStream() : nextToken(0) { } bool TokenStream::hasToken() const { return nextToken < bytes.size(); } std::string TokenStream::nextTokenString() { std::string t = "("; // line unsigned int line = nextLine(); t += std::to_string(line); t += ", "; // tokentype TokenType type = nextTokenType(); t += TokenTypeUtils::getEnumName(type); if(type == TokenType::STRING || type == TokenType::LITERAL || type == TokenType::LABEL) { t += ", \""; t += nextString(); t += "\""; } if(type == TokenType::NUMBER) { t += ", "; double d = nextDouble(); char buffer[32]; snprintf(buffer, 32, (d == (long) d) ? "%lg.0" : "%lg", d); t += buffer; } t += ")"; return t; } TokenType TokenStream::nextTokenType() { TokenType type = TokenType::EOF_TOKEN; read(&type, sizeof (TokenType)); return type; } unsigned int TokenStream::nextLine() { unsigned int line = 0; read(&line, sizeof (unsigned int)); return line; } std::string TokenStream::nextString() { std::string text; char c; while(true) { read(&c, 1); if(c == '\0') { break; } text += c; } return text; } double TokenStream::nextDouble() { double d; read(&d, sizeof (double)); return d; } void TokenStream::write(const void* data, size_t length) { const char* chars = reinterpret_cast (data); for(size_t i = 0; i < length; i++) { bytes.push_back(chars[i]); } } void TokenStream::read(void* data, size_t length) { memcpy(data, bytes.data() + nextToken, length); nextToken += length; } void TokenStream::add(TokenType type, unsigned int line) { write(&line, sizeof (unsigned int)); write(&type, sizeof (TokenType)); } void TokenStream::add(TokenType type, unsigned int line, double d) { add(type, line); write(&d, sizeof (double)); } void TokenStream::add(TokenType type, unsigned int line, const std::string& text) { add(type, line); write(text.data(), text.length()); write("\0", 1); }