#include #include #include #include "tokenizer/TokenStream.h" TokenStream* newTokenStream() { TokenStream* tokenStream = malloc(sizeof (TokenStream)); tokenStream->readIndex = 0; tokenStream->index = 0; tokenStream->length = 1024; tokenStream->buffer = malloc(sizeof (char) * tokenStream->length); return tokenStream; } void deleteTokenStream(TokenStream** tokenStream) { if(tokenStream == NULL || (*tokenStream) == NULL) { return; } free((*tokenStream)->buffer); free(*tokenStream); *tokenStream = NULL; } bool hasToken(TokenStream* tokenStream) { return tokenStream->readIndex < tokenStream->index; } static size_t tokenstream_concat(char* dst, const char* src, size_t bufferLength) { size_t length = strlen(src); strncat(dst, src, bufferLength); if(length > bufferLength) { return 0; } return bufferLength - length; } void nextTokenString(TokenStream* tokenStream, char* buffer, size_t bufferLength) { if(bufferLength == 0) { return; } buffer[0] = '\0'; bufferLength--; bufferLength = tokenstream_concat(buffer, "(", bufferLength); // line char lineBuffer[32]; unsigned int line = nextLine(tokenStream); snprintf(lineBuffer, 32, "%u, ", line); bufferLength = tokenstream_concat(buffer, lineBuffer, bufferLength); // tokentype Token token = nextToken(tokenStream); bufferLength = tokenstream_concat(buffer, getTokenEnumName(token), bufferLength); if(token == STRING || token == LITERAL || token == LABEL) { bufferLength = tokenstream_concat(buffer, ", \"", bufferLength); bufferLength = tokenstream_concat(buffer, nextString(tokenStream), bufferLength); bufferLength = tokenstream_concat(buffer, "\"", bufferLength); } if(token == NUMBER) { char doubleBuffer[32]; double d = nextDouble(tokenStream); snprintf(doubleBuffer, 32, (d == (long) d) ? ", %lg.0" : ", %lg", d); bufferLength = tokenstream_concat(buffer, doubleBuffer, bufferLength); } bufferLength = tokenstream_concat(buffer, ")", bufferLength); } void tokenstream_read(TokenStream* tokenStream, void* data, size_t length) { memcpy(data, tokenStream->buffer + tokenStream->readIndex, length); tokenStream->readIndex += length; } Token nextToken(TokenStream* tokenStream) { Token token = EOF_TOKEN; tokenstream_read(tokenStream, &token, sizeof (Token)); return token; } unsigned int nextLine(TokenStream* tokenStream) { unsigned int line = 0; tokenstream_read(tokenStream, &line, sizeof (unsigned int)); return line; } const char* nextString(TokenStream* tokenStream) { size_t offset = tokenStream->readIndex; tokenStream->readIndex += strlen(tokenStream->buffer + tokenStream->readIndex) + 1; return tokenStream->buffer + offset; } double nextDouble(TokenStream* tokenStream) { double d; tokenstream_read(tokenStream, &d, sizeof (double)); return d; } static void tokenstream_write(TokenStream* tokenStream, const void* data, size_t length) { if(tokenStream->index + length >= tokenStream->length) { size_t newSize = tokenStream->length; while(newSize < tokenStream->index + length) { newSize *= 2; } tokenStream->buffer = realloc(tokenStream->buffer, sizeof (char) * newSize); tokenStream->length = newSize; } const char* chars = (const char*) data; for(size_t i = 0; i < length; i++) { tokenStream->buffer[tokenStream->index++] = chars[i]; } } void addToken(TokenStream* tokenStream, Token token, unsigned int line) { tokenstream_write(tokenStream, &line, sizeof (unsigned int)); tokenstream_write(tokenStream, &token, sizeof (Token)); } void addDoubleToken(TokenStream* tokenStream, Token token, unsigned int line, double d) { addToken(tokenStream, token, line); tokenstream_write(tokenStream, &d, sizeof (double)); } void addStringToken(TokenStream* tokenStream, Token token, unsigned int line, const char* text) { addToken(tokenStream, token, line); tokenstream_write(tokenStream, text, strlen(text) + 1); }