|
@@ -1,30 +1,33 @@
|
|
|
#include <limits.h>
|
|
|
#include <setjmp.h>
|
|
|
-#include <stdarg.h>
|
|
|
+//#include <stdarg.h>
|
|
|
#include <stdio.h>
|
|
|
#include <stdlib.h>
|
|
|
#include <string.h>
|
|
|
|
|
|
-#include "tokenizer/File.h"
|
|
|
+#include "tokenizer/FileReader.h"
|
|
|
#include "tokenizer/Tokenizer.h"
|
|
|
#include "utils/SnuviUtils.h"
|
|
|
|
|
|
#define TOKEN_BUFFER_LENGTH (1024 * 1024)
|
|
|
#define ERROR_LENGTH 256
|
|
|
|
|
|
+static FileReader fileReader;
|
|
|
static jmp_buf errorJump;
|
|
|
+static Error* error;
|
|
|
+
|
|
|
static char tokenBuffer[TOKEN_BUFFER_LENGTH];
|
|
|
static int writeIndex = 0;
|
|
|
static int readIndex = 0;
|
|
|
static int16 line = 1;
|
|
|
-static char error[ERROR_LENGTH] = {'\0'};
|
|
|
|
|
|
static void tError(const char* format, ...) {
|
|
|
- va_list args;
|
|
|
+ /*va_list args;
|
|
|
va_start(args, format);
|
|
|
vsnprintf(error, ERROR_LENGTH, format, args);
|
|
|
va_end(args);
|
|
|
- longjmp(errorJump, 0);
|
|
|
+ longjmp(errorJump, 0);*/
|
|
|
+ (void)format;
|
|
|
}
|
|
|
|
|
|
static void tAdd(const void* data, int length) {
|
|
@@ -43,27 +46,60 @@ static void tAddToken(Token token) {
|
|
|
|
|
|
static bool tReadTokens(void* dest, int length) {
|
|
|
if(readIndex + length > writeIndex) {
|
|
|
- return false;
|
|
|
+ return true;
|
|
|
}
|
|
|
memcpy(dest, tokenBuffer + readIndex, length);
|
|
|
readIndex += length;
|
|
|
- return true;
|
|
|
+ return false;
|
|
|
}
|
|
|
|
|
|
static void tParseLiteral(int c) {
|
|
|
int index = 1;
|
|
|
char buffer[64];
|
|
|
buffer[0] = c;
|
|
|
- while(isAllowedInName(fPeek())) {
|
|
|
+ while(isAllowedInName(frPeek(&fileReader))) {
|
|
|
if(index >= 63) {
|
|
|
tError("literal is too long");
|
|
|
}
|
|
|
- buffer[index++] = fRead();
|
|
|
+ buffer[index++] = frRead(&fileReader);
|
|
|
}
|
|
|
buffer[index] = '\0';
|
|
|
- Token t = tFromName(buffer);
|
|
|
- if(t != T_END) {
|
|
|
- tAddToken(t);
|
|
|
+ if(strcmp(buffer, "null") == 0) {
|
|
|
+ tAddToken(T_NULL);
|
|
|
+ } else if(strcmp(buffer, "return") == 0) {
|
|
|
+ tAddToken(T_RETURN);
|
|
|
+ } else if(strcmp(buffer, "if") == 0) {
|
|
|
+ tAddToken(T_IF);
|
|
|
+ } else if(strcmp(buffer, "else") == 0) {
|
|
|
+ tAddToken(T_ELSE);
|
|
|
+ } else if(strcmp(buffer, "while") == 0) {
|
|
|
+ tAddToken(T_WHILE);
|
|
|
+ } else if(strcmp(buffer, "for") == 0) {
|
|
|
+ tAddToken(T_FOR);
|
|
|
+ } else if(strcmp(buffer, "break") == 0) {
|
|
|
+ tAddToken(T_BREAK);
|
|
|
+ } else if(strcmp(buffer, "continue") == 0) {
|
|
|
+ tAddToken(T_CONTINUE);
|
|
|
+ } else if(strcmp(buffer, "int") == 0) {
|
|
|
+ tAddToken(T_INT);
|
|
|
+ } else if(strcmp(buffer, "void") == 0) {
|
|
|
+ tAddToken(T_VOID);
|
|
|
+ } else if(strcmp(buffer, "float") == 0) {
|
|
|
+ tAddToken(T_FLOAT);
|
|
|
+ } else if(strcmp(buffer, "struct") == 0) {
|
|
|
+ tAddToken(T_STRUCT);
|
|
|
+ } else if(strcmp(buffer, "new") == 0) {
|
|
|
+ tAddToken(T_NEW);
|
|
|
+ } else if(strcmp(buffer, "length") == 0) {
|
|
|
+ tAddToken(T_LENGTH);
|
|
|
+ } else if(strcmp(buffer, "true") == 0) {
|
|
|
+ int32 i = 1;
|
|
|
+ tAddToken(T_INT_VALUE);
|
|
|
+ tAdd(&i, sizeof(int32));
|
|
|
+ } else if(strcmp(buffer, "false") == 0) {
|
|
|
+ int32 i = 0;
|
|
|
+ tAddToken(T_INT_VALUE);
|
|
|
+ tAdd(&i, sizeof(int32));
|
|
|
} else {
|
|
|
tAddToken(T_LITERAL);
|
|
|
tAdd(buffer, index + 1);
|
|
@@ -92,7 +128,7 @@ static void tParseNumber(int c) {
|
|
|
buffer[0] = c;
|
|
|
bool point = false;
|
|
|
while(true) {
|
|
|
- int c = fPeek();
|
|
|
+ int c = frPeek(&fileReader);
|
|
|
if(c == '.') {
|
|
|
point = true;
|
|
|
} else if(!isNumber(c)) {
|
|
@@ -100,28 +136,16 @@ static void tParseNumber(int c) {
|
|
|
} else if(index >= 63) {
|
|
|
tError("number is too long");
|
|
|
}
|
|
|
- buffer[index++] = fRead();
|
|
|
+ buffer[index++] = frRead(&fileReader);
|
|
|
}
|
|
|
buffer[index] = '\0';
|
|
|
- if(fPeek() == 'L' || fPeek() == 'l') {
|
|
|
- fRead();
|
|
|
- if(point) {
|
|
|
- tError("invalid mix of long and float", line);
|
|
|
- }
|
|
|
- long long l;
|
|
|
- if(tParseInt(buffer, index, &l) || l > INT64_MAX) {
|
|
|
- tError("invalid long on line %d", line);
|
|
|
- }
|
|
|
- int64 i = l;
|
|
|
- tAddToken(T_CONST_INT64);
|
|
|
- tAdd(&i, sizeof(int64));
|
|
|
- } else if(point) {
|
|
|
+ if(point) {
|
|
|
char* end = NULL;
|
|
|
float f = strtof(buffer, &end);
|
|
|
if(end[0] != '\0') {
|
|
|
tError("invalid float on line %d", line);
|
|
|
}
|
|
|
- tAddToken(T_CONST_FLOAT);
|
|
|
+ tAddToken(T_FLOAT_VALUE);
|
|
|
tAdd(&f, sizeof(float));
|
|
|
} else {
|
|
|
long long l;
|
|
@@ -129,14 +153,14 @@ static void tParseNumber(int c) {
|
|
|
tError("invalid int on line %d", line);
|
|
|
}
|
|
|
int32 i = l;
|
|
|
- tAddToken(T_CONST_INT32);
|
|
|
+ tAddToken(T_INT_VALUE);
|
|
|
tAdd(&i, sizeof(int32));
|
|
|
}
|
|
|
}
|
|
|
|
|
|
static int32 tUnicode(int32 c) {
|
|
|
if(c == '\\') {
|
|
|
- switch(fRead()) {
|
|
|
+ switch(frRead(&fileReader)) {
|
|
|
case '"': c = '"'; break;
|
|
|
case '\\': c = '\\'; break;
|
|
|
case 'n': c = '\n'; break;
|
|
@@ -146,14 +170,14 @@ static int32 tUnicode(int32 c) {
|
|
|
}
|
|
|
}
|
|
|
if((c & 0xE0) == 0xC0) {
|
|
|
- c = ((c & 0x1F) << 6) | (fRead() & 0x3F);
|
|
|
+ c = ((c & 0x1F) << 6) | (frRead(&fileReader) & 0x3F);
|
|
|
} else if((c & 0xF0) == 0xE0) {
|
|
|
- c = ((c & 0xF) << 12) | ((fRead() & 0x3F) << 6);
|
|
|
- c |= fRead() & 0x3F;
|
|
|
+ c = ((c & 0xF) << 12) | ((frRead(&fileReader) & 0x3F) << 6);
|
|
|
+ c |= frRead(&fileReader) & 0x3F;
|
|
|
} else if((c & 0xF8) == 0xF0) {
|
|
|
- c = ((c & 0x7) << 18) | ((fRead() & 0x3F) << 12);
|
|
|
- c |= (fRead() & 0x3F) << 6;
|
|
|
- c |= fRead() & 0x3F;
|
|
|
+ c = ((c & 0x7) << 18) | ((frRead(&fileReader) & 0x3F) << 12);
|
|
|
+ c |= (frRead(&fileReader) & 0x3F) << 6;
|
|
|
+ c |= frRead(&fileReader) & 0x3F;
|
|
|
}
|
|
|
return c;
|
|
|
}
|
|
@@ -161,7 +185,7 @@ static int32 tUnicode(int32 c) {
|
|
|
static void tAddString() {
|
|
|
tAddToken(T_TEXT);
|
|
|
while(true) {
|
|
|
- int32 c = fRead();
|
|
|
+ int32 c = frRead(&fileReader);
|
|
|
if(c == '"') {
|
|
|
break;
|
|
|
} else if(c == EOF) {
|
|
@@ -175,17 +199,17 @@ static void tAddString() {
|
|
|
}
|
|
|
|
|
|
static void tAddUnicode() {
|
|
|
- int32 c = fRead();
|
|
|
+ int32 c = frRead(&fileReader);
|
|
|
c = tUnicode(c);
|
|
|
- tAddToken(T_CONST_INT32);
|
|
|
+ tAddToken(T_INT_VALUE);
|
|
|
tAdd(&c, sizeof(int32));
|
|
|
- if(fRead() != '\'') {
|
|
|
+ if(frRead(&fileReader) != '\'') {
|
|
|
tError("expecting unicode end");
|
|
|
}
|
|
|
}
|
|
|
|
|
|
static void tAddToken2(Token te, Token t) {
|
|
|
- if(fReadIf('=')) {
|
|
|
+ if(frReadIf(&fileReader, '=')) {
|
|
|
tAddToken(te);
|
|
|
} else {
|
|
|
tAddToken(t);
|
|
@@ -193,7 +217,7 @@ static void tAddToken2(Token te, Token t) {
|
|
|
}
|
|
|
|
|
|
static void tAddToken3(int c, Token tc, Token te, Token t) {
|
|
|
- if(fReadIf(c)) {
|
|
|
+ if(frReadIf(&fileReader, c)) {
|
|
|
tAddToken(tc);
|
|
|
} else {
|
|
|
tAddToken2(te, t);
|
|
@@ -201,20 +225,16 @@ static void tAddToken3(int c, Token tc, Token te, Token t) {
|
|
|
}
|
|
|
|
|
|
static void tAddToken4(int c, Token tce, Token tc, Token te, Token t) {
|
|
|
- if(fReadIf(c)) {
|
|
|
+ if(frReadIf(&fileReader, c)) {
|
|
|
tAddToken2(tce, tc);
|
|
|
} else {
|
|
|
tAddToken2(te, t);
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-static void tAddTokenMinus() {
|
|
|
- tAddToken3('-', T_DECREMENT, T_SUB_SET, fReadIf('>') ? T_ARROW : T_SUB);
|
|
|
-}
|
|
|
-
|
|
|
static void tLineComment() {
|
|
|
while(true) {
|
|
|
- int c = fRead();
|
|
|
+ int c = frRead(&fileReader);
|
|
|
if(c == EOF || c == '\n') {
|
|
|
line++;
|
|
|
return;
|
|
@@ -224,21 +244,21 @@ static void tLineComment() {
|
|
|
|
|
|
static void tMultipleLineComment() {
|
|
|
while(true) {
|
|
|
- int c = fRead();
|
|
|
+ int c = frRead(&fileReader);
|
|
|
if(c == EOF) {
|
|
|
tError("unclosed comment at line %d", line);
|
|
|
} else if(c == '\n') {
|
|
|
line++;
|
|
|
- } else if(c == '*' && fReadIf('/')) {
|
|
|
+ } else if(c == '*' && frReadIf(&fileReader, '/')) {
|
|
|
return;
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
|
|
|
static void tSlash() {
|
|
|
- if(fReadIf('/')) {
|
|
|
+ if(frReadIf(&fileReader, '/')) {
|
|
|
tLineComment();
|
|
|
- } else if(fReadIf('*')) {
|
|
|
+ } else if(frReadIf(&fileReader, '*')) {
|
|
|
tMultipleLineComment();
|
|
|
} else {
|
|
|
tAddToken2(T_DIV_SET, T_DIV);
|
|
@@ -257,7 +277,7 @@ static void tParseToken(int c) {
|
|
|
case ' ': return;
|
|
|
case '\n': line++; return;
|
|
|
case '+': tAddToken3('+', T_INCREMENT, T_ADD_SET, T_ADD); return;
|
|
|
- case '-': tAddTokenMinus(); return;
|
|
|
+ case '-': tAddToken3('-', T_DECREMENT, T_SUB_SET, T_SUB); return;
|
|
|
case '*': tAddToken2(T_MUL_SET, T_MUL); return;
|
|
|
case '/': tSlash(); return;
|
|
|
case '%': tAddToken2(T_MOD_SET, T_MOD); return;
|
|
@@ -294,9 +314,8 @@ static void tParseFile() {
|
|
|
readIndex = 0;
|
|
|
writeIndex = 0;
|
|
|
line = 1;
|
|
|
- error[0] = '\0';
|
|
|
while(true) {
|
|
|
- int c = fRead();
|
|
|
+ int c = frRead(&fileReader);
|
|
|
if(c == EOF) {
|
|
|
return;
|
|
|
}
|
|
@@ -304,25 +323,13 @@ static void tParseFile() {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-bool tTokenize(const char* path) {
|
|
|
- if(!setjmp(errorJump)) {
|
|
|
- fOpen(path, tError);
|
|
|
+void tTokenize(const char* path, Error* e) {
|
|
|
+ error = e;
|
|
|
+ frInit(path, &fileReader, e);
|
|
|
+ if(!eHasError(e) && !setjmp(errorJump)) {
|
|
|
tParseFile();
|
|
|
}
|
|
|
- fClose();
|
|
|
- return error[0] != '\0';
|
|
|
-}
|
|
|
-
|
|
|
-const char* tGetError() {
|
|
|
- return error;
|
|
|
-}
|
|
|
-
|
|
|
-int tGetLine() {
|
|
|
- return line;
|
|
|
-}
|
|
|
-
|
|
|
-void tResetReader() {
|
|
|
- readIndex = 0;
|
|
|
+ frDelete(&fileReader);
|
|
|
}
|
|
|
|
|
|
Token tPeekToken() {
|
|
@@ -332,25 +339,18 @@ Token tPeekToken() {
|
|
|
return tokenBuffer[readIndex];
|
|
|
}
|
|
|
|
|
|
-Token tReadToken() {
|
|
|
+bool tReadTokenAndLine(Token* t, int16* line) {
|
|
|
if(readIndex >= writeIndex) {
|
|
|
- return T_END;
|
|
|
+ return true;
|
|
|
}
|
|
|
- return tokenBuffer[readIndex++];
|
|
|
-}
|
|
|
-
|
|
|
-bool tReadInt16(int16* i) {
|
|
|
- return tReadTokens(i, sizeof(int16));
|
|
|
+ *t = tokenBuffer[readIndex++];
|
|
|
+ return tReadTokens(line, sizeof(int16));
|
|
|
}
|
|
|
|
|
|
bool tReadInt32(int32* i) {
|
|
|
return tReadTokens(i, sizeof(int32));
|
|
|
}
|
|
|
|
|
|
-bool tReadInt64(int64* i) {
|
|
|
- return tReadTokens(i, sizeof(int64));
|
|
|
-}
|
|
|
-
|
|
|
bool tReadFloat(float* f) {
|
|
|
return tReadTokens(f, sizeof(float));
|
|
|
}
|