Tokenizer.c 8.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370
  1. #include <limits.h>
  2. #include <setjmp.h>
  3. #include <stdarg.h>
  4. #include <stdio.h>
  5. #include <stdlib.h>
  6. #include <string.h>
  7. #include "tokenizer/File.h"
  8. #include "tokenizer/Tokenizer.h"
  9. #include "utils/Utils.h"
  10. #define TOKEN_BUFFER_LENGTH (1024 * 1024)
  11. #define ERROR_LENGTH 256
  12. static jmp_buf errorJump;
  13. static char tokenBuffer[TOKEN_BUFFER_LENGTH];
  14. static int writeIndex = 0;
  15. static int readIndex = 0;
  16. static int16 line = 1;
  17. static char error[ERROR_LENGTH] = {'\0'};
  18. static void tError(const char* format, ...) {
  19. va_list args;
  20. va_start(args, format);
  21. vsnprintf(error, ERROR_LENGTH, format, args);
  22. va_end(args);
  23. longjmp(errorJump, 0);
  24. }
  25. static void tAdd(const void* data, int length) {
  26. if(writeIndex + length > TOKEN_BUFFER_LENGTH) {
  27. tError("the token buffer is too small");
  28. }
  29. memcpy(tokenBuffer + writeIndex, data, length);
  30. writeIndex += length;
  31. }
  32. static void tAddToken(Token token) {
  33. unsigned char c = token;
  34. tAdd(&c, 1);
  35. tAdd(&line, sizeof(line));
  36. }
  37. static bool tReadTokens(void* dest, int length) {
  38. if(readIndex + length > writeIndex) {
  39. return false;
  40. }
  41. memcpy(dest, tokenBuffer + readIndex, length);
  42. readIndex += length;
  43. return true;
  44. }
  45. static void tParseLiteral(int c) {
  46. int index = 1;
  47. char buffer[64];
  48. buffer[0] = c;
  49. while(isAllowedInName(fPeek())) {
  50. if(index >= 63) {
  51. tError("literal is too long");
  52. }
  53. buffer[index++] = fRead();
  54. }
  55. buffer[index] = '\0';
  56. Token t = tFromName(buffer);
  57. if(t != T_END) {
  58. tAddToken(t);
  59. } else {
  60. tAddToken(T_LITERAL);
  61. tAdd(buffer, index + 1);
  62. }
  63. }
  64. static bool tParseInt(char* s, int length, long long* l) {
  65. *l = 0;
  66. for(int i = 0; i < length; i++) {
  67. if(*l > (LLONG_MAX / 10)) {
  68. return true;
  69. }
  70. *l *= 10;
  71. int digit = s[i] - '0';
  72. if(*l > LLONG_MAX - digit) {
  73. return true;
  74. }
  75. *l += digit;
  76. }
  77. return false;
  78. }
  79. static void tParseNumber(int c) {
  80. int index = 1;
  81. char buffer[64];
  82. buffer[0] = c;
  83. bool point = false;
  84. while(true) {
  85. int c = fPeek();
  86. if(c == '.') {
  87. point = true;
  88. } else if(!isNumber(c)) {
  89. break;
  90. } else if(index >= 63) {
  91. tError("number is too long");
  92. }
  93. buffer[index++] = fRead();
  94. }
  95. buffer[index] = '\0';
  96. if(fPeek() == 'L' || fPeek() == 'l') {
  97. fRead();
  98. if(point) {
  99. tError("invalid mix of long and float", line);
  100. }
  101. long long l;
  102. if(tParseInt(buffer, index, &l) || l > INT64_MAX) {
  103. tError("invalid long on line %d", line);
  104. }
  105. int64 i = l;
  106. tAddToken(T_CONST_INT64);
  107. tAdd(&i, sizeof(int64));
  108. } else if(point) {
  109. char* end = NULL;
  110. float f = strtof(buffer, &end);
  111. if(end[0] != '\0') {
  112. tError("invalid float on line %d", line);
  113. }
  114. tAddToken(T_CONST_FLOAT);
  115. tAdd(&f, sizeof(float));
  116. } else {
  117. long long l;
  118. if(tParseInt(buffer, index, &l) || l > INT32_MAX) {
  119. tError("invalid int on line %d", line);
  120. }
  121. int32 i = l;
  122. tAddToken(T_CONST_INT32);
  123. tAdd(&i, sizeof(int32));
  124. }
  125. }
  126. static int32 tUnicode(int32 c) {
  127. if((c & 0xE0) == 0xC0) {
  128. c = ((c & 0x1F) << 6) | (fRead() & 0x3F);
  129. } else if((c & 0xF0) == 0xE0) {
  130. c = ((c & 0xF) << 12) | ((fRead() & 0x3F) << 6);
  131. c |= fRead() & 0x3F;
  132. } else if((c & 0xF8) == 0xF0) {
  133. c = ((c & 0x7) << 18) | ((fRead() & 0x3F) << 12);
  134. c |= (fRead() & 0x3F) << 6;
  135. c |= fRead() & 0x3F;
  136. }
  137. return c;
  138. }
  139. static void tAddString() {
  140. tAddToken(T_TEXT);
  141. while(true) {
  142. int32 c = fRead();
  143. if(c == '"') {
  144. break;
  145. } else if(c == '\\') {
  146. switch(fRead()) {
  147. case '"': c = '"'; break;
  148. case '\\': c = '\\'; break;
  149. default: tError("unknown escaped character at line %d", line);
  150. }
  151. } else if(c == EOF) {
  152. tError("unclosed string starting at line %d", line);
  153. }
  154. c = tUnicode(c);
  155. tAdd(&c, sizeof(int32));
  156. }
  157. int32 c = 0;
  158. tAdd(&c, sizeof(int32));
  159. }
  160. static void tAddUnicode() {
  161. int32 c = fRead();
  162. c = tUnicode(c);
  163. tAddToken(T_CONST_INT32);
  164. tAdd(&c, sizeof(int32));
  165. if(fRead() != '\'') {
  166. tError("expecting unicode end");
  167. }
  168. }
  169. static void tAddToken2(Token te, Token t) {
  170. if(fReadIf('=')) {
  171. tAddToken(te);
  172. } else {
  173. tAddToken(t);
  174. }
  175. }
  176. static void tAddToken3(int c, Token tc, Token te, Token t) {
  177. if(fReadIf(c)) {
  178. tAddToken(tc);
  179. } else {
  180. tAddToken2(te, t);
  181. }
  182. }
  183. static void tAddToken4(int c, Token tce, Token tc, Token te, Token t) {
  184. if(fReadIf(c)) {
  185. tAddToken2(tce, tc);
  186. } else {
  187. tAddToken2(te, t);
  188. }
  189. }
  190. static void tAddTokenMinus() {
  191. tAddToken3('-', T_DECREMENT, T_SUB_SET, fReadIf('>') ? T_ARROW : T_SUB);
  192. }
  193. static void tLineComment() {
  194. while(true) {
  195. int c = fRead();
  196. if(c == EOF || c == '\n') {
  197. line++;
  198. return;
  199. }
  200. }
  201. }
  202. static void tMultipleLineComment() {
  203. while(true) {
  204. int c = fRead();
  205. if(c == EOF) {
  206. tError("unclosed comment at line %d", line);
  207. } else if(c == '\n') {
  208. line++;
  209. } else if(c == '*' && fReadIf('/')) {
  210. return;
  211. }
  212. }
  213. }
  214. static void tSlash() {
  215. if(fReadIf('/')) {
  216. tLineComment();
  217. } else if(fReadIf('*')) {
  218. tMultipleLineComment();
  219. } else {
  220. tAddToken2(T_DIV_SET, T_DIV);
  221. }
  222. }
  223. static void tParseToken(int c) {
  224. if(isLetter(c)) {
  225. tParseLiteral(c);
  226. return;
  227. } else if(isNumber(c)) {
  228. tParseNumber(c);
  229. return;
  230. }
  231. switch(c) {
  232. case ' ': return;
  233. case '\n': line++; return;
  234. case '+': tAddToken3('+', T_INCREMENT, T_ADD_SET, T_ADD); return;
  235. case '-': tAddTokenMinus(); return;
  236. case '*': tAddToken2(T_MUL_SET, T_MUL); return;
  237. case '/': tSlash(); return;
  238. case '%': tAddToken2(T_MOD_SET, T_MOD); return;
  239. case '<':
  240. tAddToken4('<', T_LEFT_SHIFT_SET, T_LEFT_SHIFT, T_LESS_EQUAL,
  241. T_LESS);
  242. return;
  243. case '>':
  244. tAddToken4('>', T_RIGHT_SHIFT_SET, T_RIGHT_SHIFT, T_GREATER_EQUAL,
  245. T_GREATER);
  246. return;
  247. case '=': tAddToken2(T_EQUAL, T_SET); return;
  248. case '!': tAddToken2(T_NOT_EQUAL, T_NOT); return;
  249. case '&': tAddToken3('&', T_AND, T_BIT_AND_SET, T_BIT_AND); return;
  250. case '|': tAddToken3('|', T_OR, T_BIT_OR_SET, T_BIT_OR); return;
  251. case '~': tAddToken(T_BIT_NOT); return;
  252. case '^': tAddToken2(T_BIT_XOR_SET, T_BIT_XOR); return;
  253. case ',': tAddToken(T_COMMA); return;
  254. case ';': tAddToken(T_SEMICOLON); return;
  255. case '(': tAddToken(T_OPEN_BRACKET); return;
  256. case ')': tAddToken(T_CLOSE_BRACKET); return;
  257. case '{': tAddToken(T_OPEN_CURVED_BRACKET); return;
  258. case '}': tAddToken(T_CLOSE_CURVED_BRACKET); return;
  259. case '"': tAddString(); return;
  260. case '\'': tAddUnicode(); return;
  261. case '.': tAddToken(T_POINT); return;
  262. case '[': tAddToken(T_OPEN_SQUARE_BRACKET); return;
  263. case ']': tAddToken(T_CLOSE_SQUARE_BRACKET); return;
  264. }
  265. tError("unknown character on line %d: %c", line, c);
  266. }
  267. static void tParseFile() {
  268. readIndex = 0;
  269. writeIndex = 0;
  270. line = 1;
  271. error[0] = '\0';
  272. while(true) {
  273. int c = fRead();
  274. if(c == EOF) {
  275. return;
  276. }
  277. tParseToken(c);
  278. }
  279. }
  280. bool tTokenize(const char* path) {
  281. if(!setjmp(errorJump)) {
  282. fOpen(path, tError);
  283. tParseFile();
  284. }
  285. fClose();
  286. return error[0] != '\0';
  287. }
  288. const char* tGetError() {
  289. return error;
  290. }
  291. int tGetLine() {
  292. return line;
  293. }
  294. void tResetReader() {
  295. readIndex = 0;
  296. }
  297. Token tPeekToken() {
  298. if(readIndex >= writeIndex) {
  299. return T_END;
  300. }
  301. return tokenBuffer[readIndex];
  302. }
  303. Token tReadToken() {
  304. if(readIndex >= writeIndex) {
  305. return T_END;
  306. }
  307. return tokenBuffer[readIndex++];
  308. }
  309. bool tReadInt16(int16* i) {
  310. return tReadTokens(i, sizeof(int16));
  311. }
  312. bool tReadInt32(int32* i) {
  313. return tReadTokens(i, sizeof(int32));
  314. }
  315. bool tReadInt64(int64* i) {
  316. return tReadTokens(i, sizeof(int64));
  317. }
  318. bool tReadFloat(float* f) {
  319. return tReadTokens(f, sizeof(float));
  320. }
  321. const char* tReadString() {
  322. const char* s = tokenBuffer + readIndex;
  323. while(readIndex <= writeIndex) {
  324. if(tokenBuffer[readIndex++] == '\0') {
  325. return s;
  326. }
  327. }
  328. return NULL;
  329. }
  330. int tGetMarker() {
  331. return readIndex;
  332. }
  333. void tReset(int marker) {
  334. readIndex = marker;
  335. }