Tokenizer.c 7.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297
  1. #include <limits.h>
  2. #include <stdarg.h>
  3. #include <stdio.h>
  4. #include <stdlib.h>
  5. #include <string.h>
  6. #include "tokenizer/File.h"
  7. #include "tokenizer/Tokenizer.h"
  8. #include "utils/Utils.h"
  9. #define TOKEN_BUFFER_LENGTH (1024 * 1024)
  10. #define ERROR_LENGTH 256
  11. static char tokenBuffer[TOKEN_BUFFER_LENGTH];
  12. static int writeIndex = 0;
  13. static int readIndex = 0;
  14. static int16 line = 1;
  15. static char error[ERROR_LENGTH] = {'\0'};
  16. static void tError(const char* format, ...) {
  17. va_list args;
  18. va_start(args, format);
  19. vsnprintf(error, ERROR_LENGTH, format, args);
  20. va_end(args);
  21. }
  22. static bool tAdd(const void* data, int length) {
  23. if(writeIndex + length > TOKEN_BUFFER_LENGTH) {
  24. tError("the token buffer is too small");
  25. return false;
  26. }
  27. memcpy(tokenBuffer + writeIndex, data, length);
  28. writeIndex += length;
  29. return true;
  30. }
  31. static bool tAddToken(Token token) {
  32. unsigned char c = token;
  33. return tAdd(&c, 1) && tAdd(&line, sizeof(line));
  34. }
  35. static bool tReadTokens(void* dest, int length) {
  36. if(readIndex + length > writeIndex) {
  37. return false;
  38. }
  39. memcpy(dest, tokenBuffer + readIndex, length);
  40. readIndex += length;
  41. return true;
  42. }
  43. static bool tParseLiteral(int c) {
  44. int index = 1;
  45. char buffer[64];
  46. buffer[0] = c;
  47. while(isLetter(fPeek()) || fPeek() == '.') {
  48. if(index >= 63) {
  49. tError("literal is too long");
  50. return false;
  51. }
  52. buffer[index++] = fRead();
  53. }
  54. buffer[index] = '\0';
  55. Token t = tFromName(buffer);
  56. if(t != T_END) {
  57. return tAddToken(t);
  58. }
  59. return tAddToken(T_LITERAL) && tAdd(buffer, index + 1);
  60. }
  61. static bool tParseNumber(int c) {
  62. int index = 1;
  63. char buffer[64];
  64. buffer[0] = c;
  65. bool point = false;
  66. while(true) {
  67. int c = fPeek();
  68. if(c == '.') {
  69. point = true;
  70. } else if(!isNumber(c)) {
  71. break;
  72. } else if(index >= 63) {
  73. tError("number is too long");
  74. return false;
  75. }
  76. buffer[index++] = fRead();
  77. }
  78. buffer[index] = '\0';
  79. if(point) {
  80. char* end = NULL;
  81. float f = strtof(buffer, &end);
  82. if(end[0] != '\0') {
  83. tError("invalid float on line %d", line);
  84. return false;
  85. }
  86. return tAddToken(T_CONST_FLOAT) && tAdd(&f, sizeof(float));
  87. } else {
  88. char* end = NULL;
  89. long l = strtol(buffer, &end, 10);
  90. if(end[0] != '\0' || l > INT_MAX) {
  91. tError("invalid int on line %d", line);
  92. return false;
  93. }
  94. int i = l;
  95. return tAddToken(T_CONST_INT) && tAdd(&i, sizeof(int));
  96. }
  97. }
  98. static bool tAddString() {
  99. if(!tAddToken(T_TEXT)) {
  100. return false;
  101. }
  102. while(true) {
  103. int c = fRead();
  104. if(c == '"') {
  105. break;
  106. } else if(c == '\\') {
  107. switch(fRead()) {
  108. case '"': c = '"'; break;
  109. case '\\': c = '\\'; break;
  110. default:
  111. tError("unknown escaped character at line %d", line);
  112. return false;
  113. }
  114. } else if(c == EOF) {
  115. tError("unclosed string starting at line %d", line);
  116. return false;
  117. }
  118. if(!tAdd(&c, 1)) {
  119. return false;
  120. }
  121. }
  122. char c = '\0';
  123. return tAdd(&c, 1);
  124. }
  125. static bool tAddToken2(Token te, Token t) {
  126. return fReadIf('=') ? tAddToken(te) : tAddToken(t);
  127. }
  128. static bool tAddToken3(int c, Token tc, Token te, Token t) {
  129. return fReadIf(c) ? tAddToken(tc) : tAddToken2(te, t);
  130. }
  131. static bool tAddToken4(int c, Token tce, Token tc, Token te, Token t) {
  132. return fReadIf(c) ? tAddToken2(tce, tc) : tAddToken2(te, t);
  133. }
  134. static bool tLineComment() {
  135. while(true) {
  136. int c = fRead();
  137. if(c == EOF || c == '\n') {
  138. line++;
  139. return true;
  140. }
  141. }
  142. }
  143. static bool tMultipleLineComment() {
  144. while(true) {
  145. int c = fRead();
  146. if(c == EOF) {
  147. tError("unclosed comment at line %d", line);
  148. return false;
  149. } else if(c == '\n') {
  150. line++;
  151. } else if(c == '*' && fReadIf('/')) {
  152. return true;
  153. }
  154. }
  155. }
  156. static bool tSlash() {
  157. if(fReadIf('/')) {
  158. return tLineComment();
  159. } else if(fReadIf('*')) {
  160. return tMultipleLineComment();
  161. }
  162. return tAddToken2(T_DIV_SET, T_DIV);
  163. }
  164. static bool tParseToken() {
  165. int c = fRead();
  166. if(c == EOF) {
  167. return false;
  168. } else if(isLetter(c)) {
  169. return tParseLiteral(c);
  170. } else if(isNumber(c)) {
  171. return tParseNumber(c);
  172. }
  173. switch(c) {
  174. case ' ': return true;
  175. case '\n': line++; return true;
  176. case '+': return tAddToken3('+', T_INCREMENT, T_ADD_SET, T_ADD);
  177. case '-': return tAddToken3('-', T_DECREMENT, T_SUB_SET, T_SUB);
  178. case '*': return tAddToken2(T_MUL_SET, T_MUL);
  179. case '/': return tSlash();
  180. case '%': return tAddToken2(T_MOD_SET, T_MOD);
  181. case '<':
  182. return tAddToken4('<', T_LEFT_SHIFT_SET, T_LEFT_SHIFT, T_LESS_EQUAL,
  183. T_LESS);
  184. case '>':
  185. return tAddToken4('>', T_RIGHT_SHIFT_SET, T_RIGHT_SHIFT,
  186. T_GREATER_EQUAL, T_GREATER);
  187. case '=': return tAddToken2(T_EQUAL, T_SET);
  188. case '!': return tAddToken2(T_NOT_EQUAL, T_NOT);
  189. case '&': return tAddToken3('&', T_AND, T_BIT_AND_SET, T_BIT_AND);
  190. case '|': return tAddToken3('|', T_OR, T_BIT_OR_SET, T_BIT_OR);
  191. case '~': return tAddToken(T_BIT_NOT);
  192. case '^': return tAddToken2(T_BIT_XOR_SET, T_BIT_XOR);
  193. case ',': return tAddToken(T_COMMA);
  194. case ';': return tAddToken(T_SEMICOLON);
  195. case '(': return tAddToken(T_OPEN_BRACKET);
  196. case ')': return tAddToken(T_CLOSE_BRACKET);
  197. case '{': return tAddToken(T_OPEN_CURVED_BRACKET);
  198. case '}': return tAddToken(T_CLOSE_CURVED_BRACKET);
  199. case '"': return tAddString();
  200. case '.': return tAddToken(T_POINT);
  201. case '[': return tAddToken(T_OPEN_SQUARE_BRACKET);
  202. case ']': return tAddToken(T_CLOSE_SQUARE_BRACKET);
  203. }
  204. tError("unknown character on line %d: %c", line, c);
  205. return false;
  206. }
  207. static void tParseFile() {
  208. readIndex = 0;
  209. writeIndex = 0;
  210. line = 1;
  211. error[0] = '\0';
  212. while(tParseToken()) {
  213. }
  214. }
  215. bool tTokenize(const char* path) {
  216. if(fOpen(path)) {
  217. tError("cannot read file '%s'", path);
  218. return true;
  219. }
  220. tParseFile();
  221. fClose();
  222. return error[0] != '\0';
  223. }
  224. const char* tGetError() {
  225. return error;
  226. }
  227. void tResetReader() {
  228. readIndex = 0;
  229. }
  230. Token tPeekToken() {
  231. if(readIndex >= writeIndex) {
  232. return T_END;
  233. }
  234. return tokenBuffer[readIndex];
  235. }
  236. Token tReadToken() {
  237. if(readIndex >= writeIndex) {
  238. return T_END;
  239. }
  240. return tokenBuffer[readIndex++];
  241. }
  242. bool tReadInt(int* i) {
  243. if(tReadTokens(i, sizeof(int))) {
  244. return true;
  245. }
  246. return false;
  247. }
  248. bool tReadInt16(int16* i) {
  249. if(tReadTokens(i, sizeof(int16))) {
  250. return true;
  251. }
  252. return false;
  253. }
  254. bool tReadFloat(float* f) {
  255. if(tReadTokens(f, sizeof(float))) {
  256. return true;
  257. }
  258. return false;
  259. }
  260. const char* tReadString(int* length) {
  261. *length = 0;
  262. const char* s = tokenBuffer + readIndex;
  263. while(readIndex <= writeIndex) {
  264. (*length)++;
  265. if(tokenBuffer[readIndex++] == '\0') {
  266. return s;
  267. }
  268. }
  269. return NULL;
  270. }