#include #include #include #include #include "tokenizer/FileTokens.h" #include "utils/SnuviUtils.h" static Error* error; typedef char String[64]; #define STRING_LENGTH ((int)sizeof(String)) typedef struct { String name; FileTokens tokens; } Define; #define DEFINES 256 static Define defines[DEFINES]; static int defineIndex = 0; static int ftGetLine(const FileTokens* ft, int index) { int fileLayer = 0; int lines = 1; for(int i = index; i >= 0; i--) { FileToken* t = ft->tokens + i; if(t->type == FT_PATH) { if(fileLayer == 0) { return lines; } fileLayer--; } else if(t->type == FT_END_PATH) { fileLayer++; } else if(t->type == FT_NEWLINE) { lines += fileLayer == 0; } } return 0; } static const char* ftGetPath(const FileTokens* ft, int index) { int fileLayer = 0; for(int i = index; i >= 0; i--) { FileToken* t = ft->tokens + i; if(t->type == FT_PATH) { if(fileLayer == 0) { return t->literal; } fileLayer--; } else if(t->type == FT_END_PATH) { fileLayer++; } } return "unknown path"; } static void ftError(const FileTokens* ft, int index, const char* format, ...) { va_list args; va_start(args, format); eInitErrorV(error, "", ftGetLine(ft, index), format, args); int fileLayer = 0; for(int i = index; i >= 0; i--) { FileToken* t = ft->tokens + i; if(t->type == FT_PATH) { if(fileLayer == 0) { eAddPath(error, t->literal); } else { fileLayer--; } } else if(t->type == FT_END_PATH) { fileLayer++; } } va_end(args); } static void ftSystemError(const char* msg, const FileTokens* parent, const FileTokens* ft, int index) { ftError(parent, index, "%s '%s': %s", msg, ftGetPath(ft, 0), strerror(errno)); } static void ftToSpace(FileToken* t) { free(t->literal); if(t->type != FT_NEWLINE) { t->type = FT_SPACE; } t->literal = NULL; t->single = 0; } static FileToken* ftAddToken(FileTokens* ft, FileTokenType type) { if(ft->length >= ft->capacity) { ft->capacity = (ft->capacity * 5) / 4 + 8; ft->tokens = realloc(ft->tokens, ft->capacity * sizeof(FileToken)); } FileToken* t = ft->tokens + ft->length++; t->type = type; t->literal = NULL; t->single = 0; return t; } static FileToken* ftAddCopyToken(FileTokens* ft, const FileToken* t) { FileToken* added = ftAddToken(ft, t->type); added->literal = t->literal == NULL ? NULL : strdup(t->literal); added->single = t->single; return added; } static void ftAddPath(FileTokens* ft, const char* path) { ftAddToken(ft, FT_PATH)->literal = strdup(path); } static void ftAddEndPath(FileTokens* ft) { ftAddToken(ft, FT_END_PATH); } static void ftAddNewline(FileTokens* ft) { ftAddToken(ft, FT_NEWLINE); } static void ftAddLiteral(FileTokens* ft, const char* path, int length) { ftAddToken(ft, FT_LITERAL)->literal = strndup(path, length); } static void ftAddSingle(FileTokens* ft, int single) { ftAddToken(ft, FT_SINGLE)->single = single; } static void ftAddSpace(FileTokens* ft) { ftAddToken(ft, FT_SPACE); } void ftPrint(FileTokens* ft) { for(int i = 0; i < ft->length; i++) { switch(ft->tokens[i].type) { case FT_PATH: printf("\033[0;34mPath: %s\n", ft->tokens[i].literal); break; case FT_END_PATH: puts("\033[0;34mEnd Path"); break; case FT_NEWLINE: putchar('\n'); break; case FT_LITERAL: printf("\033[0;33m%s", ft->tokens[i].literal); break; case FT_SINGLE: printf("\033[0;31m%c", (char)ft->tokens[i].single); break; case FT_SPACE: putchar(' '); break; } } printf("\033[0m"); } static bool ftHasError() { return eHasError(error); } static bool ftStringCompare(const char* a, const char* b) { return strcmp(a, b) == 0; } static void ftInitFileTokens(FileTokens* ft) { ft->tokens = NULL; ft->capacity = 0; ft->length = 0; } static void ftInitFileTokensWithPath(FileTokens* ft, const char* path) { ftInitFileTokens(ft); ftAddPath(ft, path); } static void ftTokenize(FileTokens* ft, unsigned char* data) { int index = 0; int oldIndex = 0; bool inString = false; while(true) { unsigned char c = data[index]; if(isAllowedInName(c) || (inString && c != '"' && c != '\0')) { index++; continue; } if(oldIndex < index) { ftAddLiteral(ft, (const char*)(data + oldIndex), index - oldIndex); } if(c == '"') { inString = !inString; } if(c == '\0') { break; } else if(c == '\n') { ftAddNewline(ft); } else if(c == ' ') { ftAddSpace(ft); } else if(c == '/' && data[index + 1] == '/') { while(data[index] != '\0' && data[index] != '\n') { index++; } ftAddNewline(ft); } else if(c == '/' && data[index + 1] == '*') { int startIndex = ft->length - 1; index += 2; while(true) { if(data[index] == '\0') { ftError(ft, startIndex, "test"); return; } else if(data[index] == '\n') { ftAddNewline(ft); } else if(data[index] == '*' && data[index + 1] == '/') { index++; break; } index++; } } else { ftAddSingle(ft, c); } index++; oldIndex = index; } } static void ftReadFull(FILE* file, FileTokens* parent, FileTokens* ft, int index) { if(fseek(file, 0, SEEK_END)) { ftSystemError("cannot seek end of file", parent, ft, index); return; } long length = ftell(file); if(length < 0) { ftSystemError("cannot tell end of file", parent, ft, index); return; } rewind(file); unsigned char* data = malloc(length + 1); int readValues = fread(data, 1, length, file); if(readValues != length) { ftSystemError("cannot read file", parent, ft, index); free(data); return; } data[length] = '\0'; ftTokenize(ft, data); ftAddEndPath(ft); free(data); } static void ftReadFullFile(FileTokens* parent, FileTokens* ft, int index) { FILE* file = fopen(ftGetPath(ft, 0), "r"); if(file == NULL) { ftSystemError("cannot open file", parent, ft, index); return; } ftReadFull(file, parent, ft, index); if(fclose(file) < 0) { ftSystemError("cannot close file", parent, ft, index); } } static void ftReplace(int from, int toEx, FileTokens* ft, const FileTokens* replacement) { int newCapacity = ft->capacity + replacement->capacity; FileToken* newTokens = malloc(newCapacity * sizeof(FileToken)); // these will be overwritten for(int i = from; i < toEx; i++) { free(ft->tokens[i].literal); ft->tokens[i].literal = NULL; } memcpy(newTokens, ft->tokens, from * sizeof(FileToken)); for(int i = 0; i < replacement->length; i++) { FileToken* src = replacement->tokens + i; FileToken* dst = newTokens + from + i; dst->literal = src->literal == NULL ? NULL : strdup(src->literal); dst->single = src->single; dst->type = src->type; } memcpy(newTokens + from + replacement->length, ft->tokens + toEx, (ft->length - toEx) * sizeof(FileToken)); free(ft->tokens); ft->tokens = newTokens; ft->capacity = newCapacity; ft->length = from + replacement->length + ft->length - toEx; } static void ftSkipSpaces(int* index, FileTokens* ft) { while(*index < ft->length) { FileToken* t = ft->tokens + *index; if(t->type != FT_SPACE) { return; } (*index)++; } } const char* INCLUDE_ERROR = "include path must be a string"; static bool ftIncludeDoubleQuote(int* index, FileTokens* ft) { if(*index >= ft->length || ft->tokens[*index].type != FT_SINGLE || ft->tokens[*index].single != '"') { ftError(ft, *index - 1, INCLUDE_ERROR); return true; } (*index)++; return false; } static void ftHandleInclude(int* index, int start, FileTokens* ft) { ftSkipSpaces(index, ft); if(ftIncludeDoubleQuote(index, ft)) { return; } if(*index >= ft->length || ft->tokens[*index].type != FT_LITERAL) { ftError(ft, *index - 1, INCLUDE_ERROR); return; } const char* path = ft->tokens[(*index)++].literal; if(ftIncludeDoubleQuote(index, ft)) { return; } const char* parentPath = ftGetPath(ft, *index - 1); int afterLastSlash = 0; for(int i = 0; parentPath[i] != '\0'; i++) { afterLastSlash = parentPath[i] == '/' ? i + 1 : afterLastSlash; } int pathLength = strlen(path) + 1; char* fullPath = malloc(afterLastSlash + pathLength); memcpy(fullPath, parentPath, afterLastSlash); memcpy(fullPath + afterLastSlash, path, pathLength); FileTokens include; ftInitFileTokensWithPath(&include, fullPath); free(fullPath); ftReadFullFile(ft, &include, *index - 1); if(!ftHasError()) { ftReplace(start, *index, ft, &include); *index = start; } ftDelete(&include); } static int ftGetDefineIndex(const char* s) { for(int i = 0; i < defineIndex; i++) { if(ftStringCompare(defines[i].name, s)) { return i; } } return -1; } static bool ftIsDefined(const char* s) { return ftGetDefineIndex(s) != -1; } static int ftFindEndIf(int from, FileTokens* ft) { int layer = 0; for(int i = from; i < ft->length - 1; i++) { if(ft->tokens[i].type != FT_SINGLE || ft->tokens[i].single != '#') { continue; } else if(ft->tokens[i + 1].type != FT_LITERAL) { continue; } if(ftStringCompare(ft->tokens[i + 1].literal, "ifndef")) { layer++; } else if(ftStringCompare(ft->tokens[i + 1].literal, "endif")) { if(layer == 0) { return i; } layer--; } } return -1; } static void ftHandleIfNotDefined(int* index, int start, FileTokens* ft) { ftSkipSpaces(index, ft); if(*index >= ft->length || ft->tokens[*index].type != FT_LITERAL) { ftError(ft, start, "ifndef expects a literal"); return; } const char* check = ft->tokens[(*index)++].literal; int endIf = ftFindEndIf(*index, ft); if(endIf == -1) { ftError(ft, start, "cannot find #endif"); return; } if(ftIsDefined(check)) { for(int i = start; i < endIf + 2; i++) { ftToSpace(ft->tokens + i); } } else { ftToSpace(ft->tokens + start); // # ftToSpace(ft->tokens + start + 1); // ifndef ftToSpace(ft->tokens + (*index - 1)); // DEFINITION ftToSpace(ft->tokens + endIf); // # ftToSpace(ft->tokens + endIf + 1); // endif } } static void ftHandleDefine(int* index, int start, FileTokens* ft) { if(defineIndex >= DEFINES) { ftError(ft, *index - 1, "too much defines"); return; } ftSkipSpaces(index, ft); if(*index >= ft->length || ft->tokens[*index].type != FT_LITERAL) { ftError(ft, *index - 1, "define expects a literal"); return; } const char* defineName = ft->tokens[(*index)++].literal; ftSkipSpaces(index, ft); int defineStart = *index; bool skip = false; while(*index < ft->length) { FileToken* t = ft->tokens + *index; if(!skip && t->type == FT_SINGLE && t->single == '\\') { skip = true; ftToSpace(t); } else if(t->type == FT_NEWLINE) { if(skip) { skip = false; } else { break; } } else if(skip) { ftError(ft, *index - 1, "expected newline after \\"); return; } (*index)++; } int foundIndex = ftGetDefineIndex(defineName); if(foundIndex != -1) { ftError(ft, *index - 1, "'%s' redefined", defineName); return; } Define* define = NULL; if(foundIndex == -1) { define = defines + defineIndex++; strncpy(define->name, defineName, STRING_LENGTH - 1); define->name[STRING_LENGTH - 1] = '\0'; } else { define = defines + foundIndex; ftDelete(&define->tokens); } ftInitFileTokens(&define->tokens); for(int i = defineStart; i < *index; i++) { ftAddCopyToken(&define->tokens, ft->tokens + i); if(ft->tokens[i].type == FT_NEWLINE) { FileToken* t = define->tokens.tokens + (define->tokens.length - 1); ftToSpace(t); t->type = FT_SPACE; } } for(int i = start; i < *index; i++) { ftToSpace(ft->tokens + i); } } static void ftHandleUndefine(int* index, int start, FileTokens* ft) { ftSkipSpaces(index, ft); if(*index >= ft->length || ft->tokens[*index].type != FT_LITERAL) { ftError(ft, *index - 1, "undefine expects a literal"); return; } const char* defineName = ft->tokens[(*index)++].literal; int foundIndex = ftGetDefineIndex(defineName); if(foundIndex != -1) { Define* define = defines + foundIndex; ftDelete(&define->tokens); defineIndex--; if(defineIndex != foundIndex) { memcpy(define, defines + defineIndex, sizeof(Define)); } } for(int i = start; i < *index; i++) { ftToSpace(ft->tokens + i); } } static void ftParseInstruction(int* index, FileTokens* ft) { int start = *index; (*index)++; if(*index >= ft->length || ft->tokens[*index].type != FT_LITERAL) { ftError(ft, *index - 1, "expected literal after #"); return; } const char* name = ft->tokens[(*index)++].literal; if(ftStringCompare(name, "include")) { ftHandleInclude(index, start, ft); } else if(ftStringCompare(name, "ifndef")) { ftHandleIfNotDefined(index, start, ft); } else if(ftStringCompare(name, "define")) { ftHandleDefine(index, start, ft); } else if(ftStringCompare(name, "undef")) { ftHandleUndefine(index, start, ft); } else if(ftStringCompare(name, "endif")) { ftError(ft, *index - 1, "endif without if"); } else { ftError(ft, *index - 1, "unknown preprocessor literal '%s'", name); } } static bool ftReplaceMakros(FileTokens* ft, int index) { FileToken* t = ft->tokens + index; int foundIndex = ftGetDefineIndex(t->literal); if(foundIndex != -1) { ftReplace(index, index + 1, ft, &(defines[foundIndex].tokens)); return true; } return false; } static void ftSearchInstruction(FileTokens* ft) { for(int i = 0; i < ft->length && !ftHasError(); i++) { if(ft->tokens[i].type == FT_SINGLE && ft->tokens[i].single == '#') { ftParseInstruction(&i, ft); } else if(ft->tokens[i].type == FT_LITERAL) { if(ftReplaceMakros(ft, i)) { i--; } } } } void ftInit(const char* path, FileTokens* ft, Error* e) { defineIndex = 0; error = e; eInitSuccess(error); ftInitFileTokensWithPath(ft, path); ftReadFullFile(ft, ft, 0); if(!ftHasError()) { ftSearchInstruction(ft); } for(int i = 0; i < defineIndex; i++) { ftDelete(&(defines[i].tokens)); } } void ftDelete(FileTokens* ft) { for(int i = 0; i < ft->length; i++) { free(ft->tokens[i].literal); } free(ft->tokens); ft->tokens = NULL; ft->capacity = 0; ft->length = 0; }