partial move back to cpp

Main.c → Main.cpp

@@ -4,11 +4,8 @@
 #include "test/Test.h"
 
 int main(int argc, char** argv) {
-    if(argc <= 0) {
-        return 0;
-    }
     if(argc >= 3 && strcmp(argv[1], "test") == 0) {
-        startTests(argv[2]);
+        Test::start(argv[2]);
     }
     return 0;
 }

meson.build

@@ -1,7 +1,7 @@
-project('lonely tiger', 'c')
+project('lonely tiger', 'cpp')
 
-src = ['Main.c', 'test/Test.c', 'utils/Path.c', 'tokenizer/Token.c', 'tokenizer/TokenStream.c', 'tokenizer/Tokenizer.c']
+src = ['Main.cpp', 'test/Test.cpp', 'utils/String.cpp', 'tokenizer/Token.cpp', 'tokenizer/TokenStream.cpp', 'tokenizer/Tokenizer.cpp']
 
 executable('lonely_tiger', 
     sources: src,
-    c_args: ['-Wall', '-Wextra', '-pedantic', '-Werror'])
+    cpp_args: ['-Wall', '-Wextra', '-pedantic', '-Werror'])

test/Test.c

@@ -1,154 +0,0 @@
-#include <stdio.h>
-#include <stdbool.h>
-#include <stdlib.h>
-#include <dirent.h>
-#include <string.h>
-#include <unistd.h>
-
-#include "test/Test.h"
-#include "utils/Path.h"
-#include "tokenizer/TokenStream.h"
-#include "tokenizer/Tokenizer.h"
-
-static unsigned int done = 0;
-static unsigned int tests = 0;
-
-static bool test_checkPath(Path* path, const char* ending, bool(*f) (const char*, const char*)) {
-    (void) ending;
-    DIR* dir;
-    dir = opendir(path->path);
-    if(dir == NULL) {
-        fprintf(stderr, "cannot open '%s': ", path->path);
-        perror("");
-        return true;
-    }
-    struct dirent* entry = NULL;
-    while(true) {
-        entry = readdir(dir);
-        if(entry == NULL) {
-            break;
-        } else if(strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) {
-            continue;
-        }
-        if(entry->d_type == DT_DIR) {
-            if(enterPath(path, entry->d_name)) {
-                return true;
-            }
-            if(test_checkPath(path, ending, f)) {
-                leavePath(path);
-                return true;
-            }
-            leavePath(path);
-        } else if(entry->d_type == DT_REG && strchr(entry->d_name, '.') == NULL) {
-            Path inputPath;
-            initPath(&inputPath, path->path);
-            enterFile(&inputPath, entry->d_name, "");
-            Path outputPath;
-            initPath(&outputPath, path->path);
-            enterFile(&outputPath, entry->d_name, ending);
-            if(f(inputPath.path, outputPath.path)) {
-                return true;
-            }
-        }
-    }
-    if(closedir(dir)) {
-        fprintf(stderr, "cannot close '%s': ", path->path);
-        perror("");
-        return true;
-    }
-    return false;
-}
-
-static bool test_forEachFile(const char* strPath, const char* ending, bool(*f) (const char*, const char*)) {
-    Path path;
-    initPath(&path, strPath);
-    return test_checkPath(&path, ending, f);
-}
-
-static void test_readLine(char* buffer, size_t capacity, FILE* f) {
-    size_t index = 0;
-    while(index < capacity - 1) {
-        int c = fgetc(f);
-        if(c == EOF || c == '\n') {
-            break;
-        }
-        buffer[index++] = c;
-    }
-    buffer[index] = '\0';
-}
-
-static bool test_tokenizerTester(const char* input, const char* output) {
-    FILE* f = fopen(output, "r");
-    if(f == NULL) {
-        perror("cannot open file");
-        return false;
-    }
-
-    TokenStream* tokenStream = newTokenStream();
-    bool b = tokenize(tokenStream, input);
-
-    while(hasToken(tokenStream)) {
-        const size_t bufferLength = 64;
-        char buffer[bufferLength];
-        nextTokenString(tokenStream, buffer, bufferLength);
-
-        char expected[1024];
-        test_readLine(expected, 1024, f);
-        if(strchr(buffer, '\n') != NULL) {
-            size_t length = strlen(expected);
-            expected[length] = '\n';
-            test_readLine(expected + length + 1, 1024 - length - 1, f);
-        }
-        if(strcmp(buffer, expected) != 0) {
-            printf("error in '%s'\n", input);
-            printf("'%s' should be '%s'\n", buffer, expected);
-            break;
-        }
-    }
-
-    fclose(f);
-    deleteTokenStream(&tokenStream);
-    return b;
-}
-
-static void test_testTokenizer(const char* path) {
-    done = 0;
-    tests = 0;
-
-    test_forEachFile(path, ".tout", test_tokenizerTester);
-    /*forEachFile(path, ".tout", [](const std::string& input, const std::string & output) {
-        tests++;
-
-        Tokenizer::if32stream iStream;
-        iStream.open(input);
-
-        std::ifstream oStream;
-        oStream.open(output);
-
-        if(!iStream.good() || !oStream.good()) {
-            return false;
-        }
-
-        TokenStream tokens;
-        if(Tokenizer::tokenize(tokens, iStream)) {
-            return false;
-        }
-        logger.reset();
-        while(tokens.hasToken()) {
-            std::string s = tokens.nextTokenString();
-                    logger.print(&s);
-        }
-
-        if(logger.check(input, oStream)) {
-            done++;
-        }
-        return true;
-    });
-    std::cout << done << " / " << tests << " tokenizer tests succeeded" << std::endl;*/
-}
-
-void startTests(const char* path) {
-    test_testTokenizer(path);
-    //testCompiler();
-    //testOutput();
-}

test/Test.cpp

@@ -0,0 +1,99 @@
+#include <iostream>
+#include <fstream>
+
+#include <stdio.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <dirent.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "test/Test.h"
+#include "utils/String.h"
+#include "tokenizer/TokenStream.h"
+#include "tokenizer/Tokenizer.h"
+
+static unsigned int done = 0;
+static unsigned int tests = 0;
+
+static bool checkPath(const String& path, const char* ending, bool(*f) (const String&, const String&)) {
+    DIR* dir = opendir(path);
+    if(dir == NULL) {
+        std::cout << "cannot open '" << path << "': " << strerror(errno) << "\n";
+        return true;
+    }
+    struct dirent* entry = nullptr;
+    while(true) {
+        entry = readdir(dir);
+        if(entry == nullptr) {
+            break;
+        } else if(strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) {
+            continue;
+        } else if(entry->d_type == DT_DIR) {
+            checkPath(path + "/" + entry->d_name, ending, f);
+        } else if(entry->d_type == DT_REG && strchr(entry->d_name, '.') == NULL) {
+            if(f(path + "/" + entry->d_name, path + "/" + entry->d_name + ending)) {
+                return true;
+            }
+        }
+    }
+    if(closedir(dir)) {
+        std::cout << "cannot close '" << path << "': " << strerror(errno) << "\n";
+        return true;
+    }
+    return false;
+}
+
+static String readLine(std::ifstream& f) {
+    String s;
+    while(true) {
+        char c = f.get();
+        if(!f.good() || c == '\n') {
+            break;
+        }
+        s += c;
+    }
+    return s;
+}
+
+static bool testTokenizer(const String& input, const String& output) {
+    tests++;
+    std::ifstream oStream;
+    oStream.open(output);
+    if(!oStream.good()) {
+        std::cout << "cannot open file '" << output << "'\n";
+        return true;
+    }
+    TokenStream tokenStream;
+    bool b = tokenize(&tokenStream, input);
+    if(!b) {
+        done++;
+    }
+    while(tokenStream.hasToken()) {
+        String buffer = tokenStream.nextTokenString();
+        String expected = readLine(oStream);
+        if(strchr(buffer, '\n') != NULL) {
+            expected += '\n';
+            expected += readLine(oStream);
+        }
+        if(strcmp(buffer, expected) != 0) {
+            std::cout << "error in '" << input << "\n'" << buffer << "' should be '" << expected << "'\n";
+            done--;
+            break;
+        }
+    }
+    return b;
+}
+
+static void test_testTokenizer(const char* path) {
+    done = 0;
+    tests = 0;
+    checkPath(path, ".tout", testTokenizer);
+    std::cout << done << " / " << tests << " tokenizer tests succeeded" << std::endl;
+}
+
+void Test::start(const char* path) {
+    test_testTokenizer(path);
+    //testCompiler();
+    //testOutput();
+}

test/Test.h

@@ -1,6 +1,8 @@
 #ifndef TEST_H
 #define TEST_H
 
-void startTests(const char* path);
+namespace Test {
+    void start(const char* path);
+}
 
 #endif

tokenizer/Token.c

@@ -1,133 +0,0 @@
-#include "tokenizer/Token.h"
-
-const char* getTokenName(Token token) {
-    switch(token) {
-        case NUMBER: return "number";
-        case STRING: return "string";
-        case LITERAL: return "literal";
-        case LABEL: return "label";
-        case TRUE: return "true";
-        case FALSE: return "false";
-        case NULL_TOKEN: return "null";
-        case OPEN_BRACKET: return "(";
-        case CLOSE_BRACKET: return ")";
-        case OPEN_SQUARE_BRACKET: return "[";
-        case CLOSE_SQUARE_BRACKET: return "]";
-        case OPEN_CURVED_BRACKET: return "{";
-        case CLOSE_CURVED_BRACKET: return "}";
-        case SEMICOLON: return ";";
-        case COMMA: return ",";
-        case INC: return "++";
-        case DEC: return "--";
-        case INVERT: return "!";
-        case BIT_INVERT: return "~";
-        case MUL: return "*";
-        case DIV: return "/";
-        case MOD: return "%";
-        case ADD: return "+";
-        case SUB: return "-";
-        case ADD_SET: return "+=";
-        case SUB_SET: return "-=";
-        case MUL_SET: return "*=";
-        case DIV_SET: return "/=";
-        case MOD_SET: return "%=";
-        case LEFT_SHIFT: return "<<";
-        case RIGHT_SHIFT: return ">>";
-        case LEFT_SHIFT_SET: return "<<=";
-        case RIGHT_SHIFT_SET: return ">>=";
-        case BIT_AND_SET: return "&=";
-        case BIT_XOR_SET: return "^=";
-        case BIT_OR_SET: return "|=";
-        case LESS: return "<";
-        case LESS_EQUAL: return "<=";
-        case GREATER: return ">";
-        case GREATER_EQUAL: return ">=";
-        case EQUAL: return "==";
-        case NOT_EQUAL: return "!=";
-        case BIT_AND: return "&";
-        case BIT_XOR: return "^";
-        case BIT_OR: return "|";
-        case AND: return "&&";
-        case OR: return "||";
-        case SET: return "=";
-        case IF: return "if";
-        case ELSE: return "else";
-        case ELSEIF: return "else if";
-        case WHILE: return "while";
-        case TRY: return "try";
-        case CATCH: return "catch";
-        case FOR: return "for";
-        case FUNCTION: return "function";
-        case BREAK: return "break";
-        case CONTINUE: return "continue";
-        case RETURN: return "return";
-        case EOF_TOKEN: return "end of file";
-    }
-    return "Unknown TokenType";
-}
-
-const char* getTokenEnumName(Token token) {
-    switch(token) {
-        case NUMBER: return "NUMBER";
-        case STRING: return "STRING";
-        case LITERAL: return "LITERAL";
-        case LABEL: return "LABEL";
-        case TRUE: return "TRUE";
-        case FALSE: return "FALSE";
-        case NULL_TOKEN: return "NULL_TOKEN";
-        case OPEN_BRACKET: return "OPEN_BRACKET";
-        case CLOSE_BRACKET: return "CLOSE_BRACKET";
-        case OPEN_SQUARE_BRACKET: return "OPEN_SQUARE_BRACKET";
-        case CLOSE_SQUARE_BRACKET: return "CLOSE_SQUARE_BRACKET";
-        case OPEN_CURVED_BRACKET: return "OPEN_CURVED_BRACKET";
-        case CLOSE_CURVED_BRACKET: return "CLOSE_CURVED_BRACKET";
-        case SEMICOLON: return "SEMICOLON";
-        case COMMA: return "COMMA";
-        case INC: return "INC";
-        case DEC: return "DEC";
-        case INVERT: return "INVERT";
-        case BIT_INVERT: return "BIT_INVERT";
-        case MUL: return "MUL";
-        case DIV: return "DIV";
-        case MOD: return "MOD";
-        case ADD: return "ADD";
-        case SUB: return "SUB";
-        case ADD_SET: return "ADD_SET";
-        case SUB_SET: return "SUB_SET";
-        case MUL_SET: return "MUL_SET";
-        case DIV_SET: return "DIV_SET";
-        case MOD_SET: return "MOD_SET";
-        case LEFT_SHIFT: return "LEFT_SHIFT";
-        case RIGHT_SHIFT: return "RIGHT_SHIFT";
-        case LEFT_SHIFT_SET: return "LEFT_SHIFT_SET";
-        case RIGHT_SHIFT_SET: return "RIGHT_SHIFT_SET";
-        case BIT_AND_SET: return "BIT_AND_SET";
-        case BIT_XOR_SET: return "BIT_XOR_SET";
-        case BIT_OR_SET: return "BIT_OR_SET";
-        case LESS: return "LESS";
-        case LESS_EQUAL: return "LESS_EQUAL";
-        case GREATER: return "GREATER";
-        case GREATER_EQUAL: return "GREATER_EQUAL";
-        case EQUAL: return "EQUAL";
-        case NOT_EQUAL: return "NOT_EQUAL";
-        case BIT_AND: return "BIT_AND";
-        case BIT_XOR: return "BIT_XOR";
-        case BIT_OR: return "BIT_OR";
-        case AND: return "AND";
-        case OR: return "OR";
-        case SET: return "SET";
-        case IF: return "IF";
-        case ELSE: return "ELSE";
-        case ELSEIF: return "ELSEIF";
-        case WHILE: return "WHILE";
-        case TRY: return "TRY";
-        case CATCH: return "CATCH";
-        case FOR: return "FOR";
-        case FUNCTION: return "FUNCTION";
-        case BREAK: return "BREAK";
-        case CONTINUE: return "CONTINUE";
-        case RETURN: return "RETURN";
-        case EOF_TOKEN: return "EOF";
-    }
-    return "UNKNOWN";
-}

tokenizer/Token.cpp

@@ -0,0 +1,24 @@
+#include "tokenizer/Token.h"
+
+const char* getTokenName(Token token) {
+    static const char* data[] = {
+        "number", "string", "literal", "label", "true", "false", "null", "(", ")", "[", "]", "{", "}", ";", ",", "++",
+        "--", "!", "~", "*", "/", "%", "+", "-", "+=", "-=", "*=", "/=", "%=", "<<", ">>", "<<=", ">>=", "&=", "^=",
+        "|=", "<", "<=", ">", ">=", "==", "!=", "&", "^", "|", "&&", "||", "=", "if", "else", "else if", "while", "try",
+        "catch", "for", "function", "break", "continue", "return", "end of file"
+    };
+    return data[token];
+}
+
+const char* getTokenEnumName(Token token) {
+    static const char* data[] = {
+        "NUMBER", "STRING", "LITERAL", "LABEL", "TRUE", "FALSE", "NULL_TOKEN", "OPEN_BRACKET", "CLOSE_BRACKET",
+        "OPEN_SQUARE_BRACKET", "CLOSE_SQUARE_BRACKET", "OPEN_CURVED_BRACKET", "CLOSE_CURVED_BRACKET", "SEMICOLON",
+        "COMMA", "INC", "DEC", "INVERT", "BIT_INVERT", "MUL", "DIV", "MOD", "ADD", "SUB", "ADD_SET", "SUB_SET",
+        "MUL_SET", "DIV_SET", "MOD_SET", "LEFT_SHIFT", "RIGHT_SHIFT", "LEFT_SHIFT_SET", "RIGHT_SHIFT_SET",
+        "BIT_AND_SET", "BIT_XOR_SET", "BIT_OR_SET", "LESS", "LESS_EQUAL", "GREATER", "GREATER_EQUAL", "EQUAL",
+        "NOT_EQUAL", "BIT_AND", "BIT_XOR", "BIT_OR", "AND", "OR", "SET", "IF", "ELSE", "ELSEIF", "WHILE", "TRY",
+        "CATCH", "FOR", "FUNCTION", "BREAK", "CONTINUE", "RETURN", "EOF"
+    };
+    return data[token];
+}

tokenizer/Token.h

@@ -2,12 +2,12 @@
 #define TOKENTYPE_H
 
 typedef enum Token {
-    NUMBER, 
-    STRING, 
-    LITERAL, 
+    NUMBER,
+    STRING,
+    LITERAL,
     LABEL,
-    TRUE, 
-    FALSE, 
+    TRUE,
+    FALSE,
     NULL_TOKEN,
 
     OPEN_BRACKET, CLOSE_BRACKET,

tokenizer/TokenStream.c

@@ -1,129 +0,0 @@
-#include <stdlib.h>
-#include <string.h>
-#include <stdio.h>
-
-#include "tokenizer/TokenStream.h"
-
-TokenStream* newTokenStream() {
-    TokenStream* tokenStream = malloc(sizeof (TokenStream));
-    tokenStream->readIndex = 0;
-    tokenStream->index = 0;
-    tokenStream->length = 1024;
-    tokenStream->buffer = malloc(sizeof (char) * tokenStream->length);
-    return tokenStream;
-}
-
-void deleteTokenStream(TokenStream** tokenStream) {
-    if(tokenStream == NULL || (*tokenStream) == NULL) {
-        return;
-    }
-    free((*tokenStream)->buffer);
-    free(*tokenStream);
-    *tokenStream = NULL;
-}
-
-bool hasToken(TokenStream* tokenStream) {
-    return tokenStream->readIndex < tokenStream->index;
-}
-
-static size_t tokenstream_concat(char* dst, const char* src, size_t bufferLength) {
-    size_t length = strlen(src);
-    strncat(dst, src, bufferLength);
-    if(length > bufferLength) {
-        return 0;
-    }
-    return bufferLength - length;
-}
-
-void nextTokenString(TokenStream* tokenStream, char* buffer, size_t bufferLength) {
-    if(bufferLength == 0) {
-        return;
-    }
-    buffer[0] = '\0';
-    bufferLength--;
-
-    bufferLength = tokenstream_concat(buffer, "(", bufferLength);
-    // line
-    char lineBuffer[32];
-    unsigned int line = nextLine(tokenStream);
-    snprintf(lineBuffer, 32, "%u, ", line);
-    bufferLength = tokenstream_concat(buffer, lineBuffer, bufferLength);
-
-    // tokentype
-    Token token = nextToken(tokenStream);
-    bufferLength = tokenstream_concat(buffer, getTokenEnumName(token), bufferLength);
-
-    if(token == STRING || token == LITERAL || token == LABEL) {
-        bufferLength = tokenstream_concat(buffer, ", \"", bufferLength);
-        bufferLength = tokenstream_concat(buffer, nextString(tokenStream), bufferLength);
-        bufferLength = tokenstream_concat(buffer, "\"", bufferLength);
-    }
-
-    if(token == NUMBER) {
-        char doubleBuffer[32];
-        double d = nextDouble(tokenStream);
-        snprintf(doubleBuffer, 32, (d == (long) d) ? ", %lg.0" : ", %lg", d);
-        bufferLength = tokenstream_concat(buffer, doubleBuffer, bufferLength);
-    }
-
-    bufferLength = tokenstream_concat(buffer, ")", bufferLength);
-}
-
-void tokenstream_read(TokenStream* tokenStream, void* data, size_t length) {
-    memcpy(data, tokenStream->buffer + tokenStream->readIndex, length);
-    tokenStream->readIndex += length;
-}
-
-Token nextToken(TokenStream* tokenStream) {
-    Token token = EOF_TOKEN;
-    tokenstream_read(tokenStream, &token, sizeof (Token));
-    return token;
-}
-
-unsigned int nextLine(TokenStream* tokenStream) {
-    unsigned int line = 0;
-    tokenstream_read(tokenStream, &line, sizeof (unsigned int));
-    return line;
-}
-
-const char* nextString(TokenStream* tokenStream) {
-    size_t offset = tokenStream->readIndex;
-    tokenStream->readIndex += strlen(tokenStream->buffer + tokenStream->readIndex) + 1;
-    return tokenStream->buffer + offset;
-}
-
-double nextDouble(TokenStream* tokenStream) {
-    double d;
-    tokenstream_read(tokenStream, &d, sizeof (double));
-    return d;
-}
-
-static void tokenstream_write(TokenStream* tokenStream, const void* data, size_t length) {
-    if(tokenStream->index + length >= tokenStream->length) {
-        size_t newSize = tokenStream->length;
-        while(newSize < tokenStream->index + length) {
-            newSize *= 2;
-        }
-        tokenStream->buffer = realloc(tokenStream->buffer, sizeof (char) * newSize);
-        tokenStream->length = newSize;
-    }
-    const char* chars = (const char*) data;
-    for(size_t i = 0; i < length; i++) {
-        tokenStream->buffer[tokenStream->index++] = chars[i];
-    }
-}
-
-void addToken(TokenStream* tokenStream, Token token, unsigned int line) {
-    tokenstream_write(tokenStream, &line, sizeof (unsigned int));
-    tokenstream_write(tokenStream, &token, sizeof (Token));
-}
-
-void addDoubleToken(TokenStream* tokenStream, Token token, unsigned int line, double d) {
-    addToken(tokenStream, token, line);
-    tokenstream_write(tokenStream, &d, sizeof (double));
-}
-
-void addStringToken(TokenStream* tokenStream, Token token, unsigned int line, const char* text) {
-    addToken(tokenStream, token, line);
-    tokenstream_write(tokenStream, text, strlen(text) + 1);
-}

tokenizer/TokenStream.cpp

@@ -0,0 +1,81 @@
+#include <cstring>
+
+#include "tokenizer/TokenStream.h"
+
+TokenStream::TokenStream() : readIndex(0) {
+}
+
+bool TokenStream::hasToken() const {
+    return readIndex < buffer.size();
+}
+
+String TokenStream::nextTokenString() {
+    String s;
+    s += '(';
+    s += nextLine();
+    s += ", ";
+    Token token = nextToken();
+    s += getTokenEnumName(token);
+    if(token == STRING || token == LITERAL || token == LABEL) {
+        s += ", \"";
+        s += nextString();
+        s += '"';
+    }
+    if(token == NUMBER) {
+        s += ", ";
+        s += nextDouble();
+    }
+    s += ')';
+    return s;
+}
+
+void TokenStream::read(void* data, size_t length) {
+    memcpy(data, buffer.data() + readIndex, length);
+    readIndex += length;
+}
+
+Token TokenStream::nextToken() {
+    Token token = EOF_TOKEN;
+    read(&token, sizeof (unsigned int));
+    return token;
+}
+
+unsigned int TokenStream::nextLine() {
+    unsigned int line = 0;
+    read(&line, sizeof (unsigned int));
+    return line;
+}
+
+const char* TokenStream::nextString() {
+    size_t offset = readIndex;
+    readIndex += strlen(buffer.data() + readIndex) + 1;
+    return buffer.data() + offset;
+}
+
+double TokenStream::nextDouble() {
+    double d;
+    read(&d, sizeof (double));
+    return d;
+}
+
+void TokenStream::write(const void* data, size_t length) {
+    const char* chars = static_cast<const char*> (data);
+    for(size_t i = 0; i < length; i++) {
+        buffer.push_back(chars[i]);
+    }
+}
+
+void TokenStream::add(Token token, unsigned int line) {
+    write(&line, sizeof (unsigned int));
+    write(&token, sizeof (Token));
+}
+
+void TokenStream::add(Token token, unsigned int line, double d) {
+    add(token, line);
+    write(&d, sizeof (double));
+}
+
+void TokenStream::add(Token token, unsigned int line, const char* text) {
+    add(token, line);
+    write(text, strlen(text) + 1);
+}

tokenizer/TokenStream.h

@@ -1,31 +1,32 @@
 #ifndef TOKENSTREAM_H
 #define TOKENSTREAM_H
 
-#include <stddef.h>
-#include <stdbool.h>
+#include <vector> 
 
 #include "tokenizer/Token.h"
-
-typedef struct TokenStream {
+#include "utils/String.h"
+
+class TokenStream {
+public:
+    TokenStream();
+
+    bool hasToken() const;
+    String nextTokenString();
+    Token nextToken();
+    unsigned int nextLine();
+    const char* nextString();
+    double nextDouble();
+
+    void add(Token token, unsigned int line);
+    void add(Token token, unsigned int line, double d);
+    void add(Token token, unsigned int line, const char* text);
+
+private:
+    void read(void* data, size_t length);
+    void write(const void* data, size_t length);
+    
     size_t readIndex;
-    size_t index;
-    size_t length;
-    char* buffer;
-} TokenStream;
-
-TokenStream* newTokenStream();
-void deleteTokenStream(TokenStream** tokenStream);
-
-bool hasToken(TokenStream* tokenStream);
-void nextTokenString(TokenStream* tokenStream, char* buffer, size_t bufferLength);
-
-Token nextToken(TokenStream* tokenStream);
-unsigned int nextLine(TokenStream* tokenStream);
-const char* nextString(TokenStream* tokenStream);
-double nextDouble(TokenStream* tokenStream);
-
-void addToken(TokenStream* tokenStream, Token token, unsigned int line);
-void addDoubleToken(TokenStream* tokenStream, Token token, unsigned int line, double d);
-void addStringToken(TokenStream* tokenStream, Token token, unsigned int line, const char* text);
+    std::vector<char> buffer;
+};
 
 #endif

tokenizer/Tokenizer.c → tokenizer/Tokenizer.cpp

@@ -99,13 +99,11 @@ static bool tokenizer_nextIf(char32_t c) {
 }
 
 static void tokenizer_addToken(Token token) {
-
-    addToken(tokens, token, line);
+    tokens->add(token, line);
 }
 
 static void tokenizer_addStringToken(Token token, const char* text) {
-
-    addStringToken(tokens, token, line, text);
+    tokens->add(token, line, text);
 }
 
 static Token tokenizer_chooseToken(char c, Token aCharEqual, Token aChar, Token aEqual, Token other) {
@@ -138,35 +136,33 @@ static bool tokenizer_handleLiteral(char32_t c, Token token) {
     buffer[index] = '\0';
 
     if(strcmp(buffer, "if") == 0) {
-        tokenizer_addToken(IF);
-    } else if(strcmp(buffer, "if") == 0) {
-        tokenizer_addToken(IF);
+        tokenizer_addToken(Token::IF);
     } else if(strcmp(buffer, "else") == 0) {
-        tokenizer_addToken(ELSE);
+        tokenizer_addToken(Token::ELSE);
     } else if(strcmp(buffer, "elseif") == 0) {
-        tokenizer_addToken(ELSEIF);
+        tokenizer_addToken(Token::ELSEIF);
     } else if(strcmp(buffer, "while") == 0) {
-        tokenizer_addToken(WHILE);
+        tokenizer_addToken(Token::WHILE);
     } else if(strcmp(buffer, "try") == 0) {
-        tokenizer_addToken(TRY);
+        tokenizer_addToken(Token::TRY);
     } else if(strcmp(buffer, "catch") == 0) {
-        tokenizer_addToken(CATCH);
+        tokenizer_addToken(Token::CATCH);
     } else if(strcmp(buffer, "for") == 0) {
-        tokenizer_addToken(FOR);
+        tokenizer_addToken(Token::FOR);
     } else if(strcmp(buffer, "function") == 0) {
-        tokenizer_addToken(FUNCTION);
+        tokenizer_addToken(Token::FUNCTION);
     } else if(strcmp(buffer, "break") == 0) {
-        tokenizer_addToken(BREAK);
+        tokenizer_addToken(Token::BREAK);
     } else if(strcmp(buffer, "continue") == 0) {
-        tokenizer_addToken(CONTINUE);
+        tokenizer_addToken(Token::CONTINUE);
     } else if(strcmp(buffer, "return") == 0) {
-        tokenizer_addToken(RETURN);
+        tokenizer_addToken(Token::RETURN);
     } else if(strcmp(buffer, "true") == 0) {
-        tokenizer_addToken(TRUE);
+        tokenizer_addToken(Token::TRUE);
     } else if(strcmp(buffer, "false") == 0) {
-        tokenizer_addToken(FALSE);
+        tokenizer_addToken(Token::FALSE);
     } else if(strcmp(buffer, "null") == 0) {
-        tokenizer_addToken(NULL_TOKEN);
+        tokenizer_addToken(Token::NULL_TOKEN);
     } else {
 
         tokenizer_addStringToken(token, buffer);
@@ -194,8 +190,7 @@ static bool tokenizer_handleNumber(char32_t c) {
         number = (number * 10) + (data - '0');
         tokenizer_next(&data);
     }
-    addDoubleToken(tokens, NUMBER, line, number);
-
+    tokens->add(Token::NUMBER, line, number);
     return false;
 }
 
@@ -213,7 +208,7 @@ static bool tokenizer_handleString() {
         }
         if(data == '"') {
             buffer[index] = '\0';
-            tokenizer_addStringToken(STRING, buffer);
+            tokenizer_addStringToken(Token::STRING, buffer);
             return false;
         }
         if(data == '\n') {
@@ -276,10 +271,10 @@ static bool tokenizer_handleSlash() {
     } else if(tokenizer_nextIf('*')) {
         return tokenizer_handleMultiLineComment();
     } else if(tokenizer_nextIf('=')) {
-        tokenizer_addToken(DIV_SET);
+        tokenizer_addToken(Token::DIV_SET);
         return false;
     }
-    tokenizer_addToken(DIV);
+    tokenizer_addToken(Token::DIV);
 
     return false;
 }
@@ -294,51 +289,51 @@ static bool tokenizer_handleSpecial(char32_t c) {
             return false;
         case '"':
             return tokenizer_handleString();
-        case '(': tokenizer_addToken(OPEN_BRACKET);
+        case '(': tokenizer_addToken(Token::OPEN_BRACKET);
             return false;
-        case ')': tokenizer_addToken(CLOSE_BRACKET);
+        case ')': tokenizer_addToken(Token::CLOSE_BRACKET);
             return false;
-        case '[': tokenizer_addToken(OPEN_SQUARE_BRACKET);
+        case '[': tokenizer_addToken(Token::OPEN_SQUARE_BRACKET);
             return false;
-        case ']': tokenizer_addToken(CLOSE_SQUARE_BRACKET);
+        case ']': tokenizer_addToken(Token::CLOSE_SQUARE_BRACKET);
             return false;
-        case '{': tokenizer_addToken(OPEN_CURVED_BRACKET);
+        case '{': tokenizer_addToken(Token::OPEN_CURVED_BRACKET);
             return false;
-        case '}': tokenizer_addToken(CLOSE_CURVED_BRACKET);
+        case '}': tokenizer_addToken(Token::CLOSE_CURVED_BRACKET);
             return false;
         case '$':
-            return tokenizer_handleLiteral(c, LITERAL);
+            return tokenizer_handleLiteral(c, Token::LITERAL);
         case '@':
-            return tokenizer_handleLiteral(c, LABEL);
-        case ';': tokenizer_addToken(SEMICOLON);
+            return tokenizer_handleLiteral(c, Token::LABEL);
+        case ';': tokenizer_addToken(Token::SEMICOLON);
             return false;
-        case ',': tokenizer_addToken(COMMA);
+        case ',': tokenizer_addToken(Token::COMMA);
             return false;
-        case '~': tokenizer_addToken(BIT_INVERT);
+        case '~': tokenizer_addToken(Token::BIT_INVERT);
             return false;
-        case '+': tokenizer_addToken(tokenizer_nextIf('=') ? ADD_SET: (tokenizer_nextIf('+') ? INC: ADD));
+        case '+': tokenizer_addToken(tokenizer_nextIf('=') ? Token::ADD_SET: (tokenizer_nextIf('+') ? Token::INC: Token::ADD));
             return false;
-        case '-': tokenizer_addToken(tokenizer_nextIf('=') ? SUB_SET: (tokenizer_nextIf('-') ? DEC: SUB));
+        case '-': tokenizer_addToken(tokenizer_nextIf('=') ? Token::SUB_SET: (tokenizer_nextIf('-') ? Token::DEC: Token::SUB));
             return false;
-        case '!': tokenizer_addToken(tokenizer_nextIf('=') ? NOT_EQUAL: INVERT);
+        case '!': tokenizer_addToken(tokenizer_nextIf('=') ? Token::NOT_EQUAL: Token::INVERT);
             break;
-        case '=': tokenizer_addToken(tokenizer_nextIf('=') ? EQUAL: SET);
+        case '=': tokenizer_addToken(tokenizer_nextIf('=') ? Token::EQUAL: Token::SET);
             return false;
-        case '*': tokenizer_addToken(tokenizer_nextIf('=') ? MUL_SET: MUL);
+        case '*': tokenizer_addToken(tokenizer_nextIf('=') ? Token::MUL_SET: Token::MUL);
             return false;
         case '/':
             return tokenizer_handleSlash();
-        case '%': tokenizer_addToken(tokenizer_nextIf('=') ? MOD_SET: MOD);
+        case '%': tokenizer_addToken(tokenizer_nextIf('=') ? Token::MOD_SET: Token::MOD);
             return false;
-        case '&': tokenizer_addToken(tokenizer_nextIf('=') ? BIT_AND_SET: (tokenizer_nextIf('&') ? AND: BIT_AND));
+        case '&': tokenizer_addToken(tokenizer_nextIf('=') ? Token::BIT_AND_SET: (tokenizer_nextIf('&') ? Token::AND: Token::BIT_AND));
             return false;
-        case '|': tokenizer_addToken(tokenizer_nextIf('=') ? BIT_OR_SET: (tokenizer_nextIf('|') ? OR: BIT_OR));
+        case '|': tokenizer_addToken(tokenizer_nextIf('=') ? Token::BIT_OR_SET: (tokenizer_nextIf('|') ? Token::OR: Token::BIT_OR));
             return false;
-        case '^': tokenizer_addToken(tokenizer_nextIf('=') ? BIT_XOR_SET: BIT_XOR);
+        case '^': tokenizer_addToken(tokenizer_nextIf('=') ? Token::BIT_XOR_SET: Token::BIT_XOR);
             return false;
-        case '<': tokenizer_addToken(tokenizer_chooseToken('<', LEFT_SHIFT_SET, LEFT_SHIFT, LESS_EQUAL, LESS));
+        case '<': tokenizer_addToken(tokenizer_chooseToken('<', Token::LEFT_SHIFT_SET, Token::LEFT_SHIFT, Token::LESS_EQUAL, Token::LESS));
             return false;
-        case '>': tokenizer_addToken(tokenizer_chooseToken('>', RIGHT_SHIFT_SET, RIGHT_SHIFT, GREATER_EQUAL, GREATER));
+        case '>': tokenizer_addToken(tokenizer_chooseToken('>', Token::RIGHT_SHIFT_SET, Token::RIGHT_SHIFT, Token::GREATER_EQUAL, Token::GREATER));
             return false;
     }
     char buffer[32];
@@ -354,7 +349,7 @@ static bool tokenizer_handleSpecial(char32_t c) {
 
 static bool tokenizer_handleChar(char32_t c) {
     if(tokenizer_isValidNameStart(c)) {
-        return tokenizer_handleLiteral(c, LITERAL);
+        return tokenizer_handleLiteral(c, Token::LITERAL);
     } else if(tokenizer_isDigit(c)) {
 
         return tokenizer_handleNumber(c);
@@ -377,7 +372,7 @@ bool tokenize(TokenStream* tokenStream, const char* inputPath) {
             return true;
         }
     }
-    tokenizer_addToken(EOF_TOKEN);
+    tokenizer_addToken(Token::EOF_TOKEN);
 
     fclose(input);
     input = NULL;

utils/Path.c

@@ -1,41 +0,0 @@
-#include <stdlib.h>
-#include <string.h>
-#include <stdio.h>
-
-#include "utils/Path.h"
-
-void initPath(Path* path, const char* strPath) {
-    path->capacity = PATH_MAX;
-    path->usedCapacity = strlen(strPath) + 1;
-    if(path->usedCapacity >= path->capacity) {
-        path->usedCapacity = path->capacity;
-    }
-    strncpy(path->path, strPath, path->usedCapacity);
-}
-
-static bool path_add(Path* path, const char* add) {
-    size_t addedLength = strlen(add);
-    size_t freeSpace = path->capacity - path->usedCapacity;
-    if(freeSpace < addedLength) {
-        fprintf(stderr, "cannot add '%s' to '%s', buffer too small\n", add, path->path);
-        return true;
-    }
-    strncpy(path->path + path->usedCapacity - 1, add, addedLength + 1);
-    path->usedCapacity += addedLength;
-    return false;
-}
-
-bool enterPath(Path* path, const char* object) {
-    return path_add(path, "/") || path_add(path, object);
-}
-
-bool enterFile(Path* path, const char* object, const char* ending) {
-    return path_add(path, "/") || path_add(path, object) || path_add(path, ending);
-}
-
-void leavePath(Path* path) {
-    while(path->usedCapacity > 1 && path->path[path->usedCapacity - 1] != '/') {
-        path->usedCapacity--;
-    }
-    path->path[path->usedCapacity - 1] = '\0';
-}

utils/Path.h

@@ -1,18 +0,0 @@
-#ifndef PATH_H
-#define PATH_H
-
-#include <limits.h>
-#include <stdbool.h>
-
-typedef struct {
-    size_t capacity;
-    size_t usedCapacity;
-    char path[PATH_MAX];
-} Path;
-
-void initPath(Path* path, const char* strPath);
-bool enterPath(Path* path, const char* object);
-bool enterFile(Path* path, const char* object, const char* ending);
-void leavePath(Path* path);
-
-#endif

utils/String.cpp

@@ -0,0 +1,50 @@
+#include <cstdio>
+
+#include "utils/String.h"
+
+String::String() : String("") {
+}
+
+String::String(const char* str) : usedCapacity(1) {
+    *this += str;
+}
+
+String& String::operator+=(const char* str) {
+    usedCapacity--;
+    size_t start = usedCapacity;
+    while(usedCapacity + 1 < capacity && str[usedCapacity - start] != '\0') {
+        path[usedCapacity] = str[usedCapacity - start];
+        usedCapacity++;
+    }
+    path[usedCapacity++] = '\0';
+    return *this;
+}
+
+String& String::operator+=(char c) {
+    if(usedCapacity + 1 < capacity) {
+        path[usedCapacity - 1] = c;
+        path[usedCapacity] = '\0';
+        usedCapacity++;
+    }
+    return *this;
+}
+
+String& String::operator+=(unsigned int i) {
+    usedCapacity += snprintf(path + usedCapacity - 1, capacity - usedCapacity, "%u", i);
+    return *this;
+}
+
+String& String::operator+=(double d) {
+    usedCapacity += snprintf(path + usedCapacity - 1, capacity - usedCapacity, (d == (long) d) ? "%lg.0" : "%lg", d);
+    return *this;
+}
+
+String String::operator+(const char* str) const {
+    String s(this->path);
+    s += str;
+    return s;
+}
+
+String::operator const char*() const {
+    return path;
+}

utils/String.h

@@ -0,0 +1,23 @@
+#ifndef STRING_H
+#define STRING_H
+
+#include <cstddef>
+
+class String final {
+public:
+    String();
+    String(const char* str);
+    String& operator+=(const char* str);
+    String& operator+=(char c);
+    String& operator+=(unsigned int i);
+    String& operator+=(double d);
+    String operator+(const char* str) const;
+    operator const char*() const;
+    
+private:
+    static constexpr size_t capacity = 4096;
+    size_t usedCapacity;
+    char path[capacity];
+};
+
+#endif