everything moved from cpp to c

Main.cpp → Main.c

@@ -1,5 +1,5 @@
-#include <cstring>
-#include <iostream>
+#include <stdio.h>
+#include <string.h>
 
 #include "test/Test.h"
 
@@ -8,7 +8,7 @@ int main(int argc, char** argv) {
         return 0;
     }
     if(argc >= 3 && strcmp(argv[1], "test") == 0) {
-        Test::start(argv[2]);
+        startTests(argv[2]);
     }
     return 0;
 }

meson.build

@@ -1,7 +1,7 @@
-project('lonely tiger', 'cpp')
+project('lonely tiger', 'c')
 
-src = ['Main.cpp', 'test/Test.cpp', 'test/TestLogger.cpp', 'tokenizer/TokenType.cpp', 'tokenizer/Tokenizer.cpp', 'tokenizer/TokenStream.cpp']
+src = ['Main.c', 'test/Test.c', 'utils/Path.c', 'tokenizer/Token.c', 'tokenizer/TokenStream.c', 'tokenizer/Tokenizer.c']
 
 executable('lonely_tiger', 
     sources: src,
-    cpp_args: ['-Wall', '-Wextra', '-pedantic', '-Werror'])
+    c_args: ['-Wall', '-Wextra', '-pedantic', '-Werror'])

test/Test.c

@@ -0,0 +1,154 @@
+#include <stdio.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <dirent.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "test/Test.h"
+#include "utils/Path.h"
+#include "tokenizer/TokenStream.h"
+#include "tokenizer/Tokenizer.h"
+
+static unsigned int done = 0;
+static unsigned int tests = 0;
+
+static bool test_checkPath(Path* path, const char* ending, bool(*f) (const char*, const char*)) {
+    (void) ending;
+    DIR* dir;
+    dir = opendir(path->path);
+    if(dir == NULL) {
+        fprintf(stderr, "cannot open '%s': ", path->path);
+        perror("");
+        return true;
+    }
+    struct dirent* entry = NULL;
+    while(true) {
+        entry = readdir(dir);
+        if(entry == NULL) {
+            break;
+        } else if(strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) {
+            continue;
+        }
+        if(entry->d_type == DT_DIR) {
+            if(enterPath(path, entry->d_name)) {
+                return true;
+            }
+            if(test_checkPath(path, ending, f)) {
+                leavePath(path);
+                return true;
+            }
+            leavePath(path);
+        } else if(entry->d_type == DT_REG && strchr(entry->d_name, '.') == NULL) {
+            Path inputPath;
+            initPath(&inputPath, path->path);
+            enterFile(&inputPath, entry->d_name, "");
+            Path outputPath;
+            initPath(&outputPath, path->path);
+            enterFile(&outputPath, entry->d_name, ending);
+            if(f(inputPath.path, outputPath.path)) {
+                return true;
+            }
+        }
+    }
+    if(closedir(dir)) {
+        fprintf(stderr, "cannot close '%s': ", path->path);
+        perror("");
+        return true;
+    }
+    return false;
+}
+
+static bool test_forEachFile(const char* strPath, const char* ending, bool(*f) (const char*, const char*)) {
+    Path path;
+    initPath(&path, strPath);
+    return test_checkPath(&path, ending, f);
+}
+
+static void test_readLine(char* buffer, size_t capacity, FILE* f) {
+    size_t index = 0;
+    while(index < capacity - 1) {
+        int c = fgetc(f);
+        if(c == EOF || c == '\n') {
+            break;
+        }
+        buffer[index++] = c;
+    }
+    buffer[index] = '\0';
+}
+
+static bool test_tokenizerTester(const char* input, const char* output) {
+    FILE* f = fopen(output, "r");
+    if(f == NULL) {
+        perror("cannot open file");
+        return false;
+    }
+
+    TokenStream* tokenStream = newTokenStream();
+    bool b = tokenize(tokenStream, input);
+
+    while(hasToken(tokenStream)) {
+        const size_t bufferLength = 64;
+        char buffer[bufferLength];
+        nextTokenString(tokenStream, buffer, bufferLength);
+
+        char expected[1024];
+        test_readLine(expected, 1024, f);
+        if(strchr(buffer, '\n') != NULL) {
+            size_t length = strlen(expected);
+            expected[length] = '\n';
+            test_readLine(expected + length + 1, 1024 - length - 1, f);
+        }
+        if(strcmp(buffer, expected) != 0) {
+            printf("error in '%s'\n", input);
+            printf("'%s' should be '%s'\n", buffer, expected);
+            break;
+        }
+    }
+
+    fclose(f);
+    deleteTokenStream(&tokenStream);
+    return b;
+}
+
+static void test_testTokenizer(const char* path) {
+    done = 0;
+    tests = 0;
+
+    test_forEachFile(path, ".tout", test_tokenizerTester);
+    /*forEachFile(path, ".tout", [](const std::string& input, const std::string & output) {
+        tests++;
+
+        Tokenizer::if32stream iStream;
+        iStream.open(input);
+
+        std::ifstream oStream;
+        oStream.open(output);
+
+        if(!iStream.good() || !oStream.good()) {
+            return false;
+        }
+
+        TokenStream tokens;
+        if(Tokenizer::tokenize(tokens, iStream)) {
+            return false;
+        }
+        logger.reset();
+        while(tokens.hasToken()) {
+            std::string s = tokens.nextTokenString();
+                    logger.print(&s);
+        }
+
+        if(logger.check(input, oStream)) {
+            done++;
+        }
+        return true;
+    });
+    std::cout << done << " / " << tests << " tokenizer tests succeeded" << std::endl;*/
+}
+
+void startTests(const char* path) {
+    test_testTokenizer(path);
+    //testCompiler();
+    //testOutput();
+}

test/Test.cpp

@@ -1,139 +0,0 @@
-#include <iostream>
-#include <fstream>
-#include <cstring>
-#include <vector>
-
-#include <dirent.h>
-
-#include "test/Test.h"
-#include "test/TestLogger.h"
-#include "tokenizer/Tokenizer.h"
-#include "tokenizer/TokenStream.h"
-
-static unsigned int done = 0;
-static unsigned int tests = 0;
-static TestLogger logger;
-static bool run = true;
-
-static void forEachFile(const std::string& path, const std::string& ending, bool (*f) (const std::string&, const std::string&)) {
-    DIR* dir;
-    dir = opendir(path.c_str());
-    struct dirent* entry = nullptr;
-    if(dir != nullptr) {
-        while(run && (entry = readdir(dir)) != nullptr) {
-            if(strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) {
-                continue;
-            }
-            if(entry->d_type == DT_DIR) {
-                forEachFile(path + "/" + entry->d_name, ending, f);
-            } else if(entry->d_type == DT_REG) {
-                if(strchr(entry->d_name, '.') == nullptr) {
-                    std::string pathInputFile = path + "/" + entry->d_name;
-                    std::string pathOutputFile = pathInputFile + ending;
-                    run = f(pathInputFile, pathOutputFile);
-                }
-            }
-        }
-        closedir(dir);
-    }
-}
-
-static void testTokenizer(const char* path) {
-    done = 0;
-    tests = 0;
-    run = true;
-    forEachFile(path, ".tout", [](const std::string& input, const std::string & output) {
-        tests++;
-
-        Tokenizer::if32stream iStream;
-        iStream.open(input);
-
-        std::ifstream oStream;
-        oStream.open(output);
-
-        if(!iStream.good() || !oStream.good()) {
-            return false;
-        }
-
-        TokenStream tokens;
-        if(Tokenizer::tokenize(tokens, iStream)) {
-            return false;
-        }
-        logger.reset();
-        while(tokens.hasToken()) {
-            std::string s = tokens.nextTokenString();
-                    logger.print(&s);
-        }
-
-        if(logger.check(input, oStream)) {
-            done++;
-        }
-        return true;
-    });
-    std::cout << done << " / " << tests << " tokenizer tests succeeded" << std::endl;
-}
-
-//void Test::testCompiler()
-//{
-//     done = 0;
-//        tests = 0; 
-//        final Compiler c = new Compiler();
-//        forEachFile(new File("./test"), ".cout", (inFile, checkFile) -> 
-//        {
-//            tests++;
-//            try
-//            {
-//                try(FileInputStream in = new FileInputStream(inFile))
-//                {
-//                    Tokenizer tokenizer = new Tokenizer();
-//                    LOGGER.reset();
-//                    Instruction[] instr = c.compile(tokenizer.tokenize(in), 
-//                            new HashMap<>(), new HashMap<>(), new HashMap<>(), 
-//                            new HashMap<>());
-//                    for(Instruction i : instr)
-//                    {
-//                        LOGGER.print(i.toString(), null, null, null, null, -1);
-//                    }
-//                    if(LOGGER.check(checkFile))
-//                    {
-//                        done++;
-//                    }
-//                }
-//            }
-//            catch(Exception ex)
-//            {
-//                System.out.println("_________________________________________");
-//                System.out.println(inFile + " failed:");
-//                System.out.println(ex.getMessage());
-//                ex.printStackTrace();
-//            }
-//        });
-//        System.out.println(String.format("%d / %d compiler tests succeeded", done, tests));
-//}
-//
-//void Test::testOutput()
-//{
-//     done = 0;
-//        tests = 0;  
-//        forEachFile(new File("./test"), ".out", (inFile, checkFile) -> 
-//        {
-//            tests++;
-//                
-//            LOGGER.reset();
-//            
-//            Script sc = new Script(PARSER, null, null, inFile.getName(), inFile.getPath());
-//            sc.run();
-//
-//            if(LOGGER.check(checkFile))
-//            {
-//                done++;
-//            }
-//        });
-//        System.out.println(String.format("%d / %d output tests succeeded", done, tests));
-//}
-
-void Test::start(const char* path) {
-    testTokenizer(path);
-    //testCompiler();
-    //testOutput();
-}

test/Test.h

@@ -1,8 +1,6 @@
 #ifndef TEST_H
 #define TEST_H
 
-namespace Test {
-    void start(const char* path);
-}
+void startTests(const char* path);
 
 #endif

test/TestLogger.cpp

@@ -1,73 +0,0 @@
-#include "TestLogger.h"
-
-TestLogger::TestLogger() {
-}
-
-TestLogger::~TestLogger() {
-}
-
-void TestLogger::print(const std::string* message, const std::exception* ex,
-        const std::string* function, const std::string* scriptname, const Script* sc, int line) {
-    (void) function;
-    (void) scriptname;
-    (void) sc;
-    (void) line;
-    if(ex == nullptr) {
-        if(message != nullptr) {
-            int start = 0;
-            while(true) {
-                int newLine = message->find('\n', start);
-                if(newLine == -1) {
-                    output.push_back(message->substr(start, message->size() - start));
-                    break;
-                }
-                output.push_back(message->substr(start, newLine - start));
-                start = newLine + 1;
-            }
-        }
-    } else {
-        output.push_back(ex->what());
-    }
-}
-
-void TestLogger::reset() {
-    output.clear();
-}
-
-bool TestLogger::check(const std::string& name, std::ifstream& check) {
-    std::vector<std::string> file;
-
-    while(!check.eof()) {
-        std::string line;
-        std::getline(check, line);
-        if(!check.eof()) {
-            file.push_back(line);
-        }
-    }
-
-    if(file.size() != output.size()) {
-        std::cout << file.size() << " " << output.size() << std::endl;
-        printNoMatch(name, file, 0);
-        return false;
-    }
-    for(size_t i = 0; i < file.size(); i++) {
-        if(file[i] != output[i]) {
-            printNoMatch(name, file, i + 1);
-            return false;
-        }
-    }
-    return true;
-}
-
-void TestLogger::printNoMatch(const std::string& name, std::vector<std::string>& file, unsigned int line) {
-    std::cout << "error checking " << name << ", error starting at " << line << "\n";
-    std::cout << "Expected ---------------------------------------------\n";
-    for(unsigned int i = 0; i < file.size(); i++) {
-        std::cout << file[i] << "\n";
-    }
-    std::cout << "Actual -----------------------------------------------\n";
-    for(unsigned int i = 0; i < output.size(); i++) {
-        std::cout << output[i] << "\n";
-    }
-    std::cout << "------------------------------------------------------\n";
-}

test/TestLogger.h

@@ -1,33 +0,0 @@
-#ifndef TESTLOGGER_H
-#define TESTLOGGER_H
-
-#include <iostream>
-#include <fstream>
-#include <vector>
-
-#include "code/ISnuviLogger.h"
-
-class TestLogger : public ISnuviLogger
-{
-public:
-    TestLogger();
-    ~TestLogger();
-    
-    void print(
-            const std::string* message = nullptr, 
-            const std::exception* ex = nullptr, 
-            const std::string* function = nullptr, 
-            const std::string* scriptname = nullptr, 
-            const Script* sc = nullptr,
-            int line = -1) override;
-    
-    void reset();
-    bool check(const std::string& name, std::ifstream& check);
-
-private:
-    void printNoMatch(const std::string& name, std::vector<std::string>& file, unsigned int line);
-    
-    std::vector<std::string> output;
-};
-
-#endif

tokenizer/TokenType.cpp → tokenizer/Token.c

@@ -1,7 +1,7 @@
-#include "TokenType.h"
+#include "tokenizer/Token.h"
 
-std::string TokenTypeUtils::getName(TokenType tt) {
-    switch(tt) {
+const char* getTokenName(Token token) {
+    switch(token) {
         case NUMBER: return "number";
         case STRING: return "string";
         case LITERAL: return "literal";
@@ -66,8 +66,8 @@ std::string TokenTypeUtils::getName(TokenType tt) {
     return "Unknown TokenType";
 }
 
-std::string TokenTypeUtils::getEnumName(TokenType tt) {
-    switch(tt) {
+const char* getTokenEnumName(Token token) {
+    switch(token) {
         case NUMBER: return "NUMBER";
         case STRING: return "STRING";
         case LITERAL: return "LITERAL";

tokenizer/Token.h

@@ -0,0 +1,43 @@
+#ifndef TOKENTYPE_H
+#define TOKENTYPE_H
+
+typedef enum Token {
+    NUMBER, 
+    STRING, 
+    LITERAL, 
+    LABEL,
+    TRUE, 
+    FALSE, 
+    NULL_TOKEN,
+
+    OPEN_BRACKET, CLOSE_BRACKET,
+    OPEN_SQUARE_BRACKET, CLOSE_SQUARE_BRACKET,
+    OPEN_CURVED_BRACKET, CLOSE_CURVED_BRACKET,
+
+    SEMICOLON, COMMA,
+
+    INC, DEC,
+
+    INVERT, BIT_INVERT,
+
+    MUL, DIV, MOD, ADD, SUB,
+    ADD_SET, SUB_SET, MUL_SET, DIV_SET, MOD_SET,
+
+    LEFT_SHIFT, RIGHT_SHIFT,
+    LEFT_SHIFT_SET, RIGHT_SHIFT_SET, BIT_AND_SET, BIT_XOR_SET, BIT_OR_SET,
+
+    LESS, LESS_EQUAL, GREATER, GREATER_EQUAL, EQUAL, NOT_EQUAL,
+    BIT_AND, BIT_XOR, BIT_OR,
+    AND, OR, SET,
+
+    IF, ELSE, ELSEIF, WHILE, TRY,
+    CATCH, FOR, FUNCTION, BREAK,
+    CONTINUE, RETURN,
+
+    EOF_TOKEN
+} Token;
+
+const char* getTokenName(Token token);
+const char* getTokenEnumName(Token token);
+
+#endif

tokenizer/TokenStream.c

@@ -0,0 +1,129 @@
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+#include "tokenizer/TokenStream.h"
+
+TokenStream* newTokenStream() {
+    TokenStream* tokenStream = malloc(sizeof (TokenStream));
+    tokenStream->readIndex = 0;
+    tokenStream->index = 0;
+    tokenStream->length = 1024;
+    tokenStream->buffer = malloc(sizeof (char) * tokenStream->length);
+    return tokenStream;
+}
+
+void deleteTokenStream(TokenStream** tokenStream) {
+    if(tokenStream == NULL || (*tokenStream) == NULL) {
+        return;
+    }
+    free((*tokenStream)->buffer);
+    free(*tokenStream);
+    *tokenStream = NULL;
+}
+
+bool hasToken(TokenStream* tokenStream) {
+    return tokenStream->readIndex < tokenStream->index;
+}
+
+static size_t tokenstream_concat(char* dst, const char* src, size_t bufferLength) {
+    size_t length = strlen(src);
+    strncat(dst, src, bufferLength);
+    if(length > bufferLength) {
+        return 0;
+    }
+    return bufferLength - length;
+}
+
+void nextTokenString(TokenStream* tokenStream, char* buffer, size_t bufferLength) {
+    if(bufferLength == 0) {
+        return;
+    }
+    buffer[0] = '\0';
+    bufferLength--;
+
+    bufferLength = tokenstream_concat(buffer, "(", bufferLength);
+    // line
+    char lineBuffer[32];
+    unsigned int line = nextLine(tokenStream);
+    snprintf(lineBuffer, 32, "%u, ", line);
+    bufferLength = tokenstream_concat(buffer, lineBuffer, bufferLength);
+
+    // tokentype
+    Token token = nextToken(tokenStream);
+    bufferLength = tokenstream_concat(buffer, getTokenEnumName(token), bufferLength);
+
+    if(token == STRING || token == LITERAL || token == LABEL) {
+        bufferLength = tokenstream_concat(buffer, ", \"", bufferLength);
+        bufferLength = tokenstream_concat(buffer, nextString(tokenStream), bufferLength);
+        bufferLength = tokenstream_concat(buffer, "\"", bufferLength);
+    }
+
+    if(token == NUMBER) {
+        char doubleBuffer[32];
+        double d = nextDouble(tokenStream);
+        snprintf(doubleBuffer, 32, (d == (long) d) ? ", %lg.0" : ", %lg", d);
+        bufferLength = tokenstream_concat(buffer, doubleBuffer, bufferLength);
+    }
+
+    bufferLength = tokenstream_concat(buffer, ")", bufferLength);
+}
+
+void tokenstream_read(TokenStream* tokenStream, void* data, size_t length) {
+    memcpy(data, tokenStream->buffer + tokenStream->readIndex, length);
+    tokenStream->readIndex += length;
+}
+
+Token nextToken(TokenStream* tokenStream) {
+    Token token = EOF_TOKEN;
+    tokenstream_read(tokenStream, &token, sizeof (Token));
+    return token;
+}
+
+unsigned int nextLine(TokenStream* tokenStream) {
+    unsigned int line = 0;
+    tokenstream_read(tokenStream, &line, sizeof (unsigned int));
+    return line;
+}
+
+const char* nextString(TokenStream* tokenStream) {
+    size_t offset = tokenStream->readIndex;
+    tokenStream->readIndex += strlen(tokenStream->buffer + tokenStream->readIndex) + 1;
+    return tokenStream->buffer + offset;
+}
+
+double nextDouble(TokenStream* tokenStream) {
+    double d;
+    tokenstream_read(tokenStream, &d, sizeof (double));
+    return d;
+}
+
+static void tokenstream_write(TokenStream* tokenStream, const void* data, size_t length) {
+    if(tokenStream->index + length >= tokenStream->length) {
+        size_t newSize = tokenStream->length;
+        while(newSize < tokenStream->index + length) {
+            newSize *= 2;
+        }
+        tokenStream->buffer = realloc(tokenStream->buffer, sizeof (char) * newSize);
+        tokenStream->length = newSize;
+    }
+    const char* chars = (const char*) data;
+    for(size_t i = 0; i < length; i++) {
+        tokenStream->buffer[tokenStream->index++] = chars[i];
+    }
+}
+
+void addToken(TokenStream* tokenStream, Token token, unsigned int line) {
+    tokenstream_write(tokenStream, &line, sizeof (unsigned int));
+    tokenstream_write(tokenStream, &token, sizeof (Token));
+}
+
+void addDoubleToken(TokenStream* tokenStream, Token token, unsigned int line, double d) {
+    addToken(tokenStream, token, line);
+    tokenstream_write(tokenStream, &d, sizeof (double));
+}
+
+void addStringToken(TokenStream* tokenStream, Token token, unsigned int line, const char* text) {
+    addToken(tokenStream, token, line);
+    tokenstream_write(tokenStream, text, strlen(text) + 1);
+}

tokenizer/TokenStream.cpp

@@ -1,97 +0,0 @@
-#include <cstring>
-
-#include "TokenStream.h"
-
-TokenStream::TokenStream() : nextToken(0) {
-}
-
-bool TokenStream::hasToken() const {
-    return nextToken < bytes.size();
-}
-
-std::string TokenStream::nextTokenString() {
-    std::string t = "(";
-    // line
-    unsigned int line = nextLine();
-    t += std::to_string(line);
-    t += ", ";
-    // tokentype
-    TokenType type = nextTokenType();
-    t += TokenTypeUtils::getEnumName(type);
-
-    if(type == TokenType::STRING || type == TokenType::LITERAL || type == TokenType::LABEL) {
-        t += ", \"";
-        t += nextString();
-        t += "\"";
-    }
-
-    if(type == TokenType::NUMBER) {
-        t += ", ";
-        double d = nextDouble();
-        char buffer[32];
-        snprintf(buffer, 32, (d == (long) d) ? "%lg.0" : "%lg", d);
-        t += buffer;
-    }
-
-    t += ")";
-    return t;
-}
-
-TokenType TokenStream::nextTokenType() {
-    TokenType type = TokenType::EOF_TOKEN;
-    read(&type, sizeof (TokenType));
-    return type;
-}
-
-unsigned int TokenStream::nextLine() {
-    unsigned int line = 0;
-    read(&line, sizeof (unsigned int));
-    return line;
-}
-
-std::string TokenStream::nextString() {
-    std::string text;
-    char c;
-    while(true) {
-        read(&c, 1);
-        if(c == '\0') {
-            break;
-        }
-        text += c;
-    }
-    return text;
-}
-
-double TokenStream::nextDouble() {
-    double d;
-    read(&d, sizeof (double));
-    return d;
-}
-
-void TokenStream::write(const void* data, size_t length) {
-    const char* chars = reinterpret_cast<const char*> (data);
-    for(size_t i = 0; i < length; i++) {
-        bytes.push_back(chars[i]);
-    }
-}
-
-void TokenStream::read(void* data, size_t length) {
-    memcpy(data, bytes.data() + nextToken, length);
-    nextToken += length;
-}
-
-void TokenStream::add(TokenType type, unsigned int line) {
-    write(&line, sizeof (unsigned int));
-    write(&type, sizeof (TokenType));
-}
-
-void TokenStream::add(TokenType type, unsigned int line, double d) {
-    add(type, line);
-    write(&d, sizeof (double));
-}
-
-void TokenStream::add(TokenType type, unsigned int line, const std::string& text) {
-    add(type, line);
-    write(text.data(), text.length());
-    write("\0", 1);
-}

tokenizer/TokenStream.h

@@ -1,33 +1,31 @@
 #ifndef TOKENSTREAM_H
 #define TOKENSTREAM_H
 
-#include <vector>
-
-#include "tokenizer/TokenType.h"
-
-class TokenStream final {
-public:
-    TokenStream();
-
-    bool hasToken() const;
-    std::string nextTokenString();
-    
-    TokenType nextTokenType();
-    unsigned int nextLine();
-    std::string nextString();
-    double nextDouble();
-
-    void add(TokenType type, unsigned int line);
-    void add(TokenType type, unsigned int line, double d);
-    void add(TokenType type, unsigned int line, const std::string& text);
-
-private:
-    void write(const void* data, size_t length);
-    void read(void* data, size_t length);
-
-private:
-    size_t nextToken;
-    std::vector<char> bytes;
-};
+#include <stddef.h>
+#include <stdbool.h>
+
+#include "tokenizer/Token.h"
+
+typedef struct TokenStream {
+    size_t readIndex;
+    size_t index;
+    size_t length;
+    char* buffer;
+} TokenStream;
+
+TokenStream* newTokenStream();
+void deleteTokenStream(TokenStream** tokenStream);
+
+bool hasToken(TokenStream* tokenStream);
+void nextTokenString(TokenStream* tokenStream, char* buffer, size_t bufferLength);
+
+Token nextToken(TokenStream* tokenStream);
+unsigned int nextLine(TokenStream* tokenStream);
+const char* nextString(TokenStream* tokenStream);
+double nextDouble(TokenStream* tokenStream);
+
+void addToken(TokenStream* tokenStream, Token token, unsigned int line);
+void addDoubleToken(TokenStream* tokenStream, Token token, unsigned int line, double d);
+void addStringToken(TokenStream* tokenStream, Token token, unsigned int line, const char* text);
 
 #endif

tokenizer/TokenType.h

@@ -1,44 +0,0 @@
-#ifndef TOKENTYPE_H
-#define TOKENTYPE_H
-
-#include <string>
-
-enum TokenType
-{
-    NUMBER, STRING, LITERAL, LABEL, 
-    TRUE, FALSE, NULL_TOKEN,
-    
-    OPEN_BRACKET, CLOSE_BRACKET, 
-    OPEN_SQUARE_BRACKET, CLOSE_SQUARE_BRACKET, 
-    OPEN_CURVED_BRACKET, CLOSE_CURVED_BRACKET, 
-    
-    SEMICOLON, COMMA, 
-    
-    INC, DEC, 
-    
-    INVERT, BIT_INVERT, 
-    
-    MUL, DIV, MOD, ADD, SUB, 
-    ADD_SET, SUB_SET, MUL_SET, DIV_SET, MOD_SET, 
-    
-    LEFT_SHIFT, RIGHT_SHIFT, 
-    LEFT_SHIFT_SET, RIGHT_SHIFT_SET, BIT_AND_SET, BIT_XOR_SET, BIT_OR_SET,
-    
-    LESS, LESS_EQUAL, GREATER, GREATER_EQUAL, EQUAL, NOT_EQUAL, 
-    BIT_AND, BIT_XOR, BIT_OR, 
-    AND, OR, SET,
-    
-    IF, ELSE, ELSEIF, WHILE, TRY, 
-    CATCH, FOR, FUNCTION, BREAK, 
-    CONTINUE, RETURN,
-    
-    EOF_TOKEN
-};
-
-namespace TokenTypeUtils
-{
-    std::string getName(TokenType tt);
-    std::string getEnumName(TokenType tt);
-}
-
-#endif

tokenizer/Tokenizer.c

@@ -0,0 +1,385 @@
+#include <stdio.h>
+#include <uchar.h>
+#include <string.h>
+
+#include "tokenizer/Tokenizer.h"
+
+static FILE* input = NULL;
+static TokenStream* tokens = NULL;
+static unsigned int line = 1;
+static char32_t buffer = 0;
+
+static void tokenizer_onError(const char* message, unsigned int line) {
+    printf("%s Line: %u\n", message, line);
+}
+
+static size_t tokenizer_printChar(char32_t c, char* buffer) {
+    if(c <= 0x7F) {
+        buffer[0] = (char) c;
+        return 1;
+    } else if(c < 0xE00000) {
+        buffer[0] = (char) ((c >> 8) & 0xFF);
+        buffer[1] = (char) ((c >> 0) & 0xFF);
+        return 2;
+    } else if(c <= 0xF0000000) {
+        buffer[0] = (char) ((c >> 16) & 0xFF);
+        buffer[1] = (char) ((c >> 8) & 0xFF);
+        buffer[2] = (char) ((c >> 0) & 0xFF);
+        return 3;
+    }
+    buffer[0] = (char) ((c >> 24) & 0xFF);
+    buffer[1] = (char) ((c >> 16) & 0xFF);
+    buffer[2] = (char) ((c >> 8) & 0xFF);
+    buffer[3] = (char) ((c >> 0) & 0xFF);
+    return 4;
+}
+
+static bool tokenizer_isLetter(char32_t c) {
+    return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
+}
+
+static bool tokenizer_isDigit(char32_t c) {
+    return c >= '0' && c <= '9';
+}
+
+static bool tokenizer_isValidNameStart(char32_t c) {
+    return tokenizer_isLetter(c) || c == '.' || c == '_';
+}
+
+static bool tokenizer_isValidNamePart(char32_t c) {
+    return tokenizer_isDigit(c) || tokenizer_isValidNameStart(c);
+}
+
+static bool tokenizer_next(char32_t* c) {
+    if(buffer != 0) {
+        *c = buffer;
+        buffer = 0;
+        return true;
+    }
+    int in = fgetc(input);
+    if(in == EOF) {
+        return false;
+    }
+    if((in & 0x80) == 0) {
+        *c = in;
+        return true;
+    }
+    if((in >> 5) == 0x6) {
+        *c = (in << 8) | fgetc(input);
+        return true;
+    }
+    if((in >> 4) == 0xE) {
+        *c = (in << 16) | (fgetc(input) << 8) | fgetc(input);
+        return true;
+    }
+    if((in >> 3) == 0x1E) {
+        *c = (in << 24) | (fgetc(input) << 16) | (fgetc(input) << 8) | fgetc(input);
+        return true;
+    }
+    return true;
+}
+
+static bool tokenizer_peek(char32_t* c) {
+    if(buffer != 0 || tokenizer_next(&buffer)) {
+        *c = buffer;
+
+        return true;
+    }
+    return false;
+}
+
+static bool tokenizer_nextIf(char32_t c) {
+    char32_t nextChar;
+    if(tokenizer_peek(&nextChar) && c == nextChar) {
+        tokenizer_next(&nextChar);
+
+        return true;
+    }
+    return false;
+}
+
+static void tokenizer_addToken(Token token) {
+
+    addToken(tokens, token, line);
+}
+
+static void tokenizer_addStringToken(Token token, const char* text) {
+
+    addStringToken(tokens, token, line, text);
+}
+
+static Token tokenizer_chooseToken(char c, Token aCharEqual, Token aChar, Token aEqual, Token other) {
+    if(tokenizer_nextIf(c)) {
+        if(tokenizer_nextIf('=')) {
+            return aCharEqual;
+        }
+        return aChar;
+    } else if(tokenizer_nextIf('=')) {
+
+        return aEqual;
+    }
+    return other;
+}
+
+static bool tokenizer_handleLiteral(char32_t c, Token token) {
+    const size_t bufferSize = 1024;
+    char buffer[bufferSize];
+    size_t index = 1;
+    buffer[0] = c;
+
+    while(index < bufferSize - 1) {
+        char32_t data;
+        if(!tokenizer_peek(&data) || !tokenizer_isValidNamePart(data)) {
+            break;
+        }
+        buffer[index++] = data;
+        tokenizer_next(&data);
+    }
+    buffer[index] = '\0';
+
+    if(strcmp(buffer, "if") == 0) {
+        tokenizer_addToken(IF);
+    } else if(strcmp(buffer, "if") == 0) {
+        tokenizer_addToken(IF);
+    } else if(strcmp(buffer, "else") == 0) {
+        tokenizer_addToken(ELSE);
+    } else if(strcmp(buffer, "elseif") == 0) {
+        tokenizer_addToken(ELSEIF);
+    } else if(strcmp(buffer, "while") == 0) {
+        tokenizer_addToken(WHILE);
+    } else if(strcmp(buffer, "try") == 0) {
+        tokenizer_addToken(TRY);
+    } else if(strcmp(buffer, "catch") == 0) {
+        tokenizer_addToken(CATCH);
+    } else if(strcmp(buffer, "for") == 0) {
+        tokenizer_addToken(FOR);
+    } else if(strcmp(buffer, "function") == 0) {
+        tokenizer_addToken(FUNCTION);
+    } else if(strcmp(buffer, "break") == 0) {
+        tokenizer_addToken(BREAK);
+    } else if(strcmp(buffer, "continue") == 0) {
+        tokenizer_addToken(CONTINUE);
+    } else if(strcmp(buffer, "return") == 0) {
+        tokenizer_addToken(RETURN);
+    } else if(strcmp(buffer, "true") == 0) {
+        tokenizer_addToken(TRUE);
+    } else if(strcmp(buffer, "false") == 0) {
+        tokenizer_addToken(FALSE);
+    } else if(strcmp(buffer, "null") == 0) {
+        tokenizer_addToken(NULL_TOKEN);
+    } else {
+
+        tokenizer_addStringToken(token, buffer);
+    }
+    return false;
+}
+
+static bool tokenizer_handleNumber(char32_t c) {
+    double number = c - '0';
+    char32_t data;
+    while(tokenizer_peek(&data)) {
+        if(!tokenizer_isDigit(data)) {
+            if(data != '.') {
+                break;
+            }
+            tokenizer_next(&data);
+            double factor = 10;
+            while(tokenizer_peek(&data) && tokenizer_isDigit(data)) {
+                number += (data - '0') / factor;
+                factor *= 10;
+                tokenizer_next(&data);
+            }
+            break;
+        }
+        number = (number * 10) + (data - '0');
+        tokenizer_next(&data);
+    }
+    addDoubleToken(tokens, NUMBER, line, number);
+
+    return false;
+}
+
+static bool tokenizer_handleString() {
+    const size_t bufferSize = 1024;
+    char buffer[bufferSize];
+    size_t index = 0;
+
+    unsigned int oldLine = line;
+    while(index + 4 < bufferSize) {
+        char32_t data;
+        if(!tokenizer_next(&data)) {
+            tokenizer_onError("non closed string literal", oldLine);
+            return true;
+        }
+        if(data == '"') {
+            buffer[index] = '\0';
+            tokenizer_addStringToken(STRING, buffer);
+            return false;
+        }
+        if(data == '\n') {
+            line++;
+        }
+        if(data == '\\') {
+            char32_t escape;
+            if(!tokenizer_next(&escape)) {
+                tokenizer_onError("missing escaped character", line);
+                return true;
+            }
+            switch(escape) {
+                case 'n': data = '\n';
+                    break;
+                case '\\': data = '\\';
+                    break;
+                case '"': data = '"';
+                    break;
+                default:
+                    tokenizer_onError("invalid escaped character", line);
+                    return true;
+            }
+        }
+        index += tokenizer_printChar(data, buffer + index);
+    }
+    tokenizer_onError("string buffer to small", line);
+
+    return true;
+}
+
+static bool tokenizer_handleOneLineComment() {
+    char32_t data;
+    while(tokenizer_next(&data) && data != '\n');
+    line++;
+
+    return false;
+}
+
+static bool tokenizer_handleMultiLineComment() {
+    char32_t first;
+    char32_t sec = 0;
+    unsigned int oldLine = line;
+    while(true) {
+        first = sec;
+        if(!tokenizer_next(&sec)) {
+            tokenizer_onError("unclosed multiline comment", oldLine);
+            return true;
+        }
+        if(first == '*' && sec == '/') {
+
+            return false;
+        }
+        line += (sec == '\n');
+    }
+}
+
+static bool tokenizer_handleSlash() {
+    if(tokenizer_nextIf('/')) {
+        return tokenizer_handleOneLineComment();
+    } else if(tokenizer_nextIf('*')) {
+        return tokenizer_handleMultiLineComment();
+    } else if(tokenizer_nextIf('=')) {
+        tokenizer_addToken(DIV_SET);
+        return false;
+    }
+    tokenizer_addToken(DIV);
+
+    return false;
+}
+
+static bool tokenizer_handleSpecial(char32_t c) {
+    switch(c) {
+        case ' ':
+        case '\t':
+        case '\r':
+            return false;
+        case '\n': line++;
+            return false;
+        case '"':
+            return tokenizer_handleString();
+        case '(': tokenizer_addToken(OPEN_BRACKET);
+            return false;
+        case ')': tokenizer_addToken(CLOSE_BRACKET);
+            return false;
+        case '[': tokenizer_addToken(OPEN_SQUARE_BRACKET);
+            return false;
+        case ']': tokenizer_addToken(CLOSE_SQUARE_BRACKET);
+            return false;
+        case '{': tokenizer_addToken(OPEN_CURVED_BRACKET);
+            return false;
+        case '}': tokenizer_addToken(CLOSE_CURVED_BRACKET);
+            return false;
+        case '$':
+            return tokenizer_handleLiteral(c, LITERAL);
+        case '@':
+            return tokenizer_handleLiteral(c, LABEL);
+        case ';': tokenizer_addToken(SEMICOLON);
+            return false;
+        case ',': tokenizer_addToken(COMMA);
+            return false;
+        case '~': tokenizer_addToken(BIT_INVERT);
+            return false;
+        case '+': tokenizer_addToken(tokenizer_nextIf('=') ? ADD_SET: (tokenizer_nextIf('+') ? INC: ADD));
+            return false;
+        case '-': tokenizer_addToken(tokenizer_nextIf('=') ? SUB_SET: (tokenizer_nextIf('-') ? DEC: SUB));
+            return false;
+        case '!': tokenizer_addToken(tokenizer_nextIf('=') ? NOT_EQUAL: INVERT);
+            break;
+        case '=': tokenizer_addToken(tokenizer_nextIf('=') ? EQUAL: SET);
+            return false;
+        case '*': tokenizer_addToken(tokenizer_nextIf('=') ? MUL_SET: MUL);
+            return false;
+        case '/':
+            return tokenizer_handleSlash();
+        case '%': tokenizer_addToken(tokenizer_nextIf('=') ? MOD_SET: MOD);
+            return false;
+        case '&': tokenizer_addToken(tokenizer_nextIf('=') ? BIT_AND_SET: (tokenizer_nextIf('&') ? AND: BIT_AND));
+            return false;
+        case '|': tokenizer_addToken(tokenizer_nextIf('=') ? BIT_OR_SET: (tokenizer_nextIf('|') ? OR: BIT_OR));
+            return false;
+        case '^': tokenizer_addToken(tokenizer_nextIf('=') ? BIT_XOR_SET: BIT_XOR);
+            return false;
+        case '<': tokenizer_addToken(tokenizer_chooseToken('<', LEFT_SHIFT_SET, LEFT_SHIFT, LESS_EQUAL, LESS));
+            return false;
+        case '>': tokenizer_addToken(tokenizer_chooseToken('>', RIGHT_SHIFT_SET, RIGHT_SHIFT, GREATER_EQUAL, GREATER));
+            return false;
+    }
+    char buffer[32];
+    strncpy(buffer, "unknown token '", 32);
+    size_t index = strlen(buffer);
+    index += tokenizer_printChar(c, buffer + index);
+    buffer[index] = '\'';
+    buffer[index + 1] = '\0';
+    tokenizer_onError(buffer, line);
+
+    return true;
+}
+
+static bool tokenizer_handleChar(char32_t c) {
+    if(tokenizer_isValidNameStart(c)) {
+        return tokenizer_handleLiteral(c, LITERAL);
+    } else if(tokenizer_isDigit(c)) {
+
+        return tokenizer_handleNumber(c);
+    }
+    return tokenizer_handleSpecial(c);
+}
+
+bool tokenize(TokenStream* tokenStream, const char* inputPath) {
+    input = fopen(inputPath, "r");
+    if(input == NULL) {
+        return true;
+    }
+    tokens = tokenStream;
+    line = 1;
+    buffer = 0;
+
+    char32_t c;
+    while(tokenizer_next(&c)) {
+        if(tokenizer_handleChar(c)) {
+            return true;
+        }
+    }
+    tokenizer_addToken(EOF_TOKEN);
+
+    fclose(input);
+    input = NULL;
+    return false;
+}

tokenizer/Tokenizer.cpp

@@ -1,335 +0,0 @@
-#include <sstream>
-
-#include "tokenizer/Tokenizer.h"
-
-static unsigned int line = 1;
-static TokenStream* tokens = nullptr;
-static Tokenizer::i32stream* input = nullptr;
-static char32_t buffer = 0;
-
-static void onError(const std::string& message, unsigned int line) {
-    std::cout << message << " Line: " << line << std::endl;
-}
-
-static void convertChar(char32_t c, char* buffer) {
-    if(c <= 0x7F) {
-        buffer[0] = (char) c;
-        buffer[1] = '\0';
-    } else if(c <= 0x7FF) {
-        buffer[0] = (char) (0xC0 | ((c >> 6) & 0x1F));
-        buffer[1] = (char) (0x80 | ((c >> 0) & 0x3F));
-        buffer[2] = '\0';
-    } else if(c <= 0xFFFF) {
-        buffer[0] = (char) (0xE0 | ((c >> 12) & 0x0F));
-        buffer[1] = (char) (0x80 | ((c >> 6) & 0x3F));
-        buffer[2] = (char) (0x80 | ((c >> 0) & 0x3F));
-        buffer[3] = '\0';
-    } else {
-        buffer[0] = (char) (0xF0 | ((c >> 18) & 0x07));
-        buffer[1] = (char) (0x80 | ((c >> 12) & 0x3F));
-        buffer[2] = (char) (0x80 | ((c >> 6) & 0x3F));
-        buffer[3] = (char) (0x80 | ((c >> 0) & 0x3F));
-        buffer[4] = '\0';
-    }
-}
-
-static bool isLetter(char32_t c) {
-    return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
-}
-
-static bool isDigit(char32_t c) {
-    return c >= '0' && c <= '9';
-}
-
-static bool isValidNameStart(char32_t c) {
-    return isLetter(c) || c == '.' || c == '_';
-}
-
-static bool isValidNamePart(char32_t c) {
-    return isDigit(c) || isValidNameStart(c);
-}
-
-static bool next(char32_t& c) {
-    if(buffer != 0) {
-        c = buffer;
-        buffer = 0;
-        return true;
-    }
-    c = input->get();
-    return input->good();
-}
-
-static bool peek(char32_t& c) {
-    if(buffer != 0 || next(buffer)) {
-        c = buffer;
-        return true;
-    }
-    return false;
-}
-
-static bool nextIf(char32_t c) {
-    char32_t nextChar;
-    if(peek(nextChar) && c == nextChar) {
-        next(nextChar);
-        return true;
-    }
-    return false;
-}
-
-static void add(TokenType type) {
-    tokens->add(type, line);
-}
-
-static void add(TokenType type, const std::string& text) {
-    tokens->add(type, line, text);
-}
-
-static TokenType chooseTokenType(char c, TokenType aCharEqual, TokenType aChar, TokenType aEqual, TokenType other) {
-    if(nextIf(c)) {
-        if(nextIf('=')) {
-            return aCharEqual;
-        }
-        return aChar;
-    } else if(nextIf('=')) {
-        return aEqual;
-    }
-    return other;
-}
-
-static bool handleLiteral(char32_t c, TokenType type) {
-    std::stringstream sBuilder;
-    sBuilder << (char) c;
-
-    while(true) {
-        char32_t data;
-        if(!peek(data) || !isValidNamePart(data)) {
-            break;
-        }
-        sBuilder << (char) data;
-        next(data);
-    }
-
-    std::string s = sBuilder.str();
-    if(s == "if") {
-        add(TokenType::IF);
-    } else if(s == "if") {
-        add(TokenType::IF);
-    } else if(s == "else") {
-        add(TokenType::ELSE);
-    } else if(s == "elseif") {
-        add(TokenType::ELSEIF);
-    } else if(s == "while") {
-        add(TokenType::WHILE);
-    } else if(s == "try") {
-        add(TokenType::TRY);
-    } else if(s == "catch") {
-        add(TokenType::CATCH);
-    } else if(s == "for") {
-        add(TokenType::FOR);
-    } else if(s == "function") {
-        add(TokenType::FUNCTION);
-    } else if(s == "break") {
-        add(TokenType::BREAK);
-    } else if(s == "continue") {
-        add(TokenType::CONTINUE);
-    } else if(s == "return") {
-        add(TokenType::RETURN);
-    } else if(s == "true") {
-        add(TokenType::TRUE);
-    } else if(s == "false") {
-        add(TokenType::FALSE);
-    } else if(s == "null") {
-        add(TokenType::NULL_TOKEN);
-    } else {
-        add(type, s);
-    }
-    return false;
-}
-
-static bool handleNumber(char32_t c) {
-    double number = c - '0';
-    char32_t data;
-    while(peek(data)) {
-        if(!isDigit(data)) {
-            if(data != '.') {
-                break;
-            }
-            next(data);
-            double factor = 10;
-            while(peek(data) && isDigit(data)) {
-                number += (data - '0') / factor;
-                factor *= 10;
-                next(data);
-            }
-            break;
-        }
-        number = (number * 10) + (data - '0');
-        next(data);
-    }
-    tokens->add(NUMBER, line, number);
-    return false;
-}
-
-static bool handleString() {
-    std::stringstream ss;
-    unsigned int oldLine = line;
-    while(true) {
-        char32_t data;
-        if(!next(data)) {
-            onError("non closed string literal", oldLine);
-            return true;
-        }
-        if(data == '"') {
-            add(STRING, ss.str());
-            return false;
-        }
-        if(data == '\n') {
-            line++;
-        }
-        if(data == '\\') {
-            char32_t escape;
-            if(!next(escape)) {
-                onError("missing escaped character", line);
-                return true;
-            }
-            switch(escape) {
-                case 'n': data = '\n';
-                    break;
-                case '\\': data = '\\';
-                    break;
-                case '"': data = '"';
-                    break;
-                default:
-                    onError("invalid escaped character", line);
-                    return true;
-            }
-        }
-        char buffer[5];
-        convertChar(data, buffer);
-        ss << buffer;
-    }
-}
-
-static bool handleOneLineComment() {
-    char32_t data;
-    while(next(data) && data != '\n');
-    line++;
-    return false;
-}
-
-static bool handleMultiLineComment() {
-    char32_t first;
-    char32_t sec = 0;
-    unsigned int oldLine = line;
-    while(true) {
-        first = sec;
-        if(!next(sec)) {
-            onError("unclosed multiline comment", oldLine);
-            return true;
-        }
-        if(first == '*' && sec == '/') {
-            return false;
-        }
-        line += (sec == '\n');
-    }
-}
-
-static bool handleSlash() {
-    if(nextIf('/')) {
-        return handleOneLineComment();
-    } else if(nextIf('*')) {
-        return handleMultiLineComment();
-    } else if(nextIf('=')) {
-        add(DIV_SET);
-        return false;
-    }
-    add(DIV);
-    return false;
-}
-
-static bool handleSpecial(char32_t c) {
-    switch(c) {
-        case ' ':
-        case '\t':
-        case '\r':
-            return false;
-        case '\n': line++;
-            return false;
-        case '"':
-            return handleString();
-        case '(': add(OPEN_BRACKET);
-            return false;
-        case ')': add(CLOSE_BRACKET);
-            return false;
-        case '[': add(OPEN_SQUARE_BRACKET);
-            return false;
-        case ']': add(CLOSE_SQUARE_BRACKET);
-            return false;
-        case '{': add(OPEN_CURVED_BRACKET);
-            return false;
-        case '}': add(CLOSE_CURVED_BRACKET);
-            return false;
-        case '$':
-            return handleLiteral(c, LITERAL);
-        case '@':
-            return handleLiteral(c, LABEL);
-        case ';': add(SEMICOLON);
-            return false;
-        case ',': add(COMMA);
-            return false;
-        case '~': add(BIT_INVERT);
-            return false;
-        case '+': add(nextIf('=') ? ADD_SET: (nextIf('+') ? INC: ADD));
-            return false;
-        case '-': add(nextIf('=') ? SUB_SET: (nextIf('-') ? DEC: SUB));
-            return false;
-        case '!': add(nextIf('=') ? NOT_EQUAL: INVERT);
-            break;
-        case '=': add(nextIf('=') ? EQUAL: SET);
-            return false;
-        case '*': add(nextIf('=') ? MUL_SET: MUL);
-            return false;
-        case '/':
-            return handleSlash();
-        case '%': add(nextIf('=') ? MOD_SET: MOD);
-            return false;
-        case '&': add(nextIf('=') ? BIT_AND_SET: (nextIf('&') ? AND: BIT_AND));
-            return false;
-        case '|': add(nextIf('=') ? BIT_OR_SET: (nextIf('|') ? OR: BIT_OR));
-            return false;
-        case '^': add(nextIf('=') ? BIT_XOR_SET: BIT_XOR);
-            return false;
-        case '<': add(chooseTokenType('<', LEFT_SHIFT_SET, LEFT_SHIFT, LESS_EQUAL, LESS));
-            return false;
-        case '>': add(chooseTokenType('>', RIGHT_SHIFT_SET, RIGHT_SHIFT, GREATER_EQUAL, GREATER));
-            return false;
-    }
-    char buffer[5];
-    convertChar(c, buffer);
-    onError(std::string("unknown token '") + buffer + "'", line);
-    return true;
-}
-
-static bool handleChar(char32_t c) {
-    if(isValidNameStart(c)) {
-        return handleLiteral(c, TokenType::LITERAL);
-    } else if(isDigit(c)) {
-        return handleNumber(c);
-    }
-    return handleSpecial(c);
-}
-
-bool Tokenizer::tokenize(TokenStream& inTokens, i32stream& inInput) {
-    tokens = &inTokens;
-    input = &inInput;
-    line = 1;
-    buffer = 0;
-    char32_t c;
-    while(next(c)) {
-        if(handleChar(c)) {
-            return true;
-        }
-    }
-    add(EOF_TOKEN);
-    return false;
-}

tokenizer/Tokenizer.h

@@ -1,15 +1,10 @@
 #ifndef TOKENIZER_H
 #define TOKENIZER_H
 
-#include <iostream>
-#include <vector>
+#include <stdbool.h>
 
 #include "tokenizer/TokenStream.h"
 
-namespace Tokenizer {
-    typedef std::basic_istream<char32_t> i32stream;
-    typedef std::basic_ifstream<char32_t> if32stream;
-    bool tokenize(TokenStream& tokens, i32stream& input);
-}
+bool tokenize(TokenStream* tokenStream, const char* inputPath);
 
 #endif

utils/Path.c

@@ -0,0 +1,41 @@
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+#include "utils/Path.h"
+
+void initPath(Path* path, const char* strPath) {
+    path->capacity = PATH_MAX;
+    path->usedCapacity = strlen(strPath) + 1;
+    if(path->usedCapacity >= path->capacity) {
+        path->usedCapacity = path->capacity;
+    }
+    strncpy(path->path, strPath, path->usedCapacity);
+}
+
+static bool path_add(Path* path, const char* add) {
+    size_t addedLength = strlen(add);
+    size_t freeSpace = path->capacity - path->usedCapacity;
+    if(freeSpace < addedLength) {
+        fprintf(stderr, "cannot add '%s' to '%s', buffer too small\n", add, path->path);
+        return true;
+    }
+    strncpy(path->path + path->usedCapacity - 1, add, addedLength + 1);
+    path->usedCapacity += addedLength;
+    return false;
+}
+
+bool enterPath(Path* path, const char* object) {
+    return path_add(path, "/") || path_add(path, object);
+}
+
+bool enterFile(Path* path, const char* object, const char* ending) {
+    return path_add(path, "/") || path_add(path, object) || path_add(path, ending);
+}
+
+void leavePath(Path* path) {
+    while(path->usedCapacity > 1 && path->path[path->usedCapacity - 1] != '/') {
+        path->usedCapacity--;
+    }
+    path->path[path->usedCapacity - 1] = '\0';
+}

utils/Path.h

@@ -0,0 +1,18 @@
+#ifndef PATH_H
+#define PATH_H
+
+#include <limits.h>
+#include <stdbool.h>
+
+typedef struct {
+    size_t capacity;
+    size_t usedCapacity;
+    char path[PATH_MAX];
+} Path;
+
+void initPath(Path* path, const char* strPath);
+bool enterPath(Path* path, const char* object);
+bool enterFile(Path* path, const char* object, const char* ending);
+void leavePath(Path* path);
+
+#endif