#ifndef TOKENIZER_H
#define TOKENIZER_H

#include <iostream>
#include <vector>

#include "tokenizer/TokenStream.h"

namespace Tokenizer {
    typedef std::basic_istream<char32_t> i32stream;
    typedef std::basic_ifstream<char32_t> if32stream;
    bool tokenize(TokenStream& tokens, i32stream& input);
}

#endif