lonely-tiger/snuviscript/tokenizer/Tokenizer.h

#ifndef TOKENIZER_H
#define TOKENIZER_H

#include "Token.h"
#include "DoubleToken.h"
#include "StringToken.h"
#include <vector>
#include <memory>

using namespace std;

class Tokenizer
{
public:
    Tokenizer();
    
    void tokenize(vector<unique_ptr<Token>>& tokens, vector<unique_ptr<istream>>& streams);
    
private:
    int next();
    int peek();
    bool next(char c);
    void add(TokenType type);
    void add(TokenType type, double data);
    void add(TokenType type, string data);
    void add(char c, TokenType t1, TokenType t2, TokenType t3, TokenType t4);
    void handleChar(int c);
    void handleLiteral(int c, TokenType type);
    void handleNumber(int c);
    void handleSpecial(int c);
    void handleString();
    void handleSlash();
    void handleOneLineComment();
    void handleMultiLineComment();
    
    bool isLetter(int c);
    bool isDigit(int c);
    bool isValidNamePart(int c);
    
    int buffer;
    unsigned int line;
    vector<unique_ptr<Token>>* tokens;
    unsigned int streamIndex;
    vector<unique_ptr<istream>>* streams;
};

#endif