lonely-tiger/compiler/Tokenizer.h

#ifndef TOKENIZER_H
#define TOKENIZER_H

#include <string>
#include "../data/ArrayList.h"
#include "Token.h"
#include "TokenList.h"

using namespace std;

class Tokenizer 
{
public:
    Tokenizer(string s);
    Tokenizer(const Tokenizer& orig);
    virtual ~Tokenizer();
    
    void tokenize(TokenList& tokens);
private:
    string data;
    int pos;
    int line;
    int length;
    
    void tokenize(TokenList& tokens, char c, TokenType type1, TokenType type2);
    void tokenize(TokenList& tokens, TokenType type1, char c2, TokenType, char c3, TokenType type3);
    void tokenize(TokenList& tokens, TokenType type1, char c2, char c3, TokenType type2, TokenType type3, char c4, TokenType type4);
};

#endif