#ifndef TOKENIZER_H
#define TOKENIZER_H

#include <string>
#include "ArrayList.h"
#include "Token.h"

using namespace std;

class Tokenizer 
{
public:
    Tokenizer(string s);
    Tokenizer(const Tokenizer& orig);
    virtual ~Tokenizer();
    
    void tokenize(ArrayList<Token*>& tokens);
private:
    string data;
    int pos;
    int line;
    int length;
    
    void tokenize(ArrayList<Token*>& tokens, char c, Tokens::Type type1, Tokens::Type type2);
    void tokenize(ArrayList<Token*>& tokens, Tokens::Type type1, char c2, Tokens::Type type2, char c3, Tokens::Type type3);
    void tokenize(ArrayList<Token*>& tokens, Tokens::Type type1, char c2, char c3, Tokens::Type type2, Tokens::Type type3, char c4, Tokens::Type type4);
};

#endif