unit tests, tokenizer, strict syntax checker

src/me/hammerle/snuviscript/SnuviScript.java

@@ -1,67 +1,12 @@
 package me.hammerle.snuviscript;
 
-import java.io.IOException;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import me.hammerle.snuviscript.code.ISnuviLogger;
-import me.hammerle.snuviscript.code.ISnuviScheduler;
-import me.hammerle.snuviscript.code.Script;
-import me.hammerle.snuviscript.code.SnuviParser;
+import me.hammerle.snuviscript.test.Test;
 
 public class SnuviScript
 {
-    public static void main(String[] args) throws IOException 
+    public static void main(String[] args)
     {
-        ISnuviLogger logger = new ISnuviLogger() 
-        {
-            @Override
-            public void print(String message, Exception ex, String function, String scriptname, Script sc, int line) 
-            {
-                System.out.println("________________________________________");
-                System.out.println("Exception");
-                if(message != null)
-                {
-                    System.out.println(" - " + message);
-                }
-                if(ex != null)
-                {
-                    System.out.println(" - " + ex.getClass().getSimpleName());
-                }
-                if(function != null)
-                {
-                    System.out.println(" - Funktion: " + function);
-                }
-                if(scriptname != null)
-                {
-                    System.out.println(" - Script: " + scriptname);
-                }
-                if(line != -1)
-                {
-                    System.out.println(" - Line: " + line);
-                }
-            }
-        };
-        ISnuviScheduler scheduler = new ISnuviScheduler() 
-        {
-            @Override
-            public int scheduleTask(Runnable r) 
-            {
-                System.out.println("SCHEDULER");
-                return 0;
-            }
-
-            @Override
-            public int scheduleTask(Runnable r, long delay) 
-            {
-                ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();  
-                executor.schedule(r, delay, TimeUnit.MILLISECONDS);
-                executor.shutdown();
-                return 1;
-            }
-        };
-        
-        SnuviParser parser = new SnuviParser(logger, scheduler);
-        parser.startScript(true, ".sbasic", "./test");    
+        //Test.test();
+        Test.testNew();
     }  
-}
+}

src/me/hammerle/snuviscript/code/Compiler.java

@@ -447,10 +447,10 @@ public class Compiler
             switch(parts[0])
             {           
                 case "++":
-                    addCodeInstruction("p+", compileFunction(new String[] {parts[1]}, false));
+                    addCodeInstruction("p++", compileFunction(new String[] {parts[1]}, false));
                     return;
                 case "--":
-                    addCodeInstruction("p-", compileFunction(new String[] {parts[1]}, false));
+                    addCodeInstruction("p--", compileFunction(new String[] {parts[1]}, false));
                     return;
             }
             switch(parts[1])

src/me/hammerle/snuviscript/code/FunctionLoader.java

@@ -910,7 +910,7 @@ public class FunctionLoader
         });
         registerFunction("print", (sc, in) -> 
         {
-            System.out.println(SnuviUtils.connect(sc, in, 0));
+            sc.logger.print(SnuviUtils.connect(sc, in, 0), null, "print", sc.name, sc, sc.getActiveRealLine());
             return Void.TYPE;
         });
         registerFunction("waitfor", (sc, in) ->    

src/me/hammerle/snuviscript/compiler/Compiler.java

@@ -0,0 +1,374 @@
+package me.hammerle.snuviscript.compiler;
+
+import me.hammerle.snuviscript.exceptions.PreScriptException;
+import me.hammerle.snuviscript.token.Token;
+import me.hammerle.snuviscript.token.TokenType;
+
+public class Compiler
+{
+    private int index = 0;
+    private Token[] tokens = null;
+    
+    public Compiler()
+    {
+    }
+    
+    public void checkSyntax(Token[] tokens)
+    {
+        this.tokens = tokens;
+        index = 0;
+        checkLine();
+    }
+    
+    private Token consumeToken()
+    {
+        if(index >= tokens.length)
+        {
+            return null;
+        }
+        return tokens[index++];
+    }
+    
+    private Token peekOrNullToken()
+    {
+        if(index >= tokens.length)
+        {
+            return null;
+        }
+        return tokens[index];
+    }
+    
+    private Token peekToken()
+    {
+        if(index >= tokens.length)
+        {
+            throw new PreScriptException("missing token at end of file", -1);
+        }
+        return tokens[index];
+    }
+    
+    private void consumeTokenAndCheck(TokenType type)
+    {
+        Token t = consumeToken();
+        if(t == null)
+        {
+            throw new PreScriptException("missing token at end of file", -1);
+        }
+        if(t.getType() != type)
+        {
+            throw new PreScriptException("unexpected token " + t, t.getLine());
+        }
+    }
+    
+    private void checkFunctionArguments()
+    {
+        TokenType type = peekToken().getType();
+        if(type == TokenType.CLOSE_BRACKET)
+        {
+            consumeToken();
+            return;
+        }
+
+        consumeTokenAndCheck(TokenType.LITERAL);
+
+        while(true)
+        {
+            type = peekToken().getType();
+            if(type == TokenType.CLOSE_BRACKET)
+            {
+                consumeToken();
+                return;
+            }
+            consumeTokenAndCheck(TokenType.COMMA);
+            consumeTokenAndCheck(TokenType.LITERAL);
+        }
+    }
+    
+    private void checkLine()
+    {
+        while(true)
+        {
+            Token t = peekOrNullToken();
+            if(t == null || t.getType() == TokenType.CLOSE_CURVED_BRACKET)
+            {
+                break;
+            }
+            consumeToken();
+            switch(t.getType())
+            {
+                case LITERAL:
+                    switch(t.getData().toString())
+                    {
+                        case "function":
+                            consumeTokenAndCheck(TokenType.LITERAL);
+                            consumeTokenAndCheck(TokenType.OPEN_BRACKET);
+                            checkFunctionArguments();
+                            consumeTokenAndCheck(TokenType.OPEN_CURVED_BRACKET);
+                            checkLine();
+                            consumeTokenAndCheck(TokenType.CLOSE_CURVED_BRACKET);
+                            break;
+                        case "for":
+                            consumeTokenAndCheck(TokenType.OPEN_BRACKET);
+                            checkArguments(TokenType.CLOSE_BRACKET);
+                            consumeTokenAndCheck(TokenType.OPEN_CURVED_BRACKET);
+                            checkLine();
+                            consumeTokenAndCheck(TokenType.CLOSE_CURVED_BRACKET);
+                            break;
+                        case "else":
+                            consumeTokenAndCheck(TokenType.OPEN_CURVED_BRACKET);
+                            checkLine();
+                            consumeTokenAndCheck(TokenType.CLOSE_CURVED_BRACKET);
+                            break;
+                        case "elseif":
+                        case "while":
+                        case "if":
+                            consumeTokenAndCheck(TokenType.OPEN_BRACKET);
+                            checkExpression();
+                            consumeTokenAndCheck(TokenType.CLOSE_BRACKET);
+                            consumeTokenAndCheck(TokenType.OPEN_CURVED_BRACKET);
+                            checkLine();
+                            consumeTokenAndCheck(TokenType.CLOSE_CURVED_BRACKET);
+                            break;
+                        case "try":
+                            consumeTokenAndCheck(TokenType.OPEN_CURVED_BRACKET);
+                            checkLine();
+                            consumeTokenAndCheck(TokenType.CLOSE_CURVED_BRACKET);
+                            Token token = consumeToken();
+                            if(token.getType() != TokenType.LITERAL || !token.getData().equals("catch"))
+                            {
+                                throw new PreScriptException("try without catch", token.getLine());
+                            }
+                            consumeTokenAndCheck(TokenType.OPEN_CURVED_BRACKET);
+                            checkLine();
+                            consumeTokenAndCheck(TokenType.CLOSE_CURVED_BRACKET);
+                            break;
+                        case "continue":
+                        case "break":
+                            consumeTokenAndCheck(TokenType.SEMICOLON);
+                            break;
+                        case "return":
+                            if(peekToken().getType() == TokenType.SEMICOLON)
+                            {
+                                consumeToken();
+                            }
+                            else
+                            {
+                                checkExpression();
+                                consumeTokenAndCheck(TokenType.SEMICOLON);
+                            }
+                            break;
+                        default:
+                            checkAfterLiteral(true);
+                            consumeTokenAndCheck(TokenType.SEMICOLON);
+                    }
+                    break;
+                case OPEN_CURVED_BRACKET:
+                    checkLine();
+                    consumeTokenAndCheck(TokenType.CLOSE_CURVED_BRACKET);
+                    break;
+                case DOLLAR:
+                    checkVariable();
+                    checkVariableOperation(true);
+                    consumeTokenAndCheck(TokenType.SEMICOLON);
+                    break;
+                case LABEL:
+                    consumeTokenAndCheck(TokenType.LITERAL);
+                    break;
+                case SEMICOLON:
+                    break;
+                case INC:
+                    checkVariable();
+                    consumeTokenAndCheck(TokenType.SEMICOLON);
+                    break;
+                case DEC:
+                    checkVariable();
+                    consumeTokenAndCheck(TokenType.SEMICOLON);
+                    break;
+                default:
+                    throw new PreScriptException("unexpected token " + t, t.getLine());
+            }
+        }
+    }  
+    
+    private void checkVariable()
+    {
+        consumeTokenAndCheck(TokenType.LITERAL);
+        
+        Token t = peekToken();
+        if(t.getType() == TokenType.OPEN_SQUARE_BRACKET)
+        {
+            consumeToken();
+            checkArguments(TokenType.CLOSE_SQUARE_BRACKET);
+        }
+    }
+    
+    private void checkAfterLiteral(boolean line)
+    {
+        Token t = peekToken();
+        switch(t.getType())
+        {
+            case OPEN_BRACKET:
+                consumeToken();
+                checkArguments(TokenType.CLOSE_BRACKET);
+                if(!line)
+                {
+                    checkCalc();
+                }
+                return;
+            case OPEN_SQUARE_BRACKET:
+                consumeToken();
+                checkArguments(TokenType.CLOSE_SQUARE_BRACKET);
+                checkVariableOperation(line);
+                return; 
+            case INC:
+            case DEC:
+                consumeToken();
+                checkCalc();
+                return; 
+            case SET: 
+            case ADD_SET:
+            case SUB_SET: 
+            case MUL_SET:
+            case DIV_SET:
+            case MOD_SET:
+            case LEFT_SHIFT_SET:
+            case RIGHT_SHIFT_SET: 
+            case BIT_AND_SET:
+            case BIT_XOR_SET: 
+            case BIT_OR_SET:
+                consumeToken();
+                checkExpression();
+                return;
+            default:
+                if(line)
+                {
+                    throw new PreScriptException("unexpected token " + t, t.getLine());
+                }
+        }
+        
+        checkCalc();
+    }
+    
+    private void checkVariableOperation(boolean line)
+    {
+        Token t = peekToken();
+        switch(t.getType())
+        {
+            case INC:
+            case DEC:
+            case SET: 
+            case ADD_SET:
+            case SUB_SET: 
+            case MUL_SET:
+            case DIV_SET:
+            case MOD_SET:
+            case LEFT_SHIFT_SET:
+            case RIGHT_SHIFT_SET: 
+            case BIT_AND_SET:
+            case BIT_XOR_SET: 
+            case BIT_OR_SET:
+                consumeToken();
+                checkExpression();
+                break;
+            default:
+                if(line)
+                {
+                    throw new PreScriptException("unexpected token " + t, t.getLine());
+                }
+        }
+    }
+    
+    private void checkCalc()
+    {
+        Token t = peekToken();
+        switch(t.getType())
+        {
+            case MUL:
+            case DIV:
+            case MOD:
+            case ADD:
+            case SUB:
+            case LEFT_SHIFT:
+            case RIGHT_SHIFT:
+            case LESS:
+            case LESS_EQUAL:
+            case GREATER:
+            case GREATER_EQUAL:
+            case EQUAL:
+            case NOT_EQUAL:
+            case BIT_AND:
+            case BIT_XOR:
+            case BIT_OR:
+            case AND:
+            case OR:
+                consumeToken();
+                checkExpression();
+                break;
+        }
+    }
+    
+    private void checkArguments(TokenType end)
+    {
+        Token t = peekToken();
+        if(t.getType() == end)
+        {
+            consumeToken();
+            return;
+        }
+        
+        checkExpression();
+        
+        while(true)
+        {
+            t = peekToken();
+            if(t.getType() == end)
+            {
+                consumeToken();
+                return;
+            }
+            consumeTokenAndCheck(TokenType.COMMA);
+            checkExpression();
+        }
+    }
+    
+    private void checkExpression()
+    {
+        Token t = consumeToken();
+        switch(t.getType())
+        {
+            case SUB:
+                checkExpression();
+                break;
+            case NUMBER:
+            case STRING:
+                checkCalc();
+                break;
+            case LITERAL:
+                checkAfterLiteral(false);
+                break;
+            case OPEN_BRACKET:
+                checkExpression();
+                consumeTokenAndCheck(TokenType.CLOSE_BRACKET);
+                checkCalc();
+                break;
+            case DOLLAR:
+                checkVariable();
+                checkVariableOperation(false);
+                break;
+            case LABEL:
+                consumeTokenAndCheck(TokenType.LITERAL);
+                break;
+            case INC:
+            case DEC:
+                checkVariable();
+                checkCalc();
+                break;
+            case INVERT:
+            case BIT_INVERT:
+                checkExpression();
+                break;          
+            default:
+                throw new PreScriptException("unexpected token " + t, t.getLine());
+        }
+    }
+}

src/me/hammerle/snuviscript/test/Test.java

@@ -0,0 +1,113 @@
+package me.hammerle.snuviscript.test;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import me.hammerle.snuviscript.code.SnuviParser;
+import me.hammerle.snuviscript.token.Tokenizer;
+import me.hammerle.snuviscript.compiler.Compiler;
+import me.hammerle.snuviscript.exceptions.PreScriptException;
+
+public class Test
+{
+    private static final TestScheduler SCHEDULER = new TestScheduler();
+    private static final TestLogger LOGGER = new TestLogger();
+    private static final SnuviParser PARSER = new SnuviParser(LOGGER, SCHEDULER);
+    private static int done = 0;
+    private static int tests = 0;    
+    
+    public static void test()
+    {
+        //genTests(5, 15, "functions");
+        findTestFiles(new File("./test"));
+        System.out.println(String.format("%d / %d tests succeeded", done, tests));
+    }
+    
+    public static void testNew()
+    {
+        findTestFilesNew(new File("./test"));
+    }
+    
+    private static void genTests(int from, int to, String name)
+    {
+        for(int i = from; i < to; i++)
+        {
+            try
+            {
+                File f = new File(String.format("./test/%s/%s%d", name, name, i));
+                f.createNewFile();
+                
+                f = new File(String.format("./test/%s/%s%d.out", name, name, i));
+                f.createNewFile();
+            }
+            catch(Exception ex)
+            {
+            }
+        }
+    }
+    
+    private static void findTestFiles(File f)
+    {
+        if(f.isFile())
+        {
+            if(!f.getName().endsWith(".out"))
+            {
+                tests++;
+                
+                LOGGER.reset();
+                PARSER.startScript(true, "", f.getPath());
+                
+                if(LOGGER.check(new File(f.getPath() + ".out")))
+                {
+                    done++;
+                }
+            }
+        }
+        else if(f.isDirectory())
+        {
+            for(File fi : f.listFiles())
+            {
+                findTestFiles(fi);
+            }
+        }
+    }
+    
+    private static void findTestFilesNew(File f)
+    {
+        if(f.isFile())
+        {
+            if(!f.getName().endsWith(".out"))
+            {
+                System.out.println("_________________________________________________");
+                System.out.println(String.format("Tokenize \"%s\"", f.getPath()));
+                try
+                {
+                    try(FileInputStream in = new FileInputStream(f))
+                    {
+                        Tokenizer tokenizer = new Tokenizer();
+                        Compiler c = new Compiler();
+                        c.checkSyntax(tokenizer.tokenize(in));
+                    }
+                }
+                catch(IOException ex)
+                {
+                    ex.printStackTrace();
+                }
+                catch(PreScriptException ex)
+                {
+                    //ex.printStackTrace();
+                    System.out.println(ex.getMessage());
+                    System.out.println(ex.getStartLine());
+                    System.out.println(ex.getEndLine());
+                }
+            }
+        }
+        else if(f.isDirectory())
+        {
+            for(File fi : f.listFiles())
+            {
+                findTestFilesNew(fi);
+            }
+        }
+    }
+}

src/me/hammerle/snuviscript/test/TestLogger.java

@@ -0,0 +1,74 @@
+package me.hammerle.snuviscript.test;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.List;
+import me.hammerle.snuviscript.code.ISnuviLogger;
+import me.hammerle.snuviscript.code.Script;
+
+public class TestLogger implements ISnuviLogger
+{
+    private final ArrayList<String> list = new ArrayList<>();
+    
+    @Override
+    public void print(String message, Exception ex, String function, String scriptname, Script sc, int line)
+    {
+        if(ex == null)
+        {
+            list.add(message);
+        }
+        else
+        {
+            System.out.println(ex);
+            System.out.println(ex.getMessage());
+        }
+    }
+    
+    public void reset()
+    {
+        list.clear();
+    }
+    
+    public boolean check(File f)
+    {
+        if(!f.exists())
+        {
+            System.out.println(String.format("\"%s\" does not exist", f.getPath()));
+            return false;
+        }
+        try
+        {
+            List<String> file = Files.readAllLines(f.toPath());
+            if(file.size() != list.size())
+            {
+                printNoMatch(f, file);
+                return false;
+            }
+            for(int i = 0; i < file.size(); i++)
+            {
+                if(!file.get(i).equals(list.get(i)))
+                {
+                    printNoMatch(f, file);
+                    return false;
+                }
+            }
+        }
+        catch(Exception ex)
+        {
+            ex.printStackTrace();
+            return false;
+        }
+        return true;
+    }
+    
+    private void printNoMatch(File f, List<String> file)
+    {
+        System.out.println(String.format("error checking %s ", f.getPath()));
+        System.out.println("Expected ----------------------------------------");
+        file.forEach(s -> System.out.println(s));
+        System.out.println("Actual ------------------------------------------");
+        list.forEach(s -> System.out.println(s));
+        System.out.println("-------------------------------------------------");
+    }
+}

src/me/hammerle/snuviscript/test/TestScheduler.java

@@ -0,0 +1,20 @@
+package me.hammerle.snuviscript.test;
+
+import me.hammerle.snuviscript.code.ISnuviScheduler;
+
+public class TestScheduler implements ISnuviScheduler
+{
+    @Override
+    public int scheduleTask(Runnable r)
+    {
+        System.out.println("Schedule");
+        return 0;
+    }
+
+    @Override
+    public int scheduleTask(Runnable r, long delay)
+    {
+        return 0;
+    }
+    
+}

src/me/hammerle/snuviscript/token/DataToken.java

@@ -0,0 +1,24 @@
+package me.hammerle.snuviscript.token;
+
+public class DataToken extends Token
+{
+    private final Object value;
+    
+    public DataToken(TokenType type, int line,  Object value)
+    {
+        super(type, line);
+        this.value = value;
+    }
+
+    @Override
+    public Object getData()
+    {
+        return value;
+    }
+
+    @Override
+    public String toString()
+    {
+        return String.valueOf(value);
+    }
+}

src/me/hammerle/snuviscript/token/Token.java

@@ -0,0 +1,34 @@
+package me.hammerle.snuviscript.token;
+
+public class Token
+{
+    private final TokenType type;
+    private final int line;
+    
+    public Token(TokenType type, int line)
+    {
+        this.type = type;
+        this.line = line;
+    }
+
+    public TokenType getType()
+    {
+        return type;
+    }
+    
+    public Object getData()
+    {
+        return null;
+    }
+    
+    public int getLine()
+    {
+        return line;
+    }
+
+    @Override
+    public String toString()
+    {
+        return type.getName();
+    }
+}

src/me/hammerle/snuviscript/token/TokenType.java

@@ -0,0 +1,125 @@
+package me.hammerle.snuviscript.token;
+
+public enum TokenType
+{
+    UNKNOWN(""),
+    NUMBER("number"),
+    STRING("string"),
+    LITERAL("literal"),
+    
+    OPEN_BRACKET("("),
+    CLOSE_BRACKET(")"),
+    OPEN_SQUARE_BRACKET("["),
+    CLOSE_SQUARE_BRACKET("]"),
+    OPEN_CURVED_BRACKET("{"),
+    CLOSE_CURVED_BRACKET("}"),
+    
+    DOLLAR("$"),
+    LABEL("@"),
+    
+    ONE_LINE_COMMENT("//"),
+    COMMENT("/*"),
+    
+    SEMICOLON(";"),
+    COMMA(","),
+    INC("++"),
+    DEC("--"),
+    INVERT("!"),
+    BIT_INVERT("~"),
+    MUL("*"),
+    DIV("/"),
+    MOD("%"),
+    ADD("+"),
+    SUB("-"),
+    LEFT_SHIFT("<<"),
+    RIGHT_SHIFT(">>"),
+    LESS("<"),
+    LESS_EQUAL("<="),
+    GREATER(">"),
+    GREATER_EQUAL(">="),
+    EQUAL("=="),
+    NOT_EQUAL("!="),
+    BIT_AND("&"),
+    BIT_XOR("^"),
+    BIT_OR("|"),
+    AND("&&"),
+    OR("||"),
+    SET("="),
+    ADD_SET("+="),
+    SUB_SET("-="),
+    MUL_SET("*="),
+    DIV_SET("/="),
+    MOD_SET("%="),
+    LEFT_SHIFT_SET("<<="),
+    RIGHT_SHIFT_SET(">>="),
+    BIT_AND_SET("&="),
+    BIT_XOR_SET("^="),
+    BIT_OR_SET("|=");
+        
+    private final String name;
+
+    private TokenType(String name)
+    {
+        this.name = name;
+    }   
+
+    public String getName() 
+    {
+        return name;
+    }
+    
+    public int getLength()
+    {
+        return name.length();
+    }
+    
+    public static TokenType getMatching(char c1, char c2, char c3)
+    {
+        switch(c1)
+        {
+            case '(': return OPEN_BRACKET;
+            case ')': return CLOSE_BRACKET;
+            case '[': return OPEN_SQUARE_BRACKET;
+            case ']': return CLOSE_SQUARE_BRACKET;
+            case '{': return OPEN_CURVED_BRACKET;
+            case '}': return CLOSE_CURVED_BRACKET;
+            case '$': return DOLLAR;
+            case '@': return LABEL;
+            case ';': return SEMICOLON;
+            case ',': return COMMA;
+            case '~': return BIT_INVERT;
+            case '+': return c2 == '=' ? ADD_SET : (c2 == '+' ? INC : ADD);
+            case '-': return c2 == '=' ? SUB_SET : (c2 == '-' ? DEC : SUB);
+            case '!': return c2 == '=' ? NOT_EQUAL : INVERT;
+            case '=': return c2 == '=' ? EQUAL : SET;
+            case '*': return c2 == '=' ? MUL_SET : MUL;
+            case '/': 
+                switch(c2)
+                {
+                    case '/': return ONE_LINE_COMMENT;
+                    case '*': return COMMENT;
+                    case '=': return DIV_SET;
+                }
+                return DIV;
+            case '%': return c2 == '=' ? MOD_SET : MOD;
+            case '&': return c2 == '=' ? BIT_AND_SET : (c2 == '&' ? AND : BIT_AND);
+            case '|': return c2 == '=' ? BIT_OR_SET : (c2 == '|' ? OR : BIT_OR);
+            case '^': return c2 == '=' ? BIT_XOR_SET : BIT_XOR;
+            case '<': 
+                switch(c2)
+                {
+                    case '<': return c3 == '=' ? LEFT_SHIFT_SET : LEFT_SHIFT;
+                    case '=': return LESS_EQUAL;
+                }
+                return LESS;
+            case '>': 
+                switch(c2)
+                {
+                    case '>': return c3 == '=' ? RIGHT_SHIFT_SET : RIGHT_SHIFT;
+                    case '=': return GREATER_EQUAL;
+                }
+                return GREATER;    
+        }
+        return UNKNOWN;
+    }
+}

src/me/hammerle/snuviscript/token/Tokenizer.java

@@ -0,0 +1,182 @@
+package me.hammerle.snuviscript.token;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import me.hammerle.snuviscript.exceptions.PreScriptException;
+
+public class Tokenizer
+{
+    private final ArrayList<Token> tokens = new ArrayList<>();
+    private String code = "";
+    private int index = 0;
+    private int old = 0;
+    private int line = 1;
+    
+    private boolean isLiteralCharacter(char c)
+    {
+        return Character.isLetterOrDigit(c) || c == '_' || c == '.';
+    }
+    
+    private char charAt(int index)
+    {
+        if(index < 0 || index >= code.length())
+        {
+            return ' ';
+        }
+        return code.charAt(index);
+    }
+    
+    private void add(int from, int to)
+    {
+        String part = code.substring(from, to);
+        try
+        {
+            double d = Double.parseDouble(part);
+            tokens.add(new DataToken(TokenType.NUMBER, line, d));
+        }
+        catch(NumberFormatException ex)
+        {
+            tokens.add(new DataToken(TokenType.LITERAL, line, part));
+        }
+    }
+    
+    private void addIfNotEmpty(int from, int to)
+    {
+        if(to - from <= 0)
+        {
+            return;
+        }
+        add(from, to);
+    }
+    
+    private void skipOneLineComment()
+    {
+        while(index < code.length() && code.charAt(index) != '\n')
+        {
+            index++;
+        }
+        line++;
+        old = index + 1;
+    }
+    
+    private void skipComment()
+    {
+        while(index < code.length())
+        {
+            if(code.charAt(index) == '\n')
+            {
+                line++;
+            }
+            else if(code.charAt(index) == '*' && charAt(index + 1) == '/')
+            {
+                break;
+            }
+            index++;
+        }
+        old = index + 2;
+        index++;
+    }
+    
+    private void handleString()
+    {
+        addIfNotEmpty(old, index);
+        old = index + 1;
+        index++;
+        while(index < code.length() && code.charAt(index) != '"')
+        {
+            if(code.charAt(index) == '\n')
+            {
+                line++;
+            }
+            index++;
+        }
+        add(old, index);
+        old = index + 1;
+    }
+    
+    private void handleNewLine()
+    {
+        addIfNotEmpty(old, index);
+        old = index + 1;
+        line++;
+    }
+    
+    private void handleSpace()
+    {
+        addIfNotEmpty(old, index);
+        old = index + 1;
+    }
+    
+    private void handleSpecialCharacter()
+    {
+        addIfNotEmpty(old, index);
+
+        TokenType type = TokenType.getMatching(code.charAt(index), charAt(index + 1), charAt(index + 2));
+        switch(type)
+        {
+            case UNKNOWN: throw new PreScriptException("unknown token", line);
+            case ONE_LINE_COMMENT: 
+                skipOneLineComment(); 
+                break;
+            case COMMENT: 
+                skipComment(); 
+                break;
+            default:
+                tokens.add(new Token(type, line));
+                old = index + type.getLength();
+                index = old - 1;
+        }
+    }
+    
+    public Token[] tokenize(InputStream in)
+    {
+        code = readStream(in);
+        index = 0;
+        old = 0;
+        tokens.clear();
+        
+        for(; index < code.length(); index++)
+        {
+            char c = code.charAt(index);
+            if(isLiteralCharacter(c))
+            {
+                continue;
+            }
+            switch(c)
+            {
+                case '\n': handleNewLine(); break;
+                case '"': handleString(); break;
+                case '\t':
+                case ' ': handleSpace(); break;
+                default: handleSpecialCharacter();
+            }
+        }
+        return tokens.toArray(new Token[tokens.size()]);
+    }
+    
+    private String readStream(InputStream in)
+    {
+        try
+        {
+            int bufferSize = in.available();
+            StringBuilder sb = new StringBuilder(bufferSize);
+            
+            while(in.available() > 0)
+            {
+                int data = in.read();
+                if((data & 0x80) != 0)
+                {
+                    data = ((data & 0x1F) << 6) | (in.read() & 0x3F);
+                }
+                sb.append((char) data);
+            }
+            
+            return sb.toString();
+        }
+        catch(IOException ex)
+        {
+            throw new PreScriptException("cannot read stream - " + ex.getMessage(), -1);
+        }
+    }
+}

test/calc/base

@@ -0,0 +1,15 @@
+print(1 + 1);
+print(1 + -3);
+
+print(1 - 3);
+print(1 - -3);
+
+print(4 * 3);
+print(6 * -3);
+print(-6 * 4);
+print(-6 * -4);
+
+print(4 / 2);
+print(6 / -2);
+print(-6 / 3);
+print(-6 / -2);

test/calc/base.out

@@ -0,0 +1,12 @@
+2.0
+-2.0
+-2.0
+4.0
+12.0
+-18.0
+-24.0
+24.0
+2.0
+-3.0
+-2.0
+3.0

test/calc/mixed

@@ -0,0 +1,8 @@
+print(2 + 3 * 4);
+print((2 + 3) * 4);
+print(2 + 3 * 4 + 2);
+print(2 + 3 * (4 + 2));
+print(2 + 3 * (4 + 2));
+print(2 + -3 * (4 + 2));
+print(2 + 3 * (-4 + 2));
+print((2 - 3) * 4);

test/calc/mixed.out

@@ -0,0 +1,8 @@
+14.0
+20.0
+16.0
+20.0
+20.0
+-16.0
+-4.0
+-4.0

test/conditions/conditions0

@@ -0,0 +1,2 @@
+print(true);
+print(false);

test/conditions/conditions0.out

@@ -0,0 +1,2 @@
+true
+false

test/conditions/conditions1

@@ -0,0 +1,4 @@
+print(true && true);
+print(false && true);
+print(true && false);
+print(false && false);

test/conditions/conditions1.out

@@ -0,0 +1,4 @@
+true
+false
+false
+false

test/conditions/conditions2

@@ -0,0 +1,4 @@
+print(true || true);
+print(false || true);
+print(true || false);
+print(false || false);

test/conditions/conditions2.out

@@ -0,0 +1,4 @@
+true
+true
+true
+false

test/conditions/conditions3

@@ -0,0 +1,3 @@
+print(true || (true && false));
+print((true || true) && false);
+print((true || false) && false);

test/conditions/conditions3.out

@@ -0,0 +1,3 @@
+true
+false
+false

test/conditions/conditions4

@@ -0,0 +1,7 @@
+a = 3;
+
+print(false && a++ == 3);
+print(a);
+
+print(true && a++ == 4);
+print(a);

test/conditions/conditions4.out

@@ -0,0 +1,4 @@
+false
+3.0
+false
+4.0

test/conditions/conditions5

@@ -0,0 +1,7 @@
+a = 3;
+
+print(false || a++ == 3);
+print(a);
+
+print(true || a++ == 4);
+print(a);

test/conditions/conditions5.out

@@ -0,0 +1,4 @@
+true
+4.0
+true
+4.0

test/conditions/conditions6

@@ -0,0 +1,17 @@
+print(4 < 5);
+print(4 > 5);
+print(4 >= 5);
+print(4 <= 5);
+print(4 == 5);
+
+print(5 < 5);
+print(5 > 5);
+print(5 >= 5);
+print(5 <= 5);
+print(5 == 5);
+
+print(6 < 5);
+print(6 > 5);
+print(6 >= 5);
+print(6 <= 5);
+print(6 == 5);

test/conditions/conditions6.out

@@ -0,0 +1,15 @@
+true
+false
+false
+true
+false
+false
+false
+true
+true
+true
+false
+true
+true
+false
+false

test/conditions/conditions7

@@ -0,0 +1,2 @@
+print(5 <= 4 || 5 >= 4);
+print(5 <= 4 && 5 >= 4);

test/conditions/conditions7.out

@@ -0,0 +1,2 @@
+true
+false

test/conditions/conditions8

@@ -0,0 +1,4 @@
+print(1 + 1 == 3 - 1);
+print(1 + 1 == 3 - 1 && false);
+print(1 + 1 == 3 - 1 && 5 / 3 == 2 );
+print(1 + 1 == 3 - 1 || 5 / 3 == 2 );

test/conditions/conditions8.out

@@ -0,0 +1,4 @@
+true
+false
+false
+true

test/for_break_continue/for_break_continue0

@@ -0,0 +1,4 @@
+for(i, 6, 9)
+{
+    print(i);
+}

test/for_break_continue/for_break_continue0.out

@@ -0,0 +1,4 @@
+6.0
+7.0
+8.0
+9.0

test/for_break_continue/for_break_continue1

@@ -0,0 +1,4 @@
+for(i, 6, 9, 2)
+{
+    print(i);
+}

test/for_break_continue/for_break_continue1.out

@@ -0,0 +1,2 @@
+6.0
+8.0

test/for_break_continue/for_break_continue2

@@ -0,0 +1,4 @@
+for(i, 5, 15, 3)
+{
+    print(i);
+}

test/for_break_continue/for_break_continue2.out

@@ -0,0 +1,4 @@
+5.0
+8.0
+11.0
+14.0

test/for_break_continue/for_break_continue3

@@ -0,0 +1,5 @@
+for(i, 5, 15, 3)
+{
+    print(i);
+    break;
+}

test/for_break_continue/for_break_continue3.out

@@ -0,0 +1 @@
+5.0

test/for_break_continue/for_break_continue4

@@ -0,0 +1,8 @@
+for(i, 5, 15, 3)
+{
+    print(i);
+    if(i == 9)
+    {
+        break;
+    }
+}

test/for_break_continue/for_break_continue4.out

@@ -0,0 +1,4 @@
+5.0
+8.0
+11.0
+14.0

test/for_break_continue/for_break_continue5

@@ -0,0 +1,8 @@
+for(i, 5, 15, 3)
+{
+    print(i);
+    if(i == 11)
+    {
+        break;
+    }
+}

test/for_break_continue/for_break_continue5.out

@@ -0,0 +1,3 @@
+5.0
+8.0
+11.0

test/for_break_continue/for_break_continue6

@@ -0,0 +1,8 @@
+for(i, 5, 7)
+{
+    for(j, 0, 5)
+    {
+        print(j);
+    }
+    print(i);
+}

test/for_break_continue/for_break_continue6.out

@@ -0,0 +1,21 @@
+0.0
+1.0
+2.0
+3.0
+4.0
+5.0
+5.0
+0.0
+1.0
+2.0
+3.0
+4.0
+5.0
+6.0
+0.0
+1.0
+2.0
+3.0
+4.0
+5.0
+7.0

test/for_break_continue/for_break_continue7

@@ -0,0 +1,9 @@
+for(i, 5, 7)
+{
+    for(j, 0, 5)
+    {
+        break;
+        print(j);
+    }
+    print(i);
+}

test/for_break_continue/for_break_continue7.out

@@ -0,0 +1,3 @@
+5.0
+6.0
+7.0

test/for_break_continue/for_break_continue8

@@ -0,0 +1,12 @@
+for(i, 5, 7)
+{
+    for(j, 0, 5)
+    {
+        if(i == 6)
+        {
+            continue;
+        }
+        print(j);
+    }
+    print(i);
+}

test/for_break_continue/for_break_continue8.out

@@ -0,0 +1,15 @@
+0.0
+1.0
+2.0
+3.0
+4.0
+5.0
+5.0
+6.0
+0.0
+1.0
+2.0
+3.0
+4.0
+5.0
+7.0

test/for_break_continue/for_break_continue9

@@ -0,0 +1,10 @@
+for(x, 0, 2)
+{
+    for(y, 0, 2)
+    {
+        for(z, 0, 2)
+        {
+            print(x, " ", y, " ", z);
+        }
+    }
+}

test/for_break_continue/for_break_continue9.out

@@ -0,0 +1,27 @@
+0.0 0.0 0.0
+0.0 0.0 1.0
+0.0 0.0 2.0
+0.0 1.0 0.0
+0.0 1.0 1.0
+0.0 1.0 2.0
+0.0 2.0 0.0
+0.0 2.0 1.0
+0.0 2.0 2.0
+1.0 0.0 0.0
+1.0 0.0 1.0
+1.0 0.0 2.0
+1.0 1.0 0.0
+1.0 1.0 1.0
+1.0 1.0 2.0
+1.0 2.0 0.0
+1.0 2.0 1.0
+1.0 2.0 2.0
+2.0 0.0 0.0
+2.0 0.0 1.0
+2.0 0.0 2.0
+2.0 1.0 0.0
+2.0 1.0 1.0
+2.0 1.0 2.0
+2.0 2.0 0.0
+2.0 2.0 1.0
+2.0 2.0 2.0

test/functions/functions0

@@ -0,0 +1,8 @@
+print("a");
+
+function wusi()
+{
+    print(b);
+}
+
+print("c");

test/functions/functions0.out

@@ -0,0 +1,2 @@
+a
+c

test/functions/functions1

@@ -0,0 +1,8 @@
+print("a");
+
+function wusi(b)
+{
+    print(b);
+}
+
+print("c");

test/functions/functions1.out

@@ -0,0 +1,2 @@
+a
+c

test/functions/functions10

@@ -0,0 +1,11 @@
+print("a");
+
+function wusi(b)
+{
+    print("b");
+    return;
+    print("c");
+}
+wusi("HI");
+
+print("d");

test/functions/functions10.out

@@ -0,0 +1,3 @@
+a
+b
+d

test/functions/functions11

@@ -0,0 +1,11 @@
+print("a");
+
+function wusi()
+{
+    print("b");
+    return "c";
+    print("d");
+}
+print(wusi());
+
+print("e");

test/functions/functions11.out

@@ -0,0 +1,4 @@
+a
+b
+c
+e

test/functions/functions12

@@ -0,0 +1,17 @@
+print("a");
+
+function b()
+{
+    print("b");
+    c();
+}
+
+function c()
+{
+    print("c");
+}
+
+b();
+c();
+
+print("e");

test/functions/functions12.out

@@ -0,0 +1,5 @@
+a
+b
+c
+c
+e

test/functions/functions13

@@ -0,0 +1,20 @@
+print("a");
+
+function b()
+{
+    n = 7;
+    print(n);
+    c();
+    print(n);
+}
+
+function c()
+{
+    n = 9;
+    print(n);
+}
+
+b();
+c();
+
+print("e");

test/functions/functions13.out

@@ -0,0 +1,6 @@
+a
+7.0
+9.0
+7.0
+9.0
+e

test/functions/functions14

@@ -0,0 +1,13 @@
+igoto("hi");
+goto("end");
+
+function wusi()
+{
+    @hi
+    print("HI");
+    @end
+}
+
+print("a");
+
+@end

test/functions/functions2

@@ -0,0 +1,9 @@
+print("a");
+
+function wusi(b)
+{
+    print("b");
+}
+wusi("HI");
+
+print("c");

test/functions/functions2.out

@@ -0,0 +1,3 @@
+a
+b
+c

test/functions/functions3

@@ -0,0 +1,9 @@
+print("a");
+
+function wusi(b)
+{
+    print(b);
+}
+wusi("HI");
+
+print("c");

test/functions/functions3.out

@@ -0,0 +1,3 @@
+a
+HI
+c

test/functions/functions4

@@ -0,0 +1,8 @@
+a = 2;
+
+function wusi()
+{
+    a = 5;
+}
+
+print(a);

test/functions/functions4.out

@@ -0,0 +1 @@
+2.0

test/functions/functions5

@@ -0,0 +1,9 @@
+a = 2;
+
+function wusi()
+{
+    a = 5;
+}
+wusi();
+
+print(a);

test/functions/functions5.out

@@ -0,0 +1 @@
+2.0

test/functions/functions6

@@ -0,0 +1,8 @@
+a = 2;
+
+function wusi()
+{
+    $a = 5;
+}
+
+print(a);

test/functions/functions6.out

@@ -0,0 +1 @@
+2.0

test/functions/functions7

@@ -0,0 +1,9 @@
+a = 2;
+
+function wusi()
+{
+    $a = 5;
+}
+wusi();
+
+print(a);

test/functions/functions7.out

@@ -0,0 +1 @@
+5.0

test/functions/functions8

@@ -0,0 +1,10 @@
+print(fac(5));
+
+function fac(n)
+{
+    if(n <= 1)
+    {
+        return 1;
+    }
+    return n * fac(n - 1);
+}

test/functions/functions8.out

@@ -0,0 +1 @@
+120.0

test/functions/functions9

@@ -0,0 +1,10 @@
+print("a");
+
+function wusi(b)
+{
+    print("b");
+    return;
+    print("c");
+}
+
+print("d");

test/functions/functions9.out

@@ -0,0 +1,2 @@
+a
+d

test/goto

@@ -0,0 +1,21 @@
+print("a");
+invstring = "";
+goto(@label4);
+@label3
+print("e");
+goto(@label5);
+@label4
+
+goto(@label1);
+print("b");
+@label1
+
+goto("label2");
+print("c");
+@label2;
+
+goto(@label3);
+@label5
+print("d");
+
+print("b");

test/goto.out

@@ -0,0 +1,4 @@
+a
+e
+d
+b

test/if_elseif_else/if0

@@ -0,0 +1,4 @@
+if(5 < 3)
+{
+    print("a");
+}

test/if_elseif_else/if1

@@ -0,0 +1,5 @@
+a = 4;
+if(a < 3)
+{
+    print("a");
+}

test/if_elseif_else/if10

@@ -0,0 +1,32 @@
+if(3 < 2)
+{
+    if(5 < 3)
+    {
+        print("a");
+    }
+    else
+    {
+        print("b");
+    }
+    print("c");
+}
+elseif(1 < 2)
+{
+    print("d");
+}
+elseif(2 < 2)
+{
+    print("e");
+    if(3 < 5)
+    {
+        print("f");
+    }
+    else
+    {
+        print("g");
+    }
+}
+else
+{
+    print("h");
+}

test/if_elseif_else/if10.out

@@ -0,0 +1 @@
+d

test/if_elseif_else/if11

@@ -0,0 +1,32 @@
+if(3 < 2)
+{
+    if(5 < 3)
+    {
+        print("a");
+    }
+    else
+    {
+        print("b");
+    }
+    print("c");
+}
+elseif(2 < 2)
+{
+    print("d");
+}
+elseif(1 < 2)
+{
+    print("e");
+    if(3 < 5)
+    {
+        print("f");
+    }
+    else
+    {
+        print("g");
+    }
+}
+else
+{
+    print("h");
+}

test/if_elseif_else/if11.out

@@ -0,0 +1,2 @@
+e
+f

test/if_elseif_else/if2

@@ -0,0 +1,5 @@
+a = 4;
+if(a - 2 < 3)
+{
+    print("a");
+}

test/if_elseif_else/if2.out

@@ -0,0 +1 @@
+a

test/if_elseif_else/if3

@@ -0,0 +1,20 @@
+if(1 < 3)
+{
+    print("a");
+}
+if(2 < 3)
+{
+    print("b");
+}
+if(3 < 3)
+{
+    print("c");
+}
+if(4 < 3)
+{
+    print("d");
+}
+if(5 < 3)
+{
+    print("e");
+}

test/if_elseif_else/if3.out

@@ -0,0 +1,2 @@
+a
+b

test/if_elseif_else/if4

@@ -0,0 +1,22 @@
+a = 5;
+
+if(a > 1)
+{
+    if(a > 3)
+    {
+        print("a");
+    }
+    
+    if(a < 3)
+    {
+        print("b");
+    }
+}
+
+if(a < 3)
+{
+    if(a == 5)
+    {
+        print("c");
+    }
+}

test/if_elseif_else/if4.out

@@ -0,0 +1 @@
+a

test/if_elseif_else/if5

@@ -0,0 +1,22 @@
+a = 5;
+
+if(a > 1)
+{
+    if(a > 3)
+    {
+        print("a");
+    }
+    
+    if(a < 3)
+    {
+        print("b");
+    }
+}
+
+if(a > 3)
+{
+    if(a == 5)
+    {
+        print("c");
+    }
+}