luapy (15) lua parser

版权声明：本文为博主原创文章，未经博主允许不得转载。 https://blog.csdn.net/guzhou_diaoke/article/details/86437463

1.parse exp

from lua_token import TokenKind
import lua_exp
from optimizer import Optimizer
from lua_value import LuaValue


class ExpParser:
    @staticmethod
    def parse_exp_list(lexer):
        exps = [ExpParser.parse_exp(lexer)]
        while lexer.look_ahead() == TokenKind.SEP_COMMA:
            lexer.get_next_token()
            exps.append(ExpParser.parse_exp(lexer))
        return exps

    @staticmethod
    def parse_exp(lexer):
        return ExpParser.parse_exp12(lexer)

    # x or y
    @staticmethod
    def parse_exp12(lexer):
        exp = ExpParser.parse_exp11(lexer)
        while lexer.look_ahead() == TokenKind.OP_OR:
            line, op, _ = lexer.get_next_token()
            lor = lua_exp.BinopExp(line, op, exp, ExpParser.parse_exp11(lexer))
            exp = Optimizer.optimize_logical_or(lor)
        return exp

    # x and y
    @staticmethod
    def parse_exp11(lexer):
        exp = ExpParser.parse_exp10(lexer)
        while lexer.look_ahead() == TokenKind.OP_AND:
            line, op, _ = lexer.get_next_token()
            lor = lua_exp.BinopExp(line, op, exp, ExpParser.parse_exp10(lexer))
            exp = Optimizer.optimize_logical_and(lor)
        return exp

    # compare
    @staticmethod
    def parse_exp10(lexer):
        exp = ExpParser.parse_exp9(lexer)
        while True:
            kind = lexer.look_ahead()
            if kind in (TokenKind.OP_LT, TokenKind.OP_GT, TokenKind.OP_NE,
                        TokenKind.OP_LE, TokenKind.OP_GE, TokenKind.OP_EQ):
                line, op, _ = lexer.get_next_token()
                exp = lua_exp.BinopExp(line, op, exp, ExpParser.parse_exp9(lexer))
            else:
                return exp

    # x | y
    @staticmethod
    def parse_exp9(lexer):
        exp = ExpParser.parse_exp8(lexer)
        while lexer.look_ahead() == TokenKind.OP_BOR:
            line, op, _ = lexer.get_next_token()
            bor = lua_exp.BinopExp(line, op, exp, ExpParser.parse_exp8(lexer))
            exp = Optimizer.optimize_bitwise_binary_op(bor)
        return exp

    # x ~ y
    @staticmethod
    def parse_exp8(lexer):
        exp = ExpParser.parse_exp7(lexer)
        while lexer.look_ahead() == TokenKind.OP_BXOR:
            line, op, _ = lexer.get_next_token()
            bor = lua_exp.BinopExp(line, op, exp, ExpParser.parse_exp8(lexer))
            exp = Optimizer.optimize_bitwise_binary_op(bor)
        return exp

    # x & y
    @staticmethod
    def parse_exp7(lexer):
        exp = ExpParser.parse_exp6(lexer)
        while lexer.look_ahead() == TokenKind.OP_BAND:
            line, op, _ = lexer.get_next_token()
            bor = lua_exp.BinopExp(line, op, exp, ExpParser.parse_exp8(lexer))
            exp = Optimizer.optimize_bitwise_binary_op(bor)
        return exp

    # shift
    @staticmethod
    def parse_exp6(lexer):
        exp = ExpParser.parse_exp5(lexer)
        if lexer.look_ahead() in (TokenKind.OP_SHL, TokenKind.OP_SHR):
            line, op, _ = lexer.get_next_token()
            shx = lua_exp.BinopExp(line, op, exp, ExpParser.parse_exp5(lexer))
            exp = Optimizer.optimize_bitwise_binary_op(shx)
        else:
            return exp
        return exp

    # a .. b
    @staticmethod
    def parse_exp5(lexer):
        exp = ExpParser.parse_exp4(lexer)
        if lexer.look_ahead() != TokenKind.OP_CONCAT:
            return exp

        line = 0
        exps = []
        while lexer.look_ahead() == TokenKind.OP_CONCAT:
            line, _, _ = lexer.get_next_token()
            exps.append(ExpParser.parse_exp4(lexer))
        return lua_exp.ConcatExp(line, exps)

    # x +/- y
    @staticmethod
    def parse_exp4(lexer):
        exp = ExpParser.parse_exp3(lexer)
        while True:
            if lexer.look_ahead() in (TokenKind.OP_ADD, TokenKind.OP_SUB):
                line, op, _ = lexer.get_next_token()
                arith = lua_exp.BinopExp(line, op, exp, ExpParser.parse_exp3(lexer))
                exp = Optimizer.optimize_arith_binary_op(arith)
            else:
                break
        return exp

    # *, %, /, //
    @staticmethod
    def parse_exp3(lexer):
        exp = ExpParser.parse_exp2(lexer)
        while True:
            if lexer.look_ahead() in (TokenKind.OP_MUL, TokenKind.OP_MOD, TokenKind.OP_DIV, TokenKind.OP_IDIV):
                line, op, _ = lexer.get_next_token()
                arith = lua_exp.BinopExp(line, op, exp, ExpParser.parse_exp2(lexer))
                exp = Optimizer.optimize_arith_binary_op(arith)
            else:
                break
        return exp

    # unary
    @staticmethod
    def parse_exp2(lexer):
        if lexer.look_ahead() in (TokenKind.OP_UNM, TokenKind.OP_BNOT, TokenKind.OP_LEN, TokenKind.OP_NOT):
            line, op, _ = lexer.get_next_token()
            exp = lua_exp.UnopExp(line, op, ExpParser.parse_exp2(lexer))
            return Optimizer.optimize_unary_op(exp)
        return ExpParser.parse_exp1(lexer)

    # x ^ y
    @staticmethod
    def parse_exp1(lexer):
        exp = ExpParser.parse_exp0(lexer)
        if lexer.look_ahead() == TokenKind.OP_POW:
            line, op, _ = lexer.get_next_token()
            exp = lua_exp.BinopExp(line, op, exp, ExpParser.parse_exp2(lexer))
        return Optimizer.optimize_pow(exp)

    @staticmethod
    def parse_exp0(lexer):
        kind = lexer.look_ahead()
        if kind == TokenKind.VARARG:
            line, _, _ = lexer.get_next_token()
            return lua_exp.VarArgExp(line)
        if kind == TokenKind.KW_NIL:
            line, _, _ = lexer.get_next_token()
            return lua_exp.NilExp(line)
        if kind == TokenKind.KW_TRUE:
            line, _, _ = lexer.get_next_token()
            return lua_exp.TrueExp(line)
        if kind == TokenKind.KW_FALSE:
            line, _, _ = lexer.get_next_token()
            return lua_exp.FalseExp(line)
        if kind == TokenKind.STRING:
            line, _, token = lexer.get_next_token()
            return lua_exp.StringExp(line, token)
        if kind == TokenKind.NUMBER:
            return ExpParser.parse_number_exp(lexer)
        if kind == TokenKind.SEP_LCURLY:
            return ExpParser.parse_table_constructor_exp(lexer)
        if kind == TokenKind.KW_FUNCTION:
            lexer.get_next_token()
            return ExpParser.parse_func_def_exp(lexer)
        return ExpParser.parse_prefix_exp(lexer)

    @staticmethod
    def parse_number_exp(lexer):
        line, _, token = lexer.get_next_token()
        i = LuaValue.parse_integer(token)
        if i is not None:
            return lua_exp.IntegerExp(line, i)
        f = LuaValue.parse_float(token)
        if f is not None:
            return lua_exp.FloatExp(line, f)
        raise Exception('not a number: ' + token)

    # functiondef::= function funcbody
    # funcbody::= '(' [parlist] ')' block end
    @staticmethod
    def parse_func_def_exp(lexer):
        from parser import Parser
        line = lexer.get_line()
        lexer.get_next_token_of_kind(TokenKind.SEP_LPAREN)
        par_list, is_var_arg = ExpParser.parse_par_list(lexer)
        lexer.get_next_token_of_kind(TokenKind.SEP_RPAREN)
        block = Parser.parse_block(lexer)
        last_line, _ = lexer.get_next_token_of_kind(TokenKind.KW_END)
        return lua_exp.FuncDefExp(line, last_line, par_list, is_var_arg, block)

    # [parlist]
    # parlist ::= namelist [',' '...'] | '...'
    @staticmethod
    def parse_par_list(lexer):
        kind = lexer.look_ahead()
        if kind == TokenKind.SEP_RPAREN:
            return None, False
        if kind == TokenKind.VARARG:
            return None, True

        _, name = lexer.get_next_identifier()
        names = [name]
        is_var_arg = False
        while lexer.look_ahead() == TokenKind.SEP_COMMA:
            lexer.get_next_token()
            if lexer.look_ahead() == TokenKind.IDENTIFIER:
                _, name = lexer.get_next_identifier()
                names.append(name)
            else:
                lexer.get_next_token_of_kind(TokenKind.VARARG)
                is_var_arg = True
                break
        return names, is_var_arg

    # tableconstructor ::= '{' [fieldlist] '}'
    @staticmethod
    def parse_table_constructor_exp(lexer):
        line = lexer.get_line()
        lexer.get_next_token_of_kind(TokenKind.SEP_LCURLY)
        key_exps, val_exps = ExpParser.parse_field_list(lexer)
        lexer.get_next_token_of_kind(TokenKind.SEP_RCURLY)
        last_line = lexer.get_line()
        return lua_exp.TableConstructorExp(line, last_line, key_exps, val_exps)

    # fieldlist ::= field {fieldsep field} [fieldsep]
    @staticmethod
    def parse_field_list(lexer):
        ks = []
        vs = []
        if lexer.look_ahead != TokenKind.SEP_RCURLY:
            k, v = ExpParser.parse_field(lexer)
            ks.append(k)
            vs.append(v)

            while ExpParser.is_field_sep(lexer.look_ahead()):
                lexer.get_next_token()
                if lexer.look_ahead() != TokenKind.SEP_RCURLY:
                    k, v = ExpParser.parse_field(lexer)
                    ks.append(k)
                    vs.append(v)
                else:
                    break
        return ks, vs

    # fieldsep ::= ',' | ';'
    @staticmethod
    def is_field_sep(kind):
        return kind in (TokenKind.SEP_COMMA, TokenKind.SEP_SEMI)

    # field ::= '[' exp ']' '=' exp | Name '=' exp | exp
    @staticmethod
    def parse_field(lexer):
        if lexer.look_ahead() == TokenKind.SEP_LBRACK:
            lexer.get_next_token()
            k = ExpParser.parse_exp(lexer)
            lexer.get_next_token_of_kind(TokenKind.SEP_RBRACK)
            lexer.get_next_token_of_kind(TokenKind.OP_ASSIGN)
            v = ExpParser.parse_exp(lexer)
            return k, v

        exp = ExpParser.parse_exp(lexer)
        if isinstance(exp, lua_exp.NameExp):
            if lexer.look_ahead() == TokenKind.OP_ASSIGN:
                lexer.get_next_token()
                k = lua_exp.StringExp(exp.line, exp.name)
                v = ExpParser.parse_exp(lexer)
                return k, v

        return None, exp

    # prefixexp::= var | functioncall | '(' exp ')"
    # var::= Name | prefixexp '[' exp ']' | prefixexp '.' Name
    # functioncall ::=  prefixexp
    # args | prefixexp ':' Name
    # args

    # prefixexp::= Name
    #   | '(' exp ')'
    #   | prefixexp '[' exp ']'
    #   | prefixexp '.' Name
    #   | prefixexp[':' Name] args
    @staticmethod
    def parse_prefix_exp(lexer):
        if lexer.look_ahead() == TokenKind.IDENTIFIER:
            line, name = lexer.get_next_identifier()
            exp = lua_exp.NameExp(line, name)
        else:
            exp = ExpParser.parse_parens_exp(lexer)
        return ExpParser.finish_prefix_exp(lexer, exp)

    @staticmethod
    def parse_parens_exp(lexer):
        lexer.get_next_token_of_kind(TokenKind.SEP_LPAREN)
        exp = ExpParser.parse_exp(lexer)
        lexer.get_next_token_of_kind(TokenKind.SEP_RPAREN)

        if isinstance(exp, lua_exp.VarArgExp) or isinstance(exp, lua_exp.FuncCallExp) or \
                isinstance(exp, lua_exp.NameExp) or isinstance(exp, lua_exp.TableAccessExp):
            return lua_exp.ParensExp(exp)
        return exp

    @staticmethod
    def finish_prefix_exp(lexer, exp):
        while True:
            kind = lexer.look_ahead()
            if kind == TokenKind.SEP_LBRACK:
                lexer.get_next_token()
                key_exp = ExpParser.parse_exp(lexer)
                lexer.get_next_token_of_kind(TokenKind.SEP_RBRACK)
                exp = lua_exp.TableAccessExp(lexer.get_line(), exp, key_exp)
            elif kind == TokenKind.SEP_DOT:
                lexer.get_next_token()
                line, name = lexer.get_next_identifier()
                key_exp = lua_exp.StringExp(line, name)
                exp = lua_exp.TableAccessExp(line, exp, key_exp)
            elif kind in (TokenKind.SEP_COLON, TokenKind.SEP_LPAREN, TokenKind.SEP_LCURLY, TokenKind.STRING):
                exp = ExpParser.finish_func_call_exp(lexer, exp)
            return exp

    # functioncall ::= prefixexp args | prefixexp ':' Name args
    @staticmethod
    def finish_func_call_exp(lexer, prefix_exp):
        name_exp = ExpParser.parse_name_exp(lexer)
        line = lexer.get_line()
        args = ExpParser.parse_args(lexer)
        last_line = lexer.get_line()
        return lua_exp.FuncCallExp(line, last_line, prefix_exp, name_exp, args)

    @staticmethod
    def parse_name_exp(lexer):
        if lexer.look_ahead() == TokenKind.SEP_COLON:
            lexer.get_next_token()
            line, name = lexer.get_next_identifier()
            return lua_exp.StringExp(line, name)
        return None

    # args ::= '(' [explist] ')' | tableconstructor | LitreralString
    @staticmethod
    def parse_args(lexer):
        args = []
        kind = lexer.look_ahead()
        if kind == TokenKind.SEP_LPAREN:    # '(' [explist] ')'
            lexer.get_next_token()
            if lexer.look_ahead() != TokenKind.SEP_RPAREN:
                args = ExpParser.parse_exp_list(lexer)
            lexer.get_next_token_of_kind(TokenKind.SEP_RPAREN)
        elif kind == TokenKind.SEP_LCURLY:  # '{' [fieldlist] '}'
            args = [ExpParser.parse_table_constructor_exp(lexer)]
        else:                               # LiteralString
            line, s = lexer.get_next_token_of_kind(TokenKind.STRING)
            args = [lua_exp.StringExp(line, s)]
        return args

2.parse stat

from lua_token import TokenKind
import lua_stat
import lua_exp
from exp_parser import ExpParser
from parser import Parser


"""
stat ::=  ‘;’
    | break
    | ‘::’ Name ‘::’
    | goto Name
    | do block end
    | while exp do block end
    | repeat block until exp
    | if exp then block {elseif exp then block} [else block] end
    | for Name ‘=’ exp ‘,’ exp [‘,’ exp] do block end
    | for namelist in explist do block end
    | function funcname funcbody
    | local function Name funcbody
    | local namelist [‘=’ explist]
    | varlist ‘=’ explist
    | functioncall
"""


class StatParser:
    @staticmethod
    def parse_stat(lexer):
        kind = lexer.look_ahead()
        if kind == TokenKind.SEP_SEMI:
            return StatParser.parse_empty_stat(lexer)
        if kind == TokenKind.KW_BREAK:
            return StatParser.parse_break_stat(lexer)
        if kind == TokenKind.SEP_LABEL:
            return StatParser.parse_label_stat(lexer)
        if kind == TokenKind.KW_GOTO:
            return StatParser.parse_goto_stat(lexer)
        if kind == TokenKind.KW_DO:
            return StatParser.parse_do_stat(lexer)
        if kind == TokenKind.KW_WHILE:
            return StatParser.parse_while_stat(lexer)
        if kind == TokenKind.KW_REPEAT:
            return StatParser.parse_repeat_stat(lexer)
        if kind == TokenKind.KW_IF:
            return StatParser.parse_if_stat(lexer)
        if kind == TokenKind.KW_FOR:
            return StatParser.parse_for_stat(lexer)
        if kind == TokenKind.KW_FUNCTION:
            return StatParser.parse_func_def_stat(lexer)
        if kind == TokenKind.KW_LOCAL:
            return StatParser.parse_local_assign_or_func_def_stat(lexer)
        return StatParser.parse_assign_or_func_call_stat(lexer)

    # ;
    @staticmethod
    def parse_empty_stat(lexer):
        lexer.get_next_token_of_kind(TokenKind.SEP_SEMI)
        return lua_stat.EmptyStat(lexer.get_line())

    # break
    @staticmethod
    def parse_break_stat(lexer):
        lexer.get_next_token_of_kind(TokenKind.KW_BREAK)
        return lua_stat.BreakStat(lexer.get_line())

    # '::' Name '::'
    @staticmethod
    def parse_label_stat(lexer):
        lexer.get_next_token_of_kind(TokenKind.SEP_LABEL)
        _, name = lexer.get_next_identifier()
        lexer.get_next_token_of_kind(TokenKind.SEP_LABEL)
        return lua_stat.LabelStat(name)

    # goto Name
    @staticmethod
    def parse_goto_stat(lexer):
        lexer.get_next_token_of_kind(TokenKind.KW_GOTO)
        _, name = lexer.get_next_identifier()
        return lua_stat.GotoStat(name)

    # do block end
    @staticmethod
    def parse_do_stat(lexer):
        lexer.get_next_token_of_kind(TokenKind.KW_DO)
        block = Parser.parse_block(lexer)
        lexer.get_next_token_of_kind(TokenKind.KW_END)
        return lua_stat.DoStat(block)

    # while exp do block end
    @staticmethod
    def parse_while_stat(lexer):
        lexer.get_next_token_of_kind(TokenKind.KW_WHILE)
        exp = ExpParser.parse_exp(lexer)
        lexer.get_next_token_of_kind(TokenKind.KW_DO)
        block = Parser.parse_block(lexer)
        lexer.get_next_token_of_kind(TokenKind.KW_END)
        return lua_stat.WhileStat(exp, block)

    # repeat block until exp
    @staticmethod
    def parse_repeat_stat(lexer):
        lexer.get_next_token_of_kind(TokenKind.KW_REPEAT)
        block = Parser.parse_block(lexer)
        lexer.get_next_token_of_kind(TokenKind.KW_UNTIL)
        exp = ExpParser.parse_exp(lexer)
        return lua_stat.RepeatStat(block, exp)

    # if exp then block {elseif exp then block} [else block] end
    @staticmethod
    def parse_if_stat(lexer):
        exps = []
        blocks = []

        lexer.get_next_token_of_kind(TokenKind.KW_IF)
        exps.append(ExpParser.parse_exp(lexer))
        lexer.get_next_token_of_kind(TokenKind.KW_THEN)
        blocks.append(Parser.parse_block(lexer))

        while lexer.look_ahead() == TokenKind.KW_ELSEIF:
            lexer.get_next_token()
            exps.append(ExpParser.parse_exp(lexer))
            lexer.get_next_token_of_kind(TokenKind.KW_THEN)
            blocks.append(Parser.parse_block(lexer))

        if lexer.look_ahead() == TokenKind.KW_ELSE:
            lexer.get_next_token()
            exps.append(lua_exp.TrueExp(lexer.get_line()))
            blocks.append(Parser.parse_block(lexer))

        lexer.get_next_token_of_kind(TokenKind.KW_END)
        return lua_stat.IfStat(exps, blocks)

    # for Name '=' exp ',' exp [',' exp] do block end
    # for namelist in explist do block end
    @staticmethod
    def parse_for_stat(lexer):
        line_of_for, _ = lexer.get_next_token_of_kind(TokenKind.KW_FOR)
        _, name = lexer.get_next_identifier()
        if lexer.look_ahead() == TokenKind.OP_ASSIGN:
            return StatParser.finish_for_num_stat(lexer, line_of_for, name)
        else:
            return StatParser.finish_for_in_stat(lexer, name)

    # for Name '=' exp ',' exp [',' exp] do block end
    @staticmethod
    def finish_for_num_stat(lexer, line_of_for, var_name):
        lexer.get_next_token_of_kind(TokenKind.OP_ASSIGN)
        init_exp = ExpParser.parse_exp(lexer)
        lexer.get_next_token_of_kind(TokenKind.SEP_COMMA)
        limit_exp = ExpParser.parse_exp(lexer)

        if lexer.look_ahead() == TokenKind.SEP_COMMA:
            lexer.get_next_token()
            step_exp = ExpParser.parse_exp(lexer)
        else:
            step_exp = lua_exp.IntegerExp(lexer.get_line(), 1)

        line_of_do, _ = lexer.get_next_token_of_kind(TokenKind.KW_DO)
        block = Parser.parse_block(lexer)
        lexer.get_next_token_of_kind(TokenKind.KW_END)

        return lua_stat.ForNumStat(line_of_for, line_of_do, var_name, init_exp, limit_exp, step_exp, block)

    # for Name '=' exp ',' exp [',' exp] do block end
    @staticmethod
    def finish_for_in_stat(lexer, name):
        name_list = StatParser.finish_name_list(lexer, name)
        lexer.get_next_token_of_kind(TokenKind.KW_IN)
        exp_list = ExpParser.parse_exp_list(lexer)
        line_of_do, _ = lexer.get_next_token_of_kind(TokenKind.KW_DO)
        block = Parser.parse_block(lexer)
        lexer.get_next_token_of_kind(TokenKind.KW_END)
        return lua_stat.ForInStat(line_of_do, name_list, exp_list, block)

    # namelist ::= Name {',' Name}
    @staticmethod
    def finish_name_list(lexer, name0):
        names = [name0]
        while lexer.look_ahead() == TokenKind.SEP_COMMA:
            lexer.get_next_token()
            _, name = lexer.get_next_identifier()
            names.append(name)
        return names

    #
    @staticmethod
    def parse_func_def_stat(lexer):
        lexer.get_next_token_of_kind(TokenKind.KW_FUNCTION)
        fn_exp, has_colon = StatParser.parse_func_name(lexer)
        fd_exp = ExpParser.parse_func_def_exp(lexer)
        if has_colon:
            fd_exp.par_list.insert(0, 'self')

        return lua_stat.AssignStat(fd_exp.line, fn_exp, fd_exp)

    # local function Name funcbody
    # local namelist ['=' explist]
    @staticmethod
    def parse_local_assign_or_func_def_stat(lexer):
        lexer.get_next_token_of_kind(TokenKind.KW_LOCAL)
        if lexer.look_ahead() == TokenKind.KW_FUNCTION:
            return StatParser.finish_local_func_def_stat(lexer)
        else:
            return StatParser.finish_local_var_decl_stat(lexer)

    """
    http: // www.lua.org / manual / 5.3 / manual.html  # 3.4.11

    function f() end = > f = function() end
    function t.a.b.c.f() end = > t.a.b.c.f = function() end
    function t.a.b.c: f() end = > t.a.b.c.f = function(self) end
    local function f() end = > local f; f = function() end

    The statement 'local function f() body end' 
    translates to 'local f; f = function() body end`
    not to `local f = function() body end`
    (This only makes a difference when the body of the function
    contains references to f.)
    """

    # local function Name funcbody
    @staticmethod
    def finish_local_func_def_stat(lexer):
        lexer.get_next_token_of_kind(TokenKind.KW_FUNCTION)
        _, name = lexer.get_next_identifier()
        fd_exp = ExpParser.parse_func_def_exp(lexer)
        return lua_stat.LocalFuncDefStat(name, fd_exp)

    # local namelist ['=' explist]
    @staticmethod
    def finish_local_var_decl_stat(lexer):
        _, name0 = lexer.get_next_identifier()
        name_list = StatParser.finish_name_list(lexer, name0)
        exp_list = []
        if lexer.look_ahead() == TokenKind.OP_ASSIGN:
            lexer.get_next_token()
            exp_list = ExpParser.parse_exp_list(lexer)
        last_line = lexer.get_line()
        return lua_stat.LocalVarDeclStat(last_line, name_list, exp_list)

    # varlist '=' explist
    # functioncall
    @staticmethod
    def parse_assign_or_func_call_stat(lexer):
        prefix_exp = ExpParser.parse_prefix_exp(lexer)
        if isinstance(prefix_exp, lua_exp.FuncCallExp):
            return prefix_exp
        else:
            return StatParser.parse_assign_stat(lexer, prefix_exp)

    # varlist '=' explist
    @staticmethod
    def parse_assign_stat(lexer, var0):
        var_list = StatParser.finish_var_list(lexer, var0)
        lexer.get_next_token_of_kind(TokenKind.OP_ASSIGN)
        exp_list = ExpParser.parse_exp_list(lexer)
        last_line = lexer.get_line()
        return lua_stat.AssignStat(last_line, var_list, exp_list)

    # varlist ::= var {',' var}
    @staticmethod
    def finish_var_list(lexer, var0):
        var_list = [StatParser.check_var(lexer, var0)]
        while lexer.look_ahead() == TokenKind.SEP_COMMA:
            lexer.get_next_token()
            exp = ExpParser.parse_prefix_exp(lexer)
            var_list.append(StatParser.check_var(lexer, exp))
        return var_list

    # var ::= Name | prefixexp '[' exp ']' | prefixexp '.' Name
    @staticmethod
    def check_var(lexer, exp):
        if isinstance(exp, lua_exp.NameExp) or isinstance(exp, lua_exp.TableAccessExp):
            return exp
        lexer.get_next_token_of_kind(-1)
        raise Exception('unreachable!')

    # function funcname funcbody
    # funcname ::= Name {'.' Name} [':' Name]
    # funcbody ::= '(' [parlist] ')' block end
    # parlist ::= namelist [',' '...'] | '...'
    # namelist ::= Name {',' Name}
    @staticmethod
    def parse_func_def_stat(lexer):
        lexer.get_next_token_of_kind(TokenKind.KW_FUNCTION)
        fn_exp, has_colon = StatParser.parse_func_name(lexer)
        fd_exp = ExpParser.parse_func_def_exp(lexer)
        if has_colon:
            fd_exp.insert(0, 'self')
        return lua_stat.AssignStat(fd_exp.line, [fd_exp], [fn_exp])

    # funcname ::= Name {'.' Name} [':' Name]
    @staticmethod
    def parse_func_name(lexer):
        line, name = lexer.get_next_identifier()
        exp = lua_exp.NameExp(line, name)
        has_colon = False

        while lexer.look_ahead() == TokenKind.SEP_DOT:
            lexer.get_next_token()
            line, name = lexer.get_next_identifier()
            idx = lua_exp.StringExp(line, name)
            exp = lua_exp.TableAccessExp(line, exp, idx)

        if lexer.look_ahead() == TokenKind.SEP_COLON:
            lexer.get_next_token()
            line, name = lexer.get_next_identifier()
            idx = lua_exp.StringExp(line, name)
            exp = lua_exp.TableAccessExp(line, exp, idx)
            has_colon = True

        return exp, has_colon

3.optimizer

from lua_token import TokenKind
import lua_exp
from lua_value import LuaValue


class Optimizer:
    # or
    @staticmethod
    def optimize_logical_or(exp):
        if Optimizer.is_true(exp.exp1):
            # true or x => true
            return exp.exp1
        if Optimizer.is_false(exp.exp1) and not Optimizer.is_var_arg_or_func_call(exp.exp2):
            # false or x => x
            return exp.exp2
        return exp

    # and
    @staticmethod
    def optimize_logical_and(exp):
        if Optimizer.is_false(exp.exp1):
            # false and x => false
            return exp.exp1
        if Optimizer.is_true(exp.exp1) and not Optimizer.is_var_arg_or_func_call(exp.exp2):
            # true and x => x
            return exp.exp2
        return exp

    # & | ~ << >>
    @staticmethod
    def optimize_bitwise_binary_op(exp):
        i, oki = Optimizer.cast_to_int(exp.exp1)
        j, okj = Optimizer.cast_to_int(exp.exp2)
        if oki and okj:
            if exp.op == TokenKind.OP_BAND:
                return lua_exp.IntegerExp(exp.line, i & j)
            if exp.op == TokenKind.OP_BOR:
                return lua_exp.IntegerExp(exp.line, i | j)
            if exp.op == TokenKind.OP_BXOR:
                return lua_exp.IntegerExp(exp.line, i ^ j)
            if exp.op == TokenKind.OP_SHL:
                return lua_exp.IntegerExp(exp.line, i << j)
            if exp.op == TokenKind.OP_SHR:
                return lua_exp.IntegerExp(exp.line, i >> j)
        return exp

    # + - * / // %
    @staticmethod
    def optimize_arith_binary_op(exp):
        if isinstance(exp.exp1, lua_exp.IntegerExp):
            if isinstance(exp.exp2, lua_exp.IntegerExp):
                if exp.op == TokenKind.OP_ADD:
                    return lua_exp.IntegerExp(exp.line, exp.exp1.val + exp.exp2.val)
                if exp.op == TokenKind.OP_SUB:
                    return lua_exp.IntegerExp(exp.line, exp.exp1.val - exp.exp2.val)
                if exp.op == TokenKind.OP_MUL:
                    return lua_exp.IntegerExp(exp.line, exp.exp1.val * exp.exp2.val)
                if exp.op == TokenKind.OP_IDIV:
                    if exp.exp2.val != 0:
                        return lua_exp.IntegerExp(exp.line, exp.exp1.val // exp.exp2.val)
                if exp.op == TokenKind.OP_MOD:
                    if exp.exp2.val != 0:
                        return lua_exp.IntegerExp(exp.line, exp.exp1.val % exp.exp2.val)

        f, okf = Optimizer.cast_to_float(exp.exp1)
        g, okg = Optimizer.cast_to_float(exp.exp2)
        if okf and okg:
            if exp.op == TokenKind.OP_ADD:
                return lua_exp.IntegerExp(exp.line, f + g)
            if exp.op == TokenKind.OP_SUB:
                return lua_exp.IntegerExp(exp.line, f - g)
            if exp.op == TokenKind.OP_MUL:
                return lua_exp.IntegerExp(exp.line, f * g)
            if exp.op == TokenKind.OP_DIV:
                if g != 0:
                    return lua_exp.IntegerExp(exp.line, f / g)
            if exp.op == TokenKind.OP_IDIV:
                if g != 0:
                    return lua_exp.IntegerExp(exp.line, f // g)
            if exp.op == TokenKind.OP_MOD:
                if g != 0:
                    return lua_exp.IntegerExp(exp.line, f % g)
            if exp.op == TokenKind.OP_POW:
                return lua_exp.IntegerExp(exp.line, f ** g)
        return exp

    # ^
    @staticmethod
    def optimize_pow(exp):
        if isinstance(exp, lua_exp.BinopExp):
            if exp.op == TokenKind.OP_POW:
                exp.exp2 = Optimizer.optimize_pow(exp.exp2)
            return Optimizer.optimize_arith_binary_op(exp)
        return exp

    # - not ~
    @staticmethod
    def optimize_unary_op(exp):
        if exp.op == TokenKind.OP_UNM:
            return Optimizer.optimize_unm(exp)
        if exp.op == TokenKind.OP_NOT:
            return Optimizer.optimize_not(exp)
        if exp.op == TokenKind.OP_BNOT:
            return Optimizer.optimize_bnot(exp)
        return exp

    @staticmethod
    def optimize_unm(exp):
        if isinstance(exp.exp, lua_exp.IntegerExp):
            exp.exp.val = -exp.exp.val
            return exp.exp
        if isinstance(exp.exp, lua_exp.FloatExp):
            if exp.exp.val != 0:
                exp.exp.val = -exp.exp.val
                return exp.val
        return exp

    # not
    @staticmethod
    def optimize_not(exp):
        if isinstance(exp.exp, lua_exp.NilExp) or isinstance(exp.exp, lua_exp.FalseExp):
            return lua_exp.TrueExp(exp.line)
        if isinstance(exp.exp, lua_exp.TrueExp) or isinstance(exp.exp, lua_exp.FloatExp) or \
           isinstance(exp.exp, lua_exp.StringExp):
            return lua_exp.FalseExp(exp.line)
        return exp

    # ~
    @staticmethod
    def optimize_bnot(exp):
        if isinstance(exp.exp, lua_exp.IntegerExp):
            exp.exp.val = ~exp.exp.val
            return exp.exp.val
        if isinstance(exp.exp, lua_exp.FloatExp):
            i = LuaValue.float2integer(exp.exp.val)
            if i is not None:
                return lua_exp.IntegerExp(exp.exp.line, ~i)
        return exp

    # false
    @staticmethod
    def is_false(exp):
        if isinstance(exp, lua_exp.FalseExp) or isinstance(exp, lua_exp.NilExp):
            return True
        return False

    # true
    @staticmethod
    def is_true(exp):
        if isinstance(exp, lua_exp.TrueExp) or isinstance(exp, lua_exp.IntegerExp) or \
           isinstance(exp, lua_exp.FloatExp) or isinstance(exp, lua_exp.StringExp):
            return True
        return False

    @staticmethod
    def is_var_arg_or_func_call(exp):
        if isinstance(exp, lua_exp.VarArgExp) or isinstance(exp, lua_exp.FuncCallExp):
            return True
        return False

    @staticmethod
    def cast_to_int(exp):
        if isinstance(exp, lua_exp.IntegerExp):
            return exp.val, True
        if isinstance(exp, lua_exp.FloatExp):
            i = LuaValue.float2integer(exp.val)
            return i, i is not None
        return 0, False

    @staticmethod
    def cast_to_float(exp):
        if isinstance(exp, lua_exp.IntegerExp):
            return float(exp.val), True
        if isinstance(exp, lua_exp.FloatExp):
            return exp.val, True
        return 0, False

4.parser

from lua_token import TokenKind
from exp_parser import ExpParser


class Block:
    # block ::= {stat} [retstat]
    def __init__(self, lexer):
        self.stats = Block.parse_stats(lexer)
        self.ret_exps = Block.parse_ret_exps(lexer)
        self.last_line = lexer.get_line()

    def __str__(self):
        s = '"LastLine": ' + str(self.last_line) + ',\n'
        s += '"Stats": ' + '['
        for stat in self.stats:
            s += '{\n'
            for line in str(stat).split('\n'):
                if len(line):
                    s += '  ' + line + '\n'
            s += '}'
        s += ']\n'
        s += '"RetExps": ' + '\n'
        if self.ret_exps:
            for exp in self.ret_exps:
                for l in str(exp).split('\n'):
                    if len(l):
                        s += '  ' + l + '\n'
        return s

    @staticmethod
    def parse_stats(lexer):
        from stat_parser import StatParser
        stats = []
        while not TokenKind.is_return_or_block_end(lexer.look_ahead()):
            stat = StatParser.parse_stat(lexer)
            if stat is not None:
                stats.append(stat)
        return stats

    # retstat ::= return [explist] [';']
    # explist ::= exp {',' exp}
    @staticmethod
    def parse_ret_exps(lexer):
        if lexer.look_ahead() != TokenKind.KW_RETURN:
            return None

        lexer.get_next_token()
        kind = lexer.look_ahead()
        if kind in (TokenKind.EOF, TokenKind.KW_END, TokenKind.KW_ELSE, TokenKind.KW_ELSEIF, TokenKind.KW_UNTIL):
            return []
        if kind == TokenKind.SEP_SEMI:
            lexer.get_next_token()
            return []

        exps = ExpParser.parse_exp_list(lexer)
        if lexer.look_ahead() == TokenKind.SEP_SEMI:
            lexer.get_next_token()
        return exps


class Parser:
    def __init__(self):
        pass

    @staticmethod
    def parse_block(lexer):
        return Block(lexer)

5.test parser

import sys
from parser import Parser
from lexer import Lexer


def test_parser(chunk, chunkname):
    parser = Parser()
    lexer = Lexer(chunk, chunkname)
    ast = parser.parse_block(lexer)
    print(ast)


def main(file_name):
    with open(file_name, 'r') as f:
        data = f.read()
        test_parser(data, file_name)


if __name__ == '__main__':
    if len(sys.argv) == 2:
        main(sys.argv[1])
    else:
        print('Error argument')

6. test hello world

print("Hello")
print("World")

"LastLine": 2,
"Stats": [{
  "Line": 1,
  "LastLine": 1,
  "PrefixExp": {
    "Line": 1
    "Name": "print"
  },
  "NameExp": None,
  "Args": [{
    "Line": 1
    "Str": "Hello"
  }]
}{
  "Line": 2,
  "LastLine": 2,
  "PrefixExp": {
    "Line": 2
    "Name": "print"
  },
  "NameExp": None,
  "Args": [{
    "Line": 2
    "Str": "World"
  }]
}]
"RetExps": 
  nil

7.test assign

a = 1
b = 2
c = a+b
d = a & b

"LastLine": 7,
"Stats": [{
  "LastLine": 4
  "VarList": 
    "Line": 4
    "Name": "a"
  "ExpList": 
    "Line": 4
    "Val": 1
}{
  "LastLine": 5
  "VarList": 
    "Line": 5
    "Name": "b"
  "ExpList": 
    "Line": 5
    "Val": 2
}{
  "LastLine": 6
  "VarList": 
    "Line": 6
    "Name": "c"
  "ExpList": 
    "Line": 6
    "Op": 17
    "exp1": 
      "Line": 6
      "Name": "a"
    "exp2": 
      "Line": 6
      "Name": "b"
}{
  "LastLine": 7
  "VarList": 
    "Line": 7
    "Name": "d"
  "ExpList": 
    "Line": 7
    "Op": 23
    "exp1": 
      "Line": 7
      "Name": "a"
    "exp2": 
      "Line": 7
      "Name": "b"
}]
"RetExps": 
  nil

8. test if

if a < 0 then
    a = 1
elseif a == 1 then
    a = 2
elseif a == 2 then
    a = 3
elseif a == 3 then
    a = 4
else
    error("not support")
end

"LastLine": 23,
"Stats": [{
  "Exps": 
    "Line": 13
    "Op": 28
    "exp1": 
      "Line": 13
      "Name": "a"
    "exp2": 
      "Line": 13
      "Val": 0
    
    "Line": 15
    "Op": 32
    "exp1": 
      "Line": 15
      "Name": "a"
    "exp2": 
      "Line": 15
      "Val": 1
    
    "Line": 17
    "Op": 32
    "exp1": 
      "Line": 17
      "Name": "a"
    "exp2": 
      "Line": 17
      "Val": 2
    
    "Line": 19
    "Op": 32
    "exp1": 
      "Line": 19
      "Name": "a"
    "exp2": 
      "Line": 19
      "Val": 3
    
    <lua_exp.TrueExp object at 0x7fbfd31562e8>
  "Blocks": 
    "LastLine": 14,
    "Stats": [{
      "LastLine": 14
      "VarList": 
        "Line": 14
        "Name": "a"
      "ExpList": 
        "Line": 14
        "Val": 1
    }]
    "RetExps": 
      nil
    "LastLine": 16,
    "Stats": [{
      "LastLine": 16
      "VarList": 
        "Line": 16
        "Name": "a"
      "ExpList": 
        "Line": 16
        "Val": 2
    }]
    "RetExps": 
      nil
    "LastLine": 18,
    "Stats": [{
      "LastLine": 18
      "VarList": 
        "Line": 18
        "Name": "a"
      "ExpList": 
        "Line": 18
        "Val": 3
    }]
    "RetExps": 
      nil
    "LastLine": 20,
    "Stats": [{
      "LastLine": 20
      "VarList": 
        "Line": 20
        "Name": "a"
      "ExpList": 
        "Line": 20
        "Val": 4
    }]
    "RetExps": 
      nil
    "LastLine": 22,
    "Stats": [{
      "Line": 22,
      "LastLine": 22,
      "PrefixExp": {
        "Line": 22
        "Name": "error"
      },
      "NameExp": None,
      "Args": [{
        "Line": 22
        "Str": "not support"
      }]
    }]
    "RetExps": 
      nil
}]
"RetExps": 
  nil

10. test while

while b > 0 do
    b = a - 1
end

"LastLine": 11,
"Stats": [{
  "While": 
    "Exp": 
      "Line": 9
      "Op": 28
      "exp1": 
        "Line": 9
        "Name": "b"
      "exp2": 
        "Line": 9
        "Val": 0
    "Block": 
      "LastLine": 10,
      "Stats": [{
        "LastLine": 10
        "VarList": 
          "Line": 10
          "Name": "b"
        "ExpList": 
          "Line": 10
          "Op": 15
          "exp1": 
            "Line": 10
            "Name": "a"
          "exp2": 
            "Line": 10
            "Val": 1
      }]
      "RetExps": 
        nil
}]
"RetExps": 
  nil

11. test repeat

repeat
    a = a + 1
until a >= 10

"LastLine": 27,
"Stats": [{
  "Repeat": 
    "Block": 
      "LastLine": 26,
      "Stats": [{
        "LastLine": 26
        "VarList": 
          "Line": 26
          "Name": "a"
        "ExpList": 
          "Line": 26
          "Op": 17
          "exp1": 
            "Line": 26
            "Name": "a"
          "exp2": 
            "Line": 26
            "Val": 1
      }]
      "RetExps": 
        nil
    "Exp": 
      "Line": 27
      "Op": 31
      "exp1": 
        "Line": 27
        "Name": "a"
      "exp2": 
        "Line": 27
        "Val": 10
}]
"RetExps": 
  nil

13. test for

for i = 1, 10, 3 do
    sum = sum + i
end

"LastLine": 31,
"Stats": [{
  Line of for: 29
  Line of do: 29
  Var name: i
  Init exp: 
    "Line": 29
    "Val": 1
    
  Limit exp: 
    "Line": 29
    "Val": 10
    
  Step exp: 
    "Line": 29
    "Val": 3
    
  Block: 
    "LastLine": 30,
    "Stats": [{
      "LastLine": 30
      "VarList": 
        "Line": 30
        "Name": "sum"
      "ExpList": 
        "Line": 30
        "Op": 17
        "exp1": 
          "Line": 30
          "Name": "sum"
        "exp2": 
          "Line": 30
          "Name": "i"
    }]
    "RetExps": 
      nil
}]
"RetExps": 
  nil

for a in table do
    sum = sum + a
end

"LastLine": 35,
"Stats": [{
  Line of do: 33
    a  "Line": 33
    "Name": "table"
    "LastLine": 34,
    "Stats": [{
      "LastLine": 34
      "VarList": 
        "Line": 34
        "Name": "sum"
      "ExpList": 
        "Line": 34
        "Op": 17
        "exp1": 
          "Line": 34
          "Name": "sum"
        "exp2": 
          "Line": 34
          "Name": "a"
    }]
    "RetExps": 
      nil
}]
"RetExps": 
  nil

15. test function

function isok(a, n, c)
    return a + n == c
end

isok(1, 2, 3)

"LastLine": 42,
"Stats": [{
  "LastLine": 38
  "VarList": 
    Line: 38
    LastLine: 40
    ParList: 
      a
      n
      c
    IsVarArg: False
    Block:   "LastLine": 39,
      "Stats": []
      "RetExps": 
        "Line": 39
        "Op": 32
        "exp1": 
          "Line": 39
          "Op": 17
          "exp1": 
            "Line": 39
            "Name": "a"
          "exp2": 
            "Line": 39
            "Name": "n"
        "exp2": 
          "Line": 39
          "Name": "c"
  "ExpList": 
    "Line": 38
    "Name": "isok"
}{
  "Line": 42,
  "LastLine": 42,
  "PrefixExp": {
    "Line": 42
    "Name": "isok"
  },
  "NameExp": None,
  "Args": [{
    "Line": 42
    "Val": 1
  }{
    "Line": 42
    "Val": 2
  }{
    "Line": 42
    "Val": 3
  }]
}]
"RetExps": 
  nil

16.test optimizer

a = true and false and true and false
b = true or false or false

a = 3 & 2
b = 3 | 4
c = 3 ^ 4
d = 8 >> 2
e = 8 << 2

a = 3 + 4
b = 5 - 3
c = 3 * 4
d = 10 / 3
e = 10 // 3
f = 10 % 3

a = 3 ^ (1+2)

a = -(-(-(-(-4))))

a = ~4

a = not true

"LastLine": 89,
"Stats": [{
  "LastLine": 67
  "VarList": 
    "Line": 67
    "Name": "a"
  "ExpList": 
    <lua_exp.FalseExp object at 0x7f66e29b2d30>
}{
  "LastLine": 68
  "VarList": 
    "Line": 68
    "Name": "b"
  "ExpList": 
    <lua_exp.TrueExp object at 0x7f66e29b2cf8>
}{
  "LastLine": 70
  "VarList": 
    "Line": 70
    "Name": "a"
  "ExpList": 
    "Line": 70
    "Val": 2
}{
  "LastLine": 71
  "VarList": 
    "Line": 71
    "Name": "b"
  "ExpList": 
    "Line": 71
    "Val": 7
}{
  "LastLine": 72
  "VarList": 
    "Line": 72
    "Name": "c"
  "ExpList": 
    "Line": 72
    "Val": 81.0
}{
  "LastLine": 73
  "VarList": 
    "Line": 73
    "Name": "d"
  "ExpList": 
    "Line": 73
    "Val": 2
}{
  "LastLine": 74
  "VarList": 
    "Line": 74
    "Name": "e"
  "ExpList": 
    "Line": 74
    "Val": 32
}{
  "LastLine": 76
  "VarList": 
    "Line": 76
    "Name": "a"
  "ExpList": 
    "Line": 76
    "Val": 7
}{
  "LastLine": 77
  "VarList": 
    "Line": 77
    "Name": "b"
  "ExpList": 
    "Line": 77
    "Val": 2
}{
  "LastLine": 78
  "VarList": 
    "Line": 78
    "Name": "c"
  "ExpList": 
    "Line": 78
    "Val": 12
}{
  "LastLine": 79
  "VarList": 
    "Line": 79
    "Name": "d"
  "ExpList": 
    "Line": 79
    "Val": 3.3333333333333335
}{
  "LastLine": 80
  "VarList": 
    "Line": 80
    "Name": "e"
  "ExpList": 
    "Line": 80
    "Val": 3
}{
  "LastLine": 81
  "VarList": 
    "Line": 81
    "Name": "f"
  "ExpList": 
    "Line": 81
    "Val": 1
}{
  "LastLine": 83
  "VarList": 
    "Line": 83
    "Name": "a"
  "ExpList": 
    "Line": 83
    "Val": 27.0
}{
  "LastLine": 85
  "VarList": 
    "Line": 85
    "Name": "a"
  "ExpList": 
    "Line": 85
    "Val": -4
}{
  "LastLine": 87
  "VarList": 
    "Line": 87
    "Name": "a"
  "ExpList": 
    -5
}{
  "LastLine": 89
  "VarList": 
    "Line": 89
    "Name": "a"
  "ExpList": 
    <lua_exp.FalseExp object at 0x7f66e29417b8>
}]
"RetExps": 
  nil

17.test local

local a, b, c = 1, 3, 4

local function abc()
end

a = {1, 2, 3}

"LastLine": 96,
"Stats": [{
  <lua_stat.LocalVarDeclStat object at 0x7f4b91e7bd30>
}{
  <lua_stat.LocalFuncDefStat object at 0x7f4b91e7be10>
}{
  "LastLine": 96
  "VarList": 
    "Line": 96
    "Name": "a"
  "ExpList": 
    <lua_exp.TableConstructorExp object at 0x7f4b91e7bef0>
}]
"RetExps": 
  nil

18.test var param

function test(a, b, ...)
end

"LastLine": 99,
"Stats": [{
  "LastLine": 98
  "VarList": 
    Line: 98
    LastLine: 99
    ParList: 
      a
      b
    IsVarArg: True
    Block:   "LastLine": 98,
      "Stats": []
      "RetExps": 
        nil
  "ExpList": 
    "Line": 98
    "Name": "test"
}]
"RetExps": 
  nil