diff --git a/parse.py b/parse.py index e47488c..262ba99 100644 --- a/parse.py +++ b/parse.py @@ -3,6 +3,14 @@ import log +class Token: + def __init__(self, type, value): + self.type = type + self.value = value + + def __str__(self): + return "Token(type %s, value '%s')" % (self.type, self.value) + def tokenize(code): tokens = [] token = "" @@ -31,7 +39,7 @@ log.log(log.LEXER, log.TRACE, "Ending text mode") content = text[1:-8] log.log(log.LEXER, log.DEBUG, "Appending text '%s'" % (content)) - tokens.append(("text", content)) + tokens.append(Token("text", content)) mode = "normal" text = "" elif token != "": @@ -43,8 +51,9 @@ token = token.lower() else: type = "symbol" - log.log(log.LEXER, log.DEBUG, "Appending token type %s value %s" % (type, token)) - tokens.append((type, token)) + tok = Token(type, token) + log.log(log.LEXER, log.DEBUG, "Appending token %s" % (tok)) + tokens.append(tok) else: log.log(log.LEXER, log.TRACE, "Skipping token '%s'" % (token)) token = "" @@ -53,7 +62,7 @@ if mode == "text": text += symbol log.log(log.LEXER, log.TRACE, "Done lexing, added EOF") - tokens.append(('EOF', None)) + tokens.append(Token("EOF", None)) return tokens class Parser: @@ -62,25 +71,25 @@ self.pos = 0 def next(self): - (type, value) = self.tokens[self.pos] + token = self.tokens[self.pos] if self.pos < (len(self.tokens) - 1): self.pos += 1 - log.log(log.PARSER, log.TRACE, "Read %s %s" % (type, value)) - return (type, value) + log.log(log.PARSER, log.TRACE, "Read %s" % (token)) + return token def peek(self): - (type, value) = self.tokens[self.pos] - log.log(log.PARSER, log.TRACE, "Peeked %s %s" % (type, value)) - return (type, value) + token = self.tokens[self.pos] + log.log(log.PARSER, log.TRACE, "Peeked %s" % (token)) + return token def parse_version(self): - (type, value) = self.next() - if type != "keyword" or value != "newlang": + token = self.next() + if token.type != "keyword" or token.value != "newlang": log.log(log.PARSER, log.NORMAL, "Expected NewLang keyword") return None - (type, value) = self.next() - log.log(log.PARSER, log.DEBUG, "Parsed language version %s" % (value)) - return value + token = self.next() + log.log(log.PARSER, log.DEBUG, "Parsed language version %s" % (token.value)) + return token.value def parse_value(self, type, value): if type == "symbol": @@ -94,37 +103,37 @@ def parse_arguments(self, terminator): args = [] while True: - (type, value) = self.next() - if type == "keyword": - if value == terminator: + token = self.next() + if token.type == "keyword": + if token.value == terminator: return args else: - log.log(log.PARSER, log.NORMAL, "Unexpected keyword %s" % (value)) + log.log(log.PARSER, log.NORMAL, "Unexpected keyword %s" % (token.value)) return None else: - arg = self.parse_value(type, value) + arg = self.parse_value(token.type, token.value) if not arg: log.log(log.PARSER, log.NORMAL, "While parsing argument") return None args.append(arg) def parse_statement(self, terminator): - (type, value) = self.next() - subject = self.parse_value(type, value) + token = self.next() + subject = self.parse_value(token.type, token.value) if not subject: log.log(log.PARSER, log.NORMAL, "While parsing subject") return None - (type, value) = self.next() - if type == "keyword": - if value == terminator: + token = self.next() + if token.type == "keyword": + if token.value == terminator: verb = None else: - log.log(log.PARSER, log.NORMAL, "Unexpected keyword %s" % (value)) + log.log(log.PARSER, log.NORMAL, "Unexpected keyword %s" % (token.value)) return None - elif type == "symbol": - verb = value + elif token.type == "symbol": + verb = token.value else: - verb = value + verb = token.value if verb: arguments = self.parse_arguments(terminator) if arguments is None: @@ -136,14 +145,14 @@ return ('statement', subject, verb, arguments) def parse_set(self): - (type, value) = self.next() - if type != "symbol": - log.log(log.PARSER, log.NORMAL, "Expect symbol, got %s" % (type)) + token = self.next() + if token.type != "symbol": + log.log(log.PARSER, log.NORMAL, "Expect symbol, got %s" % (token.type)) return None - subject = value - (type, value) = self.next() - if type != "keyword" or value != "to": - log.log(log.PARSER, log.NORMAL, "Expected To, got %s %s" % (type, value)) + subject = token.value + token = self.next() + if token.type != "keyword" or token.value != "to": + log.log(log.PARSER, log.NORMAL, "Expected To, got %s %s" % (token.type, token.value)) return None log.log(log.PARSER, log.TRACE, "Parsing set value...") ast = self.parse_statement("endset") @@ -173,26 +182,26 @@ return ('if', test, success, failure) def parse_directive(self): - (type, value) = self.peek() - if type != "keyword" and type != "symbol": - log.log(log.PARSER, log.NORMAL, "Expected keyword or symbol here, got %s" % (type)) + token = self.peek() + if token.type != "keyword" and token.type != "symbol": + log.log(log.PARSER, log.NORMAL, "Expected keyword or symbol here, got %s" % (token.type)) return None - if type == "keyword": + if token.type == "keyword": self.next() - if value == "set": + if token.value == "set": ast = self.parse_set() if not ast: log.log(log.PARSER, log.NORMAL, "While parsing set directive") return None return ast - elif value == "if": + elif token.value == "if": ast = self.parse_if() if not ast: log.log(log.PARSER, log.NORMAL, "While parsing set directive") return None return ast else: - log.log(log.PARSER, log.NORMAL, "Unexpected keyword %s" % (value)) + log.log(log.PARSER, log.NORMAL, "Unexpected keyword %s" % (token.value)) return None else: ast = self.parse_statement("done") @@ -211,7 +220,7 @@ if version != "0": log.log(log.PARSER, log.NORMAL, "Invalid version identifier %s" % (version)) return None - while self.peek()[0] != 'EOF': + while self.peek().type != "EOF": directive = self.parse_directive() if directive == None: log.log(log.PARSER, log.NORMAL, "While parsing directive in file")