diff --git a/parse.py b/parse.py index 4ba359d..dbb82b0 100644 --- a/parse.py +++ b/parse.py @@ -65,26 +65,27 @@ return self.code[start:self.pos - len(" EndText")] def tokenize(self): + keywords = ["NewLang", "Done", "Set", "To", "EndSet", + "If", "Then", "Else", "EndIf"] tokens = [] token = self.read_token() while token: if token == "BeginNote": self.skip_note() + token = self.read_token() + continue elif token == "BeginText": - text = self.read_text() - log.log(log.LEXER, log.DEBUG, "Appending text '%s'" % (text)) - tokens.append(Token("text", text)) + type = "text" + value = self.read_text() + elif token in keywords: + type = "keyword" + value = token.lower() else: - keywords = ["NewLang", "Done", "Set", "To", "EndSet", - "If", "Then", "Else", "EndIf"] - if token in keywords: - type = "keyword" - token = token.lower() - else: - type = "symbol" - tok = Token(type, token) - log.log(log.LEXER, log.DEBUG, "Appending %s" % (tok)) - tokens.append(tok) + type = "symbol" + value = token + tok = Token(type, value) + log.log(log.LEXER, log.DEBUG, "Appending %s" % (tok)) + tokens.append(tok) token = self.read_token() log.log(log.LEXER, log.TRACE, "Done lexing, adding EOF") tokens.append(Token("EOF", None))