diff --git a/src/parse.py b/src/parse.py index e3fe56b..1b59c54 100644 --- a/src/parse.py +++ b/src/parse.py @@ -56,6 +56,20 @@ ) +class Word: + def __init__(self, value, line, column): + self.value = value + self.line = line + self.column = column + + def __repr__(self): + return "Word(Value '%s', line %i, column %i)" % ( + self.value, + self.line, + self.column, + ) + + def is_whitespace(symbol): return symbol == " " or symbol == "\t" or symbol == "\n" @@ -91,36 +105,33 @@ ) return symbol - def read_token(self): - token = "" + def read_word(self): + value = "" symbol = self.next() while is_whitespace(symbol): log.log(log.LEXER, log.TRACE, "Skipping whitespace") symbol = self.next() if not symbol: - log.log(log.LEXER, log.TRACE, "No token to read") - return (None, self.pos_line, self.pos_column) + log.log(log.LEXER, log.TRACE, "No word to read") + return None line = self.pos_line column = self.pos_column while symbol and not is_whitespace(symbol): - token += symbol + value += symbol symbol = self.next() - log.log( - log.LEXER, - log.TRACE, - "Read token '%s' at line %i column %i" % (token, line, column), - ) - return (token, line, column) + word = Word(value, line, column) + log.log(log.LEXER, log.TRACE, "Read %s" % (word)) + return word def skip_note(self, line, column): - log.log(log.LEXER, log.TRACE, "Skipping tokens until EndNote") + log.log(log.LEXER, log.TRACE, "Skipping words until EndNote") context = ParseContext( None, "reading note", ParseLocation(line, column, self.filename) ) - (token, _, _) = self.read_token() - while token and token != "EndNote": - (token, _, _) = self.read_token() - if not token: + word = self.read_word() + while word and word.value != "EndNote": + word = self.read_word() + if not word: raise ParseError(context, "Hit end of file before EndNote") def read_text(self, line, column): @@ -130,11 +141,11 @@ ) start = self.pos end_start = self.pos - (token, _, _) = self.read_token() - while token and token != "EndText": + word = self.read_word() + while word and word.value != "EndText": end_start = self.pos - (token, _, _) = self.read_token() - if not token: + word = self.read_word() + if not word: raise ParseError(context, "Hit end of file before EndText") else: return self.code[start + 1 : end_start] @@ -160,14 +171,17 @@ "EndIf", ] tokens = [] - (token, line, column) = self.read_token() - while token: + word = self.read_word() + while word: + token = word.value + line = word.line + column = word.column context = ParseContext( - None, "reading token", ParseLocation(line, column, self.filename) + None, "reading word", ParseLocation(line, column, self.filename) ) if token == "BeginNote": self.skip_note(line, column) - (token, line, column) = self.read_token() + word = self.read_word() continue elif token == "EndNote": raise ParseError(context, "Found stray EndNote") @@ -188,7 +202,7 @@ tok = Token(type, value, ParseLocation(line, column, self.filename)) log.log(log.LEXER, log.DEBUG, "Appending %s" % (tok)) tokens.append(tok) - (token, line, column) = self.read_token() + word = self.read_word() log.log(log.LEXER, log.TRACE, "Done tokenizing, adding EOF") tokens.append( Token(