diff --git a/src/tokenize.py b/src/tokenize.py index 663dc07..6f1073a 100644 --- a/src/tokenize.py +++ b/src/tokenize.py @@ -9,6 +9,16 @@ return symbol == " " or symbol == "\t" +# Checks whether a symbol is a new line +def is_newline(symbol): + return symbol == "\n" + + +# Checks whether a symbol is general whitespace +def is_whitespace(symbol): + return is_space(symbol) or is_newline(symbol) + + # Location of a token class TokenLocation: def __init__(self, line, column, file): @@ -60,17 +70,17 @@ ) -# Splits text in to a list of characters and space +# Splits text in to a list of characters and whitespace def split_tokens(input): if input == "": return [] tokens = [] current = input[0] - curr_whitespace = is_space(input[0]) or input[0] == "\n" + curr_whitespace = is_whitespace(input[0]) location = TokenLocation(1, 1, "") type = TokenType.UNKNOWN for c in input[1:]: - c_whitespace = is_space(c) or c == "\n" + c_whitespace = is_whitespace(c) if c_whitespace != curr_whitespace: # Flush current buffer and switch modes tokens.append(Token(current, location, type)) @@ -91,7 +101,7 @@ def classify_tokens(tokens): new_tokens = [] for t in tokens: - if t.value == "\n": + if is_newline(t.value): type = TokenType.NEWLINE elif is_space(t.value): type = TokenType.SPACE