diff --git a/src/tokenize.py b/src/tokenize.py index 6435be6..f36dd6d 100644 --- a/src/tokenize.py +++ b/src/tokenize.py @@ -1,11 +1,12 @@ # SPDX-License-Identifier: LGPL-2.1-only # Copyright 2022 Jookia - +# Checks whether a symbol is whitespace def is_whitespace(symbol): return symbol == " " or symbol == "\t" or symbol == "\n" +# Splits text in to a list of characters and whitespace def split_symbols(input): if input == "": return []