diff --git a/src/parse.py b/src/parse.py index 3ab3b36..93a8ebb 100644 --- a/src/parse.py +++ b/src/parse.py @@ -100,7 +100,7 @@ return (None, self.pos_line, self.pos_column) line = self.pos_line column = self.pos_column - 1 # pos_column is one ahead - while not is_whitespace(symbol): + while symbol and not is_whitespace(symbol): token += symbol symbol = self.next() log.log( diff --git a/tests/test_parse_regress.py b/tests/test_parse_regress.py new file mode 100644 index 0000000..1ed111d --- /dev/null +++ b/tests/test_parse_regress.py @@ -0,0 +1,13 @@ +from hypothesis import given +from hypothesis.strategies import binary + +from src import parse + + +# The parser had some logic along the lines of 'read token until whitespace', +# but this didn't account for hitting the end of file. +# Make sure the parser can handle tokens terminated by end of file correctly. +def test_regress_eof(): + tokenizer = parse.Tokenizer("Hello", "") + tokens = tokenizer.tokenize() + assert tokens[0].value == "Hello"