from hypothesis import given
from hypothesis.strategies import binary

from src import parse


# The parser had some logic along the lines of 'read token until whitespace',
# but this didn't account for hitting the end of file.
# Make sure the parser can handle tokens terminated by end of file correctly.
def test_regress_eof():
    tokenizer = parse.Tokenizer("Hello", "")
    tokens = tokenizer.tokenize()
    assert tokens[0].value == "Hello"