diff --git a/tests/test_parse_regress.py b/tests/test_parse_regress.py index 65ca9d9..b6b75d8 100644 --- a/tests/test_parse_regress.py +++ b/tests/test_parse_regress.py @@ -29,3 +29,17 @@ assert tokens1[0].value == text assert tokens2[0].type == "text" assert tokens2[0].value == text + + +# The parser would read text literals by reading literal text after BeginText +# to the end of the token just before EndText. +# This solved the previous bug, but cut off any whitespace between the last +# token and EndText. +# Make sure the parser can handle trailing whitespace properly now. +def test_regress_text_trailing_whitespace(): + text = "Hi there!\n\n\n" + code = "BeginText " + text + " EndText" + tokenizer = parse.Tokenizer(code, "") + tokens = tokenizer.tokenize() + assert tokens[0].type == "text" + assert tokens[0].value == text