diff --git a/src/parse.py b/src/parse.py index b541e1d..17f2f75 100644 --- a/src/parse.py +++ b/src/parse.py @@ -149,7 +149,7 @@ if not word: raise ParseError(context, "Hit end of file before EndText") else: - return self.code[start + 1 : word.position - 1] + return self.code[start + 1 : word.position - 1].strip("\n\t ") def skip_shebang(self): log.log(log.LEXER, log.TRACE, "Skipping shebang") diff --git a/tests/test_parse.py b/tests/test_parse.py index 62f5a1b..ef9eebe 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -83,7 +83,7 @@ space1 = draw(sampled_from(lexer_whitespace)) space2 = draw(sampled_from(lexer_whitespace)) code = "StartText" + space1 + value + space2 + "EndText" - return SampleToken(code, "text", value) + return SampleToken(code, "text", value.strip(lexer_whitespace)) # Generates a Bool token diff --git a/tests/test_parse_regress.py b/tests/test_parse_regress.py index 5f2a5a2..6a3fccf 100644 --- a/tests/test_parse_regress.py +++ b/tests/test_parse_regress.py @@ -29,17 +29,3 @@ assert tokens1[0].value == text assert tokens2[0].type == "text" assert tokens2[0].value == text - - -# The parser would read text literals by reading literal text after StartText -# to the end of the token just before EndText. -# This solved the previous bug, but cut off any whitespace between the last -# token and EndText. -# Make sure the parser can handle trailing whitespace properly now. -def test_regress_text_trailing_whitespace(): - text = "Hi there!\n\n\n" - code = "StartText " + text + " EndText" - tokenizer = parse.Tokenizer(code, "") - tokens = tokenizer.tokenize() - assert tokens[0].type == "text" - assert tokens[0].value == text