diff --git a/tests/test_parse.py b/tests/test_parse.py index 40af2dc..8023f48 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -133,6 +133,7 @@ assume(not symbol.startswith("#!")) # Shebangs aren't symbols return SampleToken(symbol, "symbol", symbol) + # Generates a note token @composite def draw_token_note(draw): @@ -145,6 +146,14 @@ return SampleToken(code, "note", None) +# Generates a shebang token +@composite +def draw_token_shebang(draw): + shebang = draw(text(alphabet=characters(blacklist_characters="\n"))) + code = "#!" + shebang + "\n" + return SampleToken(code, "shebang", None) + + # Generates a soup of tokens @composite def draw_token_soup(draw): @@ -155,12 +164,14 @@ draw_token_symbol(), draw_token_note(), ] + shebang = draw(lists(draw_token_shebang(), max_size=1)) tokens = draw(lists(one_of(strategies))) + all_tokens = shebang + tokens code = "" - for token in tokens: + for token in all_tokens: space = "".join(draw(lists(sampled_from(lexer_whitespace), min_size=1))) code += token.code + space - return SampleSoup(tokens, code) + return SampleSoup(all_tokens, code) # Test that we can lex tokens correctly @@ -170,9 +181,10 @@ EOF = len(tokens) - 1 in_pos = 0 out_pos = 0 - while out_pos < EOF: - if soup.tokens[in_pos].type == "note": - in_pos += 1 # Skip notes + ignore_types = ["note", "shebang"] + while out_pos < EOF and in_pos < len(soup.tokens): + if soup.tokens[in_pos].type in ignore_types: + in_pos += 1 else: assert tokens[out_pos].type == soup.tokens[in_pos].type assert tokens[out_pos].value == soup.tokens[in_pos].value @@ -183,17 +195,6 @@ assert tokens[out_pos].location.file == "" -# Test that shebangs are skipped -@given(text(alphabet=characters(blacklist_characters="\n"))) -def test_lexer_shebang(shebang): - code = "#!" + shebang + "\n" - tokens = safe_tokenize(code, "") - assert tokens[0].type == "EOF" - assert tokens[0].location.line == 2 - assert tokens[0].location.column == 0 - assert tokens[0].location.file == "" - - # General fuzz test, make sure the parser doesn't fall apart and spew # uncontrolled errors. @given(text(), text())