diff --git a/tests/test_parse.py b/tests/test_parse.py index 62d32cb..40af2dc 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -133,6 +133,17 @@ assume(not symbol.startswith("#!")) # Shebangs aren't symbols return SampleToken(symbol, "symbol", symbol) +# Generates a note token +@composite +def draw_token_note(draw): + value = draw(text()) + note_tokens = split_by(value, lexer_whitespace) + assume("StartNote" not in note_tokens and "EndNote" not in note_tokens) + space1 = draw(sampled_from(lexer_whitespace)) + space2 = draw(sampled_from(lexer_whitespace)) + code = "StartNote" + space1 + value + space2 + "EndNote" + return SampleToken(code, "note", None) + # Generates a soup of tokens @composite @@ -142,6 +153,7 @@ draw_token_bool(), draw_token_keyword(), draw_token_symbol(), + draw_token_note(), ] tokens = draw(lists(one_of(strategies))) code = "" @@ -156,24 +168,19 @@ def test_lexer_soup(soup): tokens = safe_tokenize(soup.code, "") EOF = len(tokens) - 1 - pos = 0 - while pos < EOF: - assert tokens[pos].type == soup.tokens[pos].type - assert tokens[pos].value == soup.tokens[pos].value - assert tokens[pos].location.file == "" - pos += 1 - assert tokens[pos].type == "EOF" - assert tokens[pos].location.file == "" - - -# Test that we can make notes using StartNote and EndNote syntax. -@given(text(), sampled_from(lexer_whitespace), sampled_from(lexer_whitespace)) -def test_lexer_note(text, space1, space2): - note_tokens = split_by(text, lexer_whitespace) - assume("StartNote" not in note_tokens and "EndNote" not in note_tokens) - code = "StartNote" + space1 + text + space2 + "EndNote" - tokens = safe_tokenize(code, "") - assert tokens[0].type == "EOF" + in_pos = 0 + out_pos = 0 + while out_pos < EOF: + if soup.tokens[in_pos].type == "note": + in_pos += 1 # Skip notes + else: + assert tokens[out_pos].type == soup.tokens[in_pos].type + assert tokens[out_pos].value == soup.tokens[in_pos].value + assert tokens[out_pos].location.file == "" + in_pos += 1 + out_pos += 1 + assert tokens[out_pos].type == "EOF" + assert tokens[out_pos].location.file == "" # Test that shebangs are skipped