diff --git a/tests/test_parse.py b/tests/test_parse.py index d044103..95f5330 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -122,6 +122,29 @@ return SampleToken(symbol, "symbol", symbol) +# Generates a reserved word +@composite +def draw_token_reserved(draw): + word = draw(sampled_from(reserved_words)) + return SampleToken(word, "reserved", word) + + +# Generates a token made of conjoined tokens +@composite +def draw_token_conjoined(draw): + strategies = [ + draw_token_bool(), + draw_token_reserved(), + draw_token_symbol(), + ] + symbol = "" + tokens = draw(lists(one_of(strategies), min_size=2)) + for token in tokens: + symbol += token.code + assume(symbol not in reserved_words) # Reserved words aren't symbols + return SampleToken(symbol, "symbol", symbol) + + # Generates any token @composite def draw_token(draw): @@ -130,6 +153,7 @@ draw_token_bool(), draw_token_keyword(), draw_token_symbol(), + draw_token_conjoined(), ] return draw(one_of(strategies)) @@ -156,41 +180,6 @@ assert tokens[0].type == "EOF" -# Generates a reserved word -@composite -def draw_token_reserved(draw): - word = draw(sampled_from(reserved_words)) - return SampleToken(word, "reserved", word) - - -# Generates a token made of conjoined tokens -@composite -def draw_token_conjoined(draw): - strategies = [ - draw_token_bool(), - draw_token_reserved(), - draw_token_symbol(), - ] - symbol = "" - tokens = draw(lists(one_of(strategies), min_size=2)) - for token in tokens: - symbol += token.code - assume(symbol not in reserved_words) # Reserved words aren't symbols - return SampleToken(symbol, "symbol", symbol) - - -# Test that reserved words aren't read without whitespace -@given(draw_token_conjoined()) -def test_lexer_conjoined_token(token): - tokens = safe_tokenize(token.code, "") - assert tokens[0].type == "symbol" - assert tokens[0].value == token.value - assert tokens[0].location.line == 1 - assert tokens[0].location.column == 1 - assert tokens[0].location.file == "" - assert tokens[1].type == "EOF" - - # Test that shebangs are skipped @given(text(alphabet=characters(blacklist_characters="\n"))) def test_lexer_shebang(shebang):