diff --git a/tests/test_token.py b/tests/test_token.py index f66695c..308e199 100644 --- a/tests/test_token.py +++ b/tests/test_token.py @@ -142,16 +142,33 @@ return Token(value, location) +# All strategies used to generate tokens +all_strategies = [ + draw_token_unknown(), + draw_token_space(), + draw_token_newline(), + draw_token_bool(), + draw_token_keyword(), +] + + +# Draws a token and possibly add garbage +# This is to ensure that tokens must completely match a value +@composite +def draw_token_garbled(draw): + token = draw(one_of(all_strategies)) + value = token.value + if draw(booleans()): + value = draw(text(min_size=1)) + value + if draw(booleans()): + value = value + draw(text(min_size=1)) + return Token(value, token.location) + + # Draws a random token @composite def draw_token_random(draw): - strategies = [ - draw_token_unknown(), - draw_token_space(), - draw_token_newline(), - draw_token_bool(), - draw_token_keyword(), - ] + strategies = all_strategies + [draw_token_garbled()] token = draw(one_of(strategies)) return token diff --git a/tests/test_tokenize.py b/tests/test_tokenize.py index 528af7e..b074c75 100644 --- a/tests/test_tokenize.py +++ b/tests/test_tokenize.py @@ -176,19 +176,6 @@ assert tokenize.strip_whitespace(input) == tokens -# Draws a token and possibly add garbage -# This is to ensure that tokens must completely match a value -@composite -def draw_token_random_garbled(draw): - token = draw(draw_token_random()) - value = token.value - if draw(booleans()): - value = draw(text(min_size=1)) + value - if draw(booleans()): - value = value + draw(text(min_size=1)) - return Token(value, token.location) - - # Draw a random string made of token values @composite def draw_source_fuzz(draw):