diff --git a/tests/test_token.py b/tests/test_token.py index 308e199..dd3d3c0 100644 --- a/tests/test_token.py +++ b/tests/test_token.py @@ -4,7 +4,6 @@ from hypothesis import given, assume from hypothesis.strategies import ( booleans, - characters, composite, integers, lists, @@ -67,62 +66,16 @@ return Token(value, location) -# Values considered spaces -valid_spaces = [ - "\t", # U+0009 HORIZONTAL TAB - " ", # U+0020 SPACE -] - -# Single values reserved for new line use -single_newlines = [ - "\n", # U+000A LINE FEED - "\v", # U+000B VERTICAL TAB - "\f", # U+000C FORM FEED - "\r", # U+000D CARRIAGE RETURN - "\u0085", # U+0085 NEXT LINE - "\u2028", # U+2028 LINE SEPARATOR - "\u2029", # U+2029 PARAGRAPH SEPARATOR -] - -# Multi values reserved for new line use -multi_newlines = [ - "\r\n", # U+000A U+000D CARRIAGE RETURN then LINE FEED -] - -# All values reserved for new line use -valid_newlines = single_newlines + multi_newlines - - # Draws an unknown token @composite def draw_token_unknown(draw): - reserved = valid_spaces + single_newlines location = draw(draw_token_location()) - chars = characters(blacklist_characters=reserved) - value = draw(text(alphabet=chars, min_size=1)) - for v in multi_newlines: - assume(v not in value) + value = draw(text(min_size=1)) assume(value not in literals) assume(value not in keywords) return Token(value, location) -# Draws a space token -@composite -def draw_token_space(draw): - location = draw(draw_token_location()) - value = draw(sampled_from(valid_spaces)) - return Token(value, location) - - -# Draws a new line token -@composite -def draw_token_newline(draw): - location = draw(draw_token_location()) - value = draw(sampled_from(valid_newlines)) - return Token(value, location) - - # Draws a bool token @composite def draw_token_bool(draw): @@ -145,8 +98,6 @@ # All strategies used to generate tokens all_strategies = [ draw_token_unknown(), - draw_token_space(), - draw_token_newline(), draw_token_bool(), draw_token_keyword(), ] diff --git a/tests/test_tokenize.py b/tests/test_tokenize.py index b074c75..fb9db39 100644 --- a/tests/test_tokenize.py +++ b/tests/test_tokenize.py @@ -1,26 +1,75 @@ # SPDX-License-Identifier: LGPL-2.1-only # Copyright 2022 Jookia -from hypothesis import given +from hypothesis import assume, given from hypothesis.strategies import ( booleans, + characters, composite, just, lists, one_of, + sampled_from, text, ) from src import tokenize from src.token import Token, TokenLocation -from tests.test_token import ( - draw_token_random, - draw_token_newline, - draw_token_space, - draw_token_unknown, - valid_spaces, - valid_newlines, -) +from tests.test_token import draw_token_location, draw_token_random + + +# Values considered spaces +valid_spaces = [ + "\t", # U+0009 HORIZONTAL TAB + " ", # U+0020 SPACE +] + +# Single values reserved for new line use +single_newlines = [ + "\n", # U+000A LINE FEED + "\v", # U+000B VERTICAL TAB + "\f", # U+000C FORM FEED + "\r", # U+000D CARRIAGE RETURN + "\u0085", # U+0085 NEXT LINE + "\u2028", # U+2028 LINE SEPARATOR + "\u2029", # U+2029 PARAGRAPH SEPARATOR +] + +# Multi values reserved for new line use +multi_newlines = [ + "\r\n", # U+000A U+000D CARRIAGE RETURN then LINE FEED +] + +# All values reserved for new line use +valid_newlines = single_newlines + multi_newlines + + +# Draws a space token +@composite +def draw_token_space(draw): + location = draw(draw_token_location()) + value = draw(sampled_from(valid_spaces)) + return Token(value, location) + + +# Draws a new line token +@composite +def draw_token_newline(draw): + location = draw(draw_token_location()) + value = draw(sampled_from(valid_newlines)) + return Token(value, location) + + +# Draws an random token without whitespace +@composite +def draw_token_nospace(draw): + reserved = valid_spaces + single_newlines + location = draw(draw_token_location()) + chars = characters(blacklist_characters=reserved) + value = draw(text(alphabet=chars, min_size=1)) + for v in multi_newlines: + assume(v not in value) + return Token(value, location) # Draws a token using an existing strategy but with a blank location just like split_tokens outputs @@ -66,7 +115,7 @@ spaces = draw(lists(locationed, min_size=1)) tokens += merge_crlf(spaces) else: - strategy = draw_token_unknown + strategy = draw_token_nospace locationed = draw_token_splitted(strategy()) tokens.append(draw(locationed)) drawing_whitespace = not drawing_whitespace