diff --git a/src/tokenize.py b/src/tokenize.py index 6f1073a..d4d60be 100644 --- a/src/tokenize.py +++ b/src/tokenize.py @@ -46,6 +46,7 @@ UNKNOWN = enum.auto() # pragma: no mutate SPACE = enum.auto() # pragma: no mutate NEWLINE = enum.auto() # pragma: no mutate + BOOL = enum.auto() # pragma: no mutate # Represents a tokenizer token @@ -105,6 +106,8 @@ type = TokenType.NEWLINE elif is_space(t.value): type = TokenType.SPACE + elif t.value in ["True", "False"]: + type = TokenType.BOOL else: type = TokenType.UNKNOWN new = Token(t.value, t.location, type) diff --git a/tests/test_tokenize.py b/tests/test_tokenize.py index 277a4bf..70c02ef 100644 --- a/tests/test_tokenize.py +++ b/tests/test_tokenize.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: LGPL-2.1-only # Copyright 2022 Jookia -from hypothesis import given +from hypothesis import given, assume from hypothesis.strategies import ( booleans, characters, @@ -48,6 +48,7 @@ token = draw(draw_token_random()) chars = characters(blacklist_characters=reserved) value = draw(text(alphabet=chars, min_size=1)) + assume(value not in ["True", "False"]) type = tokenize.TokenType.UNKNOWN return tokenize.Token(value, token.location, type) @@ -71,6 +72,18 @@ return tokenize.Token(value, token.location, type) +# Draws a bool token +@composite +def draw_token_bool(draw): + token = draw(draw_token_random()) + if draw(booleans()): + value = "True" + else: + value = "False" + type = tokenize.TokenType.BOOL + return tokenize.Token(value, token.location, type) + + # Draws a classified token @composite def draw_token_classified(draw): @@ -78,6 +91,7 @@ draw_token_unknown(), draw_token_space(), draw_token_newline(), + draw_token_bool(), ] token = draw(one_of(strategies)) return token