diff --git a/src/parse.py b/src/parse.py new file mode 100644 index 0000000..ba44eb4 --- /dev/null +++ b/src/parse.py @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: LGPL-2.1-only +# Copyright 2022 Jookia + +from src import tokenize + + +# Removes whitespace tokens +def remove_whitespace(tokens): + output = [] + for t in tokens: + if t.type not in [tokenize.TokenType.SPACE, tokenize.TokenType.NEWLINE]: + output.append(t) + return output diff --git a/tests/test_parse.py b/tests/test_parse.py new file mode 100644 index 0000000..0d037e6 --- /dev/null +++ b/tests/test_parse.py @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: LGPL-2.1-only +# Copyright 2022 Jookia + +from hypothesis import given +from hypothesis.strategies import ( + composite, + lists, +) + +from src import tokenize +from src import parse +from tests import test_tokenize + + +# Draws random tokens and a list without whitespace in them +@composite +def draw_tokens_whitespace(draw): + input = draw(lists(test_tokenize.draw_token_random())) + tokens = [] + for t in input: + if t.type not in [tokenize.TokenType.SPACE, tokenize.TokenType.NEWLINE]: + tokens.append(t) + return (input, tokens) + + +# Tests is remove_whitespace works correctly +# We expect the following behaviour: +# - No tokens are modified +# - Tokens of type SPACE or NEWLINE are removed from the output +@given(draw_tokens_whitespace()) +def test_parse_remove_whitespace(test_data): + (input, tokens) = test_data + assert parse.remove_whitespace(input) == tokens