diff --git a/tests/test_parse.py b/tests/test_parse.py index d7d8b91..d44efe6 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -12,29 +12,18 @@ from src import parse from src.syntax import Syntax, SyntaxStream, SyntaxType -from tests import test_tokenize - - -# Tests that a syntax stream reads items correctly -# We expect the following behaviour: -# - All items are popped in order -# - None is returned at the end of the stream -@given(lists(test_tokenize.draw_syntax_random())) -def test_parse_syntax_stream(nodes): - stream = SyntaxStream(nodes.copy()) - read = [] - node = stream.pop() - while node is not None: - read.append(node) - node = stream.pop() - assert read == nodes - assert stream.pop() is None +from tests.test_syntax import ( + draw_token_classified, + draw_syntax_random, + draw_token_location, + draw_syntax_token, +) # Draws syntax and a syntax without whitespace in it @composite def draw_syntax_whitespace(draw): - input = draw(lists(test_tokenize.draw_syntax_random())) + input = draw(lists(draw_syntax_random())) syntax = [] for s in input: if s.type != SyntaxType.TOKEN or s.value not in ["\n", " "]: @@ -55,14 +44,14 @@ # Draws a random token suitable for text building @composite def draw_text_value_token(draw): - token = draw(test_tokenize.draw_syntax_token()) + token = draw(draw_syntax_token()) assume(token.value not in ["StartText", "EndText"]) return token @composite def draw_token_by_value(draw, value): - location = draw(test_tokenize.draw_token_location()) + location = draw(draw_token_location()) type = SyntaxType.TOKEN return Syntax(value, location, type) @@ -90,7 +79,7 @@ # - The Syntax's value is the resulting text # - The Syntax's type is SyntaxType.TEXT # - The Syntax's location is the StartText location -@given(test_tokenize.draw_syntax_random(), draw_syntax_text_valid()) +@given(draw_syntax_random(), draw_syntax_text_valid()) def test_parse_text_valid(canary, test_data): (tokens, result) = test_data stream = SyntaxStream(tokens + [canary]) @@ -110,7 +99,7 @@ def draw_syntax_text_invalid_nostarttext(draw): (tokens, _) = draw(draw_syntax_text_valid()) if draw(booleans()): - token = draw(test_tokenize.draw_syntax_random()) + token = draw(draw_syntax_random()) assume(not (token.type == SyntaxType.TOKEN and token.value == "StartText")) new_tokens = [token] + tokens[1:0] return new_tokens @@ -124,7 +113,7 @@ @composite def draw_syntax_text_invalid_invalidcontent(draw): (tokens, _) = draw(draw_syntax_text_valid()) - token = draw(test_tokenize.draw_syntax_random()) + token = draw(draw_syntax_random()) assume(token.type != SyntaxType.TOKEN) pos = draw(integers(min_value=1, max_value=(len(tokens) - 1))) new_tokens = tokens[0:pos] + [token] + tokens[pos:] @@ -176,7 +165,7 @@ # Tests the parser wrapper works correctly # We expect the following behaviour: # - Whitespace tokens are stripped -@given(lists(test_tokenize.draw_token_classified())) +@given(lists(draw_token_classified())) def test_parse_fuzz(tokens): stripped = parse.strip_whitespace(tokens) parsed = parse.parse(tokens) diff --git a/tests/test_syntax.py b/tests/test_syntax.py new file mode 100644 index 0000000..80633b8 --- /dev/null +++ b/tests/test_syntax.py @@ -0,0 +1,228 @@ +# SPDX-License-Identifier: LGPL-2.1-only +# Copyright 2022 Jookia + +from hypothesis import given, assume +from hypothesis.strategies import ( + booleans, + characters, + composite, + integers, + lists, + one_of, + sampled_from, + text, +) + +from src.syntax import Syntax, SyntaxLocation, SyntaxStream, SyntaxType + +# Keywords recognized by the language +keywords = [ + "NewLang", + "Done", + "Set", + "To", + "EndSet", + "If", + "Then", + "Else", + "EndIf", + "StartNote", + "EndNote", + "StartText", + "EndText", +] + + +# Draws a random token location +@composite +def draw_token_location(draw): + line = draw(integers()) + column = draw(integers()) + filename = draw(text()) + return SyntaxLocation(line, column, filename) + + +# Draws a random syntax type +@composite +def draw_syntax_type(draw): + return draw(sampled_from(list(SyntaxType))) + + +# Draws a token syntax value +@composite +def draw_syntax_token(draw): + value = draw(draw_token_classified()) + location = draw(draw_token_location()) + type = SyntaxType.TOKEN + return Syntax(value.value, location, type) + + +# Draws a text syntax value +@composite +def draw_syntax_text(draw): + value = draw(text()) + location = draw(draw_token_location()) + type = SyntaxType.TEXT + return Syntax(value, location, type) + + +# Draws a random syntax +@composite +def draw_syntax_random(draw): + strategies = [ + draw_syntax_token(), + draw_syntax_text(), + ] + return draw(one_of(strategies)) + + +# Test syntax getters +@given(text(), draw_token_location(), draw_syntax_type()) +def test_syntax_syntax_getters(value, location, type): + # Use text as a somewhat random value + test = Syntax(value, location, type) + assert test.value == value + assert test.location == location + assert test.type == type + + +# Test syntax equals +@given(draw_syntax_random(), draw_syntax_random()) +def test_syntax_syntax_equality(syntax1, syntax2): + equals = ( + syntax1.type == syntax2.type + and syntax1.value == syntax2.value + and syntax1.location == syntax2.location + ) + assert (syntax1 == syntax2) == equals + + +# Draws a random token +@composite +def draw_token_random(draw): + value = draw(text()) + location = draw(draw_token_location()) + return Syntax(value, location, SyntaxType.TOKEN) + + +# Draws an unknown token +@composite +def draw_token_unknown(draw): + reserved = " \n\t" + token = draw(draw_token_random()) + chars = characters(blacklist_characters=reserved) + value = draw(text(alphabet=chars, min_size=1)) + assume(value not in ["True", "False"]) + assume(value not in keywords) + assume(value[0:2] != "#!") + return Syntax(value, token.location, SyntaxType.TOKEN) + + +# Draws a space token +@composite +def draw_token_space(draw): + space = " \t" + token = draw(draw_token_random()) + value = draw(sampled_from(space)) + return Syntax(value, token.location, SyntaxType.TOKEN) + + +# Draws a new line token +@composite +def draw_token_newline(draw): + token = draw(draw_token_random()) + value = "\n" + return Syntax(value, token.location, SyntaxType.TOKEN) + + +# Draws a bool token +@composite +def draw_token_bool(draw): + token = draw(draw_token_random()) + if draw(booleans()): + value = "True" + else: + value = "False" + return Syntax(value, token.location, SyntaxType.TOKEN) + + +# Draws a keyword token +@composite +def draw_token_keyword(draw): + token = draw(draw_token_random()) + value = draw(sampled_from(keywords)) + return Syntax(value, token.location, SyntaxType.TOKEN) + + +# Draws a shebang token +@composite +def draw_token_shebang(draw): + token = draw(draw_token_random()) + value = "#!" + draw(text()) + return Syntax(value, token.location, SyntaxType.TOKEN) + + +# Draws a classified token +@composite +def draw_token_classified(draw): + strategies = [ + draw_token_unknown(), + draw_token_space(), + draw_token_newline(), + draw_token_bool(), + draw_token_keyword(), + draw_token_shebang(), + ] + token = draw(one_of(strategies)) + return token + + +# Test location getters +@given(integers(), integers(), text()) +def test_syntax_location_getters(line, column, filename): + test = SyntaxLocation(line, column, filename) + assert test.line == line + assert test.column == column + assert test.file == filename + + +# Test location equals +@given(draw_token_location(), draw_token_location()) +def test_syntax_location_equality(location1, location2): + equals = ( + location1.line == location2.line + and location1.column == location2.column + and location1.file == location2.file + ) + assert (location1 == location2) == equals + + +# Test token getters +@given(text(), draw_token_location()) +def test_syntax_token_getters(value, location): + test = Syntax(value, location, SyntaxType.TOKEN) + assert test.value == value + assert test.location == location + + +# Test token equals +@given(draw_token_random(), draw_token_random()) +def test_syntax_token_equality(token1, token2): + equals = token1.value == token2.value and token1.location == token2.location + assert (token1 == token2) == equals + + +# Tests that a syntax stream reads items correctly +# We expect the following behaviour: +# - All items are popped in order +# - None is returned at the end of the stream +@given(lists(draw_syntax_random())) +def test_syntax_syntax_stream(nodes): + stream = SyntaxStream(nodes.copy()) + read = [] + node = stream.pop() + while node is not None: + read.append(node) + node = stream.pop() + assert read == nodes + assert stream.pop() is None diff --git a/tests/test_tokenize.py b/tests/test_tokenize.py index d150136..88512f3 100644 --- a/tests/test_tokenize.py +++ b/tests/test_tokenize.py @@ -1,218 +1,24 @@ # SPDX-License-Identifier: LGPL-2.1-only # Copyright 2022 Jookia -from hypothesis import given, assume +from hypothesis import given from hypothesis.strategies import ( booleans, - characters, composite, - integers, just, lists, one_of, - sampled_from, text, ) from src import tokenize from src.syntax import Syntax, SyntaxLocation, SyntaxType - - -# Keywords recognized by the language -keywords = [ - "NewLang", - "Done", - "Set", - "To", - "EndSet", - "If", - "Then", - "Else", - "EndIf", - "StartNote", - "EndNote", - "StartText", - "EndText", -] - - -# Draws a random token location -@composite -def draw_token_location(draw): - line = draw(integers()) - column = draw(integers()) - filename = draw(text()) - return SyntaxLocation(line, column, filename) - - -# Draws a random syntax type -@composite -def draw_syntax_type(draw): - return draw(sampled_from(list(SyntaxType))) - - -# Draws a token syntax value -@composite -def draw_syntax_token(draw): - value = draw(draw_token_classified()) - location = draw(draw_token_location()) - type = SyntaxType.TOKEN - return Syntax(value.value, location, type) - - -# Draws a text syntax value -@composite -def draw_syntax_text(draw): - value = draw(text()) - location = draw(draw_token_location()) - type = SyntaxType.TEXT - return Syntax(value, location, type) - - -# Draws a random syntax -@composite -def draw_syntax_random(draw): - strategies = [ - draw_syntax_token(), - draw_syntax_text(), - ] - return draw(one_of(strategies)) - - -# Test syntax getters -@given(text(), draw_token_location(), draw_syntax_type()) -def test_tokenize_syntax_getters(value, location, type): - # Use text as a somewhat random value - test = Syntax(value, location, type) - assert test.value == value - assert test.location == location - assert test.type == type - - -# Test syntax equals -@given(draw_syntax_random(), draw_syntax_random()) -def test_tokenize_syntax_equality(syntax1, syntax2): - equals = ( - syntax1.type == syntax2.type - and syntax1.value == syntax2.value - and syntax1.location == syntax2.location - ) - assert (syntax1 == syntax2) == equals - - -# Draws a random token -@composite -def draw_token_random(draw): - value = draw(text()) - location = draw(draw_token_location()) - return Syntax(value, location, SyntaxType.TOKEN) - - -# Draws an unknown token -@composite -def draw_token_unknown(draw): - reserved = " \n\t" - token = draw(draw_token_random()) - chars = characters(blacklist_characters=reserved) - value = draw(text(alphabet=chars, min_size=1)) - assume(value not in ["True", "False"]) - assume(value not in keywords) - assume(value[0:2] != "#!") - return Syntax(value, token.location, SyntaxType.TOKEN) - - -# Draws a space token -@composite -def draw_token_space(draw): - space = " \t" - token = draw(draw_token_random()) - value = draw(sampled_from(space)) - return Syntax(value, token.location, SyntaxType.TOKEN) - - -# Draws a new line token -@composite -def draw_token_newline(draw): - token = draw(draw_token_random()) - value = "\n" - return Syntax(value, token.location, SyntaxType.TOKEN) - - -# Draws a bool token -@composite -def draw_token_bool(draw): - token = draw(draw_token_random()) - if draw(booleans()): - value = "True" - else: - value = "False" - return Syntax(value, token.location, SyntaxType.TOKEN) - - -# Draws a keyword token -@composite -def draw_token_keyword(draw): - token = draw(draw_token_random()) - value = draw(sampled_from(keywords)) - return Syntax(value, token.location, SyntaxType.TOKEN) - - -# Draws a shebang token -@composite -def draw_token_shebang(draw): - token = draw(draw_token_random()) - value = "#!" + draw(text()) - return Syntax(value, token.location, SyntaxType.TOKEN) - - -# Draws a classified token -@composite -def draw_token_classified(draw): - strategies = [ - draw_token_unknown(), - draw_token_space(), - draw_token_newline(), - draw_token_bool(), - draw_token_keyword(), - draw_token_shebang(), - ] - token = draw(one_of(strategies)) - return token - - -# Test location getters -@given(integers(), integers(), text()) -def test_tokenize_location_getters(line, column, filename): - test = SyntaxLocation(line, column, filename) - assert test.line == line - assert test.column == column - assert test.file == filename - - -# Test location equals -@given(draw_token_location(), draw_token_location()) -def test_tokenize_location_equality(location1, location2): - equals = ( - location1.line == location2.line - and location1.column == location2.column - and location1.file == location2.file - ) - assert (location1 == location2) == equals - - -# Test token getters -@given(text(), draw_token_location()) -def test_tokenize_token_getters(value, location): - test = Syntax(value, location, SyntaxType.TOKEN) - assert test.value == value - assert test.location == location - - -# Test token equals -@given(draw_token_random(), draw_token_random()) -def test_tokenize_token_equality(token1, token2): - equals = token1.value == token2.value and token1.location == token2.location - assert (token1 == token2) == equals +from tests.test_syntax import ( + draw_token_classified, + draw_token_newline, + draw_token_space, + draw_token_unknown, +) # Draws a token using an existing strategy but with a blank location just like split_tokens outputs