diff --git a/tests/test_parse.py b/tests/test_parse.py index 77971d2..189ebab 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -1,5 +1,13 @@ from hypothesis import given, assume -from hypothesis.strategies import text, booleans, sampled_from, characters, lists +from hypothesis.strategies import ( + text, + booleans, + sampled_from, + one_of, + characters, + lists, + composite, +) from src import parse @@ -63,22 +71,6 @@ pass -# Test that we can make string literals using the BeginText and EndText syntax. -@given(text(), sampled_from(lexer_whitespace), sampled_from(lexer_whitespace)) -def test_lexer_text(text, space1, space2): - text_tokens = split_by(text, lexer_whitespace) - assume("BeginText" not in text_tokens and "EndText" not in text_tokens) - code = "BeginText" + space1 + text + space2 + "EndText" - tokenizer = parse.Tokenizer(code, "") - tokens = tokenizer.tokenize() - assert tokens[0].type == "text" - assert tokens[0].value == text - assert tokens[0].location.line == 1 - assert tokens[0].location.column == 1 - assert tokens[0].location.file == "" - assert tokens[1].type == "EOF" - - # Test that we can make notes using BeginNote and EndNote syntax. @given(text(), sampled_from(lexer_whitespace), sampled_from(lexer_whitespace)) def test_lexer_note(text, space1, space2): @@ -90,47 +82,84 @@ assert tokens[0].type == "EOF" -# Test that we can make booleans using True and False -@given(booleans()) -def test_lexer_boolean(bool): +class SampleToken: + def __init__(self, code, type, value): + self.code = code + self.type = type + self.value = value + + def __repr__(self): + return "SampleToken(code '%s', type '%s', value '%s')" % ( + self.code, + self.type, + self.value, + ) + + +# Generates a Text token +@composite +def draw_token_text(draw): + value = draw(text()) + text_tokens = split_by(value, lexer_whitespace) + assume("BeginText" not in text_tokens and "EndText" not in text_tokens) + space1 = draw(sampled_from(lexer_whitespace)) + space2 = draw(sampled_from(lexer_whitespace)) + code = "BeginText" + space1 + value + space2 + "EndText" + return SampleToken(code, "text", value) + + +# Generates a Bool token +@composite +def draw_token_bool(draw): + bool = draw(booleans()) if bool == True: code = "True" else: code = "False" - tokenizer = parse.Tokenizer(code, "") - tokens = tokenizer.tokenize() - assert tokens[0].type == "bool" - assert tokens[0].value == bool - assert tokens[0].location.line == 1 - assert tokens[0].location.column == 1 - assert tokens[0].location.file == "" - assert tokens[1].type == "EOF" + return SampleToken(code, "bool", bool) -# Test that we can read keywords properly -@given(sampled_from(keywords)) -def test_lexer_boolean(keyword): - code = keyword - tokenizer = parse.Tokenizer(code, "") - tokens = tokenizer.tokenize() - assert tokens[0].type == "keyword" - assert tokens[0].value == keyword - assert tokens[0].location.line == 1 - assert tokens[0].location.column == 1 - assert tokens[0].location.file == "" - assert tokens[1].type == "EOF" +# Generates a keyword token +@composite +def draw_token_keyword(draw): + keyword = draw(sampled_from(keywords)) + return SampleToken(keyword, "keyword", keyword) -# Test that we can make symbols -@given(text(alphabet=characters(blacklist_characters=lexer_whitespace), min_size=1)) -def test_lexer_symbols(symbol): +# Generates a symbol token +@composite +def draw_token_symbol(draw): + symbol = draw( + text(alphabet=characters(blacklist_characters=lexer_whitespace), min_size=1) + ) assume(symbol not in reserved_words) # Reserved words aren't symbols assume(not symbol.startswith("#!")) # Shebangs aren't symbols - code = symbol - tokenizer = parse.Tokenizer(code, "") - tokens = tokenizer.tokenize() - assert tokens[0].type == "symbol" - assert tokens[0].value == symbol + return SampleToken(symbol, "symbol", symbol) + + +# Generates any token +@composite +def draw_token(draw): + strategies = [ + draw_token_text(), + draw_token_bool(), + draw_token_keyword(), + draw_token_symbol(), + ] + return draw(one_of(strategies)) + + +# Test that we can lex tokens correctly +@given(draw_token()) +def test_lexer_token(token): + tokens = [] + try: + tokenizer = parse.Tokenizer(token.code, "") + tokens = tokenizer.tokenize() + except parse.ParseError as e: + raise AssertionError("ParseError thrown: %s" % (e)) + assert tokens[0].type == token.type + assert tokens[0].value == token.value assert tokens[0].location.line == 1 assert tokens[0].location.column == 1 assert tokens[0].location.file == ""