# SPDX-License-Identifier: LGPL-2.1-only # Copyright 2022 Jookia <contact@jookia.org> from hypothesis import given from hypothesis.strategies import ( booleans, characters, composite, integers, just, lists, sampled_from, text, ) from src import tokenize # Whitespace that separates lexer words whitespace = " \n\t" # Draws a random symbol location @composite def draw_symbol_location(draw): line = draw(integers()) column = draw(integers()) filename = draw(text()) return tokenize.SymbolLocation(line, column, filename) # Test location getters @given(integers(), integers(), text()) def test_tokenize_location_getters(line, column, filename): test = tokenize.SymbolLocation(line, column, filename) assert test.line == line assert test.column == column assert test.file == filename # Test location equals @given(draw_symbol_location(), draw_symbol_location()) def test_tokenize_location_equality(location1, location2): equals = ( location1.line == location2.line and location1.column == location2.column and location1.file == location2.file ) assert (location1 == location2) == equals # Draws a random symbol @composite def draw_symbol(draw): value = draw(text()) location = draw(draw_symbol_location()) return tokenize.Symbol(value, location) # Test symbol getters @given(text(), draw_symbol_location()) def test_tokenize_symbol_getters(value, location): test = tokenize.Symbol(value, location) assert test.value == value assert test.location == location # Test symbol equals @given(draw_symbol(), draw_symbol()) def test_tokenize_symbol_equality(symbol1, symbol2): equals = symbol1.value == symbol2.value and symbol1.location == symbol2.location assert (symbol1 == symbol2) == equals # Draws a tokenizer non-whitespace symbol @composite def draw_symbol_nonwhitespace(draw): chars = characters(blacklist_characters=whitespace) value = draw(text(alphabet=chars, min_size=1)) location = draw(draw_symbol_location()) return tokenize.Symbol(value, location) # Draws a tokenizer whitespace symbol @composite def draw_symbol_whitespace(draw): value = draw(sampled_from(whitespace)) location = draw(draw_symbol_location()) return tokenize.Symbol(value, location) # Draws a symbol with a set location @composite def draw_symbol_with_location(draw, strategy, location): symbol = draw(strategy()) return tokenize.Symbol(symbol.value, location) # Generates an alternating sequence of symbols @composite def draw_symbols_list(draw): output = [] elements = draw(lists(just(True))) drawing_whitespace = draw(booleans()) location = tokenize.SymbolLocation(1, 1, "") for _ in elements: if drawing_whitespace: strategy = draw_symbol_whitespace locationed = draw_symbol_with_location(strategy, location) output += draw(lists(locationed, min_size=1)) else: strategy = draw_symbol_nonwhitespace locationed = draw_symbol_with_location(strategy, location) output.append(draw(locationed)) drawing_whitespace = not drawing_whitespace return output # Test that we the tokenizer can split symbols properly @given(draw_symbols_list()) def test_tokenize_split_symbols(symbols): input = "" for s in symbols: input += s.value assert tokenize.split_symbols(input) == symbols # Generates a list of symbols with locations @composite def draw_symbols_locations(draw): symbols = draw(draw_symbols_list()) filename = draw(text()) new_symbols = [] line = 1 column = 1 for s in symbols: location = tokenize.SymbolLocation(line, column, filename) new = tokenize.Symbol(s.value, location) new_symbols.append(new) if s.value == "\n": line = line + 1 column = 1 else: column += len(s.value) return new_symbols # Test that we the tokenizer can determine locations @given(draw_symbols_locations()) def test_tokenize_locations(symbols): input = [] filename = "" location = tokenize.SymbolLocation(1, 1, "") for s in symbols: input.append(tokenize.Symbol(s.value, location)) filename = s.location.file assert tokenize.locate_symbols(input, filename) == symbols