diff --git a/src/tokenize.py b/src/tokenize.py index 62a045b..2680c89 100644 --- a/src/tokenize.py +++ b/src/tokenize.py @@ -68,3 +68,20 @@ current += c symbols.append(Symbol(current, location)) return symbols + + +# Generates a list of symbols with locations +def locate_symbols(symbols, filename): + new_symbols = [] + line = 1 + column = 1 + for s in symbols: + location = SymbolLocation(line, column, filename) + new = Symbol(s.value, location) + new_symbols.append(new) + if s.value == "\n": + line = line + 1 + column = 1 + else: + column += len(s.value) + return new_symbols diff --git a/tests/test_tokenize.py b/tests/test_tokenize.py index a5f5d41..e6db9cd 100644 --- a/tests/test_tokenize.py +++ b/tests/test_tokenize.py @@ -112,3 +112,35 @@ for s in symbols: input += s.value assert tokenize.split_symbols(input) == symbols + + +# Generates a list of symbols with locations +@composite +def draw_symbols_locations(draw): + symbols = draw(draw_symbols_list()) + filename = draw(text()) + new_symbols = [] + line = 1 + column = 1 + for s in symbols: + location = tokenize.SymbolLocation(line, column, filename) + new = tokenize.Symbol(s.value, location) + new_symbols.append(new) + if s.value == "\n": + line = line + 1 + column = 1 + else: + column += len(s.value) + return new_symbols + + +# Test that we the tokenizer can determine locations +@given(draw_symbols_locations()) +def test_tokenize_locations(symbols): + input = [] + filename = "" + location = tokenize.SymbolLocation(1, 1, "") + for s in symbols: + input.append(tokenize.Symbol(s.value, location)) + filename = s.location.file + assert tokenize.locate_symbols(input, filename) == symbols