diff --git a/src/syntax.py b/src/syntax.py index 6e03c6f..f331ee0 100644 --- a/src/syntax.py +++ b/src/syntax.py @@ -34,22 +34,22 @@ # Location of a syntax node class SyntaxLocation: - def __init__(self, line, column, file): + def __init__(self, line, offset, file): self.line = line - self.column = column + self.offset = offset self.file = file def __repr__(self): - return "SyntaxLocation(line %i, column %i, file '%s')" % ( # pragma: no mutate + return "SyntaxLocation(line %i, offset %i, file '%s')" % ( # pragma: no mutate self.line, - self.column, + self.offset, self.file, ) def __eq__(self, other): return ( self.line == other.line - and self.column == other.column + and self.offset == other.offset and self.file == other.file ) diff --git a/src/tokenize.py b/src/tokenize.py index 2547784..bb73f63 100644 --- a/src/tokenize.py +++ b/src/tokenize.py @@ -49,16 +49,16 @@ def locate_tokens(tokens, filename): new_tokens = [] line = 1 - column = 1 + offset = 1 for t in tokens: - location = SyntaxLocation(line, column, filename) + location = SyntaxLocation(line, offset, filename) new = Syntax(t.value, location, SyntaxType.TOKEN) new_tokens.append(new) if is_newline(t.value): line = line + 1 - column = 1 + offset = 1 else: - column += len(t.value) + offset += len(t.value) return new_tokens diff --git a/tests/test_syntax.py b/tests/test_syntax.py index 5accd3e..519ce88 100644 --- a/tests/test_syntax.py +++ b/tests/test_syntax.py @@ -37,17 +37,17 @@ @composite def draw_syntax_location(draw): line = draw(integers()) - column = draw(integers()) + offset = draw(integers()) filename = draw(text()) - return SyntaxLocation(line, column, filename) + return SyntaxLocation(line, offset, filename) # Test location getters @given(integers(), integers(), text()) -def test_syntax_location_getters(line, column, filename): - test = SyntaxLocation(line, column, filename) +def test_syntax_location_getters(line, offset, filename): + test = SyntaxLocation(line, offset, filename) assert test.line == line - assert test.column == column + assert test.offset == offset assert test.file == filename @@ -56,7 +56,7 @@ def test_syntax_location_equality(location1, location2): equals = ( location1.line == location2.line - and location1.column == location2.column + and location1.offset == location2.offset and location1.file == location2.file ) assert (location1 == location2) == equals diff --git a/tests/test_tokenize.py b/tests/test_tokenize.py index 88512f3..abc9333 100644 --- a/tests/test_tokenize.py +++ b/tests/test_tokenize.py @@ -60,7 +60,7 @@ # - Whitespace and non-whitespace are separated in to tokens # - Whitespace characters are split in to multiple tokens # - Non-whitespace characters are combined in to a single token -# - Each token location is line 1 column 1 of file "" +# - Each token location is line 1 offset 1 of file "" @given(draw_tokens_to_split()) def test_tokenize_split_tokens(test_data): (source, tokens) = test_data @@ -74,16 +74,16 @@ filename = draw(text()) located = [] line = 1 - column = 1 + offset = 1 for t in tokens: - location = SyntaxLocation(line, column, filename) + location = SyntaxLocation(line, offset, filename) new = Syntax(t.value, location, SyntaxType.TOKEN) located.append(new) if t.value == "\n": line = line + 1 - column = 1 + offset = 1 else: - column += len(t.value) + offset += len(t.value) return (tokens, located, filename) @@ -91,10 +91,10 @@ # We expect the following behaviour: # - Only the token location is modified # - Each token's location filename is the generated filename -# - A token's line is equal to 1 plus the number of tokens with the +# - A location's line is equal to 1 plus the number of tokens with the # value '\n' before the token -# - A token's column is equal to 1 plus the sum of previous token's -# value lengths 'before' it, where 'before' is defined as any +# - A location's offset is equal to 1 plus the count of previous token's +# value code points 'before' it, where 'before' is defined as any # token between the last '\n' (or start of file) before the token # and the token itself @given(draw_tokens_locations())