diff --git a/src/parse.py b/src/parse.py index cf6e11a..79fe65b 100644 --- a/src/parse.py +++ b/src/parse.py @@ -3,7 +3,7 @@ import enum from src.ast_types import Bool, Text -from src.syntax import SyntaxStream, SyntaxType +from src.syntax import SyntaxStream # Tasks that happen during parsing @@ -44,7 +44,6 @@ class ParseError(enum.Enum): TEST_ERROR = enum.auto() # pragma: no mutate NO_TOKEN = enum.auto() # pragma: no mutate - NOT_TOKEN = enum.auto() # pragma: no mutate WRONG_TOKEN = enum.auto() # pragma: no mutate FOUND_STARTTEXT = enum.auto() # pragma: no mutate FOUND_STARTNOTE = enum.auto() # pragma: no mutate @@ -85,8 +84,6 @@ s = stream.pop() if s is None: raise ParseErrorException(ParseError.NO_TOKEN, None, None, context) - elif s.type != SyntaxType.TOKEN: - raise ParseErrorException(ParseError.NOT_TOKEN, s, None, context) elif value is not None and s.value != value: raise ParseErrorException(ParseError.WRONG_TOKEN, s, value, context) return s @@ -115,10 +112,10 @@ token = stream.peek() while token is not None: # Found a note, skip it - if token.type == SyntaxType.TOKEN and token.value == "StartNote": + if token.value == "StartNote": self.skip_note(stream, context) # EndNote found outside note - elif token.type == SyntaxType.TOKEN and token.value == "EndNote": + elif token.value == "EndNote": raise ParseErrorException( ParseError.FOUND_ENDNOTE, token, None, context ) diff --git a/src/syntax.py b/src/syntax.py index e728582..4a17f8b 100644 --- a/src/syntax.py +++ b/src/syntax.py @@ -1,36 +1,21 @@ # SPDX-License-Identifier: LGPL-2.1-only # Copyright 2022 Jookia -import enum - - -# The type of syntax -class SyntaxType(enum.Enum): - TOKEN = enum.auto() # pragma: no mutate - TEXT = enum.auto() # pragma: no mutate - BOOL = enum.auto() # pragma: no mutate - # Represents a syntax node class Syntax: - def __init__(self, value, location, type): + def __init__(self, value, location): self.value = value self.location = location - self.type = type def __repr__(self): - return "Syntax(value %s, location %s, type %s)" % ( # pragma: no mutate + return "Syntax(value %s, location %s)" % ( # pragma: no mutate repr(self.value), repr(self.location), - str(self.type), ) def __eq__(self, other): - return ( - self.type == other.type - and self.value == other.value - and self.location == other.location - ) + return self.value == other.value and self.location == other.location # Location of a syntax node diff --git a/src/tokenize.py b/src/tokenize.py index 3534977..f6c0886 100644 --- a/src/tokenize.py +++ b/src/tokenize.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: LGPL-2.1-only # Copyright 2022 Jookia -from src.syntax import Syntax, SyntaxLocation, SyntaxType +from src.syntax import Syntax, SyntaxLocation # Valid space code points spaces = [ @@ -45,11 +45,11 @@ # Don't flush if we're in the middle of a CR LF sequence flush = switching or (curr_space and not crlf) if flush: - tokens.append(Syntax(buffer, location, SyntaxType.TOKEN)) + tokens.append(Syntax(buffer, location)) buffer = "" buffer += curr prev = curr - tokens.append(Syntax(buffer, location, SyntaxType.TOKEN)) + tokens.append(Syntax(buffer, location)) return tokens @@ -60,7 +60,7 @@ offset = 1 for t in tokens: location = SyntaxLocation(line, offset, filename) - new = Syntax(t.value, location, SyntaxType.TOKEN) + new = Syntax(t.value, location) new_tokens.append(new) if t.value in newlines: line = line + 1 @@ -74,7 +74,7 @@ def strip_whitespace(syntax): output = [] for s in syntax: - if s.type == SyntaxType.TOKEN and not is_whitespace(s.value): + if not is_whitespace(s.value): output.append(s) return output diff --git a/tests/parse/templates.py b/tests/parse/templates.py index 7e2695a..7ae2f59 100644 --- a/tests/parse/templates.py +++ b/tests/parse/templates.py @@ -23,7 +23,6 @@ # - Only the supplied tokens are parsed # - The supplied tokens parse to the expected value # - The Syntax's value is the expected value -# - The Syntax's type is expected type # - The Syntax's location is the first token's location def template_parse_valid(parser, draw): @given(draw_syntax_random(), draw) diff --git a/tests/parse/test_bool.py b/tests/parse/test_bool.py index 53e3927..aa60817 100644 --- a/tests/parse/test_bool.py +++ b/tests/parse/test_bool.py @@ -6,7 +6,6 @@ from src.ast_types import Bool from src.parse import ParseContext, ParseError, ParseErrorException, ParseTask, Parser -from src.syntax import SyntaxType from tests.parse.templates import template_parse_valid, template_parse_invalid from tests.test_syntax import draw_token_bool, draw_syntax_random @@ -23,7 +22,7 @@ @composite def draw_syntax_not_bool(draw): token = draw(draw_syntax_random()) - assume(not (token.type == SyntaxType.TOKEN and token.value in ["True", "False"])) + assume(token.value not in ["True", "False"]) return token @@ -31,7 +30,6 @@ # We expect the following behaviour: # - The resulting boolean is True if the first token is True # - The resulting boolean is False if the first token is False -# - The Syntax's type is SyntaxType.BOOL # template_parse_valid provides general parsing properties @template_parse_valid(Parser().parse_bool, draw_syntax_bool_valid()) def test_parse_bool_valid(): @@ -40,16 +38,12 @@ # Tests parsing of invalid booleans # We expect the following behaviour: -# - Error if the token is not a SyntaxType.TOKEN # - Error if the token is not True or False @template_parse_invalid(Parser().parse_bool) def test_parse_bool_invalid_incorrect(draw, parent_context): token = draw(draw_syntax_not_bool()) context = ParseContext(ParseTask.PARSE_BOOL, token, parent_context) - if token.type == SyntaxType.TOKEN: - error = ParseErrorException(ParseError.NOT_BOOL, token, None, context) - else: - error = ParseErrorException(ParseError.NOT_TOKEN, token, None, context) + error = ParseErrorException(ParseError.NOT_BOOL, token, None, context) return ([token], error) diff --git a/tests/parse/test_clear_notes.py b/tests/parse/test_clear_notes.py index 13546a6..8f088eb 100644 --- a/tests/parse/test_clear_notes.py +++ b/tests/parse/test_clear_notes.py @@ -11,7 +11,7 @@ ParseErrorException, ParseTask, ) -from src.syntax import SyntaxStream, SyntaxType +from src.syntax import SyntaxStream from tests.parse.templates import template_parse_invalid from tests.test_syntax import draw_token_by_value, draw_syntax_random @@ -50,7 +50,7 @@ @composite def draw_clear_notes_value(draw): token = draw(draw_syntax_random()) - assume(not (token.type == SyntaxType.TOKEN and token.value == "EndNote")) + assume(token.value != "EndNote") return token @@ -62,7 +62,7 @@ for token in tokens: # Our modified parse_note only parses the StartNote, so we expect # the output to only remove StartNote values - if token.type != SyntaxType.TOKEN or token.value != "StartNote": + if token.value != "StartNote": output.append(token) return (tokens, output) @@ -88,7 +88,7 @@ new_tokens = tokens + [start] context = ParseContext(ParseTask.CLEAR_NOTES, new_tokens[0], parent_context) for token in new_tokens: - if token.type == SyntaxType.TOKEN and token.value == value: + if token.value == value: error = ParseErrorException(error, token, None, context) return (new_tokens, error) diff --git a/tests/parse/test_note.py b/tests/parse/test_note.py index e33d35b..886618f 100644 --- a/tests/parse/test_note.py +++ b/tests/parse/test_note.py @@ -14,7 +14,6 @@ ParseErrorException, ParseTask, ) -from src.syntax import SyntaxType from tests.parse.templates import ( insert_random_within, template_parse_invalid, @@ -22,15 +21,14 @@ ) from tests.test_syntax import ( draw_token_by_value, - draw_syntax_not_token, - draw_syntax_token, + draw_syntax_random, ) # Draws a random token suitable for note building @composite def draw_note_value_token(draw): - token = draw(draw_syntax_token()) + token = draw(draw_syntax_random()) assume(token.value not in ["StartNote", "EndNote"]) return token @@ -38,7 +36,7 @@ # Draws a random syntax that isn't a StartNote token @composite def draw_syntax_not_startnote(draw): - token = draw(draw_syntax_token()) + token = draw(draw_syntax_random()) assume(token.value != "StartNote") return token @@ -64,7 +62,6 @@ # Tests parsing notes without StartNote # We expect the following behaviour: -# - Error if StartNote is not a SyntaxType.TOKEN # - Error if StartNote's token value is not "StartNote" @template_parse_invalid(NoteSkipper().skip_note) def test_parse_note_invalid_nostartnote(draw, parent_context): @@ -72,10 +69,7 @@ token = draw(draw_syntax_not_startnote()) new_tokens = [token] + tokens[1:0] context = ParseContext(ParseTask.PARSE_NOTE, new_tokens[0], parent_context) - if token.type == SyntaxType.TOKEN: - error = ParseErrorException(ParseError.WRONG_TOKEN, token, "StartNote", context) - else: - error = ParseErrorException(ParseError.NOT_TOKEN, token, None, context) + error = ParseErrorException(ParseError.WRONG_TOKEN, token, "StartNote", context) return (new_tokens, error) @@ -89,19 +83,6 @@ return ([], error) -# Tests parsing note with invalid skipped tokens -# We expect the following behaviour: -# - Error if a skipped token is not a SyntaxType.TOKEN -@template_parse_invalid(NoteSkipper().skip_note) -def test_parse_note_invalid_invalidcontent(draw, parent_context): - (tokens, _) = draw(draw_syntax_note_valid()) - token = draw(draw_syntax_not_token()) - new_tokens = insert_random_within(draw, tokens, token) - context = ParseContext(ParseTask.PARSE_NOTE, new_tokens[0], parent_context) - error = ParseErrorException(ParseError.NOT_TOKEN, token, None, context) - return (new_tokens, error) - - # Tests parsing a note with a StartNote token in it # We expect the following behaviour: # - Error if a StartNote token is in the note content diff --git a/tests/parse/test_parse.py b/tests/parse/test_parse.py index 02a0217..cdee204 100644 --- a/tests/parse/test_parse.py +++ b/tests/parse/test_parse.py @@ -19,7 +19,7 @@ ) from src.syntax import SyntaxStream from tests.templates import template_test_structure -from tests.test_syntax import draw_syntax_token, draw_syntax_random +from tests.test_syntax import draw_syntax_random # Draws a random parse task @@ -80,7 +80,7 @@ # Tests the parser wrapper works correctly # We expect the following behaviour: # - Notes to be removed from the tokens -@given(lists(draw_syntax_token()), draw_parse_context()) +@given(lists(draw_syntax_random()), draw_parse_context()) def test_parse_fuzz(tokens, context): result = None parsed = None diff --git a/tests/parse/test_text.py b/tests/parse/test_text.py index 828405e..3d19890 100644 --- a/tests/parse/test_text.py +++ b/tests/parse/test_text.py @@ -15,7 +15,6 @@ ParseTask, Parser, ) -from src.syntax import SyntaxType from tests.parse.templates import ( insert_random_within, template_parse_invalid, @@ -24,15 +23,13 @@ from tests.test_syntax import ( draw_token_by_value, draw_syntax_random, - draw_syntax_not_token, - draw_syntax_token, ) # Draws a random token suitable for text building @composite def draw_text_value_token(draw): - token = draw(draw_syntax_token()) + token = draw(draw_syntax_random()) assume(token.value not in ["StartText", "EndText"]) return token @@ -41,7 +38,7 @@ @composite def draw_syntax_not_starttext(draw): token = draw(draw_syntax_random()) - assume(not (token.type == SyntaxType.TOKEN and token.value == "StartText")) + assume(token.value != "StartText") return token @@ -64,7 +61,6 @@ # - The resulting text is the value of tokens between StartText and EndText # - The value of the tokens is joined by U+0020 SPACE code points # - The Syntax's value is the resulting text -# - The Syntax's type is SyntaxType.TEXT # template_parse_valid provides general parsing properties @template_parse_valid(Parser().parse_text, draw_syntax_text_valid()) def test_parse_text_valid(): @@ -73,7 +69,6 @@ # Test parsing text without StartText # We expect the following behaviour: -# - Error if StartText is not a SyntaxType.TOKEN # - Error if StartText's token value is not "StartText" @template_parse_invalid(Parser().parse_text) def test_parse_text_invalid_nostarttext(draw, parent_context): @@ -81,10 +76,7 @@ token = draw(draw_syntax_not_starttext()) new_tokens = [token] + tokens[1:0] context = ParseContext(ParseTask.PARSE_TEXT, new_tokens[0], parent_context) - if token.type == SyntaxType.TOKEN: - error = ParseErrorException(ParseError.WRONG_TOKEN, token, "StartText", context) - else: - error = ParseErrorException(ParseError.NOT_TOKEN, token, None, context) + error = ParseErrorException(ParseError.WRONG_TOKEN, token, "StartText", context) return (new_tokens, error) @@ -98,19 +90,6 @@ return ([], error) -# Tests parsing text with invalid content tokens -# We expect the following behaviour: -# - Error if a content token is not a SyntaxType.TOKEN -@template_parse_invalid(Parser().parse_text) -def test_parse_text_invalid_invalidcontent(draw, parent_context): - (tokens, _) = draw(draw_syntax_text_valid()) - token = draw(draw_syntax_not_token()) - new_tokens = insert_random_within(draw, tokens, token) - context = ParseContext(ParseTask.PARSE_TEXT, new_tokens[0], parent_context) - error = ParseErrorException(ParseError.NOT_TOKEN, token, None, context) - return (new_tokens, error) - - # Tests parsing text with a StartText token in it # We expect the following behaviour: # - Error if a StartText token is in the text content diff --git a/tests/test_syntax.py b/tests/test_syntax.py index aaee057..2b02b33 100644 --- a/tests/test_syntax.py +++ b/tests/test_syntax.py @@ -13,7 +13,7 @@ text, ) -from src.syntax import Syntax, SyntaxLocation, SyntaxStream, SyntaxType +from src.syntax import Syntax, SyntaxLocation, SyntaxStream from tests.templates import template_test_structure # Keywords recognized by the language @@ -60,27 +60,11 @@ pass -# Draws a random syntax type -@composite -def draw_syntax_type(draw): - return draw(sampled_from(list(SyntaxType))) - - -# Draws a text syntax value -@composite -def draw_syntax_text(draw): - value = draw(text()) - location = draw(draw_syntax_location()) - type = SyntaxType.TEXT - return Syntax(value, location, type) - - # Draws a token with a specific value but random location @composite def draw_token_by_value(draw, value): location = draw(draw_syntax_location()) - type = SyntaxType.TOKEN - return Syntax(value, location, type) + return Syntax(value, location) # Values considered spaces @@ -120,7 +104,7 @@ assume(v not in value) assume(value not in literals) assume(value not in keywords) - return Syntax(value, location, SyntaxType.TOKEN) + return Syntax(value, location) # Draws a space token @@ -128,7 +112,7 @@ def draw_token_space(draw): location = draw(draw_syntax_location()) value = draw(sampled_from(valid_spaces)) - return Syntax(value, location, SyntaxType.TOKEN) + return Syntax(value, location) # Draws a new line token @@ -136,7 +120,7 @@ def draw_token_newline(draw): location = draw(draw_syntax_location()) value = draw(sampled_from(valid_newlines)) - return Syntax(value, location, SyntaxType.TOKEN) + return Syntax(value, location) # Draws a bool token @@ -147,7 +131,7 @@ value = "True" else: value = "False" - return Syntax(value, location, SyntaxType.TOKEN) + return Syntax(value, location) # Draws a keyword token @@ -155,12 +139,12 @@ def draw_token_keyword(draw): location = draw(draw_syntax_location()) value = draw(sampled_from(keywords)) - return Syntax(value, location, SyntaxType.TOKEN) + return Syntax(value, location) -# Draws a syntax token +# Draws a random syntax token @composite -def draw_syntax_token(draw): +def draw_syntax_random(draw): strategies = [ draw_token_unknown(), draw_token_space(), @@ -172,42 +156,12 @@ return token -# Draws a text syntax with a token value -@composite -def draw_syntax_texttoken(draw): - token = draw(draw_syntax_token()) - type = SyntaxType.TEXT - return Syntax(token.value, token.location, type) - - -# Draws a random syntax -@composite -def draw_syntax_random(draw): - strategies = [ - draw_syntax_token(), - draw_syntax_text(), - draw_syntax_texttoken(), - ] - return draw(one_of(strategies)) - - -# Draws a random syntax that isn't a token -@composite -def draw_syntax_not_token(draw): - strategies = [ - draw_syntax_text(), - draw_syntax_texttoken(), - ] - return draw(one_of(strategies)) - - # Test syntax structure @template_test_structure( Syntax, draw_syntax_random(), value=text(), location=draw_syntax_location(), - type=draw_syntax_type(), ) def test_syntax_syntax_structure(): pass diff --git a/tests/test_tokenize.py b/tests/test_tokenize.py index 7c89f6d..227e2d9 100644 --- a/tests/test_tokenize.py +++ b/tests/test_tokenize.py @@ -12,9 +12,9 @@ ) from src import tokenize -from src.syntax import Syntax, SyntaxLocation, SyntaxType +from src.syntax import Syntax, SyntaxLocation from tests.test_syntax import ( - draw_syntax_token, + draw_syntax_random, draw_token_newline, draw_token_space, draw_token_unknown, @@ -28,7 +28,7 @@ def draw_token_splitted(draw, strategy): token = draw(strategy) location = SyntaxLocation(1, 1, "") - return Syntax(token.value, location, SyntaxType.TOKEN) + return Syntax(token.value, location) # Merges \r and \n tokens to \r\n tokens @@ -41,7 +41,7 @@ if prev.value == "\r" and curr.value == "\n": # Previous token is \r, don't append it # Instead promote this \n token to \r\n - prev = Syntax("\r\n", prev.location, SyntaxType.TOKEN) + prev = Syntax("\r\n", prev.location) else: # Append the previous token merged.append(prev) @@ -102,14 +102,14 @@ # Generates a list of tokens with correct locations @composite def draw_tokens_locations(draw): - tokens = draw(lists(draw_syntax_token())) + tokens = draw(lists(draw_syntax_random())) filename = draw(text()) located = [] line = 1 offset = 1 for t in tokens: location = SyntaxLocation(line, offset, filename) - new = Syntax(t.value, location, SyntaxType.TOKEN) + new = Syntax(t.value, location) located.append(new) if t.value in valid_newlines: line = line + 1 @@ -147,11 +147,11 @@ # Generates two list of tokens: One with whitespace and one without @composite def draw_tokens_whitespace(draw): - input = draw(lists(draw_syntax_token())) + input = draw(lists(draw_syntax_random())) stripped = [] for s in input: is_whitespace = s.value in valid_spaces or s.value in valid_newlines - if s.type != SyntaxType.TOKEN or not is_whitespace: + if not is_whitespace: stripped.append(s) return (input, stripped) @@ -179,20 +179,20 @@ # Draws a token and possibly add garbage # This is to ensure that tokens must completely match a value @composite -def draw_syntax_token_garbled(draw): - token = draw(draw_syntax_token()) +def draw_syntax_random_garbled(draw): + token = draw(draw_syntax_random()) value = token.value if draw(booleans()): value = draw(text(min_size=1)) + value if draw(booleans()): value = value + draw(text(min_size=1)) - return Syntax(value, token.location, SyntaxType.TOKEN) + return Syntax(value, token.location) # Draw a random string made of token values @composite def draw_source_fuzz(draw): - tokens = draw(lists(draw_syntax_token())) + tokens = draw(lists(draw_syntax_random())) input = "" for t in tokens: input += t.value