diff --git a/src/ast_types.py b/src/ast_types.py index 8783255..a9b4e21 100644 --- a/src/ast_types.py +++ b/src/ast_types.py @@ -48,6 +48,13 @@ self.arguments, ) + def __eq__(self, other): + return ( + self.subject == other.subject + and self.verb == other.verb + and self.arguments == other.arguments + ) + class Set: def __init__(self, subject, statement): diff --git a/src/parse.py b/src/parse.py index c102911..81959fe 100644 --- a/src/parse.py +++ b/src/parse.py @@ -2,7 +2,7 @@ # Copyright 2022 Jookia import enum -from src.ast_types import Bool, Reference, Text +from src.ast_types import Bool, Reference, Statement, Text from src.token import TokenStream @@ -35,6 +35,10 @@ PARSE_BOOL = enum.auto() # pragma: no mutate PARSE_REFERENCE = enum.auto() # pragma: no mutate PARSE_VALUE = enum.auto() # pragma: no mutate + PARSE_STATEMENT = enum.auto() # pragma: no mutate + PARSE_SUBJECT = enum.auto() # pragma: no mutate + PARSE_VERB = enum.auto() # pragma: no mutate + PARSE_ARGUMENT = enum.auto() # pragma: no mutate # Context used for parse error exception @@ -72,6 +76,7 @@ NOT_BOOL = enum.auto() # pragma: no mutate FOUND_ENDNOTE = enum.auto() # pragma: no mutate RESERVED_NAME = enum.auto() # pragma: no mutate + FOUND_TERMINATOR = enum.auto() # pragma: no mutate # Exception thrown when a parse error is encountered @@ -204,6 +209,35 @@ else: return self.parse_reference(stream, context) + # Parses a statement until a specified terminator + def parse_statement(self, stream, parent_context, terminator): + context = ParseContext(ParseTask.PARSE_STATEMENT, stream.peek(), parent_context) + peeked_subject = stream.peek() + context_subject = ParseContext(ParseTask.PARSE_SUBJECT, peeked_subject, context) + if peeked_subject is not None and peeked_subject.value == terminator: + raise ParseErrorException( + ParseError.FOUND_TERMINATOR, peeked_subject, None, context_subject + ) + subject = self.parse_value(stream, context_subject) + context_verb = ParseContext(ParseTask.PARSE_VERB, stream.peek(), context) + verb = read_token(stream, None, context_verb) + if verb.value == terminator: + return Statement(subject, None, []) + elif verb.value in reserved_names: + raise ParseErrorException( + ParseError.RESERVED_NAME, verb, None, context_verb + ) + arguments = [] + # Parse following arguments + while True: + peeked_arg = stream.peek() + context_arg = ParseContext(ParseTask.PARSE_ARGUMENT, peeked_arg, context) + if peeked_arg is not None and peeked_arg.value == terminator: + stream.pop() + return Statement(subject, verb.value, arguments) + arg = self.parse_value(stream, context_arg) + arguments.append(arg) + # Parses tokens def parse(tokens, context): diff --git a/tests/parse/templates.py b/tests/parse/templates.py index 1049f1a..e4bda30 100644 --- a/tests/parse/templates.py +++ b/tests/parse/templates.py @@ -42,6 +42,30 @@ return lambda func: do +# Tests that something parses correctly with a custom parser +# We expect the following behaviour: +# - The decorated function supplies a parser, test data and expected data +# - Only the supplied tokens are parsed +# - The supplied tokens parse to the expected value +# - The Token's value is the expected value +# - The Token's location is the first token's location +def template_parse_valid_composite(func): + @given(draw_token_random(), composite(func)()) + def do(canary, test_data): + (parser, tokens, expected) = test_data + stream = TokenStream(tokens + [canary]) + parsed = parser(stream, None) + if expected is None: + assert parsed is None + else: + assert parsed is not None + assert parsed == expected + assert stream.pop() == canary + assert stream.pop() is None + + return do + + # Test that something parses incorrectly # We expect the following behaviour: # - The decoration supplies a parser function @@ -68,3 +92,31 @@ assert e == error return lambda func: given(wrapper(func))(do) + + +# Test that something parses incorrectly with a custom parser +# We expect the following behaviour: +# - The decorated function takes a parse context +# - The decorated function generates a parser, input tokens and an error +# - Parsing causes an error +# - The parse error is as expected +def template_parse_invalid_composite(func): + # Wrapper to add parse_context to our test_data + @composite + def wrapper(draw): + context = static_parse_context() + (parser, tokens, error) = draw(composite(func)(context)) + return (parser, tokens, error, context) + + # test_data is the output of wrapper + @given(wrapper()) + def do(test_data): + (parser, tokens, error, context) = test_data + stream = TokenStream(tokens) + try: + parsed = parser(stream, context) + raise AssertionError("Parsed invalid data: %s" % (parsed)) + except ParseErrorException as e: + assert e == error + + return do diff --git a/tests/parse/test_statement.py b/tests/parse/test_statement.py new file mode 100644 index 0000000..23602b1 --- /dev/null +++ b/tests/parse/test_statement.py @@ -0,0 +1,220 @@ +# SPDX-License-Identifier: LGPL-2.1-only +# Copyright 2022 Jookia + +import enum + +from hypothesis import assume +from hypothesis.strategies import composite, integers, lists + +from src.ast_types import Statement +from src.parse import ( + ParseContext, + ParseError, + ParseErrorException, + ParseTask, + Parser, + read_token, +) +from tests.parse.templates import ( + template_parse_valid_composite, + template_parse_invalid_composite, +) +from tests.test_token import ( + draw_token_known, + draw_token_random, + draw_token_unknown, + static_token_by_value, +) + + +# Values indicating what a parser did +class ParserMockAction(enum.Enum): + PARSE_VALUE = enum.auto() + WRONG_VALUE = enum.auto() + + +# Dummy Parser for testing statement parsing +# Return a static value of ParserMockACTION.PARSE_VALUE if a token starts with "TestValue" +# Otherwise throw an error of ParseMockAction.WRONG_VALUE +class ParserStatementMock(Parser): + def parse_value(self, stream, context): + token = read_token(stream, None, context) + if token.value.startswith("TestValue"): + return ParserMockAction.PARSE_VALUE + else: + raise ParseErrorException( + ParserMockAction.WRONG_VALUE, token, None, context + ) + + +# Creates a dummy parse function with a terminator specified by the last token +def make_test_parser(tokens): + def parser(stream, context): + if tokens == []: + terminator = "" + else: + terminator = tokens[-1].value + return ParserStatementMock().parse_statement(stream, context, terminator) + + return parser + + +# Draws a statement value with a somewhat random name +@composite +def draw_token_statement_value(draw): + number = draw(integers()) + return static_token_by_value("TestValue" + str(number)) + + +# Draws a statement name +@composite +def draw_token_statement_name(draw): + return draw(draw_token_unknown()) + + +# Draws a statement terminator +@composite +def draw_token_statement_terminator(draw): + return draw(draw_token_random()) + + +# Creates a context for a token in a statement +def make_test_context(parent_context, index, statement_token, token): + statement_context = ParseContext( + ParseTask.PARSE_STATEMENT, statement_token, parent_context + ) + if index == 0: + context = ParseTask.PARSE_SUBJECT + elif index == 1: + context = ParseTask.PARSE_VERB + else: + context = ParseTask.PARSE_ARGUMENT + context = ParseContext(context, token, statement_context) + return context + + +# Creates a context using existing tokens +def make_test_context_tokens(parent_context, index, tokens): + return make_test_context(parent_context, index, tokens[0], tokens[index]) + + +# Draws a valid statement's tokens +@composite +def draw_token_statement(draw): + values = draw(lists(draw_token_statement_value(), min_size=1)) + subject = values[0] + verb = [] + if len(values) > 1: + verb = [draw(draw_token_statement_name())] + arguments = values[2:] + terminator = draw(draw_token_statement_terminator()) + assume(terminator not in values) + assume(terminator not in verb) + tokens = [subject] + verb + arguments + [terminator] + return tokens + + +# Draws a valid statement +@composite +def draw_token_statement_valid(draw): + tokens = draw(draw_token_statement()) + subject = ParserMockAction.PARSE_VALUE + verb = None + # Account for the terminator + if len(tokens) > 2: + verb = tokens[1].value + argument_count = len(tokens) - 3 + arguments = [ParserMockAction.PARSE_VALUE] * argument_count + statement = Statement(subject, verb, arguments) + return (tokens, statement) + + +# Tests parsing a valid statement +# We expect the following behaviour: +# - A value is read as the subject +# - Optionally, a name is read as the verb +# - Optionally, any number of arguments are read as values +# - A terminator is found afterwards +# template_parse_valid_composite provides general parsing properties +@template_parse_valid_composite +def test_parse_statement_valid(draw): + (tokens, expected) = draw(draw_token_statement_valid()) + parser = make_test_parser(tokens) + return (parser, tokens, expected) + + +# Tests parsing a statement without a terminator +# This also covers cases of premature truncation for verbs and arguments +# We expect the following behaviour: +# - Error reading a verb or argument +# - Have ParseTask.PARSE_VERB or ParseTask.PARSE_ARGUMENT as the context's parse task +# - Have ParseTask.PARSE_STATEMENT as the context's parse task's parent +# - Have ParseError.NO_TOKEN as the exception code +# template_parse_invalid_composite provides general parsing properties +@template_parse_invalid_composite +def test_parse_statement_invalid_no_terminator(draw, parent_context): + tokens = draw(draw_token_statement()) + truncated = tokens[:-1] + context = make_test_context(parent_context, len(truncated), tokens[0], None) + error = ParseErrorException(ParseError.NO_TOKEN, None, None, context) + parser = make_test_parser(tokens) + return (parser, truncated, error) + + +# Tests parsing a statement with an invalid value +# We expect the following behaviour: +# - Error reading a invalid value on subject or argument +# - Have ParseTask.PARSE_SUBJECT or ParseTask.PARSE_ARGUMENT as the context's parse task +# - Have ParseTask.PARSE_STATEMENT as the context's parse task's parent +# - Have ParserMockAction.WRONG_VALUE as the exception code +# template_parse_invalid_composite provides general parsing properties +@template_parse_invalid_composite +def test_parse_statement_invalid_value(draw, parent_context): + tokens = draw(draw_token_statement()) + new_token = draw(draw_token_random()) + assume(not new_token.value.startswith("TestValue")) # Not a value + assume(new_token.value != tokens[-1].value) # Not the terminator + max_chosen = len(tokens) - 2 # Ignore Terminator and Verb + chosen = draw(integers(min_value=0, max_value=max_chosen)) + if chosen != 0: + chosen += 1 # Skip Verb + new_tokens = tokens[:chosen] + [new_token] + tokens[chosen + 1 :] + context = make_test_context_tokens(parent_context, chosen, new_tokens) + error = ParseErrorException(ParserMockAction.WRONG_VALUE, new_token, None, context) + parser = make_test_parser(tokens) + return (parser, new_tokens, error) + + +# Tests parsing a statement with an invalid verb +# We expect the following behaviour: +# - Error reading a known token as a verb +# - Have ParseTask.PARSE_VERB as the context's parse task +# - Have ParseTask.PARSE_STATEMENT as the context's parse task's parent +# - Have ParseError.RESERVED_NAME as the exception code +# template_parse_invalid_composite provides general parsing properties +@template_parse_invalid_composite +def test_parse_statement_invalid_verb(draw, parent_context): + tokens = draw(draw_token_statement()) + new_token = draw(draw_token_known()) + assume(new_token.value != tokens[-1].value) + new_tokens = tokens[:1] + [new_token] + tokens[1:] + context = make_test_context_tokens(parent_context, 1, new_tokens) + error = ParseErrorException(ParseError.RESERVED_NAME, new_token, None, context) + parser = make_test_parser(new_tokens) + return (parser, new_tokens, error) + + +# Tests parsing an empty statement +# We expect the following behaviour: +# - Error reading an empty statement +# - Have ParseTask.PARSE_SUBJECT as the context's parse task +# - Have ParseTask.PARSE_STATEMENT as the context's parse task's parent +# - Have ParserError.NO_TOKEN as the exception code +# template_parse_invalid_composite provides general parsing properties +@template_parse_invalid_composite +def test_parse_statement_invalid_empty(draw, parent_context): + tokens = [] + context = make_test_context(parent_context, 0, None, None) + error = ParseErrorException(ParseError.NO_TOKEN, None, None, context) + parser = make_test_parser(tokens) + return (parser, tokens, error)