Newer
Older
NewLang / tests / test_syntax.py
# SPDX-License-Identifier: LGPL-2.1-only
# Copyright 2022 Jookia <contact@jookia.org>

from hypothesis import given, assume
from hypothesis.strategies import (
    booleans,
    characters,
    composite,
    integers,
    lists,
    one_of,
    sampled_from,
    text,
)

from src.syntax import Syntax, SyntaxLocation, SyntaxStream, SyntaxType
from tests.templates import template_test_structure

# Keywords recognized by the language
keywords = [
    "Done",
    "Set",
    "To",
    "EndSet",
    "If",
    "Then",
    "Else",
    "EndIf",
    "StartNote",
    "EndNote",
    "StartText",
    "EndText",
]

# Literals recognized by the language
literals = [
    "True",
    "False",
]


# Draws a random syntax location
@composite
def draw_syntax_location(draw):
    line = draw(integers())
    offset = draw(integers())
    filename = draw(text())
    return SyntaxLocation(line, offset, filename)


# Test syntax location structure
@template_test_structure(
    SyntaxLocation,
    draw_syntax_location(),
    line=integers(),
    offset=integers(),
    file=text(),
)
def test_syntax_location_structure():
    pass


# Draws a random syntax type
@composite
def draw_syntax_type(draw):
    return draw(sampled_from(list(SyntaxType)))


# Draws a text syntax value
@composite
def draw_syntax_text(draw):
    value = draw(text())
    location = draw(draw_syntax_location())
    type = SyntaxType.TEXT
    return Syntax(value, location, type)


# Draws a token with a specific value but random location
@composite
def draw_token_by_value(draw, value):
    location = draw(draw_syntax_location())
    type = SyntaxType.TOKEN
    return Syntax(value, location, type)


# Values considered spaces
valid_spaces = [
    "\t",  # U+0009 HORIZONTAL TAB
    " ",  # U+0020 SPACE
]

# Single values reserved for new line use
single_newlines = [
    "\n",  # U+000A LINE FEED
    "\v",  # U+000B VERTICAL TAB
    "\f",  # U+000C FORM FEED
    "\r",  # U+000D CARRIAGE RETURN
    "\u0085",  # U+0085 NEXT LINE
    "\u2028",  # U+2028 LINE SEPARATOR
    "\u2029",  # U+2029 PARAGRAPH SEPARATOR
]

# Multi values reserved for new line use
multi_newlines = [
    "\r\n",  # U+000A U+000D CARRIAGE RETURN then LINE FEED
]

# All values reserved for new line use
valid_newlines = single_newlines + multi_newlines


# Draws an unknown token
@composite
def draw_token_unknown(draw):
    reserved = valid_spaces + single_newlines
    location = draw(draw_syntax_location())
    chars = characters(blacklist_characters=reserved)
    value = draw(text(alphabet=chars, min_size=1))
    for v in multi_newlines:
        assume(v not in value)
    assume(value not in literals)
    assume(value not in keywords)
    return Syntax(value, location, SyntaxType.TOKEN)


# Draws a space token
@composite
def draw_token_space(draw):
    location = draw(draw_syntax_location())
    value = draw(sampled_from(valid_spaces))
    return Syntax(value, location, SyntaxType.TOKEN)


# Draws a new line token
@composite
def draw_token_newline(draw):
    location = draw(draw_syntax_location())
    value = draw(sampled_from(valid_newlines))
    return Syntax(value, location, SyntaxType.TOKEN)


# Draws a bool token
@composite
def draw_token_bool(draw):
    location = draw(draw_syntax_location())
    if draw(booleans()):
        value = "True"
    else:
        value = "False"
    return Syntax(value, location, SyntaxType.TOKEN)


# Draws a keyword token
@composite
def draw_token_keyword(draw):
    location = draw(draw_syntax_location())
    value = draw(sampled_from(keywords))
    return Syntax(value, location, SyntaxType.TOKEN)


# Draws a syntax token
@composite
def draw_syntax_token(draw):
    strategies = [
        draw_token_unknown(),
        draw_token_space(),
        draw_token_newline(),
        draw_token_bool(),
        draw_token_keyword(),
    ]
    token = draw(one_of(strategies))
    return token


# Draws a text syntax with a token value
@composite
def draw_syntax_texttoken(draw):
    token = draw(draw_syntax_token())
    type = SyntaxType.TEXT
    return Syntax(token.value, token.location, type)


# Draws a random syntax
@composite
def draw_syntax_random(draw):
    strategies = [
        draw_syntax_token(),
        draw_syntax_text(),
        draw_syntax_texttoken(),
    ]
    return draw(one_of(strategies))


# Draws a random syntax that isn't a token
@composite
def draw_syntax_not_token(draw):
    strategies = [
        draw_syntax_text(),
        draw_syntax_texttoken(),
    ]
    return draw(one_of(strategies))


# Test syntax structure
@template_test_structure(
    Syntax,
    draw_syntax_random(),
    value=text(),
    location=draw_syntax_location(),
    type=draw_syntax_type(),
)
def test_syntax_syntax_structure():
    pass


# Tests that a syntax stream pops items correctly
# We expect the following behaviour:
# - All items are popped in order
# - None is returned at the end of the stream
@given(lists(draw_syntax_random()))
def test_syntax_syntax_stream_pop(nodes):
    stream = SyntaxStream(nodes.copy())
    read = []
    node = stream.pop()
    while node is not None:
        read.append(node)
        node = stream.pop()
    assert read == nodes
    assert stream.pop() is None


# Tests that a syntax stream peeks items correctly
# We expect the following behaviour:
# - Peeking does not pop any values
# - None is returned at the end of the stream
@given(lists(draw_syntax_random()), integers(min_value=0, max_value=100))
def test_syntax_syntax_stream_peek(nodes, times):
    stream = SyntaxStream(nodes.copy())
    node_count = len(stream.nodes)
    if node_count == 0:
        real_times = times
        expected = None
    else:
        real_times = times % node_count
        expected = nodes[0]
    for _ in range(0, real_times):
        node = stream.peek()
        assert node == expected


# Tests that peeking and popping don't influence each other
# We expect the following behaviour:
# - Peeking does not influence the next pop call
# - Popping does not influence the next peep call
@given(lists(draw_syntax_random()))
def test_syntax_syntax_stream_mixed(nodes):
    stream = SyntaxStream(nodes.copy())
    read = []
    node = True
    while node is not None:
        peeked = stream.peek()
        node = stream.pop()
        read.append(node)
        assert peeked == node
    assert read[:-1] == nodes  # Skip None at end
    assert stream.pop() is None