NewLang / tests /
# SPDX-License-Identifier: LGPL-2.1-only
# Copyright 2022 Jookia <>

from newlang import parse

# The parser had some logic along the lines of 'read token until whitespace',
# but this didn't account for hitting the end of file.
# Make sure the parser can handle tokens terminated by end of file correctly.
def test_regress_eof():
    tokenizer = parse.Tokenizer("Hello", "")
    tokens = tokenizer.tokenize()
    assert tokens[0].value == "Hello"

# The parser would read text literals by tracking the position just after of
# the StartText and EndText tokens, then reading the literal text between them.
# It would automatically remove EndText as well as the character after it.
# However, if EndText was the last token, this would cause the text to cut off.
# Make sure the parser can handle reading text at the end of a file.
def test_regress_text_eof():
    text = "Hi there!"
    code = "StartText " + text + " EndText"
    tokenizer1 = parse.Tokenizer(code, "")
    tokens1 = tokenizer1.tokenize()
    tokenizer2 = parse.Tokenizer(code + " ", "")
    tokens2 = tokenizer2.tokenize()
    assert tokens1[0].type == "text"
    assert tokens1[0].value == text
    assert tokens2[0].type == "text"
    assert tokens2[0].value == text