Newer
Older
NewLang / src / token.py
# SPDX-License-Identifier: LGPL-2.1-only
# Copyright 2022 Jookia <contact@jookia.org>


# Represents a token
class Token:
    def __init__(self, value, location):
        self.value = value
        self.location = location

    def __repr__(self):
        return "Token(value %s, location %s)" % (  # pragma: no mutate
            repr(self.value),
            repr(self.location),
        )

    def __eq__(self, other):
        if other is None:
            return False
        return self.value == other.value and self.location == other.location


# Location of a token
class TokenLocation:
    def __init__(self, line, offset, file):
        self.line = line
        self.offset = offset
        self.file = file

    def __repr__(self):
        return "TokenLocation(line %i, offset %i, file '%s')" % (  # pragma: no mutate
            self.line,
            self.offset,
            self.file,
        )

    def __eq__(self, other):
        if other is None:
            return False
        return (
            self.line == other.line
            and self.offset == other.offset
            and self.file == other.file
        )


# Represents a stream of consumable tokens
class TokenStream:
    def __init__(self, tokens):
        self.tokens = tokens

    def __repr__(self):
        return "TokenStream(%s)" % (self.tokens)  # pragma: no mutate

    def pop(self):
        if self.tokens:
            return self.tokens.pop(0)
        else:
            return None

    def peek(self):
        if self.tokens:
            return self.tokens[0]
        else:
            return None