#!/usr/bin/env python3 # SPDX-License-Identifier: MIT # Copyright 2021 Jookia <contact@jookia.org> def tokenizer(code): tokens = [] token = "" text = "" mode = "normal" for symbol in code: if symbol == " " or symbol == "\n": if token == "": pass elif token == "BeginNote": mode = "note" elif token == "EndNote": mode = "normal" elif token == "BeginText": mode = "text" elif token == "EndText": tokens.append(("Text", text[1:-8])) mode = "normal" text = "" elif token != "": if mode == "normal": tokens.append(("Symbol", token)) token = "" else: token += symbol if mode == "text": text += symbol return tokens def main(args): if len(args) != 2: print("Usage: main.py FILENAME") return 1 filename = args[1] testcode = open(filename).read() tokens = tokenizer(testcode) for t in tokens: print("TOKEN {0}: {1}".format(t[0], t[1])) return 0 if __name__ == "__main__": import sys sys.exit(main(sys.argv))