diff --git a/src/tokenize.py b/src/tokenize.py index d4d60be..8286a0f 100644 --- a/src/tokenize.py +++ b/src/tokenize.py @@ -47,6 +47,7 @@ SPACE = enum.auto() # pragma: no mutate NEWLINE = enum.auto() # pragma: no mutate BOOL = enum.auto() # pragma: no mutate + KEYWORD = enum.auto() # pragma: no mutate # Represents a tokenizer token @@ -98,6 +99,24 @@ return tokens +# Keywords recognized by the language +keywords = [ + "NewLang", + "Done", + "Set", + "To", + "EndSet", + "If", + "Then", + "Else", + "EndIf", + "StartNote", + "EndNote", + "StartText", + "EndText", +] + + # Classifies tokens in to types def classify_tokens(tokens): new_tokens = [] @@ -108,6 +127,8 @@ type = TokenType.SPACE elif t.value in ["True", "False"]: type = TokenType.BOOL + elif t.value in keywords: + type = TokenType.KEYWORD else: type = TokenType.UNKNOWN new = Token(t.value, t.location, type) diff --git a/tests/test_tokenize.py b/tests/test_tokenize.py index 70c02ef..c7976d1 100644 --- a/tests/test_tokenize.py +++ b/tests/test_tokenize.py @@ -17,6 +17,24 @@ from src import tokenize +# Keywords recognized by the language +keywords = [ + "NewLang", + "Done", + "Set", + "To", + "EndSet", + "If", + "Then", + "Else", + "EndIf", + "StartNote", + "EndNote", + "StartText", + "EndText", +] + + # Draws a random token location @composite def draw_token_location(draw): @@ -49,6 +67,7 @@ chars = characters(blacklist_characters=reserved) value = draw(text(alphabet=chars, min_size=1)) assume(value not in ["True", "False"]) + assume(value not in keywords) type = tokenize.TokenType.UNKNOWN return tokenize.Token(value, token.location, type) @@ -84,6 +103,15 @@ return tokenize.Token(value, token.location, type) +# Draws a keyword token +@composite +def draw_token_keyword(draw): + token = draw(draw_token_random()) + value = draw(sampled_from(keywords)) + type = tokenize.TokenType.KEYWORD + return tokenize.Token(value, token.location, type) + + # Draws a classified token @composite def draw_token_classified(draw): @@ -92,6 +120,7 @@ draw_token_space(), draw_token_newline(), draw_token_bool(), + draw_token_keyword(), ] token = draw(one_of(strategies)) return token