feat: [lexer] Added first basic implementation

Author: ASypula

.gitignore

@@ -0,0 +1 @@
+venv/

lexer/__init__.py

@@ -0,0 +1,78 @@
+from lexer.token import Token, TokenType
+
+SPECIAL_SIGNS = ["-", "_"]
+TAG_CHAR = "@"
+
+
+class Lexer:
+    curr_char = ""
+
+    def __init__(self, fp):
+        self.fp = fp
+        self.running = True
+
+    @staticmethod
+    def is_character(char: str) -> bool:
+        return char.isalnum() or char in SPECIAL_SIGNS
+
+    # TODO: a better way of taking next characters?
+    def next_char(self) -> str:
+        self.curr_char = self.fp.read(1)
+        if not self.curr_char:
+            self.running = False
+
+    def build_literal(self):
+        if not self.curr_char.isalpha():
+            return 0
+        literal = self.curr_char
+        self.next_char()
+        while Lexer.is_character(self.curr_char):
+            literal += self.curr_char
+            self.next_char()
+        return Token(TokenType.T_LITERAL, literal)
+
+    def build_tag(self):
+        if not self.curr_char == TAG_CHAR:
+            return 0
+        self.next_char()
+        token = self.build_literal()
+        if token.type != TokenType.T_LITERAL:
+            return 0
+        return Token(TokenType.T_IMAGE_SIZE_TAG, token.string)
+
+    def get_url_ending(self, string):
+        if self.curr_char != ".":
+            return 0
+        string += self.curr_char
+        self.next_char()
+        while Lexer.is_character(self.curr_char) or self.curr_char in ["/", "."]:
+            string += self.curr_char
+            self.next_char()
+        return string
+
+    def build_url(self):
+        if not self.curr_char == "(":
+            return 0
+        self.next_char()
+        string = ""
+        while Lexer.is_character(self.curr_char) or self.curr_char == "/":
+            string += self.curr_char
+            self.next_char()
+        if not (string := self.get_url_ending(string)):
+            return 0
+        if not self.curr_char == ")":
+            return 0
+        self.next_char()
+        return Token(TokenType.T_IMAGE_URL, string)
+
+    def get_token(self):
+        if self.running:
+            # watch out, the below works starting Python 3.8
+            if (
+                (token := self.build_tag())
+                or (token := self.build_url())
+                or (token := self.build_literal())
+            ):
+                return token
+        else:
+            return Token(TokenType.T_EOF)

lexer/lexer.py

@@ -0,0 +1,14 @@
+from enum import Enum
+
+
+class TokenType(Enum):
+    T_LITERAL = 0
+    T_IMAGE_URL = 1
+    T_IMAGE_SIZE_TAG = 2
+    T_EOF = 3
+
+
+class Token:
+    def __init__(self, type: TokenType, string: str = ""):
+        self.type = type
+        self.string = string

lexer/token.py

@@ -0,0 +1,2 @@
+hello
+second

lexer_examples/test1.txt

@@ -0,0 +1,2 @@
+hello @small
+(some/url.com)+word

lexer_examples/test2.txt

@@ -0,0 +1,16 @@
+from lexer.lexer import Lexer
+
+if __name__ == "__main__":
+    # filename = "lexer_examples/test1.txt"
+    filename = "lexer_examples/test2.txt"
+    with open(filename) as fp:
+        lexer = Lexer(fp)
+        lexer.next_char()
+        i = 0
+        while lexer.running:
+            token = lexer.get_token()
+            if token:
+                print(f"{i}. Token: {token.type.name}, value: {token.string}")
+                i += 1
+            else:
+                lexer.next_char()

main.py

@@ -0,0 +1,11 @@
+black==23.10.1
+click==8.1.7
+colorama==0.4.6
+flake8==6.1.0
+mccabe==0.7.0
+mypy-extensions==1.0.0
+packaging==23.2
+pathspec==0.11.2
+platformdirs==3.11.0
+pycodestyle==2.11.1
+pyflakes==3.1.0