Merge branch 'token-to-string-converter' into 'main'

Create Token to string converter

See merge request mwasilew/2023-ZPRP!19

Author: Mateusz Bronisław Wasilewski

image_formatter/image_properties_tag_replacer/image_properties_tag_replacer.py

@@ -2,14 +2,15 @@
 
 from image_formatter.lexer.lexer import Lexer
 from image_formatter.lexer.token import TokenType, Token
+from image_formatter.lexer.token_stream_processor import TokenStreamProcessor
 from image_formatter.error_handler.error_handler import ErrorHandler
 from image_formatter.error_handler.errors import UnexpectedTagException
 from mkdocs.plugins import get_plugin_logger
 
 log = get_plugin_logger(__name__)
 
 
-class ImagePropertiesTagReplacer:
+class ImagePropertiesTagReplacer(TokenStreamProcessor):
     """
     Class ImagePropertiesTagReplacer responsible for replacing image size tags with properties after the image URL.
     Focuses only on the plugin's purpose - images with added size tags

image_formatter/lexer/lexer.py

@@ -1,5 +1,6 @@
-from image_formatter.lexer.token import Token, TokenType, IntegerToken
+from image_formatter.lexer.token import Token, TokenType, IntegerToken, TagToken
 from image_formatter.lexer.position import Position
+from image_formatter.lexer.token_stream_processor import TokenStreamProcessor
 from image_formatter.error_handler.errors import InvalidConfigCharacterError
 import io
 import sys
@@ -10,7 +11,7 @@
 log = get_plugin_logger(__name__)
 
 
-class Lexer:
+class Lexer(TokenStreamProcessor):
     """
     Class representing Lexer.
     Responsible for going through the characters from source input one by one and
@@ -262,11 +263,12 @@ def build_integer(self) -> IntegerToken or None:
     def _is_number_in_range(self, number):
         return number * 10 + int(self.current_char) <= self.max_int
 
-    def build_tag(self) -> Token or None:
+    def build_tag(self) -> TagToken or None:
         """
         Tries to build an image tag token according to:
         ```
-        image_size_tag = '@', literal
+        image_size_tag = tag_character, literal
+        tag_character by default is '@'
         ```
 
         Returns:
@@ -284,7 +286,7 @@ def build_tag(self) -> Token or None:
             log.info(f"{Lexer.name()}: Failed to build a tag. Missing token 'T_LITERAL'.")
             return None
         log.info(f"{Lexer.name()}: Tag built successfully. Returning 'T_IMAGE_SIZE_TAG' token.")
-        return Token(TokenType.T_IMAGE_SIZE_TAG, position, token.string)
+        return TagToken(TokenType.T_IMAGE_SIZE_TAG, position, token.string, self.tag)
 
     def get_url_ending(self, string: str) -> str or None:
         """

image_formatter/lexer/token.py

@@ -55,3 +55,20 @@ def __init__(self, type: TokenType, position: Position, integer: int):
         """
         super(IntegerToken, self).__init__(type, position, str(integer))
         self.integer = integer
+
+
+class TagToken(Token):
+    """
+    Class representing token of type tag.
+    """
+
+    def __init__(self, type: TokenType, position: Position, string: str = "", tag_character: str = ""):
+        """
+        Args:
+            type: type of the token
+            position: position of the first character of the token
+            string: final version of token's text
+            tag_character: characteristic character for the tag
+        """
+        super(TagToken, self).__init__(type, position, string)
+        self.tag_character = tag_character

image_formatter/lexer/token_stream_processor.py

@@ -0,0 +1,13 @@
+from image_formatter.lexer.token import Token
+
+
+class TokenStreamProcessor:
+    """
+    Base class for tokens stream processing.
+    """
+
+    def get_token(self) -> Token:
+        """
+        Returns Token.
+        """
+        raise NotImplementedError

image_formatter/token_to_string_converter/__init__.py

@@ -0,0 +1,21 @@
+from image_formatter.lexer.token_stream_processor import TokenStreamProcessor
+from image_formatter.lexer.token import TagToken
+
+
+class TokenToStringConverter:
+    def __init__(self, token_stream_processor: TokenStreamProcessor):
+        self.token_stream_processor = token_stream_processor
+
+    def to_text(self) -> str:
+        text = ""
+        for token in self.get_all_tokens():
+            if type(token) == TagToken:
+                text += f"{token.tag_character}"
+            text += token.string
+        return text
+
+    def get_all_tokens(self) -> list:
+        tokens = []
+        for t in self.token_stream_processor.get_token():
+            tokens.append(t)
+        return tokens

image_formatter/token_to_string_converter/token_to_string_converter.py

@@ -1,5 +1,5 @@
 from image_formatter.lexer.lexer import Lexer
-from image_formatter.lexer.token import Token, TokenType
+from image_formatter.lexer.token import Token, TokenType, TagToken
 from image_formatter.lexer.position import Position
 from image_formatter.image_properties_tag_replacer.image_properties_tag_replacer import ImagePropertiesTagReplacer
 import io
@@ -30,12 +30,12 @@ def test_given_no_image_links_then_nothing_is_replaced():
         Token(TokenType.T_CHAR, Position(1, 14), "$"),
         Token(TokenType.T_CHAR, Position(1, 15), "$"),
         Token(TokenType.T_WHITE_CHAR, Position(1, 16), " "),
-        Token(TokenType.T_WHITE_CHAR, Position(1, 17), "tag1-tag"),
+        TagToken(TokenType.T_IMAGE_SIZE_TAG, Position(1, 17), "tag1-tag"),
         Token(TokenType.T_WHITE_CHAR, Position(1, 26), " "),
         Token(TokenType.T_WHITE_CHAR, Position(1, 27), "\n"),
         Token(TokenType.T_WHITE_CHAR, Position(2, 1), "\n"),
         Token(TokenType.T_WHITE_CHAR, Position(3, 1), " "),
-        Token(TokenType.T_WHITE_CHAR, Position(3, 2), "tag2"),
+        TagToken(TokenType.T_IMAGE_SIZE_TAG, Position(3, 2), "tag2"),
         Token(TokenType.T_WHITE_CHAR, Position(3, 7), " "),
         Token(TokenType.T_CHAR, Position(3, 8), "x"),
         Token(TokenType.T_WHITE_CHAR, Position(3, 9), " "),
@@ -138,7 +138,7 @@ def test_given_sequence_of_tokens_with_one_valid_image_tag_then_one_image_tag_is
                 Token(TokenType.T_WHITE_CHAR, Position(1, 26), " "),
                 Token(TokenType.T_LITERAL, Position(1, 27), "word"),
                 Token(TokenType.T_WHITE_CHAR, Position(1, 31), " "),
-                Token(TokenType.T_IMAGE_SIZE_TAG, Position(1, 32), "big"),
+                TagToken(TokenType.T_IMAGE_SIZE_TAG, Position(1, 32), "big"),
                 Token(TokenType.T_WHITE_CHAR, Position(1, 36), " "),
                 Token(TokenType.T_CHAR, Position(1, 37), "*"),
                 Token(TokenType.T_IMAGE_URL, Position(1, 38), "(next/longer.url.jpg)"),

tests/image_properties_tag_replacer/test_integration_image_properties_tag_replacer_lexer.py

@@ -0,0 +1,66 @@
+import io
+
+from image_formatter.lexer.lexer import Lexer
+from image_formatter.image_properties_tag_replacer.image_properties_tag_replacer import ImagePropertiesTagReplacer
+from image_formatter.token_to_string_converter.token_to_string_converter import TokenToStringConverter
+
+# @ TODO inline global var
+image_tags_properties = {
+    "small": {"height": "100px", "width": "100px"},
+    "small2": {"height": "110px", "width": "110px"},
+}
+
+
+def test_file1_literals():
+    filename = "./resources/test_files/test1.txt"
+    with open(filename, "r") as fp:
+        expected = fp.read()
+    with open(filename, "r") as fp:
+        lexer = Lexer(fp)  # noqa
+        image_tag_replacer = ImagePropertiesTagReplacer(lexer, image_tags_properties)
+        converter = TokenToStringConverter(image_tag_replacer)
+        result = converter.to_text()
+    assert expected == result
+
+
+def test_file2_mock():
+    text = """
+    1hello1 &&@small2
+@small2(some/url.com)+word
+
+    """
+    expected = """
+    1hello1 &&@small2
+(some/url.com){: style="height:110px;width:110px"}+word
+
+    """
+    fp = io.StringIO(text)
+    lexer = Lexer(fp)  # noqa
+    image_tag_replacer = ImagePropertiesTagReplacer(lexer, image_tags_properties)
+    converter = TokenToStringConverter(image_tag_replacer)
+    result = converter.to_text()
+    assert expected == result
+
+
+def test_file3_classic_macos_newline():
+    filename = "./resources/test_files/test3_classic_macos_newline.txt"
+    with open(filename, "r") as fp:
+        expected = fp.read()
+    with open(filename, "r") as fp:
+        lexer = Lexer(fp)  # noqa
+        image_tag_replacer = ImagePropertiesTagReplacer(lexer, image_tags_properties)
+        converter = TokenToStringConverter(image_tag_replacer)
+        result = converter.to_text()
+    assert expected == result
+
+
+def test_file4_unix_and_macos_newline():
+    filename = "./resources/test_files/test4_unix_and_macos_newline.txt"
+    with open(filename, "r") as fp:
+        expected = fp.read()
+    with open(filename, "r") as fp:
+        lexer = Lexer(fp)  # noqa
+        image_tag_replacer = ImagePropertiesTagReplacer(lexer, image_tags_properties)
+        converter = TokenToStringConverter(image_tag_replacer)
+        result = converter.to_text()
+    assert expected == result