Merge pull request #1221 from Tbruno25/1210_add-gzipreader

zariiii9003 · web-flow · commit d142257f0c84 · 2022-01-27T09:51:22.000+01:00
1210 Add compressed file methods
diff --git a/can/__init__.py b/can/__init__.py
@@ -25,7 +25,7 @@
 )
 
 from .io import Logger, SizedRotatingLogger, Printer, LogReader, MessageSync
-from .io import ASCWriter, ASCReader, GzipASCWriter, GzipASCReader
+from .io import ASCWriter, ASCReader
 from .io import BLFReader, BLFWriter
 from .io import CanutilsLogReader, CanutilsLogWriter
 from .io import CSVWriter, CSVReader
diff --git a/can/io/__init__.py b/can/io/__init__.py
@@ -8,7 +8,7 @@
 from .player import LogReader, MessageSync
 
 # Format specific
-from .asc import ASCWriter, ASCReader, GzipASCWriter, GzipASCReader
+from .asc import ASCWriter, ASCReader
 from .blf import BLFReader, BLFWriter
 from .canutils import CanutilsLogReader, CanutilsLogWriter
 from .csv import CSVWriter, CSVReader
diff --git a/can/io/asc.py b/can/io/asc.py
@@ -5,7 +5,6 @@
     - https://bitbucket.org/tobylorenz/vector_asc/src/47556e1a6d32c859224ca62d075e1efcc67fa690/src/Vector/ASC/tests/unittests/data/CAN_Log_Trigger_3_2.asc?at=master&fileviewer=file-view-default
     - under `test/data/logfile.asc`
 """
-import gzip
 import re
 from typing import Any, Generator, List, Optional, Dict, Union, TextIO
 
@@ -405,69 +404,3 @@ def on_message_received(self, msg: Message) -> None:
                 data=" ".join(data),
             )
         self.log_event(serialized, msg.timestamp)
-
-
-class GzipASCReader(ASCReader):
-    """Gzipped version of :class:`~can.ASCReader`"""
-
-    def __init__(
-        self,
-        file: Union[typechecking.FileLike, typechecking.StringPathLike],
-        base: str = "hex",
-        relative_timestamp: bool = True,
-    ):
-        """
-        :param file: a path-like object or as file-like object to read from
-                     If this is a file-like object, is has to opened in text
-                     read mode, not binary read mode.
-        :param base: Select the base(hex or dec) of id and data.
-                     If the header of the asc file contains base information,
-                     this value will be overwritten. Default "hex".
-        :param relative_timestamp: Select whether the timestamps are
-                     `relative` (starting at 0.0) or `absolute` (starting at
-                     the system time). Default `True = relative`.
-        """
-        self._fileobj = None
-        if file is not None and (hasattr(file, "read") and hasattr(file, "write")):
-            # file is None or some file-like object
-            self._fileobj = file
-        super(GzipASCReader, self).__init__(
-            gzip.open(file, mode="rt"), base, relative_timestamp
-        )
-
-    def stop(self) -> None:
-        super(GzipASCReader, self).stop()
-        if self._fileobj is not None:
-            self._fileobj.close()
-
-
-class GzipASCWriter(ASCWriter):
-    """Gzipped version of :class:`~can.ASCWriter`"""
-
-    def __init__(
-        self,
-        file: Union[typechecking.FileLike, typechecking.StringPathLike],
-        channel: int = 1,
-        compresslevel: int = 6,
-    ):
-        """
-        :param file: a path-like object or as file-like object to write to
-                     If this is a file-like object, is has to opened in text
-                     write mode, not binary write mode.
-        :param channel: a default channel to use when the message does not
-                        have a channel set
-        :param compresslevel: Gzip compresslevel, see
-                              :class:`~gzip.GzipFile` for details. The default is 6.
-        """
-        self._fileobj = None
-        if file is not None and (hasattr(file, "read") and hasattr(file, "write")):
-            # file is None or some file-like object
-            self._fileobj = file
-        super(GzipASCWriter, self).__init__(
-            gzip.open(file, mode="wt", compresslevel=compresslevel), channel
-        )
-
-    def stop(self) -> None:
-        super(GzipASCWriter, self).stop()
-        if self._fileobj is not None:
-            self._fileobj.close()
diff --git a/can/io/logger.py b/can/io/logger.py
@@ -6,13 +6,9 @@
 import pathlib
 from abc import ABC, abstractmethod
 from datetime import datetime
-from typing import (
-    Any,
-    Optional,
-    Callable,
-    cast,
-    Type,
-)
+import gzip
+from typing import Any, Optional, Callable, TextIO, Type, Tuple, Union, cast
+
 from types import TracebackType
 
 from typing_extensions import Literal
@@ -21,7 +17,7 @@
 from ..message import Message
 from ..listener import Listener
 from .generic import BaseIOHandler, FileIOMessageWriter
-from .asc import ASCWriter, GzipASCWriter
+from .asc import ASCWriter
 from .blf import BLFWriter
 from .canutils import CanutilsLogWriter
 from .csv import CSVWriter
@@ -34,15 +30,18 @@ class Logger(BaseIOHandler, Listener):  # pylint: disable=abstract-method
     """
     Logs CAN messages to a file.
 
-    The format is determined from the file format which can be one of:
+    The format is determined from the file suffix which can be one of:
       * .asc: :class:`can.ASCWriter`
-      * .asc.gz: :class:`can.CompressedASCWriter`
       * .blf :class:`can.BLFWriter`
       * .csv: :class:`can.CSVWriter`
       * .db: :class:`can.SqliteWriter`
       * .log :class:`can.CanutilsLogWriter`
       * .txt :class:`can.Printer`
 
+    Any of these formats can be used with gzip compression by appending
+    the suffix .gz (e.g. filename.asc.gz). However, third-party tools might not
+    be able to read these files.
+
     The **filename** may also be *None*, to fall back to :class:`can.Printer`.
 
     The log files may be incomplete until `stop()` is called due to buffering.
@@ -55,7 +54,6 @@ class Logger(BaseIOHandler, Listener):  # pylint: disable=abstract-method
     fetched_plugins = False
     message_writers = {
         ".asc": ASCWriter,
-        ".asc.gz": GzipASCWriter,
         ".blf": BLFWriter,
         ".csv": CSVWriter,
         ".db": SqliteWriter,
@@ -85,7 +83,11 @@ def __new__(  # type: ignore
             )
             Logger.fetched_plugins = True
 
-        suffix = "".join(s.lower() for s in pathlib.PurePath(filename).suffixes)
+        suffix = pathlib.PurePath(filename).suffix.lower()
+
+        if suffix == ".gz":
+            suffix, filename = Logger.compress(filename)
+
         try:
             return cast(
                 Listener, Logger.message_writers[suffix](filename, *args, **kwargs)
@@ -95,6 +97,17 @@ def __new__(  # type: ignore
                 f'No write support for this unknown log format "{suffix}"'
             ) from None
 
+    @staticmethod
+    def compress(filename: StringPathLike) -> Tuple[str, Union[str, Any]]:
+        """
+        Return the suffix and io object of the decompressed file.
+        File will automatically recompress upon close.
+        """
+        real_suffix = pathlib.Path(filename).suffixes[-2].lower()
+        mode = "ab" if real_suffix == ".blf" else "at"
+
+        return real_suffix, gzip.open(filename, mode)
+
     def on_message_received(self, msg: Message) -> None:
         pass
 
diff --git a/can/io/player.py b/can/io/player.py
@@ -3,7 +3,7 @@
 well as :class:`MessageSync` which plays back messages
 in the recorded order an time intervals.
 """
-
+import gzip
 import pathlib
 from time import time, sleep
 import typing
@@ -14,7 +14,7 @@
     import can
 
 from .generic import BaseIOHandler, MessageReader
-from .asc import ASCReader, GzipASCReader
+from .asc import ASCReader
 from .blf import BLFReader
 from .canutils import CanutilsLogReader
 from .csv import CSVReader
@@ -25,14 +25,17 @@ class LogReader(BaseIOHandler):
     """
     Replay logged CAN messages from a file.
 
-    The format is determined from the file format which can be one of:
+    The format is determined from the file suffix which can be one of:
       * .asc
-      * .asc.gz
       * .blf
       * .csv
       * .db
       * .log
 
+    Gzip compressed files can be used as long as the original
+    files suffix is one of the above (e.g. filename.asc.gz).
+
+
     Exposes a simple iterator interface, to use simply:
 
         >>> for msg in LogReader("some/path/to/my_file.log"):
@@ -50,7 +53,6 @@ class LogReader(BaseIOHandler):
     fetched_plugins = False
     message_readers = {
         ".asc": ASCReader,
-        ".asc.gz": GzipASCReader,
         ".blf": BLFReader,
         ".csv": CSVReader,
         ".db": SqliteReader,
@@ -77,7 +79,11 @@ def __new__(  # type: ignore
             )
             LogReader.fetched_plugins = True
 
-        suffix = "".join(s.lower() for s in pathlib.PurePath(filename).suffixes)
+        suffix = pathlib.PurePath(filename).suffix.lower()
+
+        if suffix == ".gz":
+            suffix, filename = LogReader.decompress(filename)
+
         try:
             return typing.cast(
                 MessageReader,
@@ -88,6 +94,18 @@ def __new__(  # type: ignore
                 f'No read support for this unknown log format "{suffix}"'
             ) from None
 
+    @staticmethod
+    def decompress(
+        filename: "can.typechecking.StringPathLike",
+    ) -> typing.Tuple[str, typing.Union[str, typing.Any]]:
+        """
+        Return the suffix and io object of the decompressed file.
+        """
+        real_suffix = pathlib.Path(filename).suffixes[-2].lower()
+        mode = "rb" if real_suffix == ".blf" else "rt"
+
+        return real_suffix, gzip.open(filename, mode)
+
 
 class MessageSync:  # pylint: disable=too-few-public-methods
     """
diff --git a/test/data/test_CanMessage.asc.gz b/test/data/test_CanMessage.asc.gz
diff --git a/test/logformats_test.py b/test/logformats_test.py
@@ -11,7 +11,6 @@
 
 TODO: correctly set preserves_channel and adds_default_channel
 """
-import gzip
 import logging
 import unittest
 import tempfile
@@ -48,6 +47,7 @@ def test_extension_matching(self):
             suffix_variants = [
                 suffix.upper(),
                 suffix.lower(),
+                f"can.msg.ext{suffix}",
                 "".join([c.upper() if i % 2 else c for i, c in enumerate(suffix)]),
             ]
             for suffix_variant in suffix_variants:
@@ -558,34 +558,6 @@ def test_ignore_comments(self):
         _msg_list = self._read_log_file("logfile.asc")
 
 
-class TestGzipASCFileFormat(ReaderWriterTest):
-    """Tests can.GzipASCWriter and can.GzipASCReader"""
-
-    def _setup_instance(self):
-        super()._setup_instance_helper(
-            can.GzipASCWriter,
-            can.GzipASCReader,
-            binary_file=True,
-            check_comments=True,
-            preserves_channel=False,
-            adds_default_channel=0,
-        )
-
-    def assertIncludesComments(self, filename):
-        """
-        Ensures that all comments are literally contained in the given file.
-
-        :param filename: the path-like object to use
-        """
-        if self.original_comments:
-            # read the entire outout file
-            with gzip.open(filename, "rt" if self.binary_file else "r") as file:
-                output_contents = file.read()
-            # check each, if they can be found in there literally
-            for comment in self.original_comments:
-                self.assertIn(comment, output_contents)
-
-
 class TestBlfFileFormat(ReaderWriterTest):
     """Tests can.BLFWriter and can.BLFReader.
 
diff --git a/test/test_logger.py b/test/test_logger.py
@@ -7,6 +7,8 @@
 import unittest
 from unittest import mock
 from unittest.mock import Mock
+import gzip
+import os
 import sys
 import can
 import can.logger
@@ -104,5 +106,39 @@ def test_log_virtual_sizedlogger(self):
         self.mock_logger_sized.assert_called_once()
 
 
+class TestLoggerCompressedFile(unittest.TestCase):
+    def setUp(self) -> None:
+        # Patch VirtualBus object
+        self.patcher_virtual_bus = mock.patch(
+            "can.interfaces.virtual.VirtualBus", spec=True
+        )
+        self.MockVirtualBus = self.patcher_virtual_bus.start()
+        self.mock_virtual_bus = self.MockVirtualBus.return_value
+
+        self.testmsg = can.Message(
+            arbitration_id=0xC0FFEE, data=[0, 25, 0, 1, 3, 1, 4, 1], is_extended_id=True
+        )
+        self.baseargs = [sys.argv[0], "-i", "virtual"]
+
+        self.testfile = open("coffee.log.gz", "w+")
+
+    def test_compressed_logfile(self):
+        """
+        Basic test to verify Logger is able to write gzip files.
+        """
+        self.mock_virtual_bus.recv = Mock(side_effect=[self.testmsg, KeyboardInterrupt])
+        sys.argv = self.baseargs + ["--file_name", self.testfile.name]
+        can.logger.main()
+        with gzip.open(self.testfile.name, "rt") as testlog:
+            last_line = testlog.readlines()[-1]
+
+        self.assertEqual(last_line, "(0.000000) vcan0 00C0FFEE#0019000103010401\n")
+
+    def tearDown(self) -> None:
+        self.testfile.close()
+        os.remove(self.testfile.name)
+        self.patcher_virtual_bus.stop()
+
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/test/test_player.py b/test/test_player.py
@@ -15,6 +15,9 @@
 
 
 class TestPlayerScriptModule(unittest.TestCase):
+
+    logfile = os.path.join(os.path.dirname(__file__), "data", "test_CanMessage.asc")
+
     def setUp(self) -> None:
         # Patch VirtualBus object
         patcher_virtual_bus = mock.patch("can.interfaces.virtual.VirtualBus", spec=True)
@@ -29,9 +32,6 @@ def setUp(self) -> None:
         self.addCleanup(patcher_sleep.stop)
 
         self.baseargs = [sys.argv[0], "-i", "virtual"]
-        self.logfile = os.path.join(
-            os.path.dirname(__file__), "data", "test_CanMessage.asc"
-        )
 
     def assertSuccessfulCleanup(self):
         self.MockVirtualBus.assert_called_once()
@@ -111,5 +111,13 @@ def test_play_error_frame(self):
         self.assertSuccessfulCleanup()
 
 
+class TestPlayerCompressedFile(TestPlayerScriptModule):
+    """
+    Re-run tests using a compressed file.
+    """
+
+    logfile = os.path.join(os.path.dirname(__file__), "data", "test_CanMessage.asc.gz")
+
+
 if __name__ == "__main__":
     unittest.main()

Original file line number	Diff line number	Diff line change
`@@ -25,7 +25,7 @@`
`25`	`25`	`)`
`26`	`26`
`27`	`27`	`from .io import Logger, SizedRotatingLogger, Printer, LogReader, MessageSync`
`28`		`-from .io import ASCWriter, ASCReader, GzipASCWriter, GzipASCReader`
	`28`	`+from .io import ASCWriter, ASCReader`
`29`	`29`	`from .io import BLFReader, BLFWriter`
`30`	`30`	`from .io import CanutilsLogReader, CanutilsLogWriter`
`31`	`31`	`from .io import CSVWriter, CSVReader`