Skip to content

Commit dc1d3e8

Browse files
committed
dvc.objects.cache: set type in a more compatible way
It turns out that diskcache.Disk saves passed arguments to the database and uses that all the time. So this means that we are not forward compatible as it breaks moving from newer version of DVC to an older one. We don't promise this, but this breaks our benchmarks which are probably not isolated enough. Regarding the use of cache, it is a bit iffy to extend it this way. We only need this type for error messages. We could consider removing this, but since the error message is documented and it's not too hard to extend this :), I decided to preserve the message.
1 parent 3f5757a commit dc1d3e8

File tree

3 files changed

+9
-7
lines changed

3 files changed

+9
-7
lines changed

dvc/data/db/index.py

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -95,9 +95,7 @@ def __init__(
9595
self.index_dir = os.path.join(tmp_dir, self.INDEX_DIR, name)
9696
makedirs(self.index_dir, exist_ok=True)
9797
self.fs = LocalFileSystem()
98-
cache = Cache(
99-
self.index_dir, eviction_policy="none", disk_type="index"
100-
)
98+
cache = Cache(self.index_dir, eviction_policy="none", type="index")
10199
self.index = Index.fromcache(cache)
102100

103101
def __iter__(self):

dvc/objects/cache.py

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,8 @@ def wrapped(self, *args, **kwargs):
3535
class Disk(disk):
3636
"""Reraise pickle-related errors as DiskError."""
3737

38-
def __init__(self, *args, type=None, **kwargs):
39-
super().__init__(*args, **kwargs)
40-
self._type = type or os.path.basename(self._directory)
38+
# we need type to differentiate cache for better error messages
39+
_type: str
4140

4241
put = translate_pickle_error(disk.put)
4342
get = translate_pickle_error(disk.get)
@@ -53,9 +52,14 @@ def __init__(
5352
directory: str = None,
5453
timeout: int = 60,
5554
disk: disk = Disk, # pylint: disable=redefined-outer-name
55+
type: str = None,
5656
**settings: Any,
5757
) -> None:
5858
settings.setdefault("disk_pickle_protocol", 4)
5959
super().__init__(
6060
directory=directory, timeout=timeout, disk=disk, **settings
6161
)
62+
self.disk._type = self._type = type or os.path.basename(self.directory)
63+
64+
def __getstate__(self):
65+
return (*super().__getstate__(), self._type)

tests/unit/objects/test_cache.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ def test_pickle_protocol_error(tmp_dir, disk_type):
1717
cache = Cache(
1818
directory,
1919
disk_pickle_protocol=pickle.HIGHEST_PROTOCOL + 1,
20-
disk_type=disk_type,
20+
type=disk_type,
2121
)
2222
with pytest.raises(DiskError) as exc, cache as cache:
2323
set_value(cache, "key", ("value1", "value2"))

0 commit comments

Comments
 (0)