Cache generated un/structure functions using linecache

Author: Tinche

Diff for: HISTORY.rst

@@ -6,6 +6,7 @@ History
 ------------------
 * Fix ``GenConverter`` mapping structuring for unannotated dicts on Python 3.8.
   (`#151 <https://github.com/Tinche/cattrs/issues/151>`_)
+* The source code for generated un/structuring functions is stored in the `linecache` cache, which enables more informative stack traces when un/structuring errors happen using the `GenConverter`. This behavior can optionally be disabled to save memory.
 
 1.7.1 (2021-05-28)
 ------------------

Diff for: src/cattr/gen.py

@@ -1,4 +1,6 @@
+import linecache
 import re
+import uuid
 from dataclasses import is_dataclass
 from typing import Any, Optional, Type, TypeVar
 
@@ -21,7 +23,13 @@ def override(omit_if_default=None, rename=None):
 _neutral = AttributeOverride()
 
 
-def make_dict_unstructure_fn(cl, converter, omit_if_default=False, **kwargs):
+def make_dict_unstructure_fn(
+    cl,
+    converter,
+    omit_if_default: bool = False,
+    _cattrs_use_linecache: bool = True,
+    **kwargs,
+):
     """Generate a specialized dict unstructuring function for an attrs class."""
     cl_name = cl.__name__
     fn_name = "unstructure_" + cl_name
@@ -31,7 +39,7 @@ def make_dict_unstructure_fn(cl, converter, omit_if_default=False, **kwargs):
 
     attrs = adapted_fields(cl)  # type: ignore
 
-    lines.append(f"def {fn_name}(i):")
+    lines.append(f"def {fn_name}(instance):")
     lines.append("    res = {")
     for a in attrs:
         attr_name = a.name
@@ -50,11 +58,11 @@ def make_dict_unstructure_fn(cl, converter, omit_if_default=False, **kwargs):
         is_identity = handler == converter._unstructure_identity
 
         if not is_identity:
-            unstruct_handler_name = f"__cattr_unstruct_handler_{attr_name}"
+            unstruct_handler_name = f"unstructure_{attr_name}"
             globs[unstruct_handler_name] = handler
-            invoke = f"{unstruct_handler_name}(i.{attr_name})"
+            invoke = f"{unstruct_handler_name}(instance.{attr_name})"
         else:
-            invoke = f"i.{attr_name}"
+            invoke = f"instance.{attr_name}"
 
         if d is not attr.NOTHING and (
             (omit_if_default and override.omit_if_default is not False)
@@ -66,14 +74,18 @@ def make_dict_unstructure_fn(cl, converter, omit_if_default=False, **kwargs):
                 globs[def_name] = d.factory
                 if d.takes_self:
                     post_lines.append(
-                        f"    if i.{attr_name} != {def_name}(i):"
+                        f"    if instance.{attr_name} != {def_name}(instance):"
                     )
                 else:
-                    post_lines.append(f"    if i.{attr_name} != {def_name}():")
+                    post_lines.append(
+                        f"    if instance.{attr_name} != {def_name}():"
+                    )
                 post_lines.append(f"        res['{kn}'] = {invoke}")
             else:
                 globs[def_name] = d
-                post_lines.append(f"    if i.{attr_name} != {def_name}:")
+                post_lines.append(
+                    f"    if instance.{attr_name} != {def_name}:"
+                )
                 post_lines.append(f"        res['{kn}'] = {invoke}")
 
         else:
@@ -82,10 +94,17 @@ def make_dict_unstructure_fn(cl, converter, omit_if_default=False, **kwargs):
     lines.append("    }")
 
     total_lines = lines + post_lines + ["    return res"]
+    script = "\n".join(total_lines)
 
-    eval(compile("\n".join(total_lines), "", "exec"), globs)
+    fname = _generate_unique_filename(
+        cl, "unstructure", reserve=_cattrs_use_linecache
+    )
+
+    eval(compile(script, fname, "exec"), globs)
 
     fn = globs[fn_name]
+    if _cattrs_use_linecache:
+        linecache.cache[fname] = len(script), None, total_lines, fname
 
     return fn
 
@@ -110,7 +129,11 @@ def generate_mapping(cl: Type, old_mapping):
 
 
 def make_dict_structure_fn(
-    cl: Type, converter, _cattrs_forbid_extra_keys: bool = False, **kwargs
+    cl: Type,
+    converter,
+    _cattrs_forbid_extra_keys: bool = False,
+    _cattrs_use_linecache: bool = True,
+    **kwargs,
 ):
     """Generate a specialized dict structuring function for an attrs class."""
 
@@ -167,20 +190,20 @@ def make_dict_structure_fn(
         else:
             handler = converter.structure
 
-        struct_handler_name = f"__cattr_struct_handler_{an}"
+        struct_handler_name = f"structure_{an}"
         globs[struct_handler_name] = handler
 
         ian = an if (is_dc or an[0] != "_") else an[1:]
         kn = an if override.rename is None else override.rename
-        globs[f"__c_t_{an}"] = type
+        globs[f"type_{an}"] = type
         if a.default is NOTHING:
             lines.append(
-                f"    '{ian}': {struct_handler_name}(o['{kn}'], __c_t_{an}),"
+                f"    '{ian}': {struct_handler_name}(o['{kn}'], type_{an}),"
             )
         else:
             post_lines.append(f"  if '{kn}' in o:")
             post_lines.append(
-                f"    res['{ian}'] = {struct_handler_name}(o['{kn}'], __c_t_{an})"
+                f"    res['{ian}'] = {struct_handler_name}(o['{kn}'], type_{an})"
             )
     lines.append("    }")
     if _cattrs_forbid_extra_keys:
@@ -196,7 +219,20 @@ def make_dict_structure_fn(
 
     total_lines = lines + post_lines + ["  return __cl(**res)"]
 
-    eval(compile("\n".join(total_lines), "", "exec"), globs)
+    fname = _generate_unique_filename(
+        cl, "structure", reserve=_cattrs_use_linecache
+    )
+    script = "\n".join(total_lines)
+    eval(
+        compile(
+            script,
+            fname,
+            "exec",
+        ),
+        globs,
+    )
+    if _cattrs_use_linecache:
+        linecache.cache[fname] = len(script), None, total_lines, fname
 
     return globs[fn_name]
 
@@ -396,3 +432,35 @@ def make_mapping_structure_fn(
     fn = globs[fn_name]
 
     return fn
+
+
+def _generate_unique_filename(cls, func_name, reserve=True):
+    """
+    Create a "filename" suitable for a function being generated.
+    """
+    unique_id = uuid.uuid4()
+    extra = ""
+    count = 1
+
+    while True:
+        unique_filename = "<cattrs generated {0} {1}.{2}{3}>".format(
+            func_name,
+            cls.__module__,
+            getattr(cls, "__qualname__", cls.__name__),
+            extra,
+        )
+        if not reserve:
+            return unique_filename
+        # To handle concurrency we essentially "reserve" our spot in
+        # the linecache with a dummy line.  The caller can then
+        # set this value correctly.
+        cache_line = (1, None, (str(unique_id),), unique_filename)
+        if (
+            linecache.cache.setdefault(unique_filename, cache_line)
+            == cache_line
+        ):
+            return unique_filename
+
+        # Looks like this spot is taken. Try again.
+        count += 1
+        extra = "-{0}".format(count)

Diff for: tests/test_gen.py

@@ -0,0 +1,72 @@
+"""Tests for functionality from the gen module."""
+import linecache
+from traceback import format_exc
+
+from attr import define
+
+from cattr import GenConverter
+from cattr.gen import make_dict_structure_fn, make_dict_unstructure_fn
+
+
+def test_structure_linecache():
+    """Linecaching for structuring should work."""
+
+    @define
+    class A:
+        a: int
+
+    c = GenConverter()
+    try:
+        c.structure({"a": "test"}, A)
+    except ValueError:
+        res = format_exc()
+        assert "'a'" in res
+
+
+def test_unstructure_linecache():
+    """Linecaching for unstructuring should work."""
+
+    @define
+    class Inner:
+        a: int
+
+    @define
+    class Outer:
+        inner: Inner
+
+    c = GenConverter()
+    try:
+        c.unstructure(Outer({}))
+    except AttributeError:
+        res = format_exc()
+        assert "'a'" in res
+
+
+def test_no_linecache():
+    """Linecaching should be disableable."""
+
+    @define
+    class A:
+        a: int
+
+    c = GenConverter()
+    before = len(linecache.cache)
+    c.structure(c.unstructure(A(1)), A)
+    after = len(linecache.cache)
+
+    assert after == before + 2
+
+    @define
+    class B:
+        a: int
+
+    before = len(linecache.cache)
+    c.register_structure_hook(
+        B, make_dict_structure_fn(B, c, _cattrs_use_linecache=False)
+    )
+    c.register_unstructure_hook(
+        B, make_dict_unstructure_fn(B, c, _cattrs_use_linecache=False)
+    )
+    c.structure(c.unstructure(B(1)), B)
+
+    assert len(linecache.cache) == before