Skip to content

Commit 64ac473

Browse files
paulrosspaulross
committed
WIP on cPickle.
1 parent 46a0946 commit 64ac473

File tree

3 files changed

+82
-13
lines changed

3 files changed

+82
-13
lines changed

‎setup.py

+6-6
Original file line numberDiff line numberDiff line change
@@ -97,12 +97,12 @@
9797
language='c',
9898
),
9999
# Legacy code, see src/cpy/cParseArgsHelper.cpp for comments.
100-
# Extension(f"{PACKAGE_NAME}.cParseArgsHelper", sources=['src/cpy/cParseArgsHelper.cpp', ],
101-
# include_dirs=['/usr/local/include', ], # os.path.join(os.getcwd(), 'include'),],
102-
# library_dirs=[os.getcwd(), ], # path to .a or .so file(s)
103-
# extra_compile_args=extra_compile_args_cpp,
104-
# language='c++11',
105-
# ),
100+
Extension(f"{PACKAGE_NAME}.cParseArgsHelper", sources=['src/cpy/cParseArgsHelper.cpp', ],
101+
include_dirs=['/usr/local/include', ], # os.path.join(os.getcwd(), 'include'),],
102+
library_dirs=[os.getcwd(), ], # path to .a or .so file(s)
103+
extra_compile_args=extra_compile_args_cpp,
104+
language='c++11',
105+
),
106106
Extension(f"{PACKAGE_NAME}.cPyRefs", sources=['src/cpy/cPyRefs.c', ],
107107
include_dirs=['/usr/local/include', ], # os.path.join(os.getcwd(), 'include'),],
108108
library_dirs=[os.getcwd(), ], # path to .a or .so file(s)

‎src/cpy/Pickle/cCustomPickle.c

+10-2
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
#include <Python.h>
1212
#include "structmember.h"
1313

14+
#define FPRINTF_DEBUG 0
15+
1416
typedef struct {
1517
PyObject_HEAD
1618
PyObject *first; /* first name */
@@ -44,7 +46,9 @@ Custom_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kw
4446
}
4547
self->number = 0;
4648
}
49+
#if FPRINTF_DEBUG
4750
fprintf(stdout, "Custom_new() reference counts first %zu last %zu\n", Py_REFCNT(self->first), Py_REFCNT(self->last));
51+
#endif
4852
return (PyObject *) self;
4953
}
5054

@@ -109,17 +113,19 @@ Custom___getstate__(CustomObject *self, PyObject *Py_UNUSED(ignored)) {
109113
"last", self->last,
110114
"number", self->number,
111115
PICKLE_VERSION_KEY, PICKLE_VERSION);
116+
#if FPRINTF_DEBUG
112117
fprintf(stdout, "Custom___getstate__ returning type %s\n", Py_TYPE(ret)->tp_name);
118+
#endif
113119
return ret;
114120
}
115121

116122

117123
static PyObject *
118124
Custom___setstate__(CustomObject *self, PyObject *state) {
125+
#if FPRINTF_DEBUG
119126
fprintf(stdout, "Custom___getstate__ getting type %s\n", Py_TYPE(state)->tp_name);
120127
PyObject *key, *value;
121128
Py_ssize_t pos = 0;
122-
123129
while (PyDict_Next(state, &pos, &key, &value)) {
124130
/* do something interesting with the values... */
125131
fprintf(stdout, "Types Key: %s Value: %s\n", Py_TYPE(key)->tp_name, Py_TYPE(value)->tp_name);
@@ -130,6 +136,7 @@ Custom___setstate__(CustomObject *self, PyObject *state) {
130136
fprintf(stdout, "\n");
131137
}
132138
fprintf(stdout, "Initial reference counts first %zu last %zu\n", Py_REFCNT(self->first), Py_REFCNT(self->last));
139+
#endif
133140

134141
// static char *kwlist[] = {"first", "last", "number", NULL};
135142
//
@@ -198,9 +205,10 @@ Custom___setstate__(CustomObject *self, PyObject *state) {
198205
return NULL;
199206
}
200207
self->number = (int) PyLong_AsLong(number);
201-
208+
#if FPRINTF_DEBUG
202209
fprintf(stdout, "Final reference counts first %zu last %zu\n", Py_REFCNT(self->first),
203210
Py_REFCNT(self->last));
211+
#endif
204212
Py_RETURN_NONE;
205213
}
206214

‎tests/unit/test_c_custom_pickle.py

+66-5
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import io
12
import pickle
23
import pickletools
34
import sys
@@ -11,13 +12,73 @@ def test_module_dir():
1112
assert dir(cPickle) == ['Custom', '__doc__', '__file__', '__loader__', '__name__', '__package__', '__spec__']
1213

1314

15+
ARGS_FOR_CUSTOM_CLASS = ('FIRST', 'LAST', 11)
16+
PICKLE_BYTES_FOR_CUSTOM_CLASS = (b'\x80\x04\x95f\x00\x00\x00\x00\x00\x00\x00\x8c\x12cPyExtPatt.cPickle\x94'
17+
b'\x8c\x06Custom\x94\x93\x94)\x81\x94}\x94(\x8c\x05first\x94\x8c\x05FIRST'
18+
b'\x94\x8c\x04last\x94\x8c\x04LAST\x94\x8c\x06number\x94K\x0b\x8c\x0f_pickle_'
19+
b'version\x94K\x01ub.')
20+
21+
1422
def test_pickle_getstate():
15-
custom = cPickle.Custom('FIRST', 'LAST', 11)
23+
custom = cPickle.Custom(*ARGS_FOR_CUSTOM_CLASS)
1624
pickled_value = pickle.dumps(custom)
1725
print()
1826
print(f'Pickled original is {pickled_value}')
19-
assert pickled_value == (b'\x80\x04\x95f\x00\x00\x00\x00\x00\x00\x00\x8c\x12cPyExtPatt.cPickle\x94'
20-
b'\x8c\x06Custom\x94\x93\x94)\x81\x94}\x94(\x8c\x05first\x94\x8c\x05FIRST'
21-
b'\x94\x8c\x04last\x94\x8c\x04LAST\x94\x8c\x06number\x94K\x0b\x8c\x0f_pickle_'
22-
b'version\x94K\x01ub.')
27+
assert pickled_value == PICKLE_BYTES_FOR_CUSTOM_CLASS
2328
# result = pickle.loads(pickled_value)
29+
30+
31+
def test_pickle_setstate():
32+
custom = pickle.loads(PICKLE_BYTES_FOR_CUSTOM_CLASS)
33+
assert custom.first == 'FIRST'
34+
assert custom.last == 'LAST'
35+
assert custom.number == 11
36+
37+
38+
def test_pickle_round_trip():
39+
custom = cPickle.Custom(*ARGS_FOR_CUSTOM_CLASS)
40+
pickled_value = pickle.dumps(custom)
41+
result = pickle.loads(pickled_value)
42+
assert id(result) != id(custom)
43+
44+
45+
def test_pickletools():
46+
outfile = io.StringIO()
47+
pickletools.dis(PICKLE_BYTES_FOR_CUSTOM_CLASS, out=outfile, annotate=1)
48+
result = outfile.getvalue()
49+
# print()
50+
# print(result)
51+
expected = """ 0: \\x80 PROTO 4 Protocol version indicator.
52+
2: \\x95 FRAME 102 Indicate the beginning of a new frame.
53+
11: \\x8c SHORT_BINUNICODE 'cPyExtPatt.cPickle' Push a Python Unicode string object.
54+
31: \\x94 MEMOIZE (as 0) Store the stack top into the memo. The stack is not popped.
55+
32: \\x8c SHORT_BINUNICODE 'Custom' Push a Python Unicode string object.
56+
40: \\x94 MEMOIZE (as 1) Store the stack top into the memo. The stack is not popped.
57+
41: \\x93 STACK_GLOBAL Push a global object (module.attr) on the stack.
58+
42: \\x94 MEMOIZE (as 2) Store the stack top into the memo. The stack is not popped.
59+
43: ) EMPTY_TUPLE Push an empty tuple.
60+
44: \\x81 NEWOBJ Build an object instance.
61+
45: \\x94 MEMOIZE (as 3) Store the stack top into the memo. The stack is not popped.
62+
46: } EMPTY_DICT Push an empty dict.
63+
47: \\x94 MEMOIZE (as 4) Store the stack top into the memo. The stack is not popped.
64+
48: ( MARK Push markobject onto the stack.
65+
49: \\x8c SHORT_BINUNICODE 'first' Push a Python Unicode string object.
66+
56: \\x94 MEMOIZE (as 5) Store the stack top into the memo. The stack is not popped.
67+
57: \\x8c SHORT_BINUNICODE 'FIRST' Push a Python Unicode string object.
68+
64: \\x94 MEMOIZE (as 6) Store the stack top into the memo. The stack is not popped.
69+
65: \\x8c SHORT_BINUNICODE 'last' Push a Python Unicode string object.
70+
71: \\x94 MEMOIZE (as 7) Store the stack top into the memo. The stack is not popped.
71+
72: \\x8c SHORT_BINUNICODE 'LAST' Push a Python Unicode string object.
72+
78: \\x94 MEMOIZE (as 8) Store the stack top into the memo. The stack is not popped.
73+
79: \\x8c SHORT_BINUNICODE 'number' Push a Python Unicode string object.
74+
87: \\x94 MEMOIZE (as 9) Store the stack top into the memo. The stack is not popped.
75+
88: K BININT1 11 Push a one-byte unsigned integer.
76+
90: \\x8c SHORT_BINUNICODE '_pickle_version' Push a Python Unicode string object.
77+
107: \\x94 MEMOIZE (as 10) Store the stack top into the memo. The stack is not popped.
78+
108: K BININT1 1 Push a one-byte unsigned integer.
79+
110: u SETITEMS (MARK at 48) Add an arbitrary number of key+value pairs to an existing dict.
80+
111: b BUILD Finish building an object, via __setstate__ or dict update.
81+
112: . STOP Stop the unpickling machine.
82+
highest protocol among opcodes = 4
83+
"""
84+
assert result == expected

0 commit comments

Comments
 (0)