GH-132983: PEP 7 and Argument Clinic changes for zstd (#133791)

Author: AA-Turner

Lib/compression/zstd/__init__.py

@@ -71,7 +71,7 @@ def get_frame_info(frame_buffer):
     the frame may or may not need a dictionary to be decoded,
     and the ID of such a dictionary is not specified.
     """
-    return FrameInfo(*_zstd._get_frame_info(frame_buffer))
+    return FrameInfo(*_zstd.get_frame_info(frame_buffer))
 
 
 def train_dict(samples, dict_size):
@@ -91,7 +91,7 @@ def train_dict(samples, dict_size):
     chunk_sizes = tuple(_nbytes(sample) for sample in samples)
     if not chunks:
         raise ValueError("samples contained no data; can't train dictionary.")
-    dict_content = _zstd._train_dict(chunks, chunk_sizes, dict_size)
+    dict_content = _zstd.train_dict(chunks, chunk_sizes, dict_size)
     return ZstdDict(dict_content)
 
 
@@ -127,7 +127,7 @@ def finalize_dict(zstd_dict, /, samples, dict_size, level):
     if not chunks:
         raise ValueError("The samples are empty content, can't finalize the"
                          "dictionary.")
-    dict_content = _zstd._finalize_dict(zstd_dict.dict_content,
+    dict_content = _zstd.finalize_dict(zstd_dict.dict_content,
                                         chunks, chunk_sizes,
                                         dict_size, level)
     return ZstdDict(dict_content)
@@ -201,7 +201,7 @@ def bounds(self):
 
         Both the lower and upper bounds are inclusive.
         """
-        return _zstd._get_param_bounds(self.value, is_compress=True)
+        return _zstd.get_param_bounds(self.value, is_compress=True)
 
 
 class DecompressionParameter(enum.IntEnum):
@@ -214,7 +214,7 @@ def bounds(self):
 
         Both the lower and upper bounds are inclusive.
         """
-        return _zstd._get_param_bounds(self.value, is_compress=False)
+        return _zstd.get_param_bounds(self.value, is_compress=False)
 
 
 class Strategy(enum.IntEnum):
@@ -237,4 +237,4 @@ class Strategy(enum.IntEnum):
 
 
 # Check validity of the CompressionParameter & DecompressionParameter types
-_zstd._set_parameter_types(CompressionParameter, DecompressionParameter)
+_zstd.set_parameter_types(CompressionParameter, DecompressionParameter)

Lib/test/test_zstd.py

@@ -1174,43 +1174,43 @@ def test_finalize_dict_arguments(self):
     def test_train_dict_c(self):
         # argument wrong type
         with self.assertRaises(TypeError):
-            _zstd._train_dict({}, (), 100)
+            _zstd.train_dict({}, (), 100)
         with self.assertRaises(TypeError):
-            _zstd._train_dict(b'', 99, 100)
+            _zstd.train_dict(b'', 99, 100)
         with self.assertRaises(TypeError):
-            _zstd._train_dict(b'', (), 100.1)
+            _zstd.train_dict(b'', (), 100.1)
 
         # size > size_t
         with self.assertRaises(ValueError):
-            _zstd._train_dict(b'', (2**64+1,), 100)
+            _zstd.train_dict(b'', (2**64+1,), 100)
 
         # dict_size <= 0
         with self.assertRaises(ValueError):
-            _zstd._train_dict(b'', (), 0)
+            _zstd.train_dict(b'', (), 0)
 
     def test_finalize_dict_c(self):
         with self.assertRaises(TypeError):
-            _zstd._finalize_dict(1, 2, 3, 4, 5)
+            _zstd.finalize_dict(1, 2, 3, 4, 5)
 
         # argument wrong type
         with self.assertRaises(TypeError):
-            _zstd._finalize_dict({}, b'', (), 100, 5)
+            _zstd.finalize_dict({}, b'', (), 100, 5)
         with self.assertRaises(TypeError):
-            _zstd._finalize_dict(TRAINED_DICT.dict_content, {}, (), 100, 5)
+            _zstd.finalize_dict(TRAINED_DICT.dict_content, {}, (), 100, 5)
         with self.assertRaises(TypeError):
-            _zstd._finalize_dict(TRAINED_DICT.dict_content, b'', 99, 100, 5)
+            _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', 99, 100, 5)
         with self.assertRaises(TypeError):
-            _zstd._finalize_dict(TRAINED_DICT.dict_content, b'', (), 100.1, 5)
+            _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100.1, 5)
         with self.assertRaises(TypeError):
-            _zstd._finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, 5.1)
+            _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, 5.1)
 
         # size > size_t
         with self.assertRaises(ValueError):
-            _zstd._finalize_dict(TRAINED_DICT.dict_content, b'', (2**64+1,), 100, 5)
+            _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (2**64+1,), 100, 5)
 
         # dict_size <= 0
         with self.assertRaises(ValueError):
-            _zstd._finalize_dict(TRAINED_DICT.dict_content, b'', (), 0, 5)
+            _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 0, 5)
 
     def test_train_buffer_protocol_samples(self):
         def _nbytes(dat):
@@ -1232,19 +1232,19 @@ def _nbytes(dat):
         # wrong size list
         with self.assertRaisesRegex(ValueError,
                 "The samples size tuple doesn't match the concatenation's size"):
-            _zstd._train_dict(concatenation, tuple(wrong_size_lst), 100*_1K)
+            _zstd.train_dict(concatenation, tuple(wrong_size_lst), 100*_1K)
 
         # correct size list
-        _zstd._train_dict(concatenation, tuple(correct_size_lst), 3*_1K)
+        _zstd.train_dict(concatenation, tuple(correct_size_lst), 3*_1K)
 
         # wrong size list
         with self.assertRaisesRegex(ValueError,
                 "The samples size tuple doesn't match the concatenation's size"):
-            _zstd._finalize_dict(TRAINED_DICT.dict_content,
+            _zstd.finalize_dict(TRAINED_DICT.dict_content,
                                   concatenation, tuple(wrong_size_lst), 300*_1K, 5)
 
         # correct size list
-        _zstd._finalize_dict(TRAINED_DICT.dict_content,
+        _zstd.finalize_dict(TRAINED_DICT.dict_content,
                               concatenation, tuple(correct_size_lst), 300*_1K, 5)
 
     def test_as_prefix(self):

Modules/_zstd/_zstdmodule.c

@@ -69,8 +69,7 @@ typedef struct {
     char parameter_name[32];
 } ParameterInfo;
 
-static const ParameterInfo cp_list[] =
-{
+static const ParameterInfo cp_list[] = {
     {ZSTD_c_compressionLevel, "compression_level"},
     {ZSTD_c_windowLog,        "window_log"},
     {ZSTD_c_hashLog,          "hash_log"},
@@ -95,8 +94,7 @@ static const ParameterInfo cp_list[] =
     {ZSTD_c_overlapLog,       "overlap_log"}
 };
 
-static const ParameterInfo dp_list[] =
-{
+static const ParameterInfo dp_list[] = {
     {ZSTD_d_windowLogMax, "window_log_max"}
 };
 
@@ -173,7 +171,7 @@ get_zstd_state(PyObject *module)
 
 
 /*[clinic input]
-_zstd._train_dict
+_zstd.train_dict
 
     samples_bytes: PyBytesObject
         Concatenation of samples.
@@ -187,9 +185,9 @@ Internal function, train a zstd dictionary on sample data.
 [clinic start generated code]*/
 
 static PyObject *
-_zstd__train_dict_impl(PyObject *module, PyBytesObject *samples_bytes,
-                       PyObject *samples_sizes, Py_ssize_t dict_size)
-/*[clinic end generated code: output=b5b4f36347c0addd input=2dce5b57d63923e2]*/
+_zstd_train_dict_impl(PyObject *module, PyBytesObject *samples_bytes,
+                      PyObject *samples_sizes, Py_ssize_t dict_size)
+/*[clinic end generated code: output=8e87fe43935e8f77 input=70fcd8937f2528b6]*/
 {
     // TODO(emmatyping): The preamble and suffix to this function and _finalize_dict
     // are pretty similar. We should see if we can refactor them to share that code.
@@ -277,7 +275,7 @@ _zstd__train_dict_impl(PyObject *module, PyBytesObject *samples_bytes,
 }
 
 /*[clinic input]
-_zstd._finalize_dict
+_zstd.finalize_dict
 
     custom_dict_bytes: PyBytesObject
         Custom dictionary content.
@@ -295,11 +293,11 @@ Internal function, finalize a zstd dictionary.
 [clinic start generated code]*/
 
 static PyObject *
-_zstd__finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes,
-                          PyBytesObject *samples_bytes,
-                          PyObject *samples_sizes, Py_ssize_t dict_size,
-                          int compression_level)
-/*[clinic end generated code: output=5dc5b520fddba37f input=8afd42a249078460]*/
+_zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes,
+                         PyBytesObject *samples_bytes,
+                         PyObject *samples_sizes, Py_ssize_t dict_size,
+                         int compression_level)
+/*[clinic end generated code: output=f91821ba5ae85bda input=130d1508adb55ba1]*/
 {
     Py_ssize_t chunks_number;
     size_t *chunk_sizes = NULL;
@@ -396,7 +394,7 @@ _zstd__finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes,
 
 
 /*[clinic input]
-_zstd._get_param_bounds
+_zstd.get_param_bounds
 
     parameter: int
         The parameter to get bounds.
@@ -407,9 +405,8 @@ Internal function, get CompressionParameter/DecompressionParameter bounds.
 [clinic start generated code]*/
 
 static PyObject *
-_zstd__get_param_bounds_impl(PyObject *module, int parameter,
-                             int is_compress)
-/*[clinic end generated code: output=9892cd822f937e79 input=884cd1a01125267d]*/
+_zstd_get_param_bounds_impl(PyObject *module, int parameter, int is_compress)
+/*[clinic end generated code: output=4acf5a876f0620ca input=84e669591e487008]*/
 {
     ZSTD_bounds bound;
     if (is_compress) {
@@ -466,7 +463,7 @@ _zstd_get_frame_size_impl(PyObject *module, Py_buffer *frame_buffer)
 }
 
 /*[clinic input]
-_zstd._get_frame_info
+_zstd.get_frame_info
 
     frame_buffer: Py_buffer
         A bytes-like object, containing the header of a zstd frame.
@@ -475,8 +472,8 @@ Internal function, get zstd frame infomation from a frame header.
 [clinic start generated code]*/
 
 static PyObject *
-_zstd__get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer)
-/*[clinic end generated code: output=5462855464ecdf81 input=67f1f8e4b7b89c4d]*/
+_zstd_get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer)
+/*[clinic end generated code: output=56e033cf48001929 input=1816f14656b6aa22]*/
 {
     uint64_t decompressed_size;
     uint32_t dict_id;
@@ -508,7 +505,7 @@ _zstd__get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer)
 }
 
 /*[clinic input]
-_zstd._set_parameter_types
+_zstd.set_parameter_types
 
     c_parameter_type: object(subclass_of='&PyType_Type')
         CompressionParameter IntEnum type object
@@ -519,9 +516,9 @@ Internal function, set CompressionParameter/DecompressionParameter types for val
 [clinic start generated code]*/
 
 static PyObject *
-_zstd__set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type,
-                                PyObject *d_parameter_type)
-/*[clinic end generated code: output=a13d4890ccbd2873 input=4535545d903853d3]*/
+_zstd_set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type,
+                               PyObject *d_parameter_type)
+/*[clinic end generated code: output=f3313b1294f19502 input=30402523871b8280]*/
 {
     _zstd_state* const mod_state = get_zstd_state(module);
 
@@ -544,14 +541,13 @@ _zstd__set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type,
 }
 
 static PyMethodDef _zstd_methods[] = {
-    _ZSTD__TRAIN_DICT_METHODDEF
-    _ZSTD__FINALIZE_DICT_METHODDEF
-    _ZSTD__GET_PARAM_BOUNDS_METHODDEF
+    _ZSTD_TRAIN_DICT_METHODDEF
+    _ZSTD_FINALIZE_DICT_METHODDEF
+    _ZSTD_GET_PARAM_BOUNDS_METHODDEF
     _ZSTD_GET_FRAME_SIZE_METHODDEF
-    _ZSTD__GET_FRAME_INFO_METHODDEF
-    _ZSTD__SET_PARAMETER_TYPES_METHODDEF
-
-    {0}
+    _ZSTD_GET_FRAME_INFO_METHODDEF
+    _ZSTD_SET_PARAMETER_TYPES_METHODDEF
+    {NULL, NULL}
 };
 
 static int
@@ -595,7 +591,7 @@ do {                                                                         \
     ADD_TYPE(mod_state->ZstdCompressor_type, zstd_compressor_type_spec);
     ADD_TYPE(mod_state->ZstdDecompressor_type, zstd_decompressor_type_spec);
     mod_state->ZstdError = PyErr_NewExceptionWithDoc(
-        "_zstd.ZstdError",
+        "compression.zstd.ZstdError",
         "An error occurred in the zstd library.",
         NULL, NULL);
     if (mod_state->ZstdError == NULL) {
@@ -732,14 +728,15 @@ static struct PyModuleDef_Slot _zstd_slots[] = {
 };
 
 struct PyModuleDef _zstdmodule = {
-    PyModuleDef_HEAD_INIT,
+    .m_base = PyModuleDef_HEAD_INIT,
     .m_name = "_zstd",
+    .m_doc = "Implementation module for Zstandard compression.",
     .m_size = sizeof(_zstd_state),
     .m_slots = _zstd_slots,
     .m_methods = _zstd_methods,
     .m_traverse = _zstd_traverse,
     .m_clear = _zstd_clear,
-    .m_free = _zstd_free
+    .m_free = _zstd_free,
 };
 
 PyMODINIT_FUNC

Modules/_zstd/_zstdmodule.h

@@ -20,7 +20,8 @@ extern PyModuleDef _zstdmodule;
 
 /* For clinic type calculations */
 static inline _zstd_state *
-get_zstd_state_from_type(PyTypeObject *type) {
+get_zstd_state_from_type(PyTypeObject *type)
+{
     PyObject *module = PyType_GetModuleByDef(type, &_zstdmodule);
     if (module == NULL) {
         return NULL;
@@ -149,7 +150,8 @@ typedef enum {
 } dictionary_type;
 
 static inline int
-mt_continue_should_break(ZSTD_inBuffer *in, ZSTD_outBuffer *out) {
+mt_continue_should_break(ZSTD_inBuffer *in, ZSTD_outBuffer *out)
+{
     return in->size == in->pos && out->size != out->pos;
 }
 
@@ -163,13 +165,3 @@ set_parameter_error(const _zstd_state* const state, int is_compress,
                     int key_v, int value_v);
 
 static const char init_twice_msg[] = "__init__ method is called twice.";
-
-extern PyObject *
-decompress_impl(ZstdDecompressor *self, ZSTD_inBuffer *in,
-                Py_ssize_t max_length,
-                Py_ssize_t initial_size,
-                decompress_type type);
-
-extern PyObject *
-compress_impl(ZstdCompressor *self, Py_buffer *data,
-              ZSTD_EndDirective end_directive);