BUG: cluster: cophenet intercept invalid linkage matrix count (scipy#22187)

* BUG: catch invalid linkage count

* Fixes scipygh-22183.

* The input data from scipygh-22183 causes an out
of bounds memory access in a 1-D `memoryview`
in the Cython `cophenetic_distances` function.
Prevent this by enforcing a check for an allowable
upper bound on the cluster membership (4th column)
of the linkage matrix `Z` received by `is_valid_linkage`.

Co-authored-by: Warren Weckesser <[email protected]>

* TST: PR 22187 revisions

* Adjust `test_gh_22183()` such that the linkage matrix is invalid
only because of an excessive observation count, rather than
also being invalid because the large observation count is
not integral.

* Adjust `test_gh_22183()` such that is properly flushes
through array API backends.

* DOC: backticks

---------

Co-authored-by: Warren Weckesser <[email protected]>
Co-authored-by: Jake Bowhay <[email protected]>

Author: 3 people

scipy/cluster/_hierarchy.pyx

@@ -365,6 +365,10 @@ def cophenetic_distances(const double[:, :] Z, double[:] d, int n):
         # back to the root of current subtree
         dist = Z[root, 2]
         right_start = left_start[k] + n_lc
+        # NOTE: an invalid linkage matrix (gh-22183)
+        # can cause an out of bounds memory access
+        # of `j` on `members` memoryview below, if not
+        # caught ahead of time by `is_valid_linkage`
         for i in range(left_start[k], right_start):
             for j in range(right_start, right_start + n_rc):
                 d[condensed_index(n, members[i], members[j])] = dist

scipy/cluster/hierarchy.py

@@ -2157,6 +2157,10 @@ def is_valid_linkage(Z, warning=False, throw=False, name=None):
     I.e., a cluster cannot join another cluster unless the cluster being joined
     has been generated.
 
+    The fourth column of `Z` represents the number of original observations
+    in a cluster, so a valid ``Z[i, 3]`` value may not exceed the number of
+    original observations.
+
     Parameters
     ----------
     Z : array_like
@@ -2244,6 +2248,9 @@ def is_valid_linkage(Z, warning=False, throw=False, name=None):
                 raise ValueError(f'Linkage {name_str}contains negative distances.')
             if xp.any(Z[:, 3] < 0):
                 raise ValueError(f'Linkage {name_str}contains negative counts.')
+            if xp.any(Z[:, 3] > (Z.shape[0] + 1)):
+                raise ValueError('Linkage matrix contains excessive observations'
+                                 'in a cluster')
         if _check_hierarchy_uses_cluster_before_formed(Z):
             raise ValueError(f'Linkage {name_str}uses non-singleton cluster before'
                              ' it is formed.')

scipy/cluster/tests/test_hierarchy.py

@@ -200,6 +200,25 @@ def test_linkage_cophenet_tdist_Z_Y(self, xp):
         xp_assert_close(c, expectedc, atol=1e-10)
         xp_assert_close(M, expectedM, atol=1e-10)
 
+    def test_gh_22183(self, xp):
+        # check for lack of segfault
+        # (out of bounds memory access)
+        # and correct interception of
+        # invalid linkage matrix
+        arr=[[0.0, 1.0, 1.0, 2.0],
+             [2.0, 12.0, 1.0, 3.0],
+             [3.0, 4.0, 1.0, 2.0],
+             [5.0, 14.0, 1.0, 3.0],
+             [6.0, 7.0, 1.0, 2.0],
+             [8.0, 16.0, 1.0, 3.0],
+             [9.0, 10.0, 1.0, 2.0],
+             [11.0, 18.0, 1.0, 3.0],
+             [13.0, 15.0, 2.0, 6.0],
+             [17.0, 20.0, 2.0, 32.0],
+             [19.0, 21.0, 2.0, 12.0]]
+        with pytest.raises(ValueError, match="excessive observations"):
+            cophenet(xp.asarray(arr))
+
 
 class TestMLabLinkageConversion: