diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md
index cd12e84f2d..0ae7724278 100644
--- a/RELEASE-NOTES.md
+++ b/RELEASE-NOTES.md
@@ -9,7 +9,8 @@
 + ...
 
 ### Maintenance
-- The `pymc3.memoize` module was removed and replaced with `cachetools`.  The `hashable` function and `WithMemoization` class were moved to `pymc3.util`.
+- The `pymc3.memoize` module was removed and replaced with `cachetools`.  The `hashable` function and `WithMemoization` class were moved to `pymc3.util` (see [#4509](https://github.com/pymc-devs/pymc3/pull/4509)).
+- Remove float128 dtype support (see [#4514](https://github.com/pymc-devs/pymc3/pull/4514)).
 
 ## PyMC3 3.11.1 (12 February 2021)
 
diff --git a/pymc3/distributions/dist_math.py b/pymc3/distributions/dist_math.py
index 7f5aeb40f5..90ff0c1908 100644
--- a/pymc3/distributions/dist_math.py
+++ b/pymc3/distributions/dist_math.py
@@ -1,4 +1,4 @@
-#   Copyright 2020 The PyMC Developers
+#   Copyright 2021 The PyMC Developers
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
@@ -17,8 +17,6 @@
 
 @author: johnsalvatier
 """
-import platform
-
 import aesara
 import aesara.tensor as aet
 import numpy as np
@@ -45,11 +43,6 @@
     dtype: (np.nextafter(0, 1, dtype=dtype), np.nextafter(1, 0, dtype=dtype))
     for dtype in ["float16", "float32", "float64"]
 }
-if platform.system() in ["Linux", "Darwin"]:
-    _beta_clip_values["float128"] = (
-        np.nextafter(0, 1, dtype="float128"),
-        np.nextafter(1, 0, dtype="float128"),
-    )
 
 
 def bound(logp, *conditions, **kwargs):
diff --git a/pymc3/tests/test_dist_math.py b/pymc3/tests/test_dist_math.py
index f3b193b8a1..df9c5ddb68 100644
--- a/pymc3/tests/test_dist_math.py
+++ b/pymc3/tests/test_dist_math.py
@@ -1,4 +1,4 @@
-#   Copyright 2020 The PyMC Developers
+#   Copyright 2021 The PyMC Developers
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
@@ -11,8 +11,6 @@
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
-import sys
-
 import aesara
 import aesara.tensor as aet
 import numpy as np
@@ -237,12 +235,7 @@ def test_grad(self):
         verify_grad(i0e, [[[0.5, -2.0]]])
 
 
-@pytest.mark.parametrize(
-    "dtype",
-    ["float16", "float32", "float64", "float128"]
-    if sys.platform != "win32"
-    else ["float16", "float32", "float64"],
-)
+@pytest.mark.parametrize("dtype", ["float16", "float32", "float64"])
 def test_clipped_beta_rvs(dtype):
     # Verify that the samples drawn from the beta distribution are never
     # equal to zero or one (issue #3898)