Fix RecyclerBytesStreamOutput allocating unlimited heap for some capacities (#90632) (#90636)

We'd allocate infinite memory here because we'd overflow to negative
capacities for newPosition close to `Integer.MAX_VALUE`.

Author: original-brownbear

docs/changelog/90632.yaml

@@ -0,0 +1,5 @@
+pr: 90632
+summary: Fix `RecyclerBytesStreamOutput` allocating unlimited heap for some capacities
+area: Network
+type: bug
+issues: []

server/src/main/java/org/elasticsearch/common/io/stream/RecyclerBytesStreamOutput.java

@@ -225,10 +225,12 @@ private void ensureCapacity(int bytesNeeded) {
     }
 
     private void ensureCapacityFromPosition(long newPosition) {
+        // Integer.MAX_VALUE is not a multiple of the page size so we can only allocate the largest multiple of the pagesize that is less
+        // than Integer.MAX_VALUE
+        if (newPosition > Integer.MAX_VALUE - (Integer.MAX_VALUE % pageSize)) {
+            throw new IllegalArgumentException(getClass().getSimpleName() + " cannot hold more than 2GB of data");
+        }
         while (newPosition > currentCapacity) {
-            if (newPosition > Integer.MAX_VALUE) {
-                throw new IllegalArgumentException(getClass().getSimpleName() + " cannot hold more than 2GB of data");
-            }
             Recycler.V<BytesRef> newPage = recycler.obtain();
             assert pageSize == newPage.v().length;
             pages.add(newPage);