Fix, calloc now fails on extra-large request. (#8482)

Added code to handle multiply overflow in calloc.
Added code to handle add overflow in umm_poison_*

Author: mhightower83

Diff for: cores/esp8266/heap.cpp

@@ -5,6 +5,7 @@
 
 #include <stdlib.h>
 #include "umm_malloc/umm_malloc.h"
+extern "C" size_t umm_umul_sat(const size_t a, const size_t b);;
 
 // Need FORCE_ALWAYS_INLINE to put HeapSelect class constructor/deconstructor in IRAM
 #define FORCE_ALWAYS_INLINE_HEAP_SELECT
@@ -153,7 +154,7 @@ void* _calloc_r(struct _reent* unused, size_t count, size_t size)
 {
     (void) unused;
     void *ret = calloc(count, size);
-    PTR_CHECK__LOG_LAST_FAIL(ret, count * size);
+    PTR_CHECK__LOG_LAST_FAIL(ret, umm_umul_sat(count, size));
     return ret;
 }
 
@@ -247,8 +248,11 @@ void* IRAM_ATTR calloc(size_t count, size_t size)
     INTEGRITY_CHECK__ABORT();
     POISON_CHECK__ABORT();
     void* ret = UMM_CALLOC(count, size);
-    PTR_CHECK__LOG_LAST_FAIL(ret, count * size);
-    OOM_CHECK__PRINT_OOM(ret, size);
+    #if defined(DEBUG_ESP_OOM)
+    size_t total_size = umm_umul_sat(count, size);// For logging purposes
+    #endif
+    PTR_CHECK__LOG_LAST_FAIL(ret, total_size);
+    OOM_CHECK__PRINT_OOM(ret, total_size);
     return ret;
 }
 
@@ -287,8 +291,11 @@ void* IRAM_ATTR heap_pvPortCalloc(size_t count, size_t size, const char* file, i
     INTEGRITY_CHECK__PANIC_FL(file, line);
     POISON_CHECK__PANIC_FL(file, line);
     void* ret = UMM_CALLOC(count, size);
-    PTR_CHECK__LOG_LAST_FAIL_FL(ret, count * size, file, line);
-    OOM_CHECK__PRINT_LOC(ret, size, file, line);
+    #if defined(DEBUG_ESP_OOM)
+    size_t total_size = umm_umul_sat(count, size);
+    #endif
+    PTR_CHECK__LOG_LAST_FAIL_FL(ret, total_size, file, line);
+    OOM_CHECK__PRINT_LOC(ret, total_size, file, line);
     return ret;
 }
 

Diff for: cores/esp8266/umm_malloc/umm_local.c

@@ -137,7 +137,7 @@ void *umm_poison_realloc_fl(void *ptr, size_t size, const char *file, int line)
 
     ptr = get_unpoisoned_check_neighbors(ptr, file, line);
 
-    size += poison_size(size);
+    add_poison_size(&size);
     ret = umm_realloc(ptr, size);
 
     ret = get_poisoned(ret, size);
@@ -296,4 +296,32 @@ size_t ICACHE_FLASH_ATTR umm_get_free_null_count(void) {
 }
 #endif // UMM_STATS_FULL
 
+#if defined(UMM_POISON_CHECK) || defined(UMM_POISON_CHECK_LITE)
+/*
+ * Saturated unsigned add
+ * Poison added to allocation size requires overflow protection.
+ */
+static size_t umm_uadd_sat(const size_t a, const size_t b) {
+    size_t r = a + b;
+    if (r < a) {
+        return SIZE_MAX;
+    }
+    return r;
+}
+#endif
+
+/*
+ * Use platform-specific functions to protect against unsigned overflow/wrap by
+ * implementing saturated unsigned multiply.
+ * The function umm_calloc requires a saturated multiply function.
+ */
+size_t umm_umul_sat(const size_t a, const size_t b) {
+    size_t r;
+    if (__builtin_mul_overflow(a, b, &r)) {
+        return SIZE_MAX;
+    }
+    return r;
+}
+
+
 #endif // BUILD_UMM_MALLOC_C

Diff for: cores/esp8266/umm_malloc/umm_local.h

@@ -15,6 +15,14 @@
 #define memset ets_memset
 
 
+/*
+ * Saturated unsigned add and unsigned multiply
+ */
+size_t umm_umul_sat(const size_t a, const size_t b);  // share with heap.cpp
+#if defined(UMM_POISON_CHECK) || defined(UMM_POISON_CHECK_LITE)
+static size_t umm_uadd_sat(const size_t a, const size_t b);
+#endif
+
 /*
  * This redefines DBGLOG_FORCE defined in dbglog/dbglog.h
  * Just for printing from umm_info() which is assumed to always be called from

Diff for: cores/esp8266/umm_malloc/umm_malloc.cpp

@@ -1214,10 +1214,14 @@ void *umm_realloc(void *ptr, size_t size) {
 void *umm_calloc(size_t num, size_t item_size) {
     void *ret;
 
-    ret = umm_malloc((size_t)(item_size * num));
+    // Use saturated multiply.
+    // Rely on umm_malloc to supply the fail response as needed.
+    size_t size = umm_umul_sat(num, item_size);
+
+    ret = umm_malloc(size);
 
     if (ret) {
-        memset(ret, 0x00, (size_t)(item_size * num));
+        memset(ret, 0x00, size);
     }
 
     return ret;

Diff for: cores/esp8266/umm_malloc/umm_poison.c

@@ -8,15 +8,19 @@
 #include <stddef.h>
 #include <stdbool.h>
 
+#define UMM_POISON_BLOCK_SIZE (UMM_POISON_SIZE_BEFORE + sizeof(UMM_POISONED_BLOCK_LEN_TYPE) + UMM_POISON_SIZE_AFTER)
+
 /*
- * Yields a size of the poison for the block of size `s`.
+ * Yields the total size of a poison block of size `s`.
  * If `s` is 0, returns 0.
+ * If result overflows/wraps, return saturation value.
  */
-static size_t poison_size(size_t s) {
-    return s ? (UMM_POISON_SIZE_BEFORE +
-        sizeof(UMM_POISONED_BLOCK_LEN_TYPE) +
-        UMM_POISON_SIZE_AFTER)
-             : 0;
+static void add_poison_size(size_t* s) {
+    if (*s == 0) {
+        return;
+    }
+
+    *s = umm_uadd_sat(*s, UMM_POISON_BLOCK_SIZE);
 }
 
 /*
@@ -158,7 +162,7 @@ static void *get_unpoisoned(void *vptr) {
 void *umm_poison_malloc(size_t size) {
     void *ret;
 
-    size += poison_size(size);
+    add_poison_size(&size);
 
     ret = umm_malloc(size);
 
@@ -171,9 +175,12 @@ void *umm_poison_malloc(size_t size) {
 
 void *umm_poison_calloc(size_t num, size_t item_size) {
     void *ret;
-    size_t size = item_size * num;
 
-    size += poison_size(size);
+    // Use saturated multiply.
+    // Rely on umm_malloc to supply the fail response as needed.
+    size_t size = umm_umul_sat(num, item_size);
+
+    add_poison_size(&size);
 
     ret = umm_malloc(size);
 
@@ -193,7 +200,7 @@ void *umm_poison_realloc(void *ptr, size_t size) {
 
     ptr = get_unpoisoned(ptr);
 
-    size += poison_size(size);
+    add_poison_size(&size);
     ret = umm_realloc(ptr, size);
 
     ret = get_poisoned(ret, size);